From f81cc4730624b1f4cee6c79f51f62c9f59c70662 Mon Sep 17 00:00:00 2001 From: Robert DeLuca Date: Thu, 21 May 2026 11:49:37 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Harden=20CLI=20release=20dogfood=20?= =?UTF-8?q?flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Proves the packaged CLI can link, run against prod, capture a screenshot, and fetch compact current-build context with isolated config. --- package.json | 1 + scripts/smoke-prod-pack.js | 97 ++++++++++++++++++++++++++---- src/commands/context.js | 2 +- tests/commands/context-cli.test.js | 86 ++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index b5bb794..b768e31 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "files": [ "bin", "dist", + "scripts", "README.md", "LICENSE" ], diff --git a/scripts/smoke-prod-pack.js b/scripts/smoke-prod-pack.js index a4733cd..94b9fe1 100644 --- a/scripts/smoke-prod-pack.js +++ b/scripts/smoke-prod-pack.js @@ -20,6 +20,21 @@ function readJson(text, label) { } } +function readCliData(text, label) { + let messages = text + .split('\n') + .map(line => line.trim()) + .filter(Boolean) + .map(line => readJson(line, label)); + let dataMessage = messages.find(message => message.status === 'data'); + + if (!dataMessage) { + throw new Error(`No data message found for ${label}:\n${text}`); + } + + return dataMessage.data; +} + async function run(command, args, options = {}) { let { stdout, stderr } = await execFileAsync(command, args, { maxBuffer: 1024 * 1024 * 20, @@ -33,17 +48,31 @@ async function run(command, args, options = {}) { return stdout; } -async function runPackaged(binPath, args, env) { - let stdout = await run(process.execPath, [binPath, ...args], { env }); - let parsed = readJson(stdout, `vizzly ${args.join(' ')}`); +async function runPackaged(binPath, args, env, options = {}) { + let stdout = await run(process.execPath, [binPath, ...args], { + env, + cwd: options.cwd, + }); + return readCliData(stdout, `vizzly ${args.join(' ')}`); +} + +async function writeSmokeCaptureScript(projectDir) { + let pngBase64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR42mO8fPr0fwYGBgYGJgYAAH4RBAvbXq5rAAAAAElFTkSuQmCC'; + let script = ` +import { vizzlyFlush, vizzlyScreenshot } from '@vizzly-testing/cli/client'; - if (parsed.status !== 'data') { - throw new Error( - `Unexpected CLI status for ${args.join(' ')}: ${parsed.status}` - ); - } +let image = Buffer.from('${pngBase64}', 'base64'); - return parsed.data; +await vizzlyScreenshot('release-readiness-smoke', image, { + browser: 'node', + viewport: { width: 2, height: 2 } +}); + +await vizzlyFlush(); +`; + + await writeFile(join(projectDir, 'smoke-capture.js'), script.trimStart()); } async function loadAuth(sourceHome) { @@ -152,6 +181,8 @@ async function main() { 'bin', 'vizzly.js' ); + await writeSmokeCaptureScript(installDir); + let auth = await loadAuth(sourceHome); await mkdir(smokeHome, { recursive: true }); await writeFile( @@ -213,7 +244,8 @@ async function main() { let builds = await runPackaged( binPath, ['builds', '--project', projectSlug, '--limit', '1', '--json'], - env + env, + { cwd: installDir } ); let build = builds.builds?.[0]; @@ -227,7 +259,8 @@ async function main() { let context = await runPackaged( binPath, ['context', 'build', build.id, '--source', 'cloud', '--agent', '--json'], - env + env, + { cwd: installDir } ); if (context.resource !== 'build_agent_context') { @@ -247,6 +280,48 @@ async function main() { throw new Error('Compact agent context did not include next actions.'); } + log('creating a cloud build through packaged vizzly run'); + let runResult = await runPackaged( + binPath, + [ + '--json', + 'run', + 'node smoke-capture.js', + '--build-name', + `prod-smoke-run-${Date.now()}`, + ], + env, + { cwd: installDir } + ); + + if (!runResult.buildId) { + throw new Error('Packaged vizzly run did not create a build.'); + } + + if ((runResult.screenshotsCaptured || 0) < 1) { + throw new Error('Packaged vizzly run did not capture a screenshot.'); + } + + log(`fetching current compact agent context for ${runResult.buildId}`); + let currentContext = await runPackaged( + binPath, + ['context', 'build', 'current', '--source', 'cloud', '--agent', '--json'], + env, + { cwd: installDir } + ); + + if (currentContext.resource !== 'build_agent_context') { + throw new Error( + `Expected current build_agent_context, got ${currentContext.resource}` + ); + } + + if (currentContext.build?.id !== runResult.buildId) { + throw new Error( + `Current context resolved ${currentContext.build?.id}, expected ${runResult.buildId}` + ); + } + log('revoking smoke token'); await revokeProjectToken({ apiUrl, diff --git a/src/commands/context.js b/src/commands/context.js index e329fa0..c20d8af 100644 --- a/src/commands/context.js +++ b/src/commands/context.js @@ -730,7 +730,7 @@ function formatAgentBuildContext(context) { lines.push(''); lines.push( - 'Use this as reviewed UI context. Treat approved baselines as visual truth, inspect meaningful diffs, and leave approval decisions to humans.' + 'Use this as reviewed visual evidence. Treat approved baselines as visual truth, inspect meaningful diffs, and leave approval decisions to humans.' ); return lines.join('\n'); diff --git a/tests/commands/context-cli.test.js b/tests/commands/context-cli.test.js index f92bfda..26df5e4 100644 --- a/tests/commands/context-cli.test.js +++ b/tests/commands/context-cli.test.js @@ -1,5 +1,6 @@ import assert from 'node:assert'; import { mkdirSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; @@ -140,6 +141,25 @@ function createWorkspaceFixture() { return cwd; } +function listen(server) { + return new Promise(resolve => { + server.listen(0, '127.0.0.1', () => resolve(server.address().port)); + }); +} + +function closeServer(server) { + return new Promise((resolve, reject) => { + server.close(error => { + if (error) { + reject(error); + return; + } + + resolve(); + }); + }); +} + describe('context CLI integration', () => { it('treats root-level --json as a flag before the context command', async () => { let result = await runCLI(['--json', 'context', 'build', 'build-123']); @@ -248,6 +268,72 @@ describe('context CLI integration', () => { assert.strictEqual(parsed.data.summary.new, 1); }); + it('uses cloud review queue when explicit project scope is present even with local data', async () => { + let cwd = createWorkspaceFixture(); + let vizzlyHome = join(cwd, '.vizzly-home'); + let requests = []; + mkdirSync(vizzlyHome, { recursive: true }); + + let server = createServer((req, res) => { + requests.push(req.url); + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + resource: 'review_queue_context', + source: 'cloud', + scope: { + organization: { slug: 'acme' }, + project: { slug: 'storybook' }, + }, + summary: { total: 0, changed: 0, new: 0, builds: 0 }, + comparisons: [], + }) + ); + }); + let port = await listen(server); + + try { + let result = await runCLI( + [ + '--json', + 'context', + 'review-queue', + '--org', + 'acme', + '--project', + 'storybook', + ], + { + cwd, + env: { + VIZZLY_HOME: vizzlyHome, + VIZZLY_API_URL: `http://127.0.0.1:${port}`, + VIZZLY_TOKEN: 'vzt_test_token', + }, + } + ); + + assert.strictEqual(result.code, 0); + let parsed = JSON.parse(result.stdout); + assert.strictEqual(parsed.status, 'data'); + assert.strictEqual(parsed.data.source, 'cloud'); + assert.strictEqual(parsed.data.summary.total, 0); + assert.ok( + requests.some(request => + request.startsWith('/api/sdk/context/review-queue?') + ) + ); + assert.ok( + requests.some(request => request.includes('organization=acme')) + ); + assert.ok( + requests.some(request => request.includes('project=storybook')) + ); + } finally { + await closeServer(server); + } + }); + it('fails clearly when local fingerprint similarity is requested', async () => { let cwd = createWorkspaceFixture(); let vizzlyHome = join(cwd, '.vizzly-home');