test: add live web platform smoke

Author: thymikee

.github/workflows/ci.yml

@@ -177,3 +177,44 @@ jobs:
 
       - name: Check Provider-backed integration architecture progress
         run: pnpm test:integration:progress:check
+
+  web-smoke:
+    name: Web Platform Smoke
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      AGENT_DEVICE_WEB_E2E: "1"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+        with:
+          node-version: "24"
+
+      - name: Install agent-browser runtime
+        run: |
+          export PNPM_HOME="$RUNNER_TEMP/pnpm-global"
+          mkdir -p "$PNPM_HOME"
+          export PATH="$PNPM_HOME:$PATH"
+          echo "PNPM_HOME=$PNPM_HOME" >> "$GITHUB_ENV"
+          echo "$PNPM_HOME" >> "$GITHUB_PATH"
+          pnpm config set global-bin-dir "$PNPM_HOME"
+          pnpm add --global agent-browser@0.27.1
+          agent-browser install --with-deps
+          agent-browser doctor --offline --quick
+
+      - name: Run live web smoke
+        run: |
+          pnpm clean:daemon
+          pnpm test:smoke:web
+
+      - name: Upload web smoke artifacts
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: web-smoke-artifacts
+          if-no-files-found: ignore
+          path: |
+            test/artifacts/web/**

docs/agents/domain.md

@@ -7,5 +7,6 @@ Before architecture, diagnosis, TDD, triage, PRD, or roadmap work, read:
 - `CONTEXT.md` for domain vocabulary, test strategy terms, and architecture language.
 - Relevant ADRs in `docs/adr/` for accepted architecture decisions.
 - This `docs/agents/` directory for issue-tracker and triage-label conventions.
+- `docs/agents/testing.md` for maintainer-only test lane notes such as the live web smoke.
 
 Use the vocabulary from `CONTEXT.md` in issue titles, refactor proposals, test names, and architecture notes. If a proposed change contradicts an ADR, call that out explicitly and explain why the decision should be reopened.

docs/agents/testing.md

@@ -0,0 +1,13 @@
+# Testing Notes
+
+## Live web smoke
+
+The live web platform smoke runs the public built CLI against a local fixture page through the `agent-browser` adapter:
+
+```bash
+agent-browser install --with-deps
+agent-browser doctor --offline --quick
+AGENT_DEVICE_WEB_E2E=1 pnpm test:smoke:web
+```
+
+The test is skipped unless `AGENT_DEVICE_WEB_E2E=1` is set. CI installs `agent-browser@0.27.1` in the dedicated web smoke job and runs the lane on Node 24 because that adapter version requires Node >= 24. Failure artifacts, daemon state, and browser config are written under `test/artifacts/web/`.

package.json

@@ -134,6 +134,7 @@
     "test:integration:provider": "vitest run --project provider-integration",
     "test:integration:progress": "node --experimental-strip-types scripts/integration-progress.ts",
     "test:integration:progress:check": "node --experimental-strip-types scripts/integration-progress.ts --check",
+    "test:smoke:web": "pnpm build && node --test test/integration/smoke-web-platform.test.ts",
     "test:skillgym": "node test/skillgym/runner-environment.ts && pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts",
     "test:skillgym:case": "node test/skillgym/runner-environment.ts && pnpm build && skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts --case",
     "test:smoke": "node --test test/integration/smoke-*.test.ts",

test/integration/cli-json.ts

@@ -0,0 +1,69 @@
+import { runCmd, runCmdSync, type ExecResult } from '../../src/utils/exec.ts';
+
+const CLI_TIMEOUT_MS = 120_000;
+
+export type CliJsonResult = {
+  status: number;
+  json?: any;
+  stdout: string;
+  stderr: string;
+};
+
+export function runSourceCliJsonSync(
+  args: string[],
+  options?: { env?: NodeJS.ProcessEnv },
+): CliJsonResult {
+  const result = runCmdSync(
+    process.execPath,
+    ['--experimental-strip-types', 'src/bin.ts', ...args],
+    {
+      allowFailure: true,
+      env: options?.env,
+      timeoutMs: CLI_TIMEOUT_MS,
+    },
+  );
+  return cliJsonResult(result);
+}
+
+export async function runBuiltCliJson(
+  args: string[],
+  env: NodeJS.ProcessEnv,
+): Promise<CliJsonResult> {
+  const result = await runCmd(process.execPath, ['bin/agent-device.mjs', ...args], {
+    allowFailure: true,
+    env,
+    timeoutMs: CLI_TIMEOUT_MS,
+  });
+  return cliJsonResult(result);
+}
+
+export function formatResultDebug(step: string, args: string[], result: CliJsonResult): string {
+  const jsonText =
+    result.json === undefined ? '(unparseable)' : JSON.stringify(result.json, null, 2);
+  return [
+    `step: ${step}`,
+    `command: agent-device ${args.join(' ')}`,
+    `status: ${result.status}`,
+    `stderr:`,
+    result.stderr || '(empty)',
+    `stdout:`,
+    result.stdout || '(empty)',
+    `json:`,
+    jsonText,
+  ].join('\n');
+}
+
+function cliJsonResult(result: ExecResult): CliJsonResult {
+  let json: any;
+  try {
+    json = JSON.parse(result.stdout ?? '');
+  } catch {
+    json = undefined;
+  }
+  return {
+    status: result.exitCode,
+    json,
+    stdout: json ? '<JSON output>' : (result.stdout ?? ''),
+    stderr: result.stderr ?? '',
+  };
+}