diff --git a/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts b/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts new file mode 100644 index 0000000000..a18108220d --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-fixture-context.test.ts @@ -0,0 +1,397 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { ArtifactSink } from "../framework/artifacts.ts"; +import { assertCleanupPassed, CleanupRegistry } from "../framework/cleanup.ts"; +import { test as e2eTest } from "../framework/e2e-test.ts"; +import { SecretStore } from "../framework/secrets.ts"; +import { ShellProbe, trustedShellCommand, type TrustedShellCommand } from "../framework/shell-probe.ts"; + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ESRCH"; + } +} + +async function expectProcessToExit(pid: number, timeoutMs = 2_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isProcessAlive(pid)) return; + await delay(25); + } + throw new Error(`process ${pid} was still alive after ${timeoutMs}ms`); +} + +describe("E2E fixture primitives", () => { + it("artifact sink writes under its root and rejects traversal", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-artifacts-")); + try { + const artifacts = new ArtifactSink(tmp); + await artifacts.ensureRoot(); + const written = await artifacts.writeText("nested/output.txt", "ok"); + expect(fs.readFileSync(written, "utf8")).toBe("ok"); + expect(() => artifacts.pathFor("../escape.txt")).toThrow(/escapes root/); + expect(() => artifacts.pathFor(path.join(tmp, "absolute.txt"))).toThrow(/must be relative/); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("cleanup registry runs callbacks in reverse order", async () => { + const cleanup = new CleanupRegistry(); + const order: string[] = []; + cleanup.add("first", () => { + order.push("first"); + }); + cleanup.add("second", () => { + order.push("second"); + }); + + const result = await cleanup.runAll(); + expect(order).toEqual(["second", "first"]); + expect(result).toEqual({ passed: ["second", "first"], failures: [] }); + }); + + it("cleanup registry redacts failures, continues, and clears callbacks", async () => { + const secret = "cleanup-secret-value"; + const cleanup = new CleanupRegistry((text) => text.split(secret).join("[REDACTED]")); + const order: string[] = []; + cleanup.add("first", () => { + order.push("first"); + }); + cleanup.add("second", () => { + order.push("second"); + throw new Error(`failed with ${secret}`); + }); + cleanup.add(`third-${secret}`, () => { + order.push("third"); + }); + + const result = await cleanup.runAll(); + expect(order).toEqual(["third", "second", "first"]); + expect(result).toEqual({ + passed: ["third-[REDACTED]", "first"], + failures: [{ name: "second", message: "failed with [REDACTED]" }], + }); + expect(() => assertCleanupPassed(result)).toThrow("failed with [REDACTED]"); + expect(() => assertCleanupPassed(result)).not.toThrow(secret); + expect(await cleanup.runAll()).toEqual({ passed: [], failures: [] }); + }); + + it("secret store redacts sensitive env values and skips missing required secrets", () => { + const canonicalToken = `${"nv"}${"api"}-${"a".repeat(24)}`; + const store = new SecretStore( + { NVIDIA_API_KEY: "nv-secret", PLAIN_VALUE: "visible" }, + (note?: string): never => { + throw new Error(note ?? "skipped"); + }, + ); + + expect(store.optional("PLAIN_VALUE")).toBe("visible"); + expect(store.redact("token=nv-secret plain=visible")).toBe("token=[REDACTED] plain=visible"); + expect(store.redact(`printed ${canonicalToken}`)).toContain(""); + expect(store.redact(`printed ${canonicalToken}`)).not.toContain(canonicalToken); + expect(() => store.required("MISSING_SECRET")).toThrow(/missing required E2E secret/); + }); + + it("shell probe requires trusted command descriptors", () => { + expectTypeOf[0]>().toEqualTypeOf(); + expect(() => + trustedShellCommand({ + command: "node", + reason: "", + }), + ).toThrow(/trusted command reason is required/); + expect(() => + trustedShellCommand({ + command: "node", + args: ["bad\0arg"], + reason: "validate arguments", + }), + ).toThrow(/argument cannot contain NUL bytes/); + }); + + it("shell probe cleans up and redacts missing command failures", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-shell-probe-")); + try { + const artifacts = new ArtifactSink(tmp); + await artifacts.ensureRoot(); + const secret = "spawn-secret-value"; + const controller = new AbortController(); + let abortAdds = 0; + let abortRemoves = 0; + const addEventListener = controller.signal.addEventListener.bind(controller.signal); + const removeEventListener = controller.signal.removeEventListener.bind(controller.signal); + const instrumentedAddEventListener = ( + type: string, + listener: EventListener | EventListenerObject, + options?: AddEventListenerOptions | boolean, + ) => { + if (type === "abort") abortAdds += 1; + return addEventListener(type, listener, options); + }; + const instrumentedRemoveEventListener = ( + type: string, + listener: EventListener | EventListenerObject, + options?: EventListenerOptions | boolean, + ) => { + if (type === "abort") abortRemoves += 1; + return removeEventListener(type, listener, options); + }; + controller.signal.addEventListener = instrumentedAddEventListener as typeof controller.signal.addEventListener; + controller.signal.removeEventListener = instrumentedRemoveEventListener as typeof controller.signal.removeEventListener; + const probe = new ShellProbe({ + artifacts, + redact: (text, extraValues = []) => + [secret, ...extraValues].reduce((redacted, value) => redacted.split(value).join("[REDACTED]"), text), + signal: controller.signal, + }); + + let thrown: unknown; + try { + await probe.run( + trustedShellCommand({ + command: `missing-command-${secret}`, + args: [secret], + reason: "exercise redacted spawn failure handling", + }), + { + artifactName: "spawn-error", + redactionValues: [secret], + timeoutMs: 10_000, + }, + ); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(Error); + const message = thrown instanceof Error ? thrown.message : String(thrown); + expect(message).toContain("[REDACTED]"); + expect(message).not.toContain(secret); + expect(abortAdds).toBe(1); + expect(abortRemoves).toBe(1); + expect(fs.readFileSync(artifacts.pathFor("shell/spawn-error.result.json"), "utf8")).not.toContain(secret); + expect(fs.readFileSync(artifacts.pathFor("shell/spawn-error.stderr.txt"), "utf8")).toContain("[REDACTED]"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("shell probe escalates abort-triggered termination", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-shell-probe-abort-")); + try { + const artifacts = new ArtifactSink(tmp); + await artifacts.ensureRoot(); + const controller = new AbortController(); + const probe = new ShellProbe({ + artifacts, + redact: (text) => text, + signal: controller.signal, + }); + + const started = Date.now(); + const run = probe.run( + trustedShellCommand({ + command: process.execPath, + args: ["-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"], + reason: "exercise abort escalation", + }), + { + artifactName: "abort-escalation", + timeoutMs: 10_000, + killGraceMs: 50, + }, + ); + setTimeout(() => controller.abort(), 50); + const result = await run; + + expect(Date.now() - started).toBeLessThan(2_000); + expect(result.timedOut).toBe(false); + expect(result.signal).toBe("SIGKILL"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("shell probe terminates pre-aborted signals immediately", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-shell-probe-pre-abort-")); + try { + const artifacts = new ArtifactSink(tmp); + await artifacts.ensureRoot(); + const controller = new AbortController(); + controller.abort(); + const probe = new ShellProbe({ + artifacts, + redact: (text) => text, + signal: controller.signal, + }); + + const started = Date.now(); + const result = await probe.run( + trustedShellCommand({ + command: process.execPath, + args: ["-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"], + reason: "exercise pre-aborted signal termination", + }), + { + artifactName: "pre-abort-escalation", + timeoutMs: 10_000, + killGraceMs: 50, + }, + ); + + expect(Date.now() - started).toBeLessThan(2_000); + expect(result.timedOut).toBe(false); + expect(result.signal).toBeTruthy(); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("shell probe reaps timed-out command process groups", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-shell-probe-pgid-")); + let grandchildPid: number | undefined; + try { + const artifacts = new ArtifactSink(tmp); + await artifacts.ensureRoot(); + const controller = new AbortController(); + const probe = new ShellProbe({ + artifacts, + redact: (text) => text, + signal: controller.signal, + }); + const pidFile = path.join(tmp, "sleep.pid"); + + const result = await probe.run( + trustedShellCommand({ + command: "bash", + args: ["-c", 'sleep 30 & echo "$!" > "$1"; wait', "e2e-shell-probe", pidFile], + reason: "exercise process-group timeout cleanup", + }), + { + artifactName: "process-group-timeout", + timeoutMs: 200, + killGraceMs: 50, + }, + ); + + grandchildPid = Number(fs.readFileSync(pidFile, "utf8").trim()); + expect(Number.isInteger(grandchildPid)).toBe(true); + expect(result.timedOut).toBe(true); + expect(result.signal).toBeTruthy(); + await expectProcessToExit(grandchildPid); + } finally { + if (grandchildPid && isProcessAlive(grandchildPid)) { + process.kill(grandchildPid, "SIGKILL"); + } + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); + +e2eTest("fixture context captures redacted shell artifacts", async ({ + artifacts, + cleanup, + shellProbe, +}) => { + const marker = await artifacts.writeText("context.txt", "fixture-ready"); + cleanup.add("write cleanup marker", async () => { + await artifacts.writeText("cleanup-marker.txt", "done"); + }); + + const secret = "shell-probe-secret-value"; + const result = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [ + "-e", + "console.log(process.env.NEMOCLAW_TEST_TOKEN); console.error(process.argv[1]);", + secret, + ], + reason: "exercise fixture shell artifact redaction", + }), + { + artifactName: "redaction-proof", + env: { NEMOCLAW_TEST_TOKEN: secret }, + redactionValues: [secret], + timeoutMs: 5_000, + }, + ); + + expect(fs.readFileSync(marker, "utf8")).toBe("fixture-ready"); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("[REDACTED]"); + expect(result.stderr).toContain("[REDACTED]"); + expect(result.stdout).not.toContain(secret); + expect(result.stderr).not.toContain(secret); + expect(fs.readFileSync(result.artifacts.result, "utf8")).not.toContain(secret); +}); + +e2eTest("shell probe uses explicit env and escalates ignored timeouts", async ({ shellProbe }) => { + const parentSecretName = "NEMOCLAW_PARENT_SECRET_FOR_PROBE_TEST"; + const parentSecret = "parent-secret-value"; + const explicitSecret = "explicit-secret-value"; + const oldParentSecret = process.env[parentSecretName]; + process.env[parentSecretName] = parentSecret; + try { + const envResult = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [ + "-e", + `console.log(process.env.${parentSecretName} ?? "missing"); console.log(process.env.NEMOCLAW_TEST_TOKEN);`, + ], + reason: "exercise explicit shell probe environment", + }), + { + artifactName: "minimal-env", + env: { NEMOCLAW_TEST_TOKEN: explicitSecret }, + redactionValues: [explicitSecret, parentSecret], + timeoutMs: 5_000, + }, + ); + + expect(envResult.exitCode).toBe(0); + expect(envResult.stdout).toContain("missing"); + expect(envResult.stdout).toContain("[REDACTED]"); + expect(envResult.stdout).not.toContain(parentSecret); + expect(envResult.stdout).not.toContain(explicitSecret); + expect(fs.readFileSync(envResult.artifacts.result, "utf8")).not.toContain(explicitSecret); + } finally { + if (oldParentSecret === undefined) { + delete process.env[parentSecretName]; + } else { + process.env[parentSecretName] = oldParentSecret; + } + } + + const started = Date.now(); + const timeoutResult = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: ["-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"], + reason: "exercise timeout escalation", + }), + { + artifactName: "timeout-escalation", + timeoutMs: 50, + killGraceMs: 50, + }, + ); + + expect(Date.now() - started).toBeLessThan(2_000); + expect(timeoutResult.timedOut).toBe(true); + expect(timeoutResult.signal).toBe("SIGKILL"); +}); diff --git a/test/e2e-scenario/framework-tests/e2e-probes.test.ts b/test/e2e-scenario/framework-tests/e2e-probes.test.ts index db90b47798..8b96ec4b2c 100644 --- a/test/e2e-scenario/framework-tests/e2e-probes.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-probes.test.ts @@ -14,6 +14,7 @@ import { } from "../scenarios/probes/registry.ts"; import type { ProbeContext, ProbeOutcome } from "../scenarios/probes/types.ts"; import { registerBuiltinProbes } from "../scenarios/probes/builtin.ts"; +import { writeProbeEvidence } from "../scenarios/probes/util.ts"; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); @@ -85,6 +86,37 @@ describe("probe registry", () => { }); }); +describe("probe evidence writer", () => { + it("writes evidence under the context dir and ignores escape paths", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "probe-evidence-root-")); + const contextDir = path.join(tmp, "ctx"); + fs.mkdirSync(contextDir, { recursive: true }); + try { + const insidePath = path.join(contextDir, "nested", "evidence.json"); + const insideCtx: ProbeContext = { + contextDir, + evidencePath: insidePath, + contextEnv: {}, + sandboxName: null, + gatewayUrl: null, + repoRoot: REPO_ROOT, + }; + writeProbeEvidence(insideCtx, { ok: true }); + expect(JSON.parse(fs.readFileSync(insidePath, "utf8"))).toEqual({ ok: true }); + + const outsidePath = path.join(tmp, "escape.json"); + const escapingCtx: ProbeContext = { + ...insideCtx, + evidencePath: outsidePath, + }; + writeProbeEvidence(escapingCtx, { ok: false }); + expect(fs.existsSync(outsidePath)).toBe(false); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); + // ───────────────────────────────────────────────────────────────────────────── // diagnosticsProbe — uses a fake `nemoclaw` on PATH so this test runs // reproducibly without depending on a real nemoclaw install. diff --git a/test/e2e-scenario/framework/artifacts.ts b/test/e2e-scenario/framework/artifacts.ts new file mode 100644 index 0000000000..99597d8e8a --- /dev/null +++ b/test/e2e-scenario/framework/artifacts.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs/promises"; +import path from "node:path"; + +export class ArtifactSink { + readonly rootDir: string; + + constructor(rootDir: string) { + this.rootDir = path.resolve(rootDir); + } + + async ensureRoot(): Promise { + await fs.mkdir(this.rootDir, { recursive: true }); + } + + pathFor(relativePath: string): string { + if (!relativePath || path.isAbsolute(relativePath)) { + throw new Error(`artifact path must be relative: ${relativePath}`); + } + const resolved = path.resolve(this.rootDir, relativePath); + if (resolved !== this.rootDir && !resolved.startsWith(`${this.rootDir}${path.sep}`)) { + throw new Error(`artifact path escapes root: ${relativePath}`); + } + return resolved; + } + + async writeText(relativePath: string, text: string): Promise { + const target = this.pathFor(relativePath); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, text, "utf8"); + return target; + } + + async writeJson(relativePath: string, value: unknown): Promise { + return this.writeText(relativePath, `${JSON.stringify(value, null, 2)}\n`); + } +} + +export function slugifyArtifactName(name: string): string { + const slug = name + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return slug || "unnamed-test"; +} + +export function createArtifactSink(testName: string, rootDir = process.cwd()): ArtifactSink { + const baseDir = process.env.E2E_ARTIFACT_DIR ?? path.join(rootDir, ".e2e", "vitest"); + return new ArtifactSink(path.join(baseDir, slugifyArtifactName(testName))); +} diff --git a/test/e2e-scenario/framework/cleanup.ts b/test/e2e-scenario/framework/cleanup.ts new file mode 100644 index 0000000000..84427cf5fe --- /dev/null +++ b/test/e2e-scenario/framework/cleanup.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface CleanupFailure { + name: string; + message: string; +} + +export interface CleanupResult { + passed: string[]; + failures: CleanupFailure[]; +} + +type CleanupFn = () => Promise | void; +type RedactFn = (text: string) => string; + +interface CleanupEntry { + name: string; + run: CleanupFn; +} + +export class CleanupRegistry { + private readonly entries: CleanupEntry[] = []; + private readonly redact: RedactFn; + + constructor(redact: RedactFn = (text) => text) { + this.redact = redact; + } + + add(name: string, run: CleanupFn): void { + if (!name.trim()) { + throw new Error("cleanup name is required"); + } + this.entries.push({ name, run }); + } + + async runAll(): Promise { + const result: CleanupResult = { passed: [], failures: [] }; + for (const entry of [...this.entries].reverse()) { + try { + await entry.run(); + result.passed.push(this.redact(entry.name)); + } catch (error) { + result.failures.push({ + name: this.redact(entry.name), + message: this.redact(error instanceof Error ? error.message : String(error)), + }); + } + } + this.entries.length = 0; + return result; + } +} + +export function assertCleanupPassed(result: CleanupResult): void { + if (result.failures.length === 0) return; + const details = result.failures.map((failure) => `${failure.name}: ${failure.message}`).join("; "); + throw new Error(`E2E cleanup failed: ${details}`); +} diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts new file mode 100644 index 0000000000..b442187f8d --- /dev/null +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, test as base } from "vitest"; + +import { createArtifactSink, type ArtifactSink } from "./artifacts.ts"; +import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; +import { SecretStore } from "./secrets.ts"; +import { ShellProbe } from "./shell-probe.ts"; + +export interface E2EScenarioFixtures { + artifacts: ArtifactSink; + cleanup: CleanupRegistry; + secrets: SecretStore; + shellProbe: ShellProbe; +} + +export const test = base.extend({ + artifacts: async ({ task }, use) => { + const artifacts = createArtifactSink(task.name); + await artifacts.ensureRoot(); + try { + await use(artifacts); + } finally { + await artifacts.writeJson("artifact-summary.json", { + test: task.name, + rootDir: artifacts.rootDir, + }); + } + }, + secrets: async ({ skip }, use) => { + await use(new SecretStore(process.env, skip)); + }, + cleanup: async ({ artifacts, secrets }, use) => { + const cleanup = new CleanupRegistry((text) => secrets.redact(text)); + try { + await use(cleanup); + } finally { + const result = await cleanup.runAll(); + await artifacts.writeJson("cleanup.json", result); + assertCleanupPassed(result); + } + }, + shellProbe: async ({ artifacts, secrets, signal }, use) => { + await use(new ShellProbe({ + artifacts, + redact: (text, extraValues) => secrets.redact(text, extraValues), + signal, + })); + }, +}); + +export { expect }; diff --git a/test/e2e-scenario/framework/secrets.ts b/test/e2e-scenario/framework/secrets.ts new file mode 100644 index 0000000000..e6ab9daf99 --- /dev/null +++ b/test/e2e-scenario/framework/secrets.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { redactString } from "../scenarios/orchestrators/redaction.ts"; + +const SENSITIVE_NAME_PATTERN = /(api[_-]?key|token|secret|password|credential)/i; +const EXPLICIT_SECRET_REDACTION = "[REDACTED]"; + +/** + * Bridge-only fixture secret helper. + * + * The Vitest fixture layer still needs a small SecretStore while the scenario + * runner migration is in flight; #4989 tracks consolidating it into shared E2E + * framework infra. Canonical secret-shaped token matching belongs to + * scenarios/orchestrators/redaction.ts. Keep explicit fixture secret-value + * replacement here and always layer the parity-tested framework redactor + * underneath it so this path does not become a second pattern source. + */ + +export function redactText(text: string, secretValues: Iterable): string { + let redacted = text; + for (const value of secretValues) { + if (!value) continue; + redacted = redacted.split(value).join(EXPLICIT_SECRET_REDACTION); + } + return redactString(redacted); +} + +export class SecretStore { + private readonly env: NodeJS.ProcessEnv; + private readonly skip: (note?: string) => never; + + constructor(env: NodeJS.ProcessEnv, skip: (note?: string) => never) { + this.env = env; + this.skip = skip; + } + + optional(name: string): string | undefined { + const value = this.env[name]; + return value && value.length > 0 ? value : undefined; + } + + required(name: string): string { + const value = this.optional(name); + if (!value) { + this.skip(`missing required E2E secret: ${name}`); + } + return value; + } + + redactionValues(extraValues: string[] = []): string[] { + const values = new Set(); + for (const [name, value] of Object.entries(this.env)) { + if (value && SENSITIVE_NAME_PATTERN.test(name)) { + values.add(value); + } + } + for (const value of extraValues) { + if (value) values.add(value); + } + return [...values]; + } + + redact(text: string, extraValues: string[] = []): string { + return redactText(text, this.redactionValues(extraValues)); + } +} diff --git a/test/e2e-scenario/framework/shell-probe.ts b/test/e2e-scenario/framework/shell-probe.ts new file mode 100644 index 0000000000..540f9bd372 --- /dev/null +++ b/test/e2e-scenario/framework/shell-probe.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; + +import type { ArtifactSink } from "./artifacts.ts"; +import { redactText } from "./secrets.ts"; + +/** + * Bridge-only host shell probe for the Vitest fixture migration. + * + * The end state is a shared spawn/evidence helper consumed by both this + * fixture layer and scenarios/orchestrators; #4988 tracks that consolidation. + * Until it lands, this probe mirrors the hardened shell boundary: trusted + * descriptors, NUL-byte rejection, explicit env by default, canonical + * redaction, and detached process-group termination for timeout/abort cleanup. + */ + +export interface ShellProbeRunOptions { + cwd?: string; + env?: NodeJS.ProcessEnv; + inheritEnv?: boolean; + timeoutMs?: number; + killGraceMs?: number; + artifactName?: string; + redactionValues?: string[]; +} + +const trustedShellCommandBrand: unique symbol = Symbol("TrustedShellCommand"); + +export interface TrustedShellCommand { + readonly command: string; + readonly args: readonly string[]; + readonly reason: string; + readonly [trustedShellCommandBrand]: true; +} + +export interface TrustedShellCommandInput { + command: string; + args?: string[]; + reason: string; + validate?: (command: string, args: readonly string[]) => void; +} + +export interface ShellProbeResult { + command: string[]; + exitCode: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; + stdout: string; + stderr: string; + artifacts: { + stdout: string; + stderr: string; + result: string; + }; +} + +export interface ShellProbeDeps { + artifacts: ArtifactSink; + redact: (text: string, extraValues?: string[]) => string; + signal: AbortSignal; +} + +const DEFAULT_TIMEOUT_MS = 60_000; +const DEFAULT_KILL_GRACE_MS = 1_000; + +function safeArtifactBase(raw: string): string { + const safe = raw + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return safe || "shell-probe"; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function redactedError(error: unknown, message: string): Error { + const next = new Error(message); + if (error instanceof Error) { + next.name = error.name; + } + return next; +} + +function validateShellToken(value: string, label: string): string { + if (value.includes("\0")) { + throw new Error(`shell probe ${label} cannot contain NUL bytes`); + } + return value; +} + +/** + * Declares a shell command as trusted at the fixture/helper boundary. + * + * Build descriptors from constants or typed fixture helpers. Do not pass + * scenario, manifest, PR, or other untrusted values as the executable command. + * Put command-specific argument validation in `validate` when arguments include + * values derived from scenario data. + */ +export function trustedShellCommand(input: TrustedShellCommandInput): TrustedShellCommand { + const command = validateShellToken(input.command.trim(), "command"); + if (!command) { + throw new Error("shell probe command is required"); + } + const reason = input.reason.trim(); + if (!reason) { + throw new Error("shell probe trusted command reason is required"); + } + const args = (input.args ?? []).map((arg) => validateShellToken(arg, "argument")); + input.validate?.(command, args); + return { + command, + args, + reason, + [trustedShellCommandBrand]: true, + }; +} + +export class ShellProbe { + private readonly artifacts: ArtifactSink; + private readonly redact: (text: string, extraValues?: string[]) => string; + private readonly signal: AbortSignal; + + constructor(deps: ShellProbeDeps) { + this.artifacts = deps.artifacts; + this.redact = deps.redact; + this.signal = deps.signal; + } + + async run(trustedCommand: TrustedShellCommand, options: ShellProbeRunOptions = {}): Promise { + const command = trustedCommand.command; + const args = [...trustedCommand.args]; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const killGraceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS; + const redactionValues = options.redactionValues ?? []; + const redactProbeText = (text: string) => this.redact(redactText(text, redactionValues)); + const redactedCommand = [command, ...args].map(redactProbeText); + const artifactBase = `shell/${safeArtifactBase(redactProbeText(options.artifactName ?? command))}`; + const writeArtifacts = async (result: Omit): Promise => ({ + stdout: await this.artifacts.writeText(`${artifactBase}.stdout.txt`, result.stdout), + stderr: await this.artifacts.writeText(`${artifactBase}.stderr.txt`, result.stderr), + result: await this.artifacts.writeJson(`${artifactBase}.result.json`, result), + }); + const child = spawn(command, args, { + cwd: options.cwd, + detached: true, + env: options.inheritEnv ? { ...process.env, ...(options.env ?? {}) } : { ...(options.env ?? {}) }, + stdio: ["ignore", "pipe", "pipe"], + }); + const pgid = child.pid; + + let stdout = ""; + let stderr = ""; + let timedOut = false; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + + let killTimer: NodeJS.Timeout | undefined; + let terminationStarted = false; + const signalProcessGroup = (signal: NodeJS.Signals) => { + if (typeof pgid === "number") { + try { + process.kill(-pgid, signal); + return; + } catch { + /* fall back to the leader below */ + } + } + try { + child.kill(signal); + } catch { + /* already gone */ + } + }; + const terminate = () => { + terminationStarted = true; + signalProcessGroup("SIGTERM"); + if (killTimer) clearTimeout(killTimer); + killTimer = setTimeout(() => { + signalProcessGroup("SIGKILL"); + }, killGraceMs); + killTimer.unref(); + }; + const abort = () => { + terminate(); + }; + const timeout = setTimeout(() => { + timedOut = true; + terminate(); + }, timeoutMs); + if (this.signal.aborted) { + abort(); + } else { + this.signal.addEventListener("abort", abort, { once: true }); + } + + let childResult: { code: number | null; signal: NodeJS.Signals | null } | undefined; + let childError: unknown; + try { + childResult = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { + child.on("error", reject); + child.on("close", (code, signal) => resolve({ code, signal })); + }); + } catch (error) { + childError = error; + } finally { + clearTimeout(timeout); + if (killTimer && !terminationStarted) clearTimeout(killTimer); + this.signal.removeEventListener("abort", abort); + } + + const redactedStdout = redactProbeText(stdout); + if (childError) { + const redactedMessage = redactProbeText(errorMessage(childError)); + const redactedStderr = redactProbeText([stderr, redactedMessage].filter(Boolean).join("\n")); + await writeArtifacts({ + command: redactedCommand, + exitCode: null, + signal: null, + timedOut, + stdout: redactedStdout, + stderr: redactedStderr, + }); + throw redactedError(childError, redactedMessage); + } + + if (!childResult) { + throw new Error("shell probe child process did not report a result"); + } + + const redactedStderr = redactProbeText(stderr); + const result: Omit = { + command: redactedCommand, + exitCode: childResult.code, + signal: childResult.signal, + timedOut, + stdout: redactedStdout, + stderr: redactedStderr, + }; + const artifacts = await writeArtifacts(result); + return { ...result, artifacts }; + } +} diff --git a/test/e2e-scenario/scenarios/probes/injection-blocked.ts b/test/e2e-scenario/scenarios/probes/injection-blocked.ts index d1acf8ab3d..e1d4f7d854 100644 --- a/test/e2e-scenario/scenarios/probes/injection-blocked.ts +++ b/test/e2e-scenario/scenarios/probes/injection-blocked.ts @@ -102,7 +102,7 @@ export const injectionBlockedProbe: ProbeFn = async (ctx: ProbeContext): Promise evidence.echoStderrTail = echoResult.stderr; if (echoResult.exitCode !== 0) { - writeProbeEvidence(ctx.evidencePath, evidence); + writeProbeEvidence(ctx, evidence); return { status: "failed", classifier: echoResult.signal === "SIGTERM" ? "gateway-transient" : undefined, @@ -112,7 +112,7 @@ export const injectionBlockedProbe: ProbeFn = async (ctx: ProbeContext): Promise evidence.payloadPreservedLiterally = echoResult.stdout.includes(payload); if (!evidence.payloadPreservedLiterally) { - writeProbeEvidence(ctx.evidencePath, evidence); + writeProbeEvidence(ctx, evidence); return { status: "failed", message: `injectionBlockedProbe: payload was not preserved literally; stdout tail: ${echoResult.stdout.slice(-300)}`, @@ -139,7 +139,7 @@ export const injectionBlockedProbe: ProbeFn = async (ctx: ProbeContext): Promise perCallSeconds: PER_CALL_SECONDS, }); - writeProbeEvidence(ctx.evidencePath, evidence); + writeProbeEvidence(ctx, evidence); if (!evidence.markerAbsent) { return { diff --git a/test/e2e-scenario/scenarios/probes/network-policy.ts b/test/e2e-scenario/scenarios/probes/network-policy.ts index c3bb50923c..33c51ac3cb 100644 --- a/test/e2e-scenario/scenarios/probes/network-policy.ts +++ b/test/e2e-scenario/scenarios/probes/network-policy.ts @@ -92,7 +92,7 @@ export const networkPolicyProbe: ProbeFn = async (ctx: ProbeContext): Promise