diff --git a/README.md b/README.md index fa69f2dc..4a5d5a7e 100644 --- a/README.md +++ b/README.md @@ -364,9 +364,10 @@ Gemini auto-discovers the `skills/` folder next to it): gemini extensions install https://github.com/zuke-build/zuke ``` -Gemini installs a GitHub extension from the repo's **latest release** snapshot -(offering a git clone as the fallback), so the extension tracks releases rather -than `master`. +Gemini installs a GitHub extension from the repo's **latest release**, so the +extension tracks releases rather than `master`. Each release carries a minimal +extension archive (the manifest plus `skills/`, attached by the `release` +target), so the install downloads two skills, not the whole monorepo. > The `SKILL.md` content is harness-agnostic (the open > [Agent Skills](https://agentskills.io) standard); each manifest above is a diff --git a/build/gemini_archive.ts b/build/gemini_archive.ts new file mode 100644 index 00000000..67f1f0b7 --- /dev/null +++ b/build/gemini_archive.ts @@ -0,0 +1,105 @@ +// Copyright (c) 2026 the Zuke contributors +// SPDX-License-Identifier: MIT + +/** + * The Gemini CLI extension archive attached to GitHub releases. + * + * `gemini extensions install ` prefers a release asset over + * cloning: it resolves the repository's **latest** release and picks an asset + * by name — `{platform}.{arch}.{name}`, then `{platform}.{name}`, then a + * single generic asset. Without one it downloads the whole source tarball, + * which for this monorepo means shipping 50+ packages to install two skill + * folders. This module builds a minimal, deterministic archive — the root + * `gemini-extension.json` plus `skills/` and the license, exactly what the + * extension serves — for the `release` target to attach to every release. + * + * The asset trio is platform-prefixed rather than one generic file on + * purpose: Gemini's fallback accepts a generic asset only when it is the + * *only* asset on the release, so a second file attached later (a checksum, an + * SBOM) would silently degrade installs back to the source tarball. Platform + * prefixes match deterministically regardless of what else the release + * carries; the extension is platform-independent, so all three names carry + * the same bytes. + * + * @module + */ + +import { createTarGzip } from "@zuke/core"; + +/** The extension manifest Gemini requires at the archive root. */ +export const GEMINI_MANIFEST = "gemini-extension.json"; + +/** The skills tree the extension serves. */ +export const GEMINI_SKILLS_DIR = "skills"; + +/** + * The asset names to attach to a release, one per platform Gemini matches + * (`os.platform()` values). All three point at identical archive bytes. + */ +export const GEMINI_ASSET_NAMES: readonly string[] = [ + "darwin.zuke.tar.gz", + "linux.zuke.tar.gz", + "win32.zuke.tar.gz", +]; + +/** Every file under `dir`, as `dir`-prefixed paths, sorted for determinism. */ +async function walk(root: string, dir: string): Promise { + const out: string[] = []; + for await (const entry of Deno.readDir(`${root}/${dir}`)) { + const path = `${dir}/${entry.name}`; + if (entry.isDirectory) out.push(...await walk(root, path)); + else if (entry.isFile) out.push(path); + else { + // Silently dropping it would ship an archive missing content that git + // (and every other harness) still carries. + throw new Error( + `the Gemini extension archive cannot pack "${root}/${path}" — it is ` + + "neither a regular file nor a directory (a symlink?). Keep " + + `${GEMINI_SKILLS_DIR}/ to real files so every harness ships the ` + + "same content.", + ); + } + } + return out.sort(); +} + +/** + * The files the extension archive packs, relative to `root`, in a stable + * order: the manifest, the license, then every file under `skills/`. + */ +export async function geminiArchiveFiles(root = "."): Promise { + for (const required of [GEMINI_MANIFEST, "LICENSE"]) { + const info = await Deno.stat(`${root}/${required}`).catch(() => null); + if (info?.isFile !== true) { + throw new Error( + `the Gemini extension archive requires ${required} at the extension ` + + `root — nothing at "${root}/${required}".`, + ); + } + } + try { + return [GEMINI_MANIFEST, "LICENSE", ...await walk(root, GEMINI_SKILLS_DIR)]; + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + throw new Error( + `the Gemini extension archive requires a ${GEMINI_SKILLS_DIR}/ tree ` + + `under "${root}" — Gemini auto-discovers skills from it.`, + ); + } + throw error; + } +} + +/** + * Write the extension archive to `dest`: a `.tar.gz` with + * `gemini-extension.json` at the root, which is where Gemini requires it. + * Returns the paths that were packed. + */ +export async function buildGeminiArchive( + dest: string, + root = ".", +): Promise { + const files = await geminiArchiveFiles(root); + await createTarGzip(files, dest, { cwd: root }); + return files; +} diff --git a/cspell.json b/cspell.json index 0e774546..20dbf905 100644 --- a/cspell.json +++ b/cspell.json @@ -160,6 +160,7 @@ "tfvars", "tmpl", "todorov", + "tokenless", "topo", "totollygeek", "transpiling", diff --git a/llms-full.txt b/llms-full.txt index 6b952cc1..b09fee03 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -9419,6 +9419,9 @@ function readWorkflowResult(state: TargetStateHandle): WorkflowResult | undefine async function tagCommit(configure?: (settings: GhTagSettings) => GhTagSettings): Promise Perform the configured tag. +async function uploadReleaseAsset(configure?: Configure): Promise + Upload the release asset the settings describe. + async function uploadSarifReport(configure?: Configure): Promise Upload the SARIF report the settings describe. @@ -9663,6 +9666,52 @@ class GhPullRequestSettings authToken_(): string The effective token, from the setting or the environment. +class GhReleaseAssetSettings + Settings for {@link GhReleaseAssetApi.uploadReleaseAsset}. + + file_?: string + The file to upload. Set by {@link file}. + name_?: string + The asset name on the release. Set by {@link name}. + contentType_?: string + The asset's `content-type`. Set by {@link contentType}. + repo_?: string + `owner/repo` to upload to. Set by {@link repo}. + tag_?: string + The release tag to attach to. Set by {@link tag}. + token_?: string + The token to authenticate with. Set by {@link token}. + baseUrl_: string + REST base URL. Set by {@link baseUrl}. + fetch_: typeof fetch + The `fetch` implementation. Set by {@link fetch}. + file(path: PathLike): this + The file to upload (required). + name(value: string): this + The asset's name on the release. Defaults to the file's base name. + contentType(value: string): this + The asset's `content-type`. Defaults by extension (`.tar.gz`/`.tgz`, + `.zip`, `.json`), then to `application/octet-stream`. + repo(slug: string): this + The `owner/repo` to upload to. Defaults to `GITHUB_REPOSITORY`. + tag(value: string): this + Attach to the release with this tag instead of the latest release. + token(value: string): this + The token to authenticate with — needs `contents: write`. Defaults to + `GITHUB_TOKEN` in the environment, so it never has to reach argv. + baseUrl(url: string): this + Use a different REST base (GitHub Enterprise Server). + fetch(fn: typeof fetch): this + Override the `fetch` implementation (a test seam). + repoSlug_(): string + The effective `owner/repo`, from the setting or the Actions environment. + filePath_(): string + The file to upload, or a friendly error naming the missing setting. + assetName_(): string + The effective asset name: the setting, or the file's base name. + effectiveContentType_(): string + The effective `content-type`: the setting, or inferred by extension. + class GhSarifSettings Settings for {@link GhSarifApi.uploadSarif}. @@ -9918,6 +9967,32 @@ interface GhPullRequestResult different things to a human reading a build log, even though neither is a failure. +interface GhReleaseAssetApi + The shape of the release-asset task, mixed into `GhTasks`. + + uploadReleaseAsset(configure?: Configure): Promise + Attach a file to a GitHub release — the latest release by default, or the + one named by `.tag(...)`. Idempotent: an asset the release already + carries under the same name is kept as-is, and a repository with no + releases resolves to `state: "no-release"` rather than throwing. Needs a + token with `contents: write`. + +interface GhReleaseAssetResult + What became of a release-asset upload. + + state: "uploaded" | "already-exists" | "no-release" + `uploaded` when the asset was sent; `already-exists` when the release + carries an asset of the same name (nothing was changed); `no-release` + when the repository has no release to attach to. + name: string + The asset name the call targeted. + releaseTag?: string + The tag of the release the asset belongs to, when one was resolved. + releaseId?: number + The id of the release the asset belongs to, when one was resolved. + url?: string + The asset's download URL, when it was uploaded or already present. + interface GhSarifApi The shape of the SARIF task, mixed into `GhTasks`. @@ -9933,7 +10008,7 @@ interface GhSarifUploadResult url: string The URL that reports whether GitHub finished processing the report. -interface GhTasksApi extends GhAppTokenApi, GhSarifApi, GhCommitApi, GhPullRequestApi, GhCheckRunApi +interface GhTasksApi extends GhAppTokenApi, GhSarifApi, GhReleaseAssetApi, GhCommitApi, GhPullRequestApi, GhCheckRunApi The shape of {@link GhTasks}: the `gh` CLI plus the GitHub operations that have no CLI subcommand (see {@link GhAppTokenApi}, {@link GhSarifApi}) and would otherwise force a build back to a marketplace action. diff --git a/packages/ai/tests/agent_fixer_test.ts b/packages/ai/tests/agent_fixer_test.ts index b722f41c..33f73cfa 100644 --- a/packages/ai/tests/agent_fixer_test.ts +++ b/packages/ai/tests/agent_fixer_test.ts @@ -441,3 +441,335 @@ Deno.test("commitFixes fails closed when the pre-snapshot cannot be taken", asyn false, ); }); + +/** A GitHub-PR env without any GITHUB_TOKEN. */ +const TOKENLESS_PR_ENV: Record = { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "o/r", + GITHUB_REF: "refs/pull/7/merge", +}; + +/** The one-hunk diff the fake agent leaves in the working tree. */ +const AGENT_DIFF = [ + "diff --git a/zuke.ts b/zuke.ts", + "--- a/zuke.ts", + "+++ b/zuke.ts", + "@@ -45,1 +45,1 @@", + '-const X = "remove me";', + '+const _X = "remove me";', +].join("\n"); + +/** A fetch faking the GitHub PR-detail and comment endpoints, recording calls. */ +function githubFetch(): { + fetch: typeof fetch; + calls: { url: string; auth: string; body: string }[]; +} { + const calls: { url: string; auth: string; body: string }[] = []; + const impl = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + calls.push({ + url, + auth: new Headers(init?.headers).get("authorization") ?? "", + body: typeof init?.body === "string" ? init.body : "", + }); + if (url.includes("/comments")) { + return Promise.resolve( + new Response(method === "GET" ? "[]" : "{}", { status: 200 }), + ); + } + return Promise.resolve( + new Response(JSON.stringify({ head: { sha: "abc123" } }), { + status: 200, + }), + ); + }) as typeof fetch; + return { fetch: impl, calls }; +} + +Deno.test("suggest mode authenticates with an explicit commentToken", async () => { + // GITHUB_TOKEN is unset, so the inline suggestions can only be posted through + // .commentToken(...) — success pins the explicit-token resolution path. + const { fetch, calls } = githubFetch(); + let statusCalls = 0; + const fixer = agentFixer(() => Promise.resolve(), (f) => f.suggest()) + .allowCI().comment().commentToken("agent-tok") + .conventions("").readFile(() => Promise.resolve(undefined)) + .env((n) => TOKENLESS_PR_ENV[n]) + .exec((argv) => { + if (argv.includes("status")) { + statusCalls++; + return Promise.resolve(statusCalls === 1 ? "" : " M zuke.ts"); + } + return Promise.resolve(argv.includes("diff") ? AGENT_DIFF : ""); + }) + .fetch(fetch).quiet(); + await fixer.remediate(CTX); + const post = calls.find((c) => + c.url.endsWith("/pulls/7/comments") && c.body.includes("```suggestion") + ); + assertEquals(post?.auth, "Bearer agent-tok"); +}); + +Deno.test("suggest mode without any token proposes nothing and calls no API", async () => { + const { fetch, calls } = githubFetch(); + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + let statusCalls = 0; + try { + const fixer = agentFixer(() => Promise.resolve(), (f) => f.suggest()) + .allowCI().conventions("").readFile(() => Promise.resolve(undefined)) + .env((n) => TOKENLESS_PR_ENV[n]) + .exec((argv) => { + if (argv.includes("status")) { + statusCalls++; + return Promise.resolve(statusCalls === 1 ? "" : " M zuke.ts"); + } + return Promise.resolve(argv.includes("diff") ? AGENT_DIFF : ""); + }) + .fetch(fetch); + await fixer.remediate(CTX); + } finally { + console.log = log; + } + // Suggestions were produced but no PR context could be resolved — nothing + // was posted anywhere (the overview comment has no token either). + assertEquals(calls.length, 0); + assertEquals( + lines.some((l) => l.includes("no committable suggestions produced")), + true, + ); +}); + +Deno.test("suggest mode fails closed when the post-agent status cannot be read", async () => { + const prEnv: Record = { + ...TOKENLESS_PR_ENV, + GITHUB_TOKEN: "tok", + }; + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + const git: string[][] = []; + let statusCalls = 0; + try { + const fixer = agentFixer(() => Promise.resolve(), (f) => f.suggest()) + .allowCI().noComment().conventions("") + .readFile(() => Promise.resolve(undefined)) + .env((n) => prEnv[n]) + .exec((argv) => { + git.push(argv); + if (argv.includes("status")) { + statusCalls++; + // The snapshot succeeds; the post-agent status read fails. + if (statusCalls === 1) return Promise.resolve(""); + return Promise.reject(new Error("index.lock exists")); + } + return Promise.resolve(""); + }); + const result = await fixer.remediate(CTX); + assertEquals(result.retry, false); + } finally { + console.log = log; + } + // Without the second status nothing can be scoped: no diff is ever taken. + assertEquals(git.some((a) => a.includes("diff")), false); + assertEquals( + lines.some((l) => l.includes("no committable suggestions produced")), + true, + ); +}); + +Deno.test("suggest mode with an idle agent takes no diff and proposes nothing", async () => { + const prEnv: Record = { + ...TOKENLESS_PR_ENV, + GITHUB_TOKEN: "tok", + }; + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + const git: string[][] = []; + try { + const fixer = agentFixer(() => Promise.resolve(), (f) => f.suggest()) + .allowCI().noComment().conventions("") + .readFile(() => Promise.resolve(undefined)) + .env((n) => prEnv[n]) + .exec((argv) => { + git.push(argv); + return Promise.resolve(""); // clean tree before and after the agent + }); + await fixer.remediate(CTX); + } finally { + console.log = log; + } + assertEquals(git.some((a) => a.includes("diff")), false); + assertEquals( + lines.some((l) => l.includes("no committable suggestions produced")), + true, + ); +}); + +Deno.test("a non-Error suggestion-post failure is stringified and swallowed", async () => { + const prEnv: Record = { + ...TOKENLESS_PR_ENV, + GITHUB_TOKEN: "tok", + }; + // The PR-detail fetch rejects with a bare string (not an Error). + const throwing = (() => Promise.reject("conn reset")) as typeof fetch; + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...a: unknown[]) => void warnings.push(a.join(" ")); + let statusCalls = 0; + try { + const fixer = agentFixer(() => Promise.resolve(), (f) => f.suggest()) + .allowCI().noComment().conventions("") + .readFile(() => Promise.resolve(undefined)) + .env((n) => prEnv[n]) + .exec((argv) => { + if (argv.includes("status")) { + statusCalls++; + return Promise.resolve(statusCalls === 1 ? "" : " M zuke.ts"); + } + return Promise.resolve(argv.includes("diff") ? AGENT_DIFF : ""); + }) + .fetch(throwing).quiet(); + const result = await fixer.remediate(CTX); + assertEquals(result.retry, false); + } finally { + console.warn = warn; + } + assertEquals( + warnings.some((w) => w.includes("could not post suggestions: conn reset")), + true, + ); +}); + +Deno.test("a non-Error agent failure is stringified into the warning", async () => { + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...a: unknown[]) => void warnings.push(a.join(" ")); + let result; + try { + // The runner rejects with a bare string, pinning the String(error) path. + const fixer = hermetic(agentFixer(() => Promise.reject("agent exploded"))); + result = await fixer.remediate(CTX); + } finally { + console.warn = warn; + } + assertEquals(result.retry, false); + assertEquals( + warnings.some((w) => w.includes("agent run failed: agent exploded")), + true, + ); +}); + +Deno.test("suggest mode skips suggestions when the pre-agent snapshot fails", async () => { + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + const r = recorder(); + let result; + try { + const fixer = agentFixer(r.run, (f) => f.suggest()) + .conventions("").readFile(() => Promise.resolve(undefined)) + .env(() => undefined) // local run — the CI gate does not apply + .exec(() => Promise.reject(new Error("index.lock exists"))); + result = await fixer.remediate(CTX); + } finally { + console.log = log; + } + assertEquals(result.retry, false); + assertEquals(r.calls.length, 1); // the agent still ran + assertEquals( + lines.some((l) => + l.includes( + "skipped suggestions: could not snapshot the working tree", + ) + ), + true, + ); +}); + +Deno.test("a non-Error commit failure is stringified into the report action", async () => { + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + let statusCalls = 0; + let result; + try { + const fixer = agentFixer(() => Promise.resolve(), (f) => f.commitFixes()) + .conventions("").readFile(() => Promise.resolve(undefined)) + .env(() => undefined) + .exec((argv) => { + if (argv.includes("status")) { + statusCalls++; + return Promise.resolve(statusCalls === 1 ? "" : " M zuke.ts"); + } + if (argv[1] === "commit") return Promise.reject("hook denied"); + return Promise.resolve(""); + }); + result = await fixer.remediate(CTX); + } finally { + console.log = log; + } + assertEquals(result.retry, true); // the agent's edit still re-runs the target + assertEquals( + lines.some((l) => l.includes("commit failed: hook denied")), + true, + ); +}); + +Deno.test("a long agent transcript is truncated in the PR comment", async () => { + const prEnv: Record = { + ...TOKENLESS_PR_ENV, + GITHUB_TOKEN: "tok", + }; + const { fetch, calls } = githubFetch(); + const fixer = agentFixer(() => Promise.resolve("x".repeat(5000))) + .allowCI().conventions("").readFile(() => Promise.resolve(undefined)) + .env((n) => prEnv[n]).fetch(fetch).quiet(); + await fixer.remediate(CTX); + const post = calls.find((c) => c.body.includes("Agent output")); + if (post === undefined) throw new Error("no comment posted"); + assertEquals(post.body.includes("… (truncated) …"), true); + // The capped transcript, not the full 5000 characters, travels. + assertEquals(post.body.includes("x".repeat(4001)), false); +}); + +Deno.test("an Error from the suggestion post is warned with its message", async () => { + const prEnv: Record = { + ...TOKENLESS_PR_ENV, + GITHUB_TOKEN: "tok", + }; + // The PR-detail fetch rejects with a real Error, whose message is reported. + const throwing = + (() => Promise.reject(new Error("api quota exhausted"))) as typeof fetch; + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...a: unknown[]) => void warnings.push(a.join(" ")); + let statusCalls = 0; + try { + const fixer = agentFixer(() => Promise.resolve(), (f) => f.suggest()) + .allowCI().noComment().conventions("") + .readFile(() => Promise.resolve(undefined)) + .env((n) => prEnv[n]) + .exec((argv) => { + if (argv.includes("status")) { + statusCalls++; + return Promise.resolve(statusCalls === 1 ? "" : " M zuke.ts"); + } + return Promise.resolve(argv.includes("diff") ? AGENT_DIFF : ""); + }) + .fetch(throwing).quiet(); + const result = await fixer.remediate(CTX); + assertEquals(result.retry, false); + } finally { + console.warn = warn; + } + assertEquals( + warnings.some((w) => + w.includes("could not post suggestions: api quota exhausted") + ), + true, + ); +}); diff --git a/packages/ai/tests/ai_test.ts b/packages/ai/tests/ai_test.ts index c4b4eac6..7dbfbe28 100644 --- a/packages/ai/tests/ai_test.ts +++ b/packages/ai/tests/ai_test.ts @@ -1184,3 +1184,36 @@ Deno.test("a failed PR comment never breaks the review", async () => { }, ); }); + +Deno.test("provider_ reflects the configured provider", () => { + const bare = securityReviewer(); + assertEquals(bare.provider_, undefined); // nothing configured yet + const configured = securityReviewer((r) => r.provider("openai")); + assertEquals(configured.provider_, "openai"); +}); + +Deno.test("the fetchBase fallback is announced on the console when not quiet", async () => { + const { fetch } = recordFetch( + claude({ score: 0, severity: "none", summary: "", findings: [] }), + ); + const lines = await captured(() => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .diff((d) => d.fetchBase("develop")) + .env(() => undefined) + .exec((argv) => { + if (argv[1] === "fetch") return Promise.reject(new Error("offline")); + return Promise.resolve("diff --git a/w b/w\n+working tree"); + }) + .fetch(fetch) + ).validate({ target: "t" }) + ); + // The silent-fallback hazard is called out where the operator can see it. + assertEquals( + lines.some((l) => + l.includes("fetchBase could not compute the base diff") && + l.includes("falling back to the working-tree diff") + ), + true, + ); +}); diff --git a/packages/ai/tests/comment_test.ts b/packages/ai/tests/comment_test.ts new file mode 100644 index 00000000..7ab3c213 --- /dev/null +++ b/packages/ai/tests/comment_test.ts @@ -0,0 +1,94 @@ +// Copyright (c) 2026 the Zuke contributors +// SPDX-License-Identifier: MIT + +/** + * Tests for the shared PR-comment poster: token resolution, the no-context + * no-op, and the best-effort error handling. + * + * @module + */ + +import { + assertEquals, + assertStringIncludes, +} from "../../core/tests/_assert.ts"; +import { postComment } from "../src/comment.ts"; + +/** A recorded request. */ +interface Call { + url: string; + auth: string; +} + +/** A fake `fetch` recording each call's URL and Authorization header. */ +function recordFetch(): { fetch: typeof fetch; calls: Call[] } { + const calls: Call[] = []; + const impl = ((input: string | URL | Request, init?: RequestInit) => { + const headers = new Headers(init?.headers); + calls.push({ + url: String(input), + auth: headers.get("authorization") ?? "", + }); + const method = init?.method ?? "GET"; + return Promise.resolve( + new Response(method === "GET" ? "[]" : "{}", { status: 200 }), + ); + }) as typeof fetch; + return { fetch: impl, calls }; +} + +const PR_ENV: Record = { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "o/r", + GITHUB_REF: "refs/pull/7/merge", +}; + +Deno.test("postComment is a no-op off any CI host", async () => { + const { fetch, calls } = recordFetch(); + await postComment("sec", "## body", { env: () => undefined, fetch }); + assertEquals(calls.length, 0); +}); + +Deno.test("an explicit commentToken wins over the host env token", async () => { + const { fetch, calls } = recordFetch(); + await postComment("sec", "## body", { + commentToken: "explicit-token", + env: (n) => n === "GITHUB_TOKEN" ? "env-token" : PR_ENV[n], + fetch, + }); + assertEquals(calls.length, 2); // list, then create + for (const call of calls) { + assertEquals(call.auth, "Bearer explicit-token"); + } +}); + +Deno.test("no token at all prepares no upsert — nothing is fetched", async () => { + const { fetch, calls } = recordFetch(); + // The host is detected, but GITHUB_TOKEN is unset, so the empty token yields + // no PR context and the comment is silently skipped. + await postComment("sec", "## body", { env: (n) => PR_ENV[n], fetch }); + assertEquals(calls.length, 0); +}); + +Deno.test("a non-Error throw from the upsert is stringified into the warning", async () => { + // Rejects with a bare string (not an Error), pinning the String(error) + // fallback in postComment's catch. + const throwing = (() => Promise.reject("socket torn down")) as typeof fetch; + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...a: unknown[]) => void warnings.push(a.join(" ")); + try { + await postComment("sec", "## body", { + commentToken: "tok", + env: (n) => PR_ENV[n], + fetch: throwing, + }); + } finally { + console.warn = warn; + } + assertEquals(warnings.length, 1); + assertStringIncludes( + warnings[0], + "[sec] could not post PR comment: socket torn down", + ); +}); diff --git a/packages/ai/tests/cost_controls_test.ts b/packages/ai/tests/cost_controls_test.ts index fe484d29..a71574eb 100644 --- a/packages/ai/tests/cost_controls_test.ts +++ b/packages/ai/tests/cost_controls_test.ts @@ -1,10 +1,11 @@ // Copyright (c) 2026 the Zuke contributors // SPDX-License-Identifier: MIT -import { assertEquals } from "../../core/tests/_assert.ts"; +import { assertEquals, assertRejects } from "../../core/tests/_assert.ts"; import { aiCache, aiFixer, + AiReviewError, type Assessment, budget, type CacheEntry, @@ -156,6 +157,53 @@ Deno.test("reviewer skips the call once the budget is exhausted", async () => { assertEquals(lines.some((l) => l.includes("AI budget exhausted")), true); }); +Deno.test("the verify pass's own usage draws down the shared budget", async () => { + const b = budget((x) => x.maxTokens(100_000)); + const finding = { + title: "weak hash", + severity: "high" as const, + file: "h.ts", + }; + const id = findingFingerprint("security", { + title: "weak hash", + severity: "high", + file: "h.ts", + }); + const responses = [ + claude({ score: 8, severity: "high", summary: "s", findings: [finding] }, { + input_tokens: 10, + output_tokens: 5, + }), + // The verify verdict also reports usage — it must be folded in too. + JSON.stringify({ + content: [{ + type: "text", + text: JSON.stringify({ + verdicts: [{ id, verdict: "refuted", reason: "not reachable" }], + }), + }], + stop_reason: "end_turn", + usage: { input_tokens: 7, output_tokens: 3 }, + }), + ]; + let served = 0; + const queued: typeof fetch = () => + Promise.resolve( + new Response(responses[Math.min(served++, responses.length - 1)], { + status: 200, + }), + ); + await captured(() => + // Passes: the sole finding was refuted, so nothing gates. + securityReviewer((r) => + r.provider("claude").apiKey("k").diff((d) => d.text(DIFF)) + .fetch(queued).budget(b).verify() + ).validate({ target: "t" }) + ); + assertEquals(b.spend_().calls, 2); // review + verify both recorded + assertEquals(b.spend_().totalTokens, 25); +}); + // ----- Cache --------------------------------------------------------------- Deno.test("reviewer caches a response and reuses it on a repeat run", async () => { @@ -319,6 +367,65 @@ Deno.test("an empty suppress list leaves the findings untouched", async () => { assertEquals(lines.some((l) => l.includes("suppressed")), false); }); +Deno.test("a suppress list that matches no finding drops nothing", async () => { + // A non-empty list whose fingerprints belong to some other finding: the + // assessment must pass through untouched, so the critical finding still + // trips the default gate. + const sup = suppressions((s) => s.add("ffffffffffffffff")); + const { fetch } = recordFetch( + claude({ + score: 9, + severity: "critical", + summary: "bad", + findings: [{ title: "rce", severity: "critical" }], + }), + ); + const lines = await captured(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k").diff((d) => d.text(DIFF)) + .fetch(fetch).suppress(sup) + ).validate({ target: "t" }), + AiReviewError, // the unrelated fingerprint muted nothing + ); + }); + assertEquals(lines.some((l) => l.includes("suppressed")), false); +}); + +Deno.test("the verify pass is skipped once the review call exhausts the budget", async () => { + // The review itself fits under the cap check (0 < 1) but its usage blows the + // cap, so the verify pass that follows must be skipped — visibly — and the + // unverified finding must keep gating. + const b = budget((x) => x.maxTokens(1)); + const { fetch, calls } = recordFetch( + claude( + { + score: 9, + severity: "critical", + summary: "bad", + findings: [{ title: "rce", severity: "critical" }], + }, + { input_tokens: 500, output_tokens: 100 }, + ), + ); + const lines = await captured(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k").diff((d) => d.text(DIFF)) + .fetch(fetch).budget(b).verify() + ).validate({ target: "t" }), + AiReviewError, // the finding was kept, not silently verified away + ); + }); + assertEquals(calls.length, 1); // the review call only — no verify call + assertEquals( + lines.some((l) => l.includes("verify pass skipped — AI budget exhausted")), + true, + ); +}); + // ----- Report rendering ---------------------------------------------------- Deno.test("consoleLines renders cache, suppressed, and budget extras", () => { diff --git a/packages/ai/tests/dedup_test.ts b/packages/ai/tests/dedup_test.ts index 31ba4755..2b6a83ec 100644 --- a/packages/ai/tests/dedup_test.ts +++ b/packages/ai/tests/dedup_test.ts @@ -302,3 +302,70 @@ Deno.test("eligible is the one gate both resolution paths share", () => { false, ); }); + +Deno.test("dedupNotes carries an empty file for a hand-built file-less pair", () => { + // planDedup never offers a file-less pair, but the serializer is its own + // contract: a missing file travels as "", never as the string "undefined". + const notes = dedupNotes({ + pairs: [{ + label: "p1", + candidate: candidate("c1", { file: undefined, detail: "why" }), + prior: prior("p1"), + }], + dropped: 0, + }); + assertEquals(notes, [{ + label: "p1", + file: "", + title: "finding c1", + detail: "why", + priorTitle: "prior p1", + }]); +}); + +Deno.test("planDedup round-robins past a candidate with fewer matches", () => { + // c1 has two eligible priors, c2 (in another file) has one. At depth two the + // round-robin must skip c2's exhausted list and still offer c1's second pair. + const plan = planDedup( + [candidate("c1"), candidate("c2", { file: "src/other.ts" })], + [prior("p1"), prior("p2"), prior("p3", { file: "src/other.ts" })], + ); + assertEquals( + plan.pairs.map((p) => [p.candidate.id, p.prior.id]), + [["c1", "p1"], ["c2", "p3"], ["c1", "p2"]], + ); + assertEquals(plan.dropped, 0); +}); + +Deno.test("sameAs ignores a hand-built pair whose candidate has no identity", () => { + // A pair can only enter a plan through planDedup today, but sameAs guards its + // own input: without a candidate id there is nothing to rename, so a "same" + // verdict on such a pair must resolve nothing. + for (const id of [undefined, ""]) { + const plan = { + pairs: [{ + label: "p1", + candidate: candidate("c1", { id }), + prior: prior("p1"), + }], + dropped: 0, + }; + const result = sameAs(plan, verdicts(["p1", "same"])); + assertEquals(result.matches.size, 0); + assertEquals(result.ambiguous, []); + } +}); + +Deno.test("adoptCanonicalIds skips a finding that has no id at all", () => { + const bare = candidate("c1", { id: undefined }); + const matched = candidate("c2"); + const target = prior("p1"); + const adoptions = adoptCanonicalIds( + [bare, matched], + new Map([["c2", target]]), + ); + // Only the identified finding adopts; the bare one is left untouched. + assertEquals(adoptions, [{ alias: "c2", prior: target }]); + assertEquals(matched.id, "p1"); + assertEquals(bare.id, undefined); +}); diff --git a/packages/ai/tests/deep_review_test.ts b/packages/ai/tests/deep_review_test.ts index c4334c48..48b1887a 100644 --- a/packages/ai/tests/deep_review_test.ts +++ b/packages/ai/tests/deep_review_test.ts @@ -311,3 +311,99 @@ Deno.test("verify is skipped cleanly when there are no findings", async () => { ).validate({ target: "t" }); assertEquals(calls.length, 1); // no second call to verify nothing }); + +/** Capture console output with the job-summary file unset (no real writes). */ +async function captured(fn: () => Promise): Promise { + const lines: string[] = []; + const { log, warn } = console; + const summary = Deno.env.get("GITHUB_STEP_SUMMARY"); + Deno.env.delete("GITHUB_STEP_SUMMARY"); + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + console.warn = (...a: unknown[]) => void lines.push(a.join(" ")); + try { + await fn(); + } finally { + console.log = log; + console.warn = warn; + if (summary !== undefined) Deno.env.set("GITHUB_STEP_SUMMARY", summary); + } + return lines; +} + +Deno.test("verify candidates carry the finding's line; a reason-less refutation renders bare", async () => { + const finding = { + title: "SQL injection", + severity: "high", + file: "db.ts", + line: 3, + }; + const id = findingFingerprint("security", { + title: "SQL injection", + severity: "high", + file: "db.ts", + }); + const { fetch, calls } = queuedFetch([ + claude({ score: 8, severity: "high", findings: [finding] }), + // A refutation with no reason — schema-valid, and must still narrow. + claude({ verdicts: [{ id, verdict: "refuted" }] }), + ]); + const lines = await captured(() => + // Passes: the sole finding was refuted, so the score recomputes to 0. + securityReviewer((r) => + r.provider("claude").apiKey("k") + .diff((d) => d.text(DIFF)).verify() + .fetch(fetch) + ).validate({ target: "t" }) + ); + // The verify prompt carried the line, so the verifier can find the code. + const verify = JSON.parse(calls[1].body); + assertEquals(verify.messages[0].content.includes('"line": 3'), true); + // The refutation is reported without inventing a reason. + const refutedLine = lines.find((l) => l.includes("refuted by verify")); + assertEquals(refutedLine?.includes("SQL injection"), true); + assertEquals(refutedLine?.includes("—"), false); +}); + +Deno.test("a failed verify pass warns on the console when not quiet", async () => { + const { fetch } = queuedFetch([ + claude({ + score: 9, + severity: "critical", + findings: [{ title: "RCE", severity: "critical" }], + }), + { status: 500 }, + ]); + const lines = await captured(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .retry({ attempts: 1 }) + .diff((d) => d.text(DIFF)).verify() + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, // the unverified finding stayed and still gates + ); + }); + assertEquals( + lines.some((l) => + l.includes("verify pass failed") && + l.includes("keeping unverified findings") + ), + true, + ); +}); + +Deno.test("an empty file context is omitted from the prompt entirely", async () => { + const { fetch, calls } = queuedFetch([claude({ score: 0, findings: [] })]); + const git = fakeGit({}); // every `git show` fails — nothing to send + await genericReviewer((r) => + r.provider("claude").apiKey("k").quiet() + .diff((d) => d.base("origin/master")) + .fileContext(1000) + .exec(git.run).fetch(fetch) + ).validate({ target: "t" }); + const body = JSON.parse(calls[0].body); + // No empty UNTRUSTED_FILES block confuses the model when nothing was read. + assertEquals(body.messages[0].content.includes("UNTRUSTED_FILES"), false); +}); diff --git a/packages/ai/tests/discussion_flow_test.ts b/packages/ai/tests/discussion_flow_test.ts index 95eea933..3e7a0c8f 100644 --- a/packages/ai/tests/discussion_flow_test.ts +++ b/packages/ai/tests/discussion_flow_test.ts @@ -2,11 +2,15 @@ // SPDX-License-Identifier: MIT import { assertEquals, assertRejects } from "../../core/tests/_assert.ts"; -import { AiReviewError, securityReviewer } from "../mod.ts"; +import { AiReviewError, budget, securityReviewer } from "../mod.ts"; import { findingFingerprint } from "../src/suppress.ts"; import { decodeState, encodeState } from "../src/state.ts"; import { commentMarker } from "../src/hosts/types.ts"; -import { findingMarker, outcomeMarker } from "../src/threads.ts"; +import { + findingMarker, + MAX_NEW_THREADS, + outcomeMarker, +} from "../src/threads.ts"; import { stableHash } from "../src/hash.ts"; const DIFF = "diff --git a/src/app.ts b/src/app.ts\n" + @@ -2481,3 +2485,765 @@ Deno.test("a quiet reviewer posts no threads either", async () => { ); assertEquals(threadPosts(calls).length, 0); }); + +// ─── Degraded paths: no host, exhausted budgets, failing passes ────────────── + +Deno.test("comment and discussion degrade to warnings outside any CI host", async () => { + // An env reader that sees nothing: no CI host markers at all (a local run). + const { fetch, calls } = discussionFetch([], [ + claude({ score: 0, severity: "none", findings: [] }), + ]); + const lines = await captured(() => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .env(() => undefined) + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }) + ); + // The discussion is declined in code, before any listing is attempted… + assertEquals( + lines.some((l) => + l.includes("discussion disabled") && + l.includes("the active host cannot list PR comments") + ), + true, + ); + // …and the comment is skipped with its own warning, not a crash. + assertEquals( + lines.some((l) => + l.includes("no PR-comment host detected — skipping comment") + ), + true, + ); + // No host traffic happened at all. + assertEquals(calls.every((c) => !c.url.startsWith(`${GITHUB_API}/`)), true); +}); + +/** Wrap a payload in a Claude response that also reports token usage. */ +function claudeWithUsage( + payload: unknown, + usage: Record, +): string { + return JSON.stringify({ + content: [{ type: "text", text: JSON.stringify(payload) }], + stop_reason: "end_turn", + usage, + }); +} + +Deno.test("an exhausted budget skips the reworded-finding check, and says so", async () => { + // The review call's own usage blows the cap, so the dedup pass that would + // resolve the rewording must be skipped — with a note, never silently — and + // the finding keeps its fresh identity and gates. + const b = budget((x) => x.maxTokens(1)); + const { fetch, calls } = discussionFetch( + priorComment(stateWith("dismissed")), + [claudeWithUsage({ score: 9, severity: "high", findings: [REWORDED] }, { + input_tokens: 500, + output_tokens: 100, + })], + ); + const lines = await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion().budget(b) + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, // reported under its own id — not silenced + ); + }) + ); + // The review call only: no dedup call was paid for. + assertEquals( + calls.filter((c) => !c.url.startsWith(`${GITHUB_API}/`)).length, + 1, + ); + assertEquals( + lines.some((l) => + l.includes("reworded-finding check skipped — AI budget exhausted") + ), + true, + ); + // The finding kept its own identity in the posted state. + assertEquals( + postedState(calls)?.findings.some((f) => f.id === REWORDED_ID), + true, + ); +}); + +Deno.test("an ambiguous rewording keeps the first match and reports the rest, with the cap named", async () => { + // Four earlier open findings in the same file are all eligible comparisons + // for one candidate; the per-candidate cap (3) drops the fourth, and the + // model then claims the candidate matches two of the three offered. + const priors = Array.from({ length: 4 }, (_, i) => ({ + id: `aaaa000${i + 1}`, + title: `Earlier concern ${i + 1}`, + severity: "high" as const, + status: "open" as const, + file: "src/app.ts", + })); + const { fetch, calls } = discussionFetch( + priorComment({ findings: priors }), + [ + claude({ score: 9, severity: "high", findings: [REWORDED] }), + claude({ + verdicts: [ + { id: "p1", verdict: "same", reason: "same defect" }, + { id: "p2", verdict: "same", reason: "also this one" }, + ], + }), + ], + ); + const lines = await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, // adopted an OPEN identity — still gating + ); + }) + ); + // First offered wins: the candidate adopted the first prior's identity. + const state = postedState(calls); + assertEquals( + state?.findings.find((f) => f.id === "aaaa0001")?.status, + "open", + ); + assertEquals( + state?.findings.find((f) => f.id === "aaaa0001")?.aliases, + [REWORDED_ID], + ); + // The double match is reported against the candidate's fresh id… + assertEquals( + lines.some((l) => + l.includes(REWORDED_ID) && + l.includes("matched more than one earlier finding") + ), + true, + ); + // …and the dropped fourth comparison is announced, not silent. + assertEquals( + lines.some((l) => + l.includes("reworded-finding check compared 3 of 4 candidate pairs") + ), + true, + ); +}); + +Deno.test("the finding's detail reaches the adjudicator, and a reason-less dismissal sticks", async () => { + const detailed = { + ...FINDING, + detail: "input flows straight into eval with no sanitisation", + }; + const comments = [ + { + id: 1, + body: `Finding ${ID} misreads the code: eval runs in a sandboxed worker`, + user: { login: "maintainer", type: "User" }, + author_association: "MEMBER", + }, + ]; + const { fetch, calls } = discussionFetch(comments, [ + claude({ score: 9, severity: "high", findings: [detailed] }), + // A dismissal verdict carrying no reason — still a valid two-key dismissal. + claude({ verdicts: [{ id: ID, verdict: "dismissed" }] }), + ]); + const lines = await captured(() => + inPr(async () => { + // Dismissed, so the gate does not trip. + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + }) + ); + const providerCalls = calls.filter((c) => + !c.url.startsWith(`${GITHUB_API}/`) + ); + assertEquals(providerCalls.length, 2); + // The adjudication prompt carried the finding's detail for context. + assertEquals( + JSON.parse(providerCalls[1].body).messages[0].content.includes( + "input flows straight into eval with no sanitisation", + ), + true, + ); + assertEquals( + lines.some((l) => l.includes("dismissed via discussion by maintainer")), + true, + ); + // Recorded dismissed with the author but no invented rationale. + const entry = postedState(calls)?.findings.find((f) => f.id === ID); + assertEquals(entry?.status, "dismissed"); + assertEquals(entry?.author, "maintainer"); + assertEquals(entry?.rationale, undefined); +}); + +Deno.test("a failed adjudication keeps contested findings open with a warning", async () => { + const calls: Call[] = []; + let served = 0; + const failing = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + calls.push({ + url, + method, + body: typeof init?.body === "string" ? init.body : "", + }); + if (url.startsWith(`${GITHUB_API}/`)) { + if (url.endsWith("/user")) { + return Promise.resolve(new Response("{}", { status: 403 })); + } + const payload = method === "GET" + ? JSON.stringify([{ + id: 1, + body: `Finding ${ID} misreads the code`, + user: { login: "maintainer", type: "User" }, + author_association: "MEMBER", + }]) + : "{}"; + return Promise.resolve(new Response(payload, { status: 200 })); + } + served++; + // The review answers; the adjudication call that follows fails outright. + return Promise.resolve( + served === 1 + ? new Response( + claude({ score: 9, severity: "high", findings: [FINDING] }), + { status: 200 }, + ) + : new Response("upstream exploded", { status: 500 }), + ); + }) as typeof fetch; + const lines = await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion().retry({ attempts: 1 }) + .diff((d) => d.text(DIFF)) + .fetch(failing) + ).validate({ target: "t" }), + AiReviewError, // the contested finding stayed open and gates + ); + }) + ); + assertEquals( + lines.some((l) => + l.includes("adjudication failed") && + l.includes("contested findings stay open") + ), + true, + ); +}); + +Deno.test("findings beyond the thread cap stay in the table, and the gap is announced", async () => { + // One more anchorable finding than the per-run thread cap. + const count = MAX_NEW_THREADS + 1; + const bigDiff = [ + "diff --git a/src/big.ts b/src/big.ts", + "--- a/src/big.ts", + "+++ b/src/big.ts", + `@@ -1,0 +1,${count} @@`, + ...Array.from({ length: count }, (_, i) => `+const v${i + 1} = ${i + 1};`), + ].join("\n"); + const many = Array.from({ length: count }, (_, i) => ({ + title: `Issue number ${i + 1}`, + severity: "high", + file: "src/big.ts", + line: i + 1, + })); + const { fetch, calls } = threadFetch([summaryComment()], [], [ + claude({ score: 9, severity: "high", findings: many }), + ]); + await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion((d) => d.threads()) + .diff((d) => d.text(bigDiff)) + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, + ); + }) + ); + // Exactly the cap was opened, and the leftover is promised to the next run. + assertEquals(threadPosts(calls).length, MAX_NEW_THREADS); + const write = calls.find((c) => + c.url.includes("/issues/") && c.method !== "GET" + ); + assertEquals( + JSON.parse(write?.body ?? "{}").body.includes( + `1 finding(s) did not get a review thread this run (cap ${MAX_NEW_THREADS})`, + ), + true, + ); +}); + +Deno.test("a thread phase that throws degrades to a note, never a failure", async () => { + // The head-commit lookup explodes (HTTP 500) after the thread listing + // succeeded — the whole phase must collapse into a report note. + const calls: Call[] = []; + const doFetch = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + calls.push({ + url, + method, + body: typeof init?.body === "string" ? init.body : "", + }); + if (url.startsWith(`${GITHUB_API}/`)) { + if (url.endsWith("/user")) { + return Promise.resolve(new Response("{}", { status: 403 })); + } + if (method !== "GET") return Promise.resolve(new Response("{}")); + if (url.includes("/pulls/7/comments")) { + return Promise.resolve(new Response("[]")); + } + if (url.includes("/pulls/7")) { + return Promise.resolve(new Response("boom", { status: 500 })); + } + return Promise.resolve(new Response(JSON.stringify([summaryComment()]))); + } + return Promise.resolve( + new Response( + claude({ score: 9, severity: "high", findings: [ANCHORED] }), + { status: 200 }, + ), + ); + }) as typeof fetch; + await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion((d) => d.threads()) + .diff((d) => d.text(ANCHORED_DIFF)) + .fetch(doFetch) + ).validate({ target: "t" }), + AiReviewError, // the gate still speaks — the finding never vanished + ); + }) + ); + assertEquals(threadPosts(calls).length, 0); + const write = calls.find((c) => + c.url.includes("/issues/") && c.method !== "GET" + ); + assertEquals( + JSON.parse(write?.body ?? "{}").body.includes( + "review threads unavailable this run", + ), + true, + ); +}); + +Deno.test("discussion on GitHub without a PR context is skipped silently", async () => { + // A push build on GitHub Actions: the host can list comments, but there is + // no pull request to list them from — the discussion just does not run. + const { fetch, calls } = discussionFetch([], [ + claude({ score: 0, severity: "none", findings: [] }), + ]); + const lines = await captured(() => + inEnv( + { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "zuke-build/zuke", + GITHUB_REF: "refs/heads/master", // a branch, not a PR + GITHUB_TOKEN: "tkn", + }, + async () => { + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + }, + ) + ); + // Not the "disabled" warning — this is the expected local/push shape. + assertEquals(lines.some((l) => l.includes("discussion disabled")), false); + // No comment listing was even attempted. + assertEquals(calls.every((c) => !c.url.startsWith(`${GITHUB_API}/`)), true); + // The comment itself is skipped with the no-PR-context warning. + assertEquals(lines.some((l) => l.includes("no GitHub PR context")), true); +}); + +Deno.test("a candidate matching a fixed and a dismissed prior reopens rather than inheriting the dismissal", async () => { + // Fixed entries are offered before dismissed ones exactly so that a + // candidate the model matches to both REOPENS (which reports) instead of + // inheriting a dismissal (which silences). + const priors = [ + { + id: "oooo0001", + title: "Still-open concern", + severity: "high" as const, + status: "open" as const, + file: "src/app.ts", + }, + { + id: "dddd0001", + title: "Dismissed concern", + severity: "high" as const, + status: "dismissed" as const, + file: "src/app.ts", + rationale: "argued away", + author: "maintainer", + }, + { + id: "ffff0001", + title: "Fixed concern", + severity: "high" as const, + status: "fixed" as const, + file: "src/app.ts", + }, + ]; + const { fetch, calls } = discussionFetch( + priorComment({ findings: priors }), + [ + claude({ score: 9, severity: "high", findings: [REWORDED] }), + // The model matches the FIRST comparison — which, by the fixed-first + // ordering, is the fixed entry even though it was listed last in state. + claude({ verdicts: [{ id: "p1", verdict: "same", reason: "same" }] }), + ], + ); + const lines = await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, // reopened — it gates again + ); + }) + ); + assertEquals( + lines.some((l) => l.includes("reopened under ffff0001")), + true, + ); + const state = postedState(calls); + assertEquals( + state?.findings.find((f) => f.id === "ffff0001")?.status, + "open", + ); + assertEquals( + state?.findings.find((f) => f.id === "ffff0001")?.aliases, + [REWORDED_ID], + ); + // The dismissal was NOT inherited and stays on its own entry. + assertEquals( + state?.findings.find((f) => f.id === "dddd0001")?.status, + "dismissed", + ); +}); + +Deno.test("an upheld verdict without a reason records no rationale", async () => { + const bare = { title: "Global prototype pollution", severity: "high" }; + const bareId = findingFingerprint("security", { + title: "Global prototype pollution", + severity: "high", + }); + const comments = [ + { + id: 1, + body: `Finding ${bareId} is fine, we own that global`, + user: { login: "maintainer", type: "User" }, + author_association: "OWNER", + }, + ]; + const { fetch, calls } = discussionFetch(comments, [ + claude({ score: 9, severity: "high", findings: [bare] }), + claude({ verdicts: [{ id: bareId, verdict: "upheld" }] }), + ]); + await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, // upheld → still gating + ); + }) + ); + const entry = postedState(calls)?.findings.find((f) => f.id === bareId); + assertEquals(entry?.status, "upheld"); + assertEquals(entry?.rationale, undefined); // no invented reasoning + assertEquals(entry?.file, undefined); // and no invented location +}); + +Deno.test("an exhausted budget skips the adjudication; contested findings stay open", async () => { + const b = budget((x) => x.maxTokens(1)); + const comments = [ + { + id: 1, + body: `Finding ${ID} misreads the code`, + user: { login: "maintainer", type: "User" }, + author_association: "MEMBER", + }, + ]; + // The review's own usage blows the cap before the adjudication would run. + const { fetch, calls } = discussionFetch(comments, [ + claudeWithUsage({ score: 9, severity: "high", findings: [FINDING] }, { + input_tokens: 500, + output_tokens: 100, + }), + ]); + await captured(() => + inPr(async () => { + await assertRejects( + () => + securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion().budget(b) + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }), + AiReviewError, // never adjudicated → still open → still gating + ); + }) + ); + // The review call only — the maintainer's rebuttal cost nothing further. + assertEquals( + calls.filter((c) => !c.url.startsWith(`${GITHUB_API}/`)).length, + 1, + ); + assertEquals( + postedState(calls)?.findings.find((f) => f.id === ID)?.status, + "open", + ); +}); + +Deno.test("a sticky dismissal with no recorded author or rationale still mutes, rendered bare", async () => { + const priorBody = `${MARKER}\nold report\n${ + encodeState({ + findings: [{ + id: ID, + title: FINDING.title, + severity: "high", + status: "dismissed", + // No author, rationale, or file recorded — a minimal state entry. + }], + }) + }`; + const comments = [{ + id: 1, + body: priorBody, + user: { login: "github-actions[bot]", type: "Bot" }, + author_association: "NONE", + }]; + const { fetch, calls } = discussionFetch(comments, [ + claude({ score: 9, severity: "high", findings: [FINDING] }), + ]); + const lines = await captured(() => + inPr(async () => { + // Passes: the sticky dismissal mutes the re-reported finding. + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + }) + ); + // Rendered without inventing an author or a reason. + assertEquals( + lines.some((l) => l.includes(`dismissed via discussion: ${FINDING.title}`)), + true, + ); + // The dismissed-findings memory rode into the prompt as the bare line. + const provider = calls.find((c) => !c.url.startsWith(`${GITHUB_API}/`)); + const user = JSON.parse(provider?.body ?? "{}").messages[0].content; + assertEquals(user.includes(`${ID} — ${FINDING.title}`), true); +}); + +Deno.test("a missing token env skips the comment with the PR-context warning", async () => { + // A PR on GitHub Actions, but GITHUB_TOKEN is absent and no .commentToken() + // was configured: the comment is skipped, never posted unauthenticated. + const { fetch, calls } = discussionFetch([], [ + claude({ score: 0, severity: "none", findings: [] }), + ]); + const lines = await captured(() => + inEnv( + { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "zuke-build/zuke", + GITHUB_REF: "refs/pull/7/merge", + // no GITHUB_TOKEN + }, + async () => { + const token = Deno.env.get("GITHUB_TOKEN"); + Deno.env.delete("GITHUB_TOKEN"); + try { + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + } finally { + if (token !== undefined) Deno.env.set("GITHUB_TOKEN", token); + } + }, + ) + ); + assertEquals( + lines.some((l) => l.includes("no GitHub PR context — skipping comment")), + true, + ); + assertEquals(calls.every((c) => !c.url.startsWith(`${GITHUB_API}/`)), true); +}); + +Deno.test("a file-less open prior is re-assessed bare and marked fixed when gone", async () => { + const priorBody = `${MARKER}\nold report\n${ + encodeState({ + findings: [{ + id: "bbbb0001", + title: "Orphan concern", + severity: "low", + status: "open", + // no file recorded + }], + }) + }`; + const comments = [{ + id: 1, + body: priorBody, + user: { login: "github-actions[bot]", type: "Bot" }, + author_association: "NONE", + }]; + const { fetch, calls } = discussionFetch(comments, [ + claude({ score: 0, severity: "none", findings: [] }), + ]); + const lines = await captured(() => + inPr(async () => { + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + }) + ); + // The prior rode into the prompt as the bare `id — title` line. + const provider = calls.find((c) => !c.url.startsWith(`${GITHUB_API}/`)); + const user = JSON.parse(provider?.body ?? "{}").messages[0].content; + assertEquals(user.includes("bbbb0001 — Orphan concern"), true); + // Not re-reported → recorded as progress, with no invented location. + assertEquals( + lines.some((l) => l.includes("fixed: Orphan concern · bbbb0001")), + true, + ); + assertEquals( + postedState(calls)?.findings.find((f) => f.id === "bbbb0001")?.status, + "fixed", + ); +}); + +Deno.test("a dismissal of a file-less finding records no location", async () => { + const bare = { title: "Global prototype pollution", severity: "high" }; + const bareId = findingFingerprint("security", { + title: "Global prototype pollution", + severity: "high", + }); + const comments = [ + { + id: 1, + body: `Finding ${bareId} cannot happen: the object is frozen at startup`, + user: { login: "maintainer", type: "User" }, + author_association: "MEMBER", + }, + ]; + const { fetch, calls } = discussionFetch(comments, [ + claude({ score: 9, severity: "high", findings: [bare] }), + claude({ + verdicts: [{ id: bareId, verdict: "dismissed", reason: "frozen object" }], + }), + ]); + await captured(() => + inPr(async () => { + // Dismissed → the gate does not trip. + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion() + .diff((d) => d.text(DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + }) + ); + const entry = postedState(calls)?.findings.find((f) => f.id === bareId); + assertEquals(entry?.status, "dismissed"); + assertEquals(entry?.rationale, "frozen object"); + assertEquals(entry?.file, undefined); // no invented location +}); + +Deno.test("a reason-less dismissal still closes its thread, without an invented reason", async () => { + const reply = { + id: 502, + body: "This runs in a sandboxed worker, so it cannot reach the host.", + in_reply_to_id: 501, + user: { login: "maintainer", type: "User" }, + author_association: "MEMBER", + }; + const { fetch, calls } = threadFetch( + [summaryComment({ + findings: [{ + id: ANCHORED_ID, + title: ANCHORED.title, + severity: "high", + status: "open", + file: "src/app.ts", + }], + })], + [threadRoot(ANCHORED_ID), reply], + [ + claude({ score: 9, severity: "high", findings: [ANCHORED] }), + // The adjudicator dismisses but offers no reason. + claude({ verdicts: [{ id: ANCHORED_ID, verdict: "dismissed" }] }), + ], + ); + await captured(() => + inPr(async () => { + await securityReviewer((r) => + r.provider("claude").apiKey("k") + .comment().discussion((d) => d.threads()) + .diff((d) => d.text(ANCHORED_DIFF)) + .fetch(fetch) + ).validate({ target: "t" }); + }) + ); + // The outcome reply landed in the thread, bare — no fabricated rationale. + const posted = calls.find((c) => c.url.includes("/comments/501/replies")); + const body = JSON.parse(posted?.body ?? "{}").body; + assertEquals( + body.startsWith(outcomeMarker(NAME_HASH, ANCHORED_ID, "dismissed")), + true, + ); + assertEquals(body.includes("**Dismissed via discussion**"), true); + assertEquals(body.includes("**Dismissed via discussion** —"), false); +}); diff --git a/packages/ai/tests/fixer_test.ts b/packages/ai/tests/fixer_test.ts index 035fbe50..e1915aa6 100644 --- a/packages/ai/tests/fixer_test.ts +++ b/packages/ai/tests/fixer_test.ts @@ -1004,3 +1004,199 @@ Deno.test("commitAndPush is a no-op when there are nothing to commit", async () }); assertEquals(git.length, 0); }); + +Deno.test("aiFixer without a configure lambda returns a bare fixer", () => { + assertEquals(aiFixer() instanceof AiFixer, true); +}); + +Deno.test("an explicit commentToken authenticates the inline suggestions", async () => { + // GITHUB_TOKEN is deliberately unset: posting can only succeed through the + // configured token, so the assertion pins the commentToken resolution path. + const withVariedLocs: Partial = { + diagnosis: "two spots", + rootCause: "r", + confidence: "high", + locations: [ + // No suggestion and no endLine: a single-line deletion suggestion. + { file: "a.ts", line: 5, code: "const dead = 1;" }, + // A replacement across a range. + { + file: "b.ts", + line: 10, + endLine: 12, + code: "let y = 2;", + suggestion: "const y = 2;", + }, + ], + edits: [], + }; + const { fetch, calls } = suggestFetch(claudeFix(withVariedLocs)); + const prEnv: Record = { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "o/r", + GITHUB_REF: "refs/pull/7/merge", + }; + const fixer = aiFixer((f) => + f.provider("claude").apiKey("k").commentToken("sug-tok") + ) + .conventions("").diff((d) => d.text("")).env((n) => prEnv[n]).fetch(fetch) + .quiet(); + await fixer.remediate(CTX); + const posts = calls.filter((c) => c.url.endsWith("/pulls/7/comments")); + const bodies = posts.map((c) => JSON.parse(c.body)); + assertEquals(bodies.length, 2); + // The suggestion-less location renders an empty (deletion) block at its line. + assertEquals(bodies[0].line, 5); + assertEquals(bodies[0].body.includes("```suggestion\n```"), true); + // The ranged location carries the replacement and spans start→end. + assertEquals(bodies[1].start_line, 10); + assertEquals(bodies[1].line, 12); + assertEquals( + bodies[1].body.includes("```suggestion\nconst y = 2;\n```"), + true, + ); +}); + +Deno.test("on GitHub without any token, no comment or suggestion is attempted", async () => { + // The CI host is detected and locations exist, but the empty token yields no + // PR context: the fixer must stay silent instead of calling the API. + const { fetch, calls } = suggestFetch(claudeFix(FIX_WITH_LOC)); + const prEnv: Record = { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "o/r", + GITHUB_REF: "refs/pull/7/merge", + }; + const fixer = aiFixer((f) => f.provider("claude").apiKey("k")) + .conventions("").diff((d) => d.text("")).env((n) => prEnv[n]).fetch(fetch) + .quiet(); + const result = await fixer.remediate(CTX); + assertEquals(result.retry, false); + assertEquals( + calls.some((c) => c.url.startsWith("https://api.github.com/")), + false, + ); +}); + +Deno.test("a non-Error suggestion-post failure is stringified and falls back", async () => { + const prEnv: Record = { + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: "o/r", + GITHUB_REF: "refs/pull/7/merge", + GITHUB_TOKEN: "tok", + }; + const calls: string[] = []; + const impl = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + calls.push(url); + if (url.startsWith("https://api.github.com/")) { + // The PR-detail fetch rejects with a bare string (not an Error); the + // issue-comment GET/POST succeeds so the fallback can land. + if (!url.includes("/comments")) return Promise.reject("conn reset"); + const method = init?.method ?? "GET"; + return Promise.resolve( + new Response(method === "GET" ? "[]" : "{}", { status: 200 }), + ); + } + return Promise.resolve( + new Response(claudeFix(FIX_WITH_LOC), { status: 200 }), + ); + }) as typeof fetch; + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...a: unknown[]) => void warnings.push(a.join(" ")); + try { + const fixer = aiFixer((f) => f.provider("claude").apiKey("k")) + .conventions("").diff((d) => d.text("")).env((n) => prEnv[n]).fetch(impl) + .quiet(); + await fixer.remediate(CTX); + } finally { + console.warn = warn; + } + assertEquals( + warnings.some((w) => w.includes("could not post suggestions: conn reset")), + true, + ); + // Fell back to the overview issue comment. + assertEquals(calls.some((u) => u.includes("/issues/")), true); +}); + +Deno.test("a non-Error provider failure is stringified into the warning", async () => { + const s = seams(); + const { fetch } = recordFetch("overloaded", 503); // always transient + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...a: unknown[]) => void warnings.push(a.join(" ")); + let result; + try { + const fixer = s.apply( + aiFixer((f) => + f.provider("claude").apiKey("k") + // The retry's sleep seam rejects with a bare string, which escapes + // retryingFetch unwrapped — pinning the String(error) fallback. + .retry({ + attempts: 2, + sleep: () => Promise.reject("sleep torn down"), + }) + ), + ).fetch(fetch).quiet(); + result = await fixer.remediate(CTX); + } finally { + console.warn = warn; + } + assertEquals(result.retry, false); + assertEquals( + warnings.some((w) => + w.includes("could not produce a fix: sleep torn down") + ), + true, + ); +}); + +Deno.test("a non-Error apply failure is stringified into the report action", async () => { + const { fetch } = recordFetch(claudeFix(ONE_EDIT)); + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + let result; + try { + const fixer = aiFixer((f) => f.provider("claude").apiKey("k").autoApply()) + .conventions("").diff((d) => d.text("")).env(() => undefined) + .write(() => Promise.reject("disk sealed")) // a bare string, not an Error + .fetch(fetch); + result = await fixer.remediate(CTX); + } finally { + console.log = log; + } + assertEquals(result.retry, false); // a failed apply never asks for a re-run + assertEquals( + lines.some((l) => l.includes("could not apply fix: disk sealed")), + true, + ); +}); + +Deno.test("a non-Error commit failure is stringified into the report action", async () => { + const { fetch } = recordFetch(claudeFix(ONE_EDIT)); + const lines: string[] = []; + const log = console.log; + console.log = (...a: unknown[]) => void lines.push(a.join(" ")); + let result; + try { + const fixer = aiFixer((f) => f.provider("claude").apiKey("k").commitFixes()) + .conventions("").diff((d) => d.text("")).env(() => undefined) + .write(() => Promise.resolve()) + .exec((argv) => + argv[1] === "commit" + ? Promise.reject("hook denied") // a bare string, not an Error + : Promise.resolve("") + ) + .fetch(fetch); + result = await fixer.remediate(CTX); + } finally { + console.log = log; + } + assertEquals(result.retry, true); // the applied fix still re-runs the target + assertEquals( + lines.some((l) => l.includes("commit failed: hook denied")), + true, + ); +}); diff --git a/packages/ai/tests/gate_test.ts b/packages/ai/tests/gate_test.ts new file mode 100644 index 00000000..4e606e5a --- /dev/null +++ b/packages/ai/tests/gate_test.ts @@ -0,0 +1,62 @@ +// Copyright (c) 2026 the Zuke contributors +// SPDX-License-Identifier: MIT + +/** + * Tests for the gate: the fluent rule builder, its human description, and the + * trip decision. + * + * @module + */ + +import { assertEquals } from "../../core/tests/_assert.ts"; +import { describeGate, GateSettings, gateTrips } from "../src/gate.ts"; +import type { Assessment } from "../src/types.ts"; + +/** An assessment with a fixed score/severity and no findings. */ +function assessed(score: number, severity: Assessment["severity"]): Assessment { + return { score, severity, summary: "", findings: [] }; +} + +Deno.test("GateSettings collects rules in order", () => { + const rules = new GateSettings().scoreAbove(8).severityAtLeast("high") + .rules_(); + assertEquals(rules, [ + { kind: "score", value: 8 }, + { kind: "severity", value: "high" }, + ]); +}); + +Deno.test("describeGate names every rule, and no rules as none", () => { + assertEquals(describeGate([]), "none"); + assertEquals(describeGate([{ kind: "score", value: 8 }]), "score>8"); + assertEquals( + describeGate([ + { kind: "score", value: 8 }, + { kind: "severity", value: "high" }, + ]), + "score>8, severity≥high", + ); +}); + +Deno.test("gateTrips fires on score strictly above and severity at least", () => { + const rules = new GateSettings().scoreAbove(8).severityAtLeast("high") + .rules_(); + // Score at the threshold does not trip — strictly above does. + assertEquals(gateTrips(assessed(8, "low"), rules).tripped, false); + assertEquals(gateTrips(assessed(9, "low"), rules), { + tripped: true, + reason: "risk score 9 exceeds 8", + }); + // Severity at (and above) the threshold trips with the severity reason. + assertEquals(gateTrips(assessed(0, "high"), rules), { + tripped: true, + reason: 'severity "high" is at least "high"', + }); + assertEquals(gateTrips(assessed(0, "critical"), rules).tripped, true); + assertEquals(gateTrips(assessed(0, "medium"), rules).tripped, false); + // No rules: nothing can trip. + assertEquals(gateTrips(assessed(10, "critical"), []), { + tripped: false, + reason: "", + }); +}); diff --git a/packages/ai/tests/hosts_test.ts b/packages/ai/tests/hosts_test.ts index d77764b6..69177d3b 100644 --- a/packages/ai/tests/hosts_test.ts +++ b/packages/ai/tests/hosts_test.ts @@ -1354,3 +1354,232 @@ Deno.test("listPullRequestComments treats a numeric system commentType as a bot" const listed = await listPullRequestComments(AZURE_CTX, fetch); assertEquals(listed[0].bot, true); }); + +Deno.test("resolveAzureContext refuses when any variable is absent or empty", () => { + // Each variable individually missing — and individually empty — is fatal. + for (const name of Object.keys(AZURE_ENV)) { + const without: Record = { ...AZURE_ENV }; + delete without[name]; + assertEquals(resolveAzureContext("azt", env(without)), undefined); + assertEquals( + resolveAzureContext("azt", env({ ...AZURE_ENV, [name]: "" })), + undefined, + ); + } +}); + +Deno.test("an Azure identity probe returning a non-string id fails closed", async () => { + // connectionData answers, but with a malformed identity — the reviewer can + // attribute nothing to itself, so it must POST fresh, never PATCH. + const threads = [{ + id: 7, + comments: [{ + id: 1, + content: "\nold", + author: { id: "build-service-id" }, + }], + }]; + const calls: Call[] = []; + const doFetch = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + calls.push({ + url, + method, + body: typeof init?.body === "string" ? init.body : "", + }); + if (method !== "GET") return Promise.resolve(new Response("{}")); + if (url.includes("connectionData")) { + return Promise.resolve( + new Response(JSON.stringify({ authenticatedUser: { id: 42 } })), + ); + } + return Promise.resolve(new Response(JSON.stringify({ value: threads }))); + }) as typeof fetch; + await upsertPullRequestThread( + AZURE_CTX, + "security review", + "## new", + doFetch, + ); + const write = calls.find((c) => c.method !== "GET"); + assertEquals(write?.method, "POST"); +}); + +Deno.test("a thrown Azure identity probe fails closed to a fresh thread", async () => { + const threads = [{ + id: 7, + comments: [{ + id: 1, + content: "\nold", + author: { id: "build-service-id" }, + }], + }]; + const calls: Call[] = []; + const doFetch = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + calls.push({ + url, + method, + body: typeof init?.body === "string" ? init.body : "", + }); + if (url.includes("connectionData")) { + return Promise.reject(new Error("dns failure")); + } + if (method !== "GET") return Promise.resolve(new Response("{}")); + return Promise.resolve(new Response(JSON.stringify({ value: threads }))); + }) as typeof fetch; + await upsertPullRequestThread( + AZURE_CTX, + "security review", + "## new", + doFetch, + ); + const write = calls.find((c) => c.method !== "GET"); + assertEquals(write?.method, "POST"); +}); + +Deno.test("an Azure author with a non-string id and no uniqueName is empty, not trusted", async () => { + const threads = [{ + id: 7, + comments: [{ + id: 1, + content: "trust me", + author: { id: 99, displayName: "Impostor" }, + }], + }]; + const { fetch } = fakeAzure(threads, { self: "build-service-id" }); + const listed = await listPullRequestComments(AZURE_CTX, fetch); + // No stable identity at all: the author is empty (and can never match a + // trustAuthors allowlist), while the display label is carried for reading. + assertEquals(listed[0].author, ""); + assertEquals(listed[0].displayName, "Impostor"); +}); + +Deno.test("upsertPullRequestThread in append mode POSTs without listing", async () => { + const { fetch, calls } = fakeAzure([], { self: "build-service-id" }); + await upsertPullRequestThread( + AZURE_CTX, + "security review", + "## round 2", + fetch, + "append", + ); + // No identity probe and no thread listing — straight to the create. + assertEquals(calls.length, 1); + assertEquals(calls[0].method, "POST"); + assertEquals(calls[0].url.endsWith("/threads?api-version=7.1"), true); +}); + +Deno.test("azureHost.prepare needs a PR context", () => { + assertEquals(azureHost.prepare("azt", env({})), undefined); + assertEquals(typeof azureHost.prepare("azt", env(AZURE_ENV)), "function"); +}); + +Deno.test("resolveBitbucketContext refuses when any variable is absent or empty", () => { + for (const name of Object.keys(BITBUCKET_ENV)) { + const without: Record = { ...BITBUCKET_ENV }; + delete without[name]; + assertEquals(resolveBitbucketContext("bbt", env(without)), undefined); + assertEquals( + resolveBitbucketContext("bbt", env({ ...BITBUCKET_ENV, [name]: "" })), + undefined, + ); + } +}); + +Deno.test("a refused or malformed Bitbucket /user probe fails closed", async () => { + const marked = [{ + id: 18, + content: { raw: "\nold" }, + user: { uuid: "{zuke}" }, + }]; + // A workspace access token: /2.0/user refuses it (401 with an error body). + const refused = fakeBitbucket(marked); + await upsertBitbucketComment( + BITBUCKET_CTX, + "security review", + "## new", + refused.fetch, + ); + const refusedWrite = refused.calls.find((c) => c.method !== "GET"); + assertEquals(refusedWrite?.method, "POST"); // cannot adopt its own comment + + // A /user body without a string uuid is the same failure. + const malformed = fakeBitbucket(marked, { self: "{zuke}" }); + const withBadUser = ((input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/2.0/user")) { + return Promise.resolve(new Response(JSON.stringify({ uuid: 5 }))); + } + return malformed.fetch(input, init); + }) as typeof fetch; + await upsertBitbucketComment( + BITBUCKET_CTX, + "security review", + "## new", + withBadUser, + ); + const write = malformed.calls.find((c) => c.method !== "GET"); + assertEquals(write?.method, "POST"); +}); + +Deno.test("a thrown Bitbucket /user probe fails closed to a fresh comment", async () => { + const marked = [{ + id: 18, + content: { raw: "\nold" }, + user: { uuid: "{zuke}" }, + }]; + const inner = fakeBitbucket(marked, { self: "{zuke}" }); + const throwing = ((input: string | URL | Request, init?: RequestInit) => { + if (String(input).endsWith("/2.0/user")) { + return Promise.reject(new Error("dns failure")); + } + return inner.fetch(input, init); + }) as typeof fetch; + await upsertBitbucketComment( + BITBUCKET_CTX, + "security review", + "## new", + throwing, + ); + const write = inner.calls.find((c) => c.method !== "GET"); + assertEquals(write?.method, "POST"); +}); + +Deno.test("listBitbucketComments drops malformed and deleted comments", async () => { + const values = [ + { id: "not-a-number", content: { raw: "x" } }, // id wrong type — dropped + { id: 2, content: { raw: 42 } }, // body wrong type — dropped + { id: 3, content: { raw: "gone" }, deleted: true }, // deleted — dropped + { id: 4, content: { raw: "kept" }, user: { uuid: "{dev}" } }, + ]; + const { fetch } = fakeBitbucket(values, { self: "{zuke}", members: [] }); + const listed = await listBitbucketComments(BITBUCKET_CTX, fetch); + assertEquals(listed.map((c) => c.id), [4]); + assertEquals(listed[0].body, "kept"); +}); + +Deno.test("upsertBitbucketComment in append mode POSTs without listing", async () => { + const { fetch, calls } = fakeBitbucket([], { self: "{zuke}" }); + await upsertBitbucketComment( + BITBUCKET_CTX, + "security review", + "## round 2", + fetch, + "append", + ); + // No /user probe and no comment listing — straight to the create. + assertEquals(calls.length, 1); + assertEquals(calls[0].method, "POST"); + assertEquals(calls[0].url.endsWith("/pullrequests/5/comments"), true); +}); + +Deno.test("bitbucketHost.prepare needs a PR context", () => { + assertEquals(bitbucketHost.prepare("bbt", env({})), undefined); + assertEquals( + typeof bitbucketHost.prepare("bbt", env(BITBUCKET_ENV)), + "function", + ); +}); diff --git a/packages/ai/tests/report_test.ts b/packages/ai/tests/report_test.ts new file mode 100644 index 00000000..7dc7a38e --- /dev/null +++ b/packages/ai/tests/report_test.ts @@ -0,0 +1,214 @@ +// Copyright (c) 2026 the Zuke contributors +// SPDX-License-Identifier: MIT + +/** + * Tests for the report renderer: console lines, the job-summary Markdown, and + * the best-effort step-summary writer. + * + * @module + */ + +import { + assertEquals, + assertStringIncludes, +} from "../../core/tests/_assert.ts"; +import { + consoleLines, + formatUsage, + reviewStartLine, + skipConsoleLine, + skipMarkdown, + toMarkdown, +} from "../src/report.ts"; +import type { Assessment } from "../src/types.ts"; + +/** A small assessment with one located, fingerprinted finding. */ +const ASSESSMENT: Assessment = { + score: 6, + severity: "medium", + summary: "one real issue", + findings: [ + { + title: "sql injection", + severity: "high", + file: "db.ts", + line: 3, + id: "abc123", + }, + { title: "vague worry", severity: "low" }, // no id, no location + ], +}; + +Deno.test("formatUsage renders only the counts the provider reported", () => { + assertEquals(formatUsage(undefined), undefined); + // A usage object with no counts at all is not an empty line — it is nothing. + assertEquals(formatUsage({}), undefined); + assertEquals(formatUsage({ inputTokens: 12 }), "12 in"); + assertEquals( + formatUsage({ inputTokens: 1, outputTokens: 2, totalTokens: 3 }), + "1 in · 2 out · 3 total", + ); +}); + +Deno.test("reviewStartLine echoes the gate and the comment flag", () => { + assertEquals( + reviewStartLine("sec", { + target: "deploy", + provider: "claude", + model: "m", + gate: "score>8", + comment: true, + }), + '[sec] reviewing "deploy" — claude/m · gate score>8 · comment', + ); + assertEquals( + reviewStartLine("sec", { + target: "deploy", + provider: "openai", + model: "m", + gate: "none", + comment: false, + }).includes("comment"), + false, + ); +}); + +Deno.test("consoleLines prints ids and locations only where they exist", () => { + const lines = consoleLines("sec", ASSESSMENT); + assertEquals(lines[0], "[sec] score 6/10 (medium) — 2 finding(s)"); + // The located, fingerprinted finding carries both suffixes … + assertEquals(lines[1], " - [high] sql injection (db.ts:3) · abc123"); + // … the bare one carries neither. + assertEquals(lines[2], " - [low] vague worry"); + assertEquals(lines[3], " one real issue"); +}); + +Deno.test("consoleLines audits suppressed, refuted, dismissed, and fixed findings", () => { + const lines = consoleLines("sec", ASSESSMENT, undefined, { + suppressed: 2, + suppressedFindings: [ + { title: "muted", severity: "low", file: "a.ts", line: 1, id: "id1" }, + { title: "muted bare", severity: "low" }, // no id, no location + ], + refuted: [ + { finding: { title: "not real", severity: "low" }, reason: "guarded" }, + { finding: { title: "unexplained", severity: "low" } }, // no reason + ], + dismissed: [ + { + finding: { title: "argued away", severity: "low" }, + author: "alice", + reason: "intended", + rewordedFrom: "earlier words", + }, + { finding: { title: "quietly gone", severity: "low" } }, // no author/reason + ], + fixed: [ + { + id: "f1", + title: "fixed one", + severity: "high", + status: "open", + file: "b.ts", + }, + { id: "f2", title: "fixed bare", severity: "low", status: "open" }, // no file + ], + notes: ["a bounded pass"], + }); + const text = lines.join("\n"); + assertStringIncludes(text, "suppressed 2 finding(s) via the suppress list"); + assertStringIncludes(text, " suppressed: [low] muted (a.ts:1) · id1"); + assertStringIncludes(text, " suppressed: [low] muted bare"); + assertStringIncludes(text, " refuted by verify: not real — guarded"); + assertStringIncludes(text, " refuted by verify: unexplained"); + assertStringIncludes( + text, + ' dismissed via discussion by alice: argued away (reworded from "earlier words") — intended', + ); + assertStringIncludes(text, " dismissed via discussion: quietly gone"); + assertStringIncludes(text, " fixed: fixed one (b.ts) · f1"); + assertStringIncludes(text, " fixed: fixed bare · f2"); + assertStringIncludes(text, " note: a bounded pass"); +}); + +Deno.test("toMarkdown fills missing audit fields with an em dash", () => { + const md = toMarkdown("sec", "deploy", ASSESSMENT, undefined, { + refuted: [{ finding: { title: "not real", severity: "low" } }], + dismissed: [{ finding: { title: "quietly gone", severity: "low" } }], + fixed: [{ id: "f2", title: "fixed bare", severity: "low", status: "open" }], + notes: ["pass skipped for budget"], + }); + // A refuted finding with no reason, and a dismissal with no author, reason, + // or finding id, render as explicit dashes — never as empty cells. + assertStringIncludes(md, "| not real | — |"); + assertStringIncludes(md, "| quietly gone | — | — | — |"); + // A fixed finding without a file gets a dash location. + assertStringIncludes(md, "| low | fixed bare | — | f2 |"); + assertStringIncludes(md, "- pass skipped for budget"); +}); + +Deno.test("toMarkdown names the dismisser, the reason, and the rewording", () => { + const md = toMarkdown("sec", "deploy", ASSESSMENT, undefined, { + dismissed: [{ + finding: { title: "argued away", severity: "low", id: "d1" }, + author: "alice", + reason: "intended", + rewordedFrom: "earlier words", + }], + }); + assertStringIncludes( + md, + '| argued away _(reworded from "earlier words")_ | alice | intended | d1 |', + ); +}); + +Deno.test("skipMarkdown neutralises the reason like any untrusted value", () => { + const md = skipMarkdown("sec", "deploy", "no key"); + assertEquals(md.includes("