diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1b35bf8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,17 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + - run: bun run typecheck + - run: bun test diff --git a/CHANGELOG.md b/CHANGELOG.md index 69283e2..b451786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 deterministic assertions, LLM judge details, artifact references, per-case verdicts, residual risks, verifier identity, and freshness without copying provider API keys or judge reasoning. +- GitHub Actions now runs the typecheck and full test suite on pull requests and + pushes to `main`. + +### Fixed +- Test-only module mocks no longer leak into later integration tests when the full + suite runs in one Bun process. ## [0.2.0] - 2026-07-27 diff --git a/src/adapters/mcp-adapter.test.ts b/src/adapters/mcp-adapter.test.ts index 9bb1114..8c490a1 100644 --- a/src/adapters/mcp-adapter.test.ts +++ b/src/adapters/mcp-adapter.test.ts @@ -1,52 +1,26 @@ -import { describe, test, expect, mock } from "bun:test"; +import { describe, test, expect } from "bun:test"; import { writeFileSync, mkdirSync } from "fs"; +import { fileURLToPath } from "url"; import { tmpdir } from "os"; import { join } from "path"; +import { callMcpAdapter } from "./mcp.js"; +import type { McpAdapterConfig } from "../types/index.js"; -// Mock the MCP SDK so we don't need a real MCP server process -mock.module("@modelcontextprotocol/sdk/client/index.js", () => ({ - Client: class MockClient { - async connect(_transport: unknown) {} - async callTool(params: { name: string; arguments: Record }, _schema: unknown, _opts: unknown) { - if (params.name === "echo") { - const input = params.arguments["input"] as string ?? ""; - return { content: [{ type: "text", text: `echo: ${input}` }] }; - } - if (params.name === "json_tool") { - return { content: [{ type: "text", text: '{"result": "ok"}' }] }; - } - if (params.name === "multi_content") { - return { content: [ - { type: "text", text: "part one" }, - { type: "text", text: "part two" }, - ]}; - } - if (params.name === "error_tool") { - throw new Error("Tool execution failed"); - } - if (params.name === "mapped_tool") { - // inputMapping test — receives the mapped key - const q = params.arguments["query"] as string ?? ""; - return { content: [{ type: "text", text: `query was: ${q}` }] }; - } - return { content: [{ type: "text", text: "unknown tool" }] }; - } - async close() {} - }, -})); +const fixturePath = fileURLToPath(new URL("./mcp-test-server.fixture.ts", import.meta.url)); -mock.module("@modelcontextprotocol/sdk/client/stdio.js", () => ({ - StdioClientTransport: class MockTransport { - constructor(_opts: unknown) {} - }, -})); - -const { callMcpAdapter } = await import("./mcp.js"); +function fixtureConfig(tool: string, inputMapping?: Record): McpAdapterConfig { + return { + type: "mcp", + command: [process.execPath, fixturePath], + tool, + inputMapping, + }; +} describe("MCP adapter", () => { test("calls named tool and returns text output", async () => { const result = await callMcpAdapter( - { type: "mcp", command: ["node", "mcp-server.js"], tool: "echo" }, + fixtureConfig("echo"), "hello world" ); expect(result.output).toBe("echo: hello world"); @@ -55,7 +29,7 @@ describe("MCP adapter", () => { test("concatenates multiple text content blocks", async () => { const result = await callMcpAdapter( - { type: "mcp", command: ["node", "mcp-server.js"], tool: "multi_content" }, + fixtureConfig("multi_content"), "x" ); expect(result.output).toContain("part one"); @@ -64,7 +38,7 @@ describe("MCP adapter", () => { test("works with JSON output", async () => { const result = await callMcpAdapter( - { type: "mcp", command: ["node", "mcp-server.js"], tool: "json_tool" }, + fixtureConfig("json_tool"), "x" ); expect(result.output).toContain('"result"'); @@ -72,12 +46,7 @@ describe("MCP adapter", () => { test("uses inputMapping to map input to named argument", async () => { const result = await callMcpAdapter( - { - type: "mcp", - command: ["node", "mcp-server.js"], - tool: "mapped_tool", - inputMapping: { query: "{{input}}" }, - }, + fixtureConfig("mapped_tool", { query: "{{input}}" }), "search term" ); expect(result.output).toContain("search term"); @@ -85,12 +54,7 @@ describe("MCP adapter", () => { test("passes static values in inputMapping", async () => { const result = await callMcpAdapter( - { - type: "mcp", - command: ["node", "mcp-server.js"], - tool: "mapped_tool", - inputMapping: { query: "fixed query" }, - }, + fixtureConfig("mapped_tool", { query: "fixed query" }), "ignored input" ); expect(result.output).toContain("fixed query"); @@ -98,7 +62,7 @@ describe("MCP adapter", () => { test("returns error on tool execution failure", async () => { const result = await callMcpAdapter( - { type: "mcp", command: ["node", "mcp-server.js"], tool: "error_tool" }, + fixtureConfig("error_tool"), "x" ); expect(result.error).toBeTruthy(); @@ -115,7 +79,7 @@ describe("MCP adapter", () => { test("tracks durationMs", async () => { const result = await callMcpAdapter( - { type: "mcp", command: ["node", "mcp-server.js"], tool: "echo" }, + fixtureConfig("echo"), "timing test" ); expect(typeof result.durationMs).toBe("number"); diff --git a/src/adapters/mcp-test-server.fixture.ts b/src/adapters/mcp-test-server.fixture.ts new file mode 100644 index 0000000..00797f5 --- /dev/null +++ b/src/adapters/mcp-test-server.fixture.ts @@ -0,0 +1,41 @@ +import { Server } from "@modelcontextprotocol/sdk/server/index.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; + +const tools = ["echo", "json_tool", "multi_content", "error_tool", "mapped_tool"].map((name) => ({ + name, + description: `Test fixture tool: ${name}`, + inputSchema: { type: "object" as const, additionalProperties: true }, +})); + +const server = new Server( + { name: "evals-mcp-adapter-test", version: "1.0.0" }, + { capabilities: { tools: {} } }, +); + +server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools })); +server.setRequestHandler(CallToolRequestSchema, async (request) => { + const args = (request.params.arguments ?? {}) as Record; + + switch (request.params.name) { + case "echo": + return { content: [{ type: "text", text: `echo: ${String(args["input"] ?? "")}` }] }; + case "json_tool": + return { content: [{ type: "text", text: '{"result": "ok"}' }] }; + case "multi_content": + return { + content: [ + { type: "text", text: "part one" }, + { type: "text", text: "part two" }, + ], + }; + case "error_tool": + throw new Error("Tool execution failed"); + case "mapped_tool": + return { content: [{ type: "text", text: `query was: ${String(args["query"] ?? "")}` }] }; + default: + throw new Error(`Unknown tool: ${request.params.name}`); + } +}); + +await server.connect(new StdioServerTransport()); diff --git a/src/core/runner.test.ts b/src/core/runner.test.ts index 61bfb04..851882b 100644 --- a/src/core/runner.test.ts +++ b/src/core/runner.test.ts @@ -1,47 +1,14 @@ -import { describe, test, expect, mock } from "bun:test"; +import { describe, test, expect } from "bun:test"; import type { EvalCase, AdapterConfig } from "../types/index.js"; +import { runEvals, runSingleCase } from "./runner.js"; -// Mock adapters -mock.module("../adapters/http.js", () => ({ - callHttpAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })), -})); -mock.module("../adapters/anthropic.js", () => ({ - callAnthropicAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })), -})); -mock.module("../adapters/openai.js", () => ({ - callOpenAIAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })), -})); -mock.module("../adapters/mcp.js", () => ({ - callMcpAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })), -})); -mock.module("../adapters/function.js", () => ({ - callFunctionAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })), -})); -mock.module("../adapters/cli.js", () => ({ - callCliAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })), -})); - -// Mock judge -mock.module("../core/judge.js", () => ({ - runJudge: mock(async () => ({ - verdict: "PASS", - reasoning: "Looks good", - durationMs: 100, - inputTokens: 50, - outputTokens: 20, - costUsd: 0.001, - })), -})); - -const { runEvals, runSingleCase } = await import("./runner.js"); - -const adapter: AdapterConfig = { type: "http", url: "http://localhost:9999" }; +const modulePath = "data:text/javascript,export%20default%20async%20function()%7Breturn%20%22mock%20response%22%7D"; +const adapter: AdapterConfig = { type: "function", modulePath }; const basicCase: EvalCase = { id: "test-001", input: "hello", assertions: [{ type: "min_length", value: 1 }], - judge: { rubric: "Should respond" }, }; describe("runSingleCase", () => { @@ -55,7 +22,11 @@ describe("runSingleCase", () => { }); test("skips judge when skipJudge=true", async () => { - const result = await runSingleCase(basicCase, adapter, true); + const result = await runSingleCase( + { ...basicCase, judge: { rubric: "Should respond" } }, + adapter, + true, + ); expect(result.judgeResult).toBeUndefined(); }); @@ -96,12 +67,11 @@ describe("runEvals", () => { }); test("redacts adapter apiKey from run metadata", async () => { - const secretAdapter: AdapterConfig = { - type: "openai", - model: "gpt-4o", - baseURL: "https://gateway.example.com/v1", + const secretAdapter = { + type: "function", + modulePath, apiKey: "provider-secret", - }; + } as unknown as AdapterConfig; const run = await runEvals( [{ id: "secret-case", input: "hello" }], @@ -109,9 +79,8 @@ describe("runEvals", () => { ); expect(run.adapterConfig).toEqual({ - type: "openai", - model: "gpt-4o", - baseURL: "https://gateway.example.com/v1", + type: "function", + modulePath, }); expect(JSON.stringify(run)).not.toContain("provider-secret"); });