ci: gate pull requests on typecheck and tests

.github/workflows/ci.yml

@@ -0,0 +1,17 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install --frozen-lockfile
+      - run: bun run typecheck
+      - run: bun test

CHANGELOG.md

@@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   deterministic assertions, LLM judge details, artifact references, per-case
   verdicts, residual risks, verifier identity, and freshness without copying
   provider API keys or judge reasoning.
+- GitHub Actions now runs the typecheck and full test suite on pull requests and
+  pushes to `main`.
+
+### Fixed
+- Test-only module mocks no longer leak into later integration tests when the full
+  suite runs in one Bun process.
 
 ## [0.2.0] - 2026-07-27
 

src/adapters/mcp-adapter.test.ts

@@ -1,52 +1,26 @@
-import { describe, test, expect, mock } from "bun:test";
+import { describe, test, expect } from "bun:test";
 import { writeFileSync, mkdirSync } from "fs";
+import { fileURLToPath } from "url";
 import { tmpdir } from "os";
 import { join } from "path";
+import { callMcpAdapter } from "./mcp.js";
+import type { McpAdapterConfig } from "../types/index.js";
 
-// Mock the MCP SDK so we don't need a real MCP server process
-mock.module("@modelcontextprotocol/sdk/client/index.js", () => ({
-  Client: class MockClient {
-    async connect(_transport: unknown) {}
-    async callTool(params: { name: string; arguments: Record<string, unknown> }, _schema: unknown, _opts: unknown) {
-      if (params.name === "echo") {
-        const input = params.arguments["input"] as string ?? "";
-        return { content: [{ type: "text", text: `echo: ${input}` }] };
-      }
-      if (params.name === "json_tool") {
-        return { content: [{ type: "text", text: '{"result": "ok"}' }] };
-      }
-      if (params.name === "multi_content") {
-        return { content: [
-          { type: "text", text: "part one" },
-          { type: "text", text: "part two" },
-        ]};
-      }
-      if (params.name === "error_tool") {
-        throw new Error("Tool execution failed");
-      }
-      if (params.name === "mapped_tool") {
-        // inputMapping test — receives the mapped key
-        const q = params.arguments["query"] as string ?? "";
-        return { content: [{ type: "text", text: `query was: ${q}` }] };
-      }
-      return { content: [{ type: "text", text: "unknown tool" }] };
-    }
-    async close() {}
-  },
-}));
+const fixturePath = fileURLToPath(new URL("./mcp-test-server.fixture.ts", import.meta.url));
 
-mock.module("@modelcontextprotocol/sdk/client/stdio.js", () => ({
-  StdioClientTransport: class MockTransport {
-    constructor(_opts: unknown) {}
-  },
-}));
-
-const { callMcpAdapter } = await import("./mcp.js");
+function fixtureConfig(tool: string, inputMapping?: Record<string, string>): McpAdapterConfig {
+  return {
+    type: "mcp",
+    command: [process.execPath, fixturePath],
+    tool,
+    inputMapping,
+  };
+}
 
 describe("MCP adapter", () => {
   test("calls named tool and returns text output", async () => {
     const result = await callMcpAdapter(
-      { type: "mcp", command: ["node", "mcp-server.js"], tool: "echo" },
+      fixtureConfig("echo"),
       "hello world"
     );
     expect(result.output).toBe("echo: hello world");
@@ -55,7 +29,7 @@ describe("MCP adapter", () => {
 
   test("concatenates multiple text content blocks", async () => {
     const result = await callMcpAdapter(
-      { type: "mcp", command: ["node", "mcp-server.js"], tool: "multi_content" },
+      fixtureConfig("multi_content"),
       "x"
     );
     expect(result.output).toContain("part one");
@@ -64,41 +38,31 @@ describe("MCP adapter", () => {
 
   test("works with JSON output", async () => {
     const result = await callMcpAdapter(
-      { type: "mcp", command: ["node", "mcp-server.js"], tool: "json_tool" },
+      fixtureConfig("json_tool"),
       "x"
     );
     expect(result.output).toContain('"result"');
   });
 
   test("uses inputMapping to map input to named argument", async () => {
     const result = await callMcpAdapter(
-      {
-        type: "mcp",
-        command: ["node", "mcp-server.js"],
-        tool: "mapped_tool",
-        inputMapping: { query: "{{input}}" },
-      },
+      fixtureConfig("mapped_tool", { query: "{{input}}" }),
       "search term"
     );
     expect(result.output).toContain("search term");
   });
 
   test("passes static values in inputMapping", async () => {
     const result = await callMcpAdapter(
-      {
-        type: "mcp",
-        command: ["node", "mcp-server.js"],
-        tool: "mapped_tool",
-        inputMapping: { query: "fixed query" },
-      },
+      fixtureConfig("mapped_tool", { query: "fixed query" }),
       "ignored input"
     );
     expect(result.output).toContain("fixed query");
   });
 
   test("returns error on tool execution failure", async () => {
     const result = await callMcpAdapter(
-      { type: "mcp", command: ["node", "mcp-server.js"], tool: "error_tool" },
+      fixtureConfig("error_tool"),
       "x"
     );
     expect(result.error).toBeTruthy();
@@ -115,7 +79,7 @@ describe("MCP adapter", () => {
 
   test("tracks durationMs", async () => {
     const result = await callMcpAdapter(
-      { type: "mcp", command: ["node", "mcp-server.js"], tool: "echo" },
+      fixtureConfig("echo"),
       "timing test"
     );
     expect(typeof result.durationMs).toBe("number");

src/adapters/mcp-test-server.fixture.ts

@@ -0,0 +1,41 @@
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+
+const tools = ["echo", "json_tool", "multi_content", "error_tool", "mapped_tool"].map((name) => ({
+  name,
+  description: `Test fixture tool: ${name}`,
+  inputSchema: { type: "object" as const, additionalProperties: true },
+}));
+
+const server = new Server(
+  { name: "evals-mcp-adapter-test", version: "1.0.0" },
+  { capabilities: { tools: {} } },
+);
+
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools }));
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const args = (request.params.arguments ?? {}) as Record<string, unknown>;
+
+  switch (request.params.name) {
+    case "echo":
+      return { content: [{ type: "text", text: `echo: ${String(args["input"] ?? "")}` }] };
+    case "json_tool":
+      return { content: [{ type: "text", text: '{"result": "ok"}' }] };
+    case "multi_content":
+      return {
+        content: [
+          { type: "text", text: "part one" },
+          { type: "text", text: "part two" },
+        ],
+      };
+    case "error_tool":
+      throw new Error("Tool execution failed");
+    case "mapped_tool":
+      return { content: [{ type: "text", text: `query was: ${String(args["query"] ?? "")}` }] };
+    default:
+      throw new Error(`Unknown tool: ${request.params.name}`);
+  }
+});
+
+await server.connect(new StdioServerTransport());

src/core/runner.test.ts

@@ -1,47 +1,14 @@
-import { describe, test, expect, mock } from "bun:test";
+import { describe, test, expect } from "bun:test";
 import type { EvalCase, AdapterConfig } from "../types/index.js";
+import { runEvals, runSingleCase } from "./runner.js";
 
-// Mock adapters
-mock.module("../adapters/http.js", () => ({
-  callHttpAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })),
-}));
-mock.module("../adapters/anthropic.js", () => ({
-  callAnthropicAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })),
-}));
-mock.module("../adapters/openai.js", () => ({
-  callOpenAIAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })),
-}));
-mock.module("../adapters/mcp.js", () => ({
-  callMcpAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })),
-}));
-mock.module("../adapters/function.js", () => ({
-  callFunctionAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })),
-}));
-mock.module("../adapters/cli.js", () => ({
-  callCliAdapter: mock(async () => ({ output: "mock response", durationMs: 50 })),
-}));
-
-// Mock judge
-mock.module("../core/judge.js", () => ({
-  runJudge: mock(async () => ({
-    verdict: "PASS",
-    reasoning: "Looks good",
-    durationMs: 100,
-    inputTokens: 50,
-    outputTokens: 20,
-    costUsd: 0.001,
-  })),
-}));
-
-const { runEvals, runSingleCase } = await import("./runner.js");
-
-const adapter: AdapterConfig = { type: "http", url: "http://localhost:9999" };
+const modulePath = "data:text/javascript,export%20default%20async%20function()%7Breturn%20%22mock%20response%22%7D";
+const adapter: AdapterConfig = { type: "function", modulePath };
 
 const basicCase: EvalCase = {
   id: "test-001",
   input: "hello",
   assertions: [{ type: "min_length", value: 1 }],
-  judge: { rubric: "Should respond" },
 };
 
 describe("runSingleCase", () => {
@@ -55,7 +22,11 @@ describe("runSingleCase", () => {
   });
 
   test("skips judge when skipJudge=true", async () => {
-    const result = await runSingleCase(basicCase, adapter, true);
+    const result = await runSingleCase(
+      { ...basicCase, judge: { rubric: "Should respond" } },
+      adapter,
+      true,
+    );
     expect(result.judgeResult).toBeUndefined();
   });
 
@@ -96,22 +67,20 @@ describe("runEvals", () => {
   });
 
   test("redacts adapter apiKey from run metadata", async () => {
-    const secretAdapter: AdapterConfig = {
-      type: "openai",
-      model: "gpt-4o",
-      baseURL: "https://gateway.example.com/v1",
+    const secretAdapter = {
+      type: "function",
+      modulePath,
       apiKey: "provider-secret",
-    };
+    } as unknown as AdapterConfig;
 
     const run = await runEvals(
       [{ id: "secret-case", input: "hello" }],
       { dataset: "test.jsonl", adapter: secretAdapter, skipJudge: true }
     );
 
     expect(run.adapterConfig).toEqual({
-      type: "openai",
-      model: "gpt-4o",
-      baseURL: "https://gateway.example.com/v1",
+      type: "function",
+      modulePath,
     });
     expect(JSON.stringify(run)).not.toContain("provider-secret");
   });