diff --git a/src/__tests__/integration/api/workspaces/evals/evals.test.ts b/src/__tests__/integration/api/workspaces/evals/evals.test.ts index 8e38f01baa..ce30422440 100644 --- a/src/__tests__/integration/api/workspaces/evals/evals.test.ts +++ b/src/__tests__/integration/api/workspaces/evals/evals.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect, beforeEach, vi } from "vitest"; import { GET as getEvalSets, POST as createEvalSet } from "@/app/api/workspaces/[slug]/evals/route"; -import { POST as createRequirement } from "@/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route"; +import { GET as getRequirements, POST as createRequirement } from "@/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/route"; import { POST as linkRuns } from "@/app/api/workspaces/[slug]/evals/[evalSetId]/requirements/[reqId]/runs/route"; import { GET as getSessions } from "@/app/api/workspaces/[slug]/evals/sessions/route"; import { @@ -360,6 +360,187 @@ describe("Evals API — Integration Tests", () => { }); }); + // --------------------------------------------------------------------------- + // GET /api/workspaces/[slug]/evals/[evalSetId]/requirements + // --------------------------------------------------------------------------- + describe("GET /api/workspaces/[slug]/evals/[evalSetId]/requirements", () => { + describe("Success", () => { + test("returns requirements with order merged from edges", async () => { + const owner = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + await createTestMembership({ workspaceId: workspace.id, userId: owner.id, role: "OWNER" }); + await createTestSwarm({ workspaceId: workspace.id, swarmApiKey: "test-key" }); + + const mockNodes = [ + { ref_id: "req-1", node_type: "EvalRequirement", properties: { name: "Req A" } }, + { ref_id: "req-2", node_type: "EvalRequirement", properties: { name: "Req B" } }, + ]; + const mockEdges = [ + { target_ref_id: "req-1", properties: { order: 0 } }, + { target_ref_id: "req-2", properties: { order: 1 } }, + ]; + + global.fetch = vi.fn().mockResolvedValueOnce({ + ok: true, + json: async () => ({ nodes: mockNodes, edges: mockEdges }), + } as any); + + const request = createAuthenticatedGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/eval-set-1/requirements`, + owner, + ); + + const response = await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "eval-set-1" }), + }); + + expect(response.status).toBe(200); + const data = await response.json(); + expect(data.success).toBe(true); + expect(data.data.nodes).toHaveLength(2); + expect(data.data.total).toBe(2); + expect(data.data.nodes[0].properties.order).toBe(0); + expect(data.data.nodes[1].properties.order).toBe(1); + }); + + test("returns empty array when eval set has no requirements", async () => { + const owner = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + await createTestMembership({ workspaceId: workspace.id, userId: owner.id, role: "OWNER" }); + await createTestSwarm({ workspaceId: workspace.id, swarmApiKey: "test-key" }); + + global.fetch = vi.fn().mockResolvedValueOnce({ + ok: true, + json: async () => ({ nodes: [], edges: [] }), + } as any); + + const request = createAuthenticatedGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/eval-set-empty/requirements`, + owner, + ); + + const response = await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "eval-set-empty" }), + }); + + expect(response.status).toBe(200); + const data = await response.json(); + expect(data.success).toBe(true); + expect(data.data.nodes).toEqual([]); + expect(data.data.total).toBe(0); + }); + + test("calls Jarvis with correct URL including Python list literal params", async () => { + const owner = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + await createTestMembership({ workspaceId: workspace.id, userId: owner.id, role: "OWNER" }); + await createTestSwarm({ workspaceId: workspace.id, swarmApiKey: "test-key" }); + + const fetchMock = vi.fn().mockResolvedValueOnce({ + ok: true, + json: async () => ({ nodes: [], edges: [] }), + } as any); + global.fetch = fetchMock; + + const request = createAuthenticatedGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/my-eval-set/requirements`, + owner, + ); + + await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "my-eval-set" }), + }); + + const calledUrl: string = fetchMock.mock.calls[0][0]; + expect(calledUrl).toContain("/v2/nodes/my-eval-set"); + expect(calledUrl).toContain("expand=edges"); + expect(calledUrl).toContain(encodeURIComponent("['HAS_REQUIREMENT']")); + expect(calledUrl).toContain(encodeURIComponent("['EvalRequirement']")); + expect(calledUrl).toContain("depth=1"); + }); + }); + + describe("Auth failures", () => { + test("rejects unauthenticated requests", async () => { + const owner = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + + const request = createGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/eval-set-1/requirements`, + ); + + const response = await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "eval-set-1" }), + }); + + await expectUnauthorized(response); + }); + + test("rejects non-member", async () => { + const owner = await createTestUser(); + const nonMember = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + await createTestSwarm({ workspaceId: workspace.id, swarmApiKey: "test-key" }); + + const request = createAuthenticatedGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/eval-set-1/requirements`, + nonMember, + ); + + const response = await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "eval-set-1" }), + }); + + await expectForbidden(response, "Access denied"); + }); + }); + + describe("Swarm not configured", () => { + test("returns 400 when workspace has no swarm", async () => { + const owner = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + await createTestMembership({ workspaceId: workspace.id, userId: owner.id, role: "OWNER" }); + + const request = createAuthenticatedGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/eval-set-1/requirements`, + owner, + ); + + const response = await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "eval-set-1" }), + }); + + await expectError(response, "Swarm not configured", 400); + }); + }); + + describe("Upstream failure", () => { + test("returns 502 when Jarvis returns non-ok", async () => { + const owner = await createTestUser(); + const workspace = await createTestWorkspace({ ownerId: owner.id }); + await createTestMembership({ workspaceId: workspace.id, userId: owner.id, role: "OWNER" }); + await createTestSwarm({ workspaceId: workspace.id, swarmApiKey: "test-key" }); + + global.fetch = vi.fn().mockResolvedValueOnce({ + ok: false, + status: 503, + text: async () => "Service Unavailable", + } as any); + + const request = createAuthenticatedGetRequest( + `http://localhost:3000/api/workspaces/${workspace.slug}/evals/eval-set-1/requirements`, + owner, + ); + + const response = await getRequirements(request, { + params: Promise.resolve({ slug: workspace.slug, evalSetId: "eval-set-1" }), + }); + + expect(response.status).toBe(502); + }); + }); + }); + // --------------------------------------------------------------------------- // POST /api/workspaces/[slug]/evals/[evalSetId]/requirements // --------------------------------------------------------------------------- diff --git a/src/__tests__/unit/api/mock/evals-requirements.test.ts b/src/__tests__/unit/api/mock/evals-requirements.test.ts new file mode 100644 index 0000000000..58fb8cce16 --- /dev/null +++ b/src/__tests__/unit/api/mock/evals-requirements.test.ts @@ -0,0 +1,103 @@ +import { describe, test, expect } from "vitest"; +import { GET, POST } from "@/app/api/mock/evals/[evalSetId]/requirements/route"; +import { NextRequest } from "next/server"; + +function makeGetRequest(evalSetId: string): NextRequest { + return new NextRequest( + `http://localhost:3000/api/mock/evals/${evalSetId}/requirements`, + { method: "GET" }, + ); +} + +function makePostRequest(evalSetId: string, body: object): NextRequest { + return new NextRequest( + `http://localhost:3000/api/mock/evals/${evalSetId}/requirements`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }, + ); +} + +describe("GET /api/mock/evals/[evalSetId]/requirements", () => { + test("returns seeded requirements for eval-set-1", async () => { + const request = makeGetRequest("eval-set-1"); + const response = await GET(request, { params: Promise.resolve({ evalSetId: "eval-set-1" }) }); + const data = await response.json(); + + expect(data.success).toBe(true); + expect(data.data.nodes.length).toBeGreaterThan(0); + expect(data.data.total).toBe(data.data.nodes.length); + + // All nodes belong to EvalRequirement type + for (const node of data.data.nodes) { + expect(node.node_type).toBe("EvalRequirement"); + expect(node.ref_id).toBeDefined(); + expect(node.properties.name).toBeDefined(); + expect(node.properties.prompt_snippet).toBeDefined(); + expect(Array.isArray(node.properties.positive_cases)).toBe(true); + expect(Array.isArray(node.properties.negative_cases)).toBe(true); + expect(typeof node.properties.order).toBe("number"); + } + }); + + test("returns seeded requirements for eval-set-2", async () => { + const request = makeGetRequest("eval-set-2"); + const response = await GET(request, { params: Promise.resolve({ evalSetId: "eval-set-2" }) }); + const data = await response.json(); + + expect(data.success).toBe(true); + expect(data.data.nodes.length).toBeGreaterThan(0); + }); + + test("eval-set-1 and eval-set-2 return different requirements", async () => { + const res1 = await GET(makeGetRequest("eval-set-1"), { params: Promise.resolve({ evalSetId: "eval-set-1" }) }); + const res2 = await GET(makeGetRequest("eval-set-2"), { params: Promise.resolve({ evalSetId: "eval-set-2" }) }); + + const data1 = await res1.json(); + const data2 = await res2.json(); + + const ids1 = data1.data.nodes.map((n: { ref_id: string }) => n.ref_id); + const ids2 = data2.data.nodes.map((n: { ref_id: string }) => n.ref_id); + + // No overlap between the two sets + const overlap = ids1.filter((id: string) => ids2.includes(id)); + expect(overlap).toHaveLength(0); + }); + + test("returns empty array for unknown eval set id", async () => { + const request = makeGetRequest("unknown-eval-set-xyz"); + const response = await GET(request, { params: Promise.resolve({ evalSetId: "unknown-eval-set-xyz" }) }); + const data = await response.json(); + + expect(data.success).toBe(true); + expect(data.data.nodes).toEqual([]); + expect(data.data.total).toBe(0); + }); + + test("returns 200 status", async () => { + const request = makeGetRequest("eval-set-1"); + const response = await GET(request, { params: Promise.resolve({ evalSetId: "eval-set-1" }) }); + expect(response.status).toBe(200); + }); +}); + +describe("POST /api/mock/evals/[evalSetId]/requirements", () => { + test("creates a new requirement node and returns its ref_id", async () => { + const body = { + name: "Test Req", + description: "A description", + prompt_snippet: "When asked to do X", + positive_cases: ["Does X correctly"], + negative_cases: ["Fails silently"], + }; + const request = makePostRequest("eval-set-1", body); + const response = await POST(request); + const data = await response.json(); + + expect(data.success).toBe(true); + expect(typeof data.data.ref_id).toBe("string"); + expect(data.data.ref_id.length).toBeGreaterThan(0); + }); +}); diff --git a/src/__tests__/unit/components/evals/EvalSetDetail.test.tsx b/src/__tests__/unit/components/evals/EvalSetDetail.test.tsx new file mode 100644 index 0000000000..b666cd063e --- /dev/null +++ b/src/__tests__/unit/components/evals/EvalSetDetail.test.tsx @@ -0,0 +1,169 @@ +/** + * @vitest-environment jsdom + */ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; + +globalThis.React = React; + +vi.mock("@/hooks/useWorkspace", () => ({ + useWorkspace: () => ({ slug: "test-ws" }), +})); + +vi.mock("sonner", () => ({ + toast: { error: vi.fn(), success: vi.fn() }, +})); + +vi.mock("@/components/evals/CreateRequirementModal", () => ({ + CreateRequirementModal: () => null, +})); + +vi.mock("@/components/evals/LinkRunModal", () => ({ + LinkRunModal: () => null, +})); + +vi.mock("@/components/ui/button", () => ({ + Button: ({ children, onClick, ...props }: any) => ( + + ), +})); + +vi.mock("@/components/ui/badge", () => ({ + Badge: ({ children }: any) => {children}, +})); + +vi.mock("@/components/ui/skeleton", () => ({ + Skeleton: () =>
, +})); + +vi.mock("lucide-react", () => ({ + ArrowLeft: () => ←, + Link2: () => 🔗, + Plus: () => +, +})); + +import { EvalSetDetail } from "@/components/evals/EvalSetDetail"; + +const EVAL_SET = { + ref_id: "eval-set-1", + node_type: "EvalSet", + properties: { name: "My Eval Set", description: "Test suite" }, +}; + +const MOCK_REQUIREMENTS = [ + { + ref_id: "req-1", + node_type: "EvalRequirement", + properties: { + name: "Req Alpha", + description: "First requirement", + prompt_snippet: "When asked to...", + positive_cases: ["Does A", "Does B"], + negative_cases: ["Does not C"], + order: 0, + }, + }, + { + ref_id: "req-2", + node_type: "EvalRequirement", + properties: { + name: "Req Beta", + description: "Second requirement", + prompt_snippet: "When instructed to...", + positive_cases: ["Does X"], + negative_cases: ["Does not Y"], + order: 1, + }, + }, +]; + +describe("EvalSetDetail", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("fetches from the correct requirements endpoint (not the list-all-evals endpoint)", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + json: async () => ({ data: { nodes: [], total: 0 } }), + }); + global.fetch = fetchMock as any; + + render(