From 3c04c62bd7449bed25d42f62035afab878e87f66 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:39:07 +0100 Subject: [PATCH 01/27] Add typecheck script and clear tsc --noEmit baseline - Add npm run typecheck (tsc --noEmit) as the source-of-truth type gate - Narrow exactOptionalPropertyTypes gaps via conditional spreads and optional-with-undefined fields across agent executor, orchestrator, observability, providers, and scorer - Make CheckItem.line/description optional to match runtime reality (reported items carry optional fields; output formatters use a separate Issue type, so display is unaffected) - Confine @ai-sdk/perplexity LanguageModelV1 -> LanguageModel skew to a single cast; ai@6 dropped V1 types and a provider upgrade is a separate, out-of-scope dependency change - No strict compiler options relaxed; src/agent/* left compiling only --- package.json | 1 + src/agent/executor.ts | 36 ++++++++++----------- src/cli/orchestrator.ts | 2 +- src/evaluators/violation-filter.ts | 4 +-- src/observability/factory.ts | 2 +- src/observability/langfuse-observability.ts | 4 +-- src/prompts/schema.ts | 4 +-- src/providers/perplexity-provider.ts | 7 +++- src/providers/vercel-ai-provider.ts | 22 +++++++------ src/scoring/scorer.ts | 5 ++- 10 files changed, 49 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 74f474d5..d9e16b86 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "tsx src/index.ts", "build": "tsup", + "typecheck": "tsc --noEmit", "start": "node dist/index.js", "lint": "eslint .", "lint:fix": "eslint . --fix", diff --git a/src/agent/executor.ts b/src/agent/executor.ts index 432c6d64..749a5685 100644 --- a/src/agent/executor.ts +++ b/src/agent/executor.ts @@ -1,7 +1,7 @@ import { readdir, readFile } from 'fs/promises'; import * as os from 'os'; import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type PromptEvaluationResult } from '../prompts/schema'; +import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; import type { PromptFile } from '../prompts/prompt-loader'; import { Type, Severity } from '../evaluators/types'; import { computeFilterDecision } from '../evaluators/violation-filter'; @@ -14,6 +14,7 @@ import { createReviewSessionStore } from './review-session-store'; import { buildAgentSystemPrompt } from './prompt-builder'; import { AgentToolError } from '../errors'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; +import type { FilePatternConfig } from '../boundaries/file-section-parser'; import { createAgentTools, listAvailableTools, @@ -53,7 +54,7 @@ export interface RunAgentExecutorParams { prompts: PromptFile[]; provider: LLMProvider; workspaceRoot: string; - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }>; + scanPaths: FilePatternConfig[]; outputFormat: OutputFormat; printMode: boolean; sessionHomeDir?: string; @@ -178,7 +179,7 @@ function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { function buildFileRuleMatches( relativeTargets: string[], prompts: PromptFile[], - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }> + scanPaths: FilePatternConfig[] ): Array<{ file: string; ruleSource: string }> { const resolver = new ScanPathResolver(); const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); @@ -301,12 +302,13 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'read_file', path, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `read_file:${path}`, }; } @@ -317,10 +319,11 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = currentProgressFile ?? defaultProgressFile; return { toolName: 'list_directory', path, - progressFile: currentProgressFile ?? defaultProgressFile, + ...(progressFile ? { progressFile } : {}), signature: `list_directory:${path}`, }; } @@ -333,14 +336,15 @@ function resolveVisibleToolContext(params: { const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); const path = resolvedPath ?? parsed.data.file; const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'lint', path, ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), ruleText, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, }; } @@ -390,11 +394,7 @@ type FindingLikeViolation = { suggestion?: string; fix?: string; confidence?: number; - checks?: { - plausible_non_violation?: boolean; - context_supports_violation?: boolean; - rule_supports_claim?: boolean; - }; + checks?: GateChecks; }; async function appendInlineFinding(params: { @@ -516,8 +516,8 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, promptBySource, targetFiles, - currentProgressFile, - defaultProgressFile: relativeTargets[0], + ...(currentProgressFile ? { currentProgressFile } : {}), + ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), }); if (visibleToolContext?.progressFile) { @@ -798,7 +798,7 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, fileRuleMatches, availableTools, - userInstructions, + ...(userInstructions ? { userInstructions } : {}), }), prompt: [ `Workspace root: ${workspaceRoot}`, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f4325095..6ae4acd4 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1194,7 +1194,7 @@ async function evaluateFilesInAgentMode( progressReporter, maxParallelToolCalls: 3, maxRetries: options.agentMaxRetries ?? 10, - userInstructions: options.userInstructionContent, + ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), }); let totalErrors = 0; diff --git a/src/evaluators/violation-filter.ts b/src/evaluators/violation-filter.ts index 2d3fc295..c4daa9b5 100644 --- a/src/evaluators/violation-filter.ts +++ b/src/evaluators/violation-filter.ts @@ -56,8 +56,8 @@ export function computeFilterDecision(v: GateViolationLike): FilterDecision { const hasFix = (v.fix ?? "").trim() !== ""; - const hasConfidence = typeof v.confidence === "number"; - const passesConfidence = hasConfidence && v.confidence >= confidenceThreshold; + const confidence = typeof v.confidence === "number" ? v.confidence : undefined; + const passesConfidence = confidence !== undefined && confidence >= confidenceThreshold; if (!passesConfidence) reasons.push(`confidence<${confidenceThreshold}`); const surface = diff --git a/src/observability/factory.ts b/src/observability/factory.ts index 84ec671a..c646fc08 100644 --- a/src/observability/factory.ts +++ b/src/observability/factory.ts @@ -19,6 +19,6 @@ export function createObservability(env: EnvConfig, logger?: Logger): AIObservab publicKey: env.LANGFUSE_PUBLIC_KEY, secretKey: env.LANGFUSE_SECRET_KEY, ...(env.LANGFUSE_BASE_URL ? { baseUrl: env.LANGFUSE_BASE_URL } : {}), - logger, + ...(logger ? { logger } : {}), }); } diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index b8898127..25f4d32c 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -12,8 +12,8 @@ export interface LangfuseObservabilityConfig { } export class LangfuseObservability implements AIObservability { - private sdk?: NodeSDK; - private initPromise?: Promise; + private sdk?: NodeSDK | undefined; + private initPromise?: Promise | undefined; private readonly logger: Logger; constructor(private readonly config: LangfuseObservabilityConfig) { diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 26c137e1..bd9fd9ee 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -337,8 +337,8 @@ export type JudgeResult = { }; export type CheckItem = { - line: number; - description: string; + line?: number; + description?: string; analysis: string; message?: string; suggestion?: string; diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts index c6a05499..432f8c09 100644 --- a/src/providers/perplexity-provider.ts +++ b/src/providers/perplexity-provider.ts @@ -1,4 +1,5 @@ import { generateText } from 'ai'; +import type { LanguageModel } from 'ai'; import { z } from 'zod'; import { createPerplexity } from '@ai-sdk/perplexity'; import type { SearchProvider } from './search-provider'; @@ -46,7 +47,11 @@ export class PerplexitySearchProvider implements SearchProvider { try { const result = await generateText({ - model: this.client('sonar-pro'), + // @ai-sdk/perplexity@1 exposes LanguageModelV1 models, while ai@6's + // generateText types require a LanguageModel (V2/V3). This is a + // third-party SDK version skew, not a repo type hole; resolve by + // upgrading @ai-sdk/perplexity to a V2 release in a follow-up. + model: this.client('sonar-pro') as unknown as LanguageModel, prompt: query, }); diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 689a19c0..5cd833fd 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -25,7 +25,7 @@ export class VercelAIProvider implements LLMProvider { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; - private observability?: AIObservability; + private observability?: AIObservability | undefined; constructor(config: VercelAIConfig, builder?: RequestBuilder) { this.config = { @@ -73,12 +73,14 @@ export class VercelAIProvider implements LLMProvider { } try { + const evaluator = this.extractContextValue(context, 'evaluatorName', 'evaluator'); + const rule = this.extractContextValue(context, 'ruleName', 'rule'); const observabilityOptions = this.getObservabilityOptions({ operation: 'structured-eval', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', - evaluator: this.extractContextValue(context, 'evaluatorName', 'evaluator'), - rule: this.extractContextValue(context, 'ruleName', 'rule'), + ...(evaluator ? { evaluator } : {}), + ...(rule ? { rule } : {}), }); const result = await generateText({ @@ -187,14 +189,14 @@ export class VercelAIProvider implements LLMProvider { } } - return { - usage: result.usage - ? { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, + return result.usage + ? { + usage: { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + }, } - : undefined, - }; + : {}; } private getObservabilityOptions(context: AIExecutionContext): Record { diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index ecea0db6..24f0ced5 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -48,7 +48,10 @@ export function calculateCheckScore( ): CheckResult { const strictness = resolveStrictness(options.strictness); - const mappedViolations = violations.map((item) => ({ ...item, criterionName: item.description })); + const mappedViolations = violations.map((item) => ({ + ...item, + ...(item.description !== undefined ? { criterionName: item.description } : {}), + })); // Density Calculation: Violations per 100 words const density = (mappedViolations.length / wordCount) * 100; From e319bba09fb1e07af044a55161da89ebb6510313 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:40:19 +0100 Subject: [PATCH 02/27] Exclude docs/research from eslint - Add docs/research/** to eslint ignores; the directory holds local, untracked throwaway research scripts that are not part of the shipped package and should not be type-checked by lint --- eslint.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 8549cd13..4bec57ff 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -10,7 +10,7 @@ import unicorn from "eslint-plugin-unicorn"; export default defineConfig([ // Ignored + housekeeping - { ignores: ["node_modules", "coverage", "dist", "build"] }, + { ignores: ["node_modules", "coverage", "dist", "build", "docs/research/**"] }, { linterOptions: { reportUnusedDisableDirectives: true } }, // Make resolver settings global From 20e0acc8204f1b67858ab7b31bf3f5b023c2799a Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:25 +0100 Subject: [PATCH 03/27] Add vitest.config.ts for stable test resolution - Project had no vitest config; vitest ran on pure defaults - Pin test discovery to tests/**/*.test.ts and inline ora, @langfuse/otel, and @opentelemetry/sdk-node so suites that transitively import agent/observability modules resolve reliably - No change to the 45 suites / 323 tests baseline --- vitest.config.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 vitest.config.ts diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 00000000..b5f7a7b4 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // Tests live under tests/ and follow the *.test.ts convention. + include: ['tests/**/*.test.ts'], + // Inline heavy/ESM-only deps so suites that transitively import the + // agent (ora) or observability (@langfuse/otel, @opentelemetry/sdk-node) + // modules resolve reliably without relying on scattered per-file mocks. + server: { + deps: { + inline: ['ora', '@langfuse/otel', '@opentelemetry/sdk-node'], + }, + }, + }, +}); From 3580e4971c2e6cfa8ce2308d350803c58d43ddba Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:59 +0100 Subject: [PATCH 04/27] Add npm run verify aggregate gate - Runs typecheck, lint, and test:run in sequence - Single command for CI and downstream refactor phases to prove the verification baseline --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d9e16b86..3a73ad4d 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint:fix": "eslint . --fix", "test": "vitest", "test:run": "vitest run", - "test:ci": "vitest run --coverage" + "test:ci": "vitest run --coverage", + "verify": "npm run typecheck && npm run lint && npm run test:run" }, "bin": { "vectorlint": "./dist/index.js", From 0fe8700b2e4d2ded0f51ef10e30e1f7fb4f39af5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:47:58 +0100 Subject: [PATCH 05/27] Add typecheck CI job and align test workflow Node version - New typecheck.yml runs tsc --noEmit on push/PR for main and release-docs - Bump test.yml from Node 18 to 20 to satisfy package.json engines>=20.6 - Catches type regressions in CI before merge, not just via ESLint --- .github/workflows/test.yml | 2 +- .github/workflows/typecheck.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/typecheck.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcf03d6f..06411499 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'npm' - name: Install dependencies diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 00000000..73dad374 --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,28 @@ +name: Typecheck + +on: + push: + branches: [main, release-docs] + pull_request: + branches: [main, release-docs] + +jobs: + typecheck: + name: tsc --noEmit + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run typecheck + run: npm run typecheck From 6c271f121b80e05e5e3d7ae2d7b6c02352755abd Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:58:45 +0100 Subject: [PATCH 06/27] Deprecate --mode agent and fall back to standard - README Agent Mode section replaced with under-review notice linking the audit - --mode agent now warns through the injected logger and runs standard evaluation; the agent executor code is retained (unreachable from the CLI) pending Phase 4 removal - Add logger to EvaluationOptions so the deprecation warning routes through the logging abstraction instead of console - Rewrite orchestrator-agent-output tests to assert deprecation + fallback BREAKING: --mode agent no longer runs the autonomous workspace-agent loop --- README.md | 29 +- src/cli/commands.ts | 3 +- src/cli/orchestrator.ts | 9 +- src/cli/types.ts | 2 + tests/orchestrator-agent-output.test.ts | 1153 ++--------------------- 5 files changed, 80 insertions(+), 1116 deletions(-) diff --git a/README.md b/README.md index 859ea51a..f6511e6a 100644 --- a/README.md +++ b/README.md @@ -145,29 +145,16 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode +## Agent Mode (under review) -Agent mode is for reviews that need context from several files such as -documentation drift and cross-file accuracy. - -Run VectorLint in autonomous agent mode: - -```bash -vectorlint doc.md --mode agent -``` - -For machine-parseable output: - -```bash -vectorlint doc.md --mode agent --output json -``` - -To suppress interactive progress in line output: - -```bash -vectorlint doc.md --mode agent --print -``` +The `--mode agent` flag is under active rework. It currently enables an +autonomous cross-file review mode that is being **removed** in favor of a +bounded harness model. See +[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) +for the decision and the refactor plan. +Until the refactor lands, `--mode agent` prints a deprecation warning and +falls back to standard mode. Do not build integrations against it. ## Contributing diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 369eda1f..50dbffc8 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -82,7 +82,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default) or agent', DEFAULT_REVIEW_MODE) + .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) .option('-p, --print', 'Suppress interactive progress output in agent mode') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') @@ -217,6 +217,7 @@ export function registerMainCommand(program: Command): void { ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, + logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, mode: cliOptions.mode, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 6ae4acd4..f8df367a 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1168,6 +1168,9 @@ async function buildAgentRuleScores( return results; } +// Retained in quarantine: unreachable from the CLI after --mode agent was +// deprecated (now falls back to standard evaluation). Removed in Phase 4. +// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine async function evaluateFilesInAgentMode( targets: string[], options: EvaluationOptions, @@ -1322,7 +1325,11 @@ export async function evaluateFiles( } if (mode === AGENT_REVIEW_MODE) { - return evaluateFilesInAgentMode(targets, options, outputFormat, jsonFormatter); + options.logger?.warn( + '--mode agent is deprecated and now falls back to standard mode. ' + + 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', + ); + // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } for (const file of targets) { diff --git a/src/cli/types.ts b/src/cli/types.ts index a73f55e6..7f95192a 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -10,6 +10,7 @@ import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; +import type { Logger } from '../logging/logger'; export enum OutputFormat { Line = "line", @@ -48,6 +49,7 @@ export interface EvaluationOptions { agentMaxRetries?: number; pricing?: PricingConfig; userInstructionContent?: string; + logger?: Logger; } export interface EvaluationResult { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index e08bbdaa..0b61ea63 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -2,28 +2,21 @@ import { mkdtempSync, rmSync, writeFileSync } from 'fs'; import * as path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; +import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; import type { PromptFile } from '../src/prompts/prompt-loader'; import type { LLMProvider } from '../src/providers/llm-provider'; +import type { Logger } from '../src/logging/logger'; import { Severity } from '../src/evaluators/types'; -function makePrompt(params?: { - id?: string; - name?: string; - source?: string; - body?: string; -}): PromptFile { - const id = params?.id ?? 'consistency'; - const name = params?.name ?? 'Consistency'; - const source = params?.source ?? 'packs/default/consistency.md'; - const body = params?.body ?? 'Find inconsistent wording'; - +function makePrompt(): PromptFile { + const id = 'consistency'; + const name = 'Consistency'; return { id, filename: `${id}.md`, - fullPath: source, + fullPath: 'packs/default/consistency.md', pack: 'Default', - body, + body: 'Find inconsistent wording', meta: { id: name, name, @@ -33,91 +26,39 @@ function makePrompt(params?: { }; } -function makeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; -} +type LoggerSpy = Logger & { warn: ReturnType }; -function makeTopLevelOnlyProvider(): LLMProvider { +function makeLogger(): LoggerSpy { return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Top-level without references', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as unknown as LoggerSpy; } -function makeCrossFileTopLevelProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'other.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; +interface StandardProviderSpies { + provider: LLMProvider; + runPromptStructured: ReturnType; + runAgentToolLoop: ReturnType; } -function makeNoFinalizeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Finding recorded before session ended', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - // intentionally omit finalize_review to simulate missing-finalize scenario - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; +function makeStandardProvider(): StandardProviderSpies { + const runPromptStructured = vi.fn().mockResolvedValue({ + data: { reasoning: 'ok', violations: [] }, + }); + const runAgentToolLoop = vi.fn().mockResolvedValue({ + usage: { inputTokens: 0, outputTokens: 0 }, + }); + const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; + return { provider, runPromptStructured, runAgentToolLoop }; } -describe('agent orchestrator output', () => { - const originalIsTTY = Object.getOwnPropertyDescriptor(process.stderr, 'isTTY'); +// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no +// longer reaches the autonomous agent executor: it warns through the injected +// logger and falls back to standard evaluation. The retained agent executor +// code is covered directly by tests/agent/* and removed in Phase 4. +describe('agent mode deprecation', () => { const tempRepos: string[] = []; function createTempRepo(): string { @@ -137,55 +78,15 @@ describe('agent orchestrator output', () => { for (const repo of tempRepos.splice(0, tempRepos.length)) { rmSync(repo, { recursive: true, force: true }); } - if (originalIsTTY) { - Object.defineProperty(process.stderr, 'isTTY', originalIsTTY); - } }); - it('produces agent-mode findings and summary when mode is set to agent', async () => { + it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalFiles).toBe(1); - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.hadOperationalErrors).toBe(false); - }); - - it('includes nested lint usage in the final agent-mode token totals', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { reasoning: 'ok', violations: [] }, - usage: { inputTokens: 7, outputTokens: 3 }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -194,338 +95,31 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.tokenUsage).toEqual({ - totalInputTokens: 18, - totalOutputTokens: 8, + logger, }); - }); - - it('keeps json output shape consistent with formatter-based structure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const payload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record; - summary?: { files?: number; errors?: number; warnings?: number }; - metadata?: { version?: string; timestamp?: string }; - }; - - expect(payload.files).toBeDefined(); - expect(payload.summary).toBeDefined(); - expect(typeof payload.summary?.files).toBe('number'); - expect(typeof payload.summary?.errors).toBe('number'); - expect(typeof payload.summary?.warnings).toBe('number'); - expect(payload.metadata?.timestamp).toBeTruthy(); - }); - - it('keeps top-level json keys consistent between standard and agent modes', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const standardPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const agentPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - expect(Object.keys(agentPayload).sort()).toEqual( - Object.keys(standardPayload).sort() + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('deprecated'), ); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), + ); + // Standard evaluation ran; the autonomous executor did not. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); - it('emits configured agent progress messages in line output mode', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain(' └ Found no issues in doc.md'); - expect(stderrOutput).not.toContain('[vectorlint]'); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders visible tool invocations and results while hiding internal agent tools', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.search_files.execute({ pattern: '**/*.md' }); - await tools.read_file.execute({ path: 'doc.md' }); - await tools.list_directory.execute({ path: '.' }); - await tools.search_content.execute({ pattern: 'bad phrase', path: '.', glob: '**/*.md' }); - await tools.finalize_review.execute({}); - - return { usage: { inputTokens: 6, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Lint("Find inconsistent wording...")'); - expect(stderrOutput).toContain('Found no issues in doc.md'); - expect(stderrOutput).toContain('Read(doc.md)'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('List(.)'); - expect(stderrOutput).toContain('Listed 1 entry in .'); - expect(stderrOutput).not.toContain('SearchFiles('); - expect(stderrOutput).not.toContain('SearchContent('); - expect(stderrOutput).not.toContain('Finalize('); - }); - - it('renders interactive tool lines without a trailing newline', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const firstToolLine = stderrSpy.mock.calls - .map((call) => String(call[0])) - .find((chunk) => chunk.includes(' └ ')); - - expect(firstToolLine).toBeDefined(); - expect(firstToolLine?.endsWith('\n')).toBe(false); - }); - - it('uses in-place progress updates for repeated tool calls instead of appending only plain lines', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/wordiness.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - makePrompt({ id: 'wordiness', name: 'Wordiness', source: 'packs/default/wordiness.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('\x1b[1A'); - }); - - it('updates progress rule labels as the active lint rule changes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('for AI Pattern'); - expect(stderrOutput).toContain('for Consistency'); - }); - - it('shows visible tool retry status after a visible tool failure', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('does not invoke the agent executor in line output mode', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); - const retriable = path.join(repo, 'retriable.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: 'retriable.md' })).rejects.toThrow(); - writeFileSync(retriable, 'hello\n', 'utf8'); - await tools.read_file.execute({ path: 'retriable.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); await evaluateFiles([file], { prompts: [makePrompt()], @@ -534,115 +128,23 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: false, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading retriable.md'); - expect(stderrOutput).toContain('Retrying Read(retriable.md)...'); - expect(stderrOutput).toContain('Read 1 line from retriable.md'); - }); - - it('shows visible-tool path errors even when path validation fails before file access', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, + logger, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading ../outside.md'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); }); - it('shows quality scores in agent line output and keeps operational failures explicit when finalize_review is missing', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('stays silent in standard mode without a logger', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 6, outputTokens: 3 } }; - }, - } as unknown as LLMProvider; + const { provider, runAgentToolLoop } = makeStandardProvider(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -650,549 +152,14 @@ describe('agent orchestrator output', () => { provider, concurrency: 1, verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(0); - expect(stdout).toContain('Use consistent wording'); - expect(stdout).toContain('Quality Scores:'); - expect(stderrSpy).toHaveBeenCalled(); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Review failed after'); - expect(stderrOutput).not.toContain('Completed review in'); - }); - - it('suppresses progress output when print mode is enabled', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('returns machine-parseable json output without progress text contamination', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stdout).not.toContain('Completed review.'); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('appends a new two-line block when agent work moves to the next file', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const fileOne = path.join(repo, 'doc.md'); - const fileTwo = path.join(repo, 'doc2.md'); - writeFileSync(fileOne, 'bad phrase\n', 'utf8'); - writeFileSync(fileTwo, 'another bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.read_file.execute({ path: 'doc2.md' }); - await tools.lint.execute({ file: 'doc2.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 4, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([fileOne, fileTwo], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('Reviewing doc2.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc2.md'); - const firstFileIndex = stderrOutput.indexOf('Reviewing doc.md for Consistency'); - const secondFileIndex = stderrOutput.indexOf('Reviewing doc2.md for Consistency'); - expect(firstFileIndex).toBeGreaterThan(-1); - expect(secondFileIndex).toBeGreaterThan(firstFileIndex); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders top-level findings without explicit references at 1:1 in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeTopLevelOnlyProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('Top-level without references'); - expect(stdout).toContain('1:1'); - }); - - it('prints lazy file headers for non-target referenced findings in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - const otherFile = path.join(repo, 'other.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - writeFileSync(otherFile, 'other content\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeCrossFileTopLevelProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('other.md'); - expect(stdout).toContain('Cross-file issue found'); - }); - - it('scores against every matched file, including clean files without findings', async () => { - const repo = createTempRepo(); - const firstFile = path.join(repo, 'doc.md'); - const secondFile = path.join(repo, 'other.md'); - writeFileSync(firstFile, 'one two three four five six seven eight nine ten\n', 'utf8'); - writeFileSync(secondFile, 'alpha beta gamma delta epsilon zeta eta theta iota kappa\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'one', - context_before: '', - context_after: '', - description: 'Issue found', - analysis: 'Needs cleanup.', - message: 'Fix the wording', - suggestion: 'Use clearer wording', - fix: 'clear wording', - rule_quote: 'Be precise', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([firstFile, secondFile], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('5.0/10'); - }); - - it('keeps canonical rule identity and warning severity aligned across json and rdjson outputs', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: DEFAULT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const jsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record }>; - }; - - const jsonIssue = - jsonPayload.files?.['doc.md']?.issues?.find( - (issue) => issue.rule === 'Default.Consistency' - ) ?? jsonPayload.files?.['doc.md']?.issues?.[0]; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const rdjsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - diagnostics?: Array<{ code?: { value?: string }; severity?: string }>; - }; - - const rdjsonDiagnostic = - rdjsonPayload.diagnostics?.find( - (diagnostic) => diagnostic.code?.value === 'Default.Consistency' - ) ?? rdjsonPayload.diagnostics?.[0]; - - expect(jsonIssue?.rule).toBe('Default.Consistency'); - expect(jsonIssue?.severity).toBe(Severity.WARNING); - expect(rdjsonDiagnostic?.code?.value).toBe('Default.Consistency'); - expect(rdjsonDiagnostic?.severity).toBe(Severity.WARNING); - }); - - it('keeps rdjson output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('keeps vale-json output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.ValeJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('surfaces findings recorded before missing finalize while still reporting an operational failure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeNoFinalizeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.requestFailures).toBe(0); - expect(result.hadOperationalErrors).toBe(true); - }); - - it('passes userInstructionContent into the agent system prompt', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let capturedSystemPrompt = ''; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const systemPrompt = params.systemPrompt; - capturedSystemPrompt = typeof systemPrompt === 'string' ? systemPrompt : ''; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - userInstructionContent: 'Always enforce concise phrasing.', - } as never); - - expect(capturedSystemPrompt).toContain('User Instructions (from VECTORLINT.md):'); - expect(capturedSystemPrompt).toContain('Always enforce concise phrasing.'); - }); - - it('counts provider tool-loop failures as request failures in agent mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop() { - return Promise.reject(new Error('provider request failed')); - }, - } as unknown as LLMProvider; - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(1); - }); - - it('passes a default agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(10); - }); - - it('passes configured agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - agentMaxRetries: 4, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(4); + // No deprecation warning and no executor invocation in standard mode. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); }); From a2e9f25c8935163285235736ac70ff5926d47b95 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:10:35 +0100 Subject: [PATCH 07/27] Append Phase 1 completion note to harness audit - Record the shifted 2026-07-13 baseline: only tsc --noEmit was failing; lint and test:run were already green, so the audit's lint and four-suite module-resolution failures are stale - Note the durable Phase 1 gates (typecheck, verify, vitest.config, docs/research lint exclusion, Node 20 typecheck/test workflows) - Point to the --mode agent deprecation and standard fallback as the precondition for Phases 2-5; record that no spec or architecture doc is superseded in this appendix --- ...0-vectorlint-harness-architecture-audit.md | 380 ++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md new file mode 100644 index 00000000..51161ce0 --- /dev/null +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -0,0 +1,380 @@ +# VectorLint Harness Architecture Audit + +Date: 2026-07-10 + +Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. + +## Product Decision + +VectorLint should not keep the current autonomous workspace-agent mode. + +The retained agent-like capability should be a constrained reader executor: a model or headless agent can read the target content in sections when that is useful for context management, but it should not search the workspace, inspect arbitrary files, perform top-level workspace checks, or rewrite the rule being evaluated. + +The intended execution model is: + +- `direct`: send target content and rule in one structured review call. +- `reader`: give the executor a read-section capability over the target content only. +- `auto`: choose `direct` for normal-sized inputs and `reader` for large inputs or explicit context-management needs. + +This is an execution strategy decision, not a return to VectorLint-as-agent. + +## Product Direction Under Review + +VectorLint should become a programmable, bounded, on-page content review harness that another caller can invoke. + +The caller can be Codex, Claude Code, another coding agent, CI, a local CLI wrapper, or a future service. That caller owns exploration, cross-page reasoning, research, and context gathering. VectorLint owns the constrained review of target content against source-backed rules and returns structured findings, scores, diagnostics, and usage metadata. + +In practical terms: + +- VectorLint reviews the target page or explicitly supplied content. +- External context is caller-supplied, not discovered by VectorLint. +- Rules define review behavior, granularity, severity, and scoring constraints. +- Executors can be API models, headless local agents, or constrained reader executors, but they receive the same review request contract. +- VectorLint should not expose a model-controlled workspace tool loop as its core product surface. + +## Objective System Audit Summary + +Separate from product direction, the current codebase has several system-level liabilities: + +- Build health is weak: `npx tsc --noEmit`, `npm run lint`, and parts of `npm run test:run` fail in the current workspace. +- Runtime contracts are duplicated across hand-written TypeScript, JSON-schema objects, Zod schemas, and formatter-specific shapes. +- Standard mode and agent mode project the same underlying findings differently. +- CLI orchestration owns too many responsibilities: config, file matching, evaluation, scoring, output formatting, debug artifacts, and agent routing. +- Evidence verification and counting are inconsistent enough to affect exit behavior. +- Observability and debug paths can record full prompts, content, and outputs. +- Documentation and implementation disagree on the current agent-mode topology and tool contracts. + +## Architecture Verdict + +The current codebase contains two execution models at once: + +1. The older evaluator path: one structured model call per rule or chunk. +2. The newer autonomous workspace-agent path: VectorLint gives a model tools to read/search the workspace and call lint as a nested tool. + +The second path should not continue as-is. The useful piece is not "agent mode" broadly; it is target-aware reading for context management. The current implementation also duplicates result projection, scoring, evidence validation, output formatting, and configuration behavior from the first path. That is why the architecture feels harder to maintain: the code is not just messy; it is serving two different ownership models. + +The highest-leverage change is to define a neutral review-domain contract and make every executor implement that contract. + +## Proposed Core Contract + +The contract should be centered on review, not providers or agents. + +```ts +interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; +} + +interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; +} +``` + +API models, reasoning models, and headless CLI agents become adapter implementations behind this contract. The CLI becomes orchestration around request creation, result formatting, and exit behavior. + +## Major Findings + +### 1. The Current Agent Surface Is Broader Than The Decided Execution Model + +Evidence: + +- `README.md:148` presents `--mode agent` as autonomous cross-file review mode. +- `src/agent/prompt-builder.ts:43` instructs the model to perform workspace-level checks. +- `src/agent/executor.ts:670` exposes `read_file`. +- `src/agent/executor.ts:715` exposes `search_content`. + +Impact: + +VectorLint still behaves like it owns exploration. That puts it in competition with external coding agents instead of becoming a harness those agents can use. It also overshoots the decided reader-executor model, which only needs target-scoped reading for context management. + +Recommendation: + +Remove the current autonomous workspace-agent surface. Preserve only a constrained reader executor if needed: + +- It may read sections of the target content. +- It may not search the workspace. +- It may not read arbitrary files. +- It may not perform top-level workspace findings. +- It may not override source-backed rules. + +### 2. The Provider Interface Mixes Structured Review With Autonomous Tool Loops + +Evidence: + +- `src/providers/llm-provider.ts:28` requires both `runPromptStructured` and `runAgentToolLoop`. +- `src/providers/vercel-ai-provider.ts:136` implements the agent loop on the same provider. +- `src/agent/executor.ts:796` depends on the provider-level agent loop. + +Impact: + +Every future executor has to pretend to be both a structured model provider and an autonomous agent runner. That is the wrong abstraction for headless CLI support and for a constrained reader executor. + +Recommendation: + +Split the contracts: + +- `StructuredModelClient` for API structured output. +- `HeadlessReviewExecutor` for local CLI agents. +- `ReaderReviewExecutor` for target-scoped read-section execution. +- `ReviewExecutor` as the stable domain-level interface. +- `TelemetrySink` or observability decoration as an optional cross-cutting concern. + +### 3. Runtime Type Contracts Are Not Strong Enough For A Harness + +Evidence: + +- `src/providers/vercel-ai-provider.ts:119` returns `output as T`. +- `src/providers/llm-provider.ts:9` defines tool input and output as `unknown`. +- `src/agent/tools-registry.ts:21` exposes tools without typed output contracts. +- `npx tsc --noEmit` currently fails in core files. + +Impact: + +The code presents types as guarantees, but several are only assertions. A programmable harness needs reliable runtime validation because external callers and headless adapters will depend on these contracts. + +Recommendation: + +Tie structured execution to concrete Zod schemas or canonical parsers. Introduce typed tool contracts only if tools remain. Add a `typecheck` script and wire it into CI and build. + +### 4. Standard Mode And Agent Mode Project Results Differently + +Evidence: + +- `src/cli/orchestrator.ts:626` derives standard check severity from scoring. +- `src/agent/executor.ts:431` stamps agent findings with prompt severity. +- `src/agent/executor.ts:598` flattens judge criteria before recording findings. +- `src/cli/orchestrator.ts:1216` builds agent scores only for line output. +- `src/cli/orchestrator.ts:1237` emits structured output without agent scores or diagnostics. + +Impact: + +The same underlying review can produce different severity, rule identity, score data, JSON shape, and exit behavior depending on mode. That makes VectorLint hard to trust as a machine-facing gate. + +Recommendation: + +Extract one shared result projection pipeline: + +```text +PromptEvaluationResult + -> verified findings + -> rule and criterion identity + -> severity and score + -> diagnostics + -> output formatter +``` + +Every executor should return into this pipeline. + +### 5. The On-Page Boundary Is Not Enforced + +Evidence: + +- `src/agent/executor.ts:585` lets `lint` resolve any file inside `workspaceRoot`. +- `src/agent/executor.ts:670` lets `read_file` read any workspace file. +- `src/agent/executor.ts:715` lets `search_content` scan workspace content. +- `src/agent/executor.ts:134` allows model-supplied `reviewInstruction`. +- `src/agent/executor.ts:583` replaces the source-backed prompt body with the model-supplied override. + +Impact: + +Prompt-injected page content can indirectly steer the model to read non-target files or rewrite the rule being evaluated. That breaks deterministic review semantics and creates privacy and security risk. + +Recommendation: + +For the new harness, enforce a target/context allowlist: + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller explicitly includes those files as context. +- Reader execution can page through target content only. +- Rule bodies are source-backed and caller-authored, not model-authored. + +### 6. Evidence Handling Can Misreport Findings + +Evidence: + +- Standard mode skips unverifiable quotes but can still count original surfaced violation totals in some paths. +- `src/agent/executor.ts:415` attempts to locate evidence. +- `src/agent/executor.ts:426` falls back to the model-provided line when location fails. + +Impact: + +Standard mode can fail a run without printing the issue that caused the failure. Agent mode can surface hallucinated evidence as if it were verified. + +Recommendation: + +Use one evidence verifier. Count only emitted, verified findings. If evidence cannot be located, return a diagnostic and either skip the finding or mark the run operationally failed. + +### 7. Cost And Work Are Not Bounded Enough + +Evidence: + +- `src/providers/vercel-ai-provider.ts:161` defaults agent loops to 1000 steps. +- `src/agent/executor.ts:577` allows repeated nested `lint` calls. +- `src/evaluators/base-evaluator.ts:213` runs one model call per chunk. +- `src/agent/executor.ts:680` and `src/agent/executor.ts:715` glob before applying result caps. + +Impact: + +A single review can multiply cost through agent steps, nested lint calls, rules, chunks, and workspace search. Even a constrained reader executor will be slower than a direct call, so `auto` must have clear thresholds and budgets. + +Recommendation: + +Add explicit budgets: + +- Max target bytes. +- Max caller context bytes. +- Max chunks per rule. +- Max model calls per review. +- Max findings per rule. +- Max wall-clock duration. +- Max headless executor retries. + +### 8. Documentation Still Points At The Old Product + +Evidence: + +- `README.md:148` documents autonomous workspace-agent mode. +- `docs/specs/2026-03-17-agentic-capabilities-design.md:24` frames the roadmap around cross-document agent mode. +- The same spec describes implementation details that are not true now: one agent per rule, final `AgentFindingSchema`, AbortSignal propagation, ripgrep JSON, and Vale JSON omission of agent findings. + +Impact: + +Future contributors will optimize toward the wrong architecture and trust contracts that the code does not implement. + +Recommendation: + +Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and migration plan. + +### 9. Secrets And Sensitive Content Need Better Handling + +Evidence: + +- `src/config/global-config.ts:13` says the global config stores API keys. +- `src/config/global-config.ts:76` creates the config directory with default permissions. +- `src/config/global-config.ts:81` writes the config file with default permissions. +- `src/observability/langfuse-observability.ts:66` records inputs. +- `src/observability/langfuse-observability.ts:67` records outputs. +- `src/providers/vercel-ai-provider.ts:59` can log full prompts and content. + +Impact: + +Provider keys, unreleased docs, caller context, and model outputs can leak through filesystem permissions, telemetry, debug logs, or persisted artifacts. + +Recommendation: + +Use private file modes for config and review artifacts. Make payload telemetry a separate opt-in from metadata telemetry. Add redaction or safe debug modes. + +## Verification Results + +Commands run from `/Users/klinsmann/Projects/TinyRocketLabs/vectorlint`: + +```bash +npm run lint +npm run test:run +npx tsc --noEmit +``` + +Results: + +- `npm run lint` failed before linting because typed ESLint rules were applied to a `.cjs` research script without parser type information. +- `npm run test:run` passed most tests but failed 4 suites during module resolution for `ora` and `@langfuse/otel`. +- `npx tsc --noEmit` failed with contract errors in agent, orchestrator, observability, logging, and provider modules. + +## Recommended Refactor Sequence + +### Phase 1: Stop The Bleeding + +1. Remove or hide the current autonomous workspace-agent mode. +2. Fix `tsc --noEmit`. +3. Fix test module resolution. +4. Fix lint configuration for non-TypeScript research scripts. +5. Add `typecheck` to CI and build validation. + +### Phase 2: Define The Harness Contract + +1. Create a neutral review-domain module. +2. Define `ReviewRequest`, `ReviewRule`, `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, and `ReviewDiagnostic`. +3. Define target/context boundary rules. +4. Define structured output shape once. +5. Define execution strategy: `direct`, `reader`, and `auto`. + +### Phase 3: Share Result Projection + +1. Extract evidence location and verification. +2. Extract filtering. +3. Extract severity and scoring. +4. Extract output formatter inputs. +5. Make standard mode and any executor mode use the same projection path. + +### Phase 4: Replace The Agent Loop With Executors + +1. Keep API model execution as direct execution. +2. Add target-scoped reader execution as another executor. +3. Add headless CLI execution behind the same review contract if still useful. +4. Pass the same review request to each executor. +5. Remove model-controlled rule overrides. +6. Remove workspace read/search tools from core review. + +### Phase 5: Update Documentation + +1. Mark the agentic capabilities spec superseded. +2. Write a new harness architecture spec. +3. Update README and CLI docs. +4. Document migration from autonomous workspace-agent mode to direct/reader/auto execution. + +## Suggested Target Architecture + +```text +CLI / API / External Agent + -> ReviewRequestBuilder + -> ReviewExecutor + -> ApiModelExecutor + -> ReaderExecutor + -> HeadlessCliExecutor + -> ResultProjection + -> EvidenceVerifier + -> ViolationFilter + -> ScoreCalculator + -> Diagnostics + -> OutputFormatter + -> line + -> json + -> rdjson + -> vale-json +``` + +The important inversion is that VectorLint no longer asks, "What tools should this agent use?" It asks, "Given this target, these rules, this context, this execution strategy, and this budget, what findings can be verified?" + +## Final Recommendation + +The decided direction is cleaner than the current architecture. The current pain is not a sign that VectorLint is too complex; it is a sign that the codebase is carrying two ownership models and several duplicated runtime contracts. + +Make VectorLint the harness. Let external agents be agents. Keep reader execution only as a bounded way to manage target-content context. + +--- + +## Appendix: Phase 1 — Completed (2026-07-13) + +Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on branch `codex/ci/harness-stop-the-bleeding`. It shifts the verification baseline recorded in "Verification Results" above and addresses Finding 1 at the CLI boundary. + +**Shifted baseline.** The 2026-07-13 pre-work capture differed from the original verification results: `npm run lint` and `npm run test:run` already passed, so the earlier lint failure and the four `ora` / `@langfuse/otel` module-resolution failures were stale. Only `npx tsc --noEmit` was still failing. Phase 1 cleared those remaining type errors (narrowed at boundaries; no strict compiler options relaxed) and added durable gates so the baseline cannot silently regress: + +- `npm run typecheck` (`tsc --noEmit`) and `npm run verify` (typecheck + lint + test:run) scripts. +- `vitest.config.ts` for stable test module resolution. +- `docs/research/**` excluded from linting. +- `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. + +**Current state.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the autonomous workspace-agent surface is deprecated at the CLI boundary: `--mode agent` emits a deprecation warning and falls back to standard evaluation. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. The deprecation notice and pointer to this audit live in the README `## Agent Mode` section and the `--mode` help text. + +This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. From 888f0271e30b0e0e1fa5d2773ca89f94cc415d58 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:22:49 +0100 Subject: [PATCH 08/27] Add root .vectorlint.ini for reproducible smoke validation - Commit a repository-native .vectorlint.ini using the bundled VectorLint preset (verbatim `vectorlint init` template). - The required validation commands `npm start -- README.md --output line` and `npm start -- README.md --mode agent` previously failed from the committed branch with "Missing configuration file"; they had relied on an untracked, deleted VECTORLINT.md. - Both smoke commands now run from a clean checkout with no untracked setup files; --mode agent still warns and falls back to standard mode. Refs: .agent-runs/2026-07-13-223741-harness-refactor/reports/01-stop-the-bleeding.md --- .vectorlint.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vectorlint.ini diff --git a/.vectorlint.ini b/.vectorlint.ini new file mode 100644 index 00000000..d5b0d845 --- /dev/null +++ b/.vectorlint.ini @@ -0,0 +1,9 @@ +# VectorLint Configuration +# Global settings +RulesPath= +Concurrency=4 +DefaultSeverity=warning + +# Default rules for all markdown files +[**/*.md] +RunRules=VectorLint From 99f73c22ae728cb3dfb904dd0208b881322ac09b Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:56:45 +0100 Subject: [PATCH 09/27] Add neutral review-domain contract module - Introduce src/review/ with typed + Zod-validated ReviewRequest/Result contracts, boundary helpers, budget defaults/enforcement, model-call selection, ReviewExecutor interface, and a PromptFile request builder - Every external shape has a paired strict Zod schema; legacy scoring-mode, rubric, and model-authored rule-override fields are rejected - modelCall is single | agent | auto; chooseModelCall() resolves auto - On-page boundary (target + caller context only) via buildScope/isInScope with pure lexical URI normalization; no filesystem reads - BudgetExceededError extends the repository VectorlintError base - Purely additive: no changes outside src/review/ and tests/review/ - tests/review/ covers surface, schemas, budget, boundary, results, model-call selection, and the request builder (46 tests) --- src/review/README.md | 42 ++++++ src/review/boundary.ts | 67 ++++++++++ src/review/budget.ts | 83 ++++++++++++ src/review/executor.ts | 53 ++++++++ src/review/index.ts | 15 +++ src/review/request-builder.ts | 88 ++++++++++++ src/review/schemas.ts | 135 +++++++++++++++++++ src/review/types.ts | 192 +++++++++++++++++++++++++++ tests/review/boundary.test.ts | 39 ++++++ tests/review/budget.test.ts | 67 ++++++++++ tests/review/executor.test.ts | 46 +++++++ tests/review/module-surface.test.ts | 34 +++++ tests/review/request-builder.test.ts | 97 ++++++++++++++ tests/review/result.test.ts | 95 +++++++++++++ tests/review/types.test.ts | 74 +++++++++++ 15 files changed, 1127 insertions(+) create mode 100644 src/review/README.md create mode 100644 src/review/boundary.ts create mode 100644 src/review/budget.ts create mode 100644 src/review/executor.ts create mode 100644 src/review/index.ts create mode 100644 src/review/request-builder.ts create mode 100644 src/review/schemas.ts create mode 100644 src/review/types.ts create mode 100644 tests/review/boundary.test.ts create mode 100644 tests/review/budget.test.ts create mode 100644 tests/review/executor.test.ts create mode 100644 tests/review/module-surface.test.ts create mode 100644 tests/review/request-builder.test.ts create mode 100644 tests/review/result.test.ts create mode 100644 tests/review/types.test.ts diff --git a/src/review/README.md b/src/review/README.md new file mode 100644 index 00000000..bc2a120b --- /dev/null +++ b/src/review/README.md @@ -0,0 +1,42 @@ +# `src/review/` — Review-Domain Contract + +The neutral, implementation-neutral contract every executor programs against. +A caller builds a `ReviewRequest`; an executor returns a `ReviewResult`. + +## What lives here + +- `types.ts` — all review-domain interfaces (`ReviewTarget`, `ReviewRule`, + `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, + `ReviewDiagnostic`, `ReviewResult`, `ReviewRequest`, `ReviewModelCall`). +- `schemas.ts` — Zod schemas mirroring every external shape (boundary + validation). All schemas are strict; legacy scoring-mode, rubric, and + model-authored rule-override fields are rejected. +- `budget.ts` — `DEFAULT_REVIEW_BUDGET`, `REVIEW_BUDGET_SCHEMA`, + `enforceBudget()`, and `BudgetExceededError` (extends `VectorlintError`). +- `boundary.ts` — `buildScope()` / `isInScope()` enforcing the on-page boundary. +- `executor.ts` — `ReviewExecutor` interface, `REVIEW_MODEL_CALLS` + (`single | agent | auto`), `chooseModelCall()`. +- `request-builder.ts` — `buildReviewRequest()` bridging `PromptFile` to + `ReviewRequest`. + +## On-page boundary (audit Finding #5) + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller includes them as context. +- Agent model calls page through target content only. +- Rule bodies are source-backed and caller-authored, never model-authored. + +## Execution strategy + +`modelCall: single | agent | auto` selects how the reviewer model is invoked, +not how rules are scored. `single` sends target + rule in one structured call; +`agent` gives the executor a target-scoped read-section capability; `auto` +picks `single` for normal-sized inputs and `agent` for large ones. + +## Wiring status + +Phase 2 (this module) is purely additive and is **not** wired into the CLI. +Phase 3 emits `ReviewResult` via shared finding processing; Phase 4 implements +executors behind `ReviewExecutor` and removes the old agent loop. See +`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`. diff --git a/src/review/boundary.ts b/src/review/boundary.ts new file mode 100644 index 00000000..8dc5bb0c --- /dev/null +++ b/src/review/boundary.ts @@ -0,0 +1,67 @@ +import type { ReviewScope } from './types'; + +const FILE_URI_RE = /^file:\/\/(?[^/]*)(?\/.*)?$/; + +/** + * Lexically resolves '.' and '..' segments in a path string. Pure: performs no + * filesystem reads. The normalization philosophy is adapted from + * src/agent/path-utils.ts (resolveWithinRoot), which is removed in Phase 4; + * copied here so src/review/ has no agent import and no filesystem access. + * + * Symlink canonicalization is intentionally NOT performed inside the contract: + * callers are responsible for canonicalizing real files into target/context + * URIs before building a ReviewRequest. + */ +function resolveDotSegments(input: string): string { + const segments: string[] = []; + for (const segment of input.split('/')) { + if (segment === '' || segment === '.') continue; + if (segment === '..') { + segments.pop(); + continue; + } + segments.push(segment); + } + return (input.startsWith('/') ? '/' : '') + segments.join('/'); +} + +/** + * Normalizes a review URI by resolving '.'/'..' segments in its path. file:// + * URIs are normalized structurally; other (virtual, in-memory) URIs are + * returned as-is after dot resolution of their path-like suffix. + */ +export function normalizeReviewUri(uri: string): string { + const match = FILE_URI_RE.exec(uri); + if (match?.groups) { + const authority = match.groups['authority'] ?? ''; + const rawPath = match.groups['path'] ?? '/'; + return `file://${authority}${resolveDotSegments(rawPath)}`; + } + return uri; +} + +/** + * Builds the on-page boundary scope (audit Finding #5) from the target URI and + * any caller-supplied context URIs. Only these URIs are in scope; arbitrary + * workspace files are out of scope unless the caller explicitly includes them. + */ +export function buildScope(params: { + targetUri: string; + contextUris?: readonly string[]; +}): ReviewScope { + const allowedUris = new Set(); + allowedUris.add(normalizeReviewUri(params.targetUri)); + for (const uri of params.contextUris ?? []) { + allowedUris.add(normalizeReviewUri(uri)); + } + return { allowedUris }; +} + +/** + * Returns true iff `uri` (after normalization) is the target or caller-supplied + * context. Path-traversal segments are resolved before comparison, so a + * traversal-style path cannot masquerade as an in-scope URI. + */ +export function isInScope(scope: ReviewScope, uri: string): boolean { + return scope.allowedUris.has(normalizeReviewUri(uri)); +} diff --git a/src/review/budget.ts b/src/review/budget.ts new file mode 100644 index 00000000..e7d4cd8b --- /dev/null +++ b/src/review/budget.ts @@ -0,0 +1,83 @@ +import { z } from 'zod'; +import type { ReviewBudget } from './types'; +import { VectorlintError } from '../errors'; + +/** + * Sensible default bounds for a single review (audit Finding #7). Frozen so a + * shared default object cannot be accidentally mutated by a caller. + * + * Note: there is intentionally no finding cap (`maxFindingsPerRule`) — + * VectorLint reports every verified issue it finds — and no headless retry + * budget (`maxHeadlessRetries`) because the decided architecture has no + * headless executor. + */ +export const DEFAULT_REVIEW_BUDGET: Readonly = Object.freeze({ + maxTargetBytes: 1_000_000, + maxCallerContextBytes: 500_000, + maxChunksPerRule: 20, + maxModelCallsPerReview: 50, + maxWallClockMs: 5 * 60_000, +}); + +/** Zod schema mirroring {@link ReviewBudget}, applying defaults for omitted fields. */ +export const REVIEW_BUDGET_SCHEMA = z + .object({ + maxTargetBytes: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxTargetBytes), + maxCallerContextBytes: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxCallerContextBytes), + maxChunksPerRule: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxChunksPerRule), + maxModelCallsPerReview: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview), + maxWallClockMs: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxWallClockMs), + }) + .strict(); + +/** Current resource usage, checked against a {@link ReviewBudget}. */ +export interface BudgetUsage { + modelCalls: number; + elapsedMs: number; +} + +/** + * Thrown by {@link enforceBudget} when current usage violates a hard limit. + * Extends the repository's canonical error base (VectorlintError). + */ +export class BudgetExceededError extends VectorlintError { + constructor( + message: string, + public readonly limit: keyof ReviewBudget, + public readonly actual: number, + ) { + super(message, 'BUDGET_EXCEEDED'); + this.name = 'BudgetExceededError'; + } +} + +/** + * Throws {@link BudgetExceededError} if current usage violates a hard limit. + * Executors call this before/after each model call and on timeout checks. + * Only runtime counters (model calls, wall clock) are enforced here; size + * limits (target/context bytes, chunks) are enforced at request-build time. + */ +export function enforceBudget(budget: ReviewBudget, usage: BudgetUsage): void { + if (usage.modelCalls > budget.maxModelCallsPerReview) { + throw new BudgetExceededError( + `model calls (${usage.modelCalls}) exceed maxModelCallsPerReview (${budget.maxModelCallsPerReview})`, + 'maxModelCallsPerReview', + usage.modelCalls, + ); + } + if (usage.elapsedMs > budget.maxWallClockMs) { + throw new BudgetExceededError( + `elapsed time (${usage.elapsedMs}ms) exceeds maxWallClockMs (${budget.maxWallClockMs}ms)`, + 'maxWallClockMs', + usage.elapsedMs, + ); + } +} diff --git a/src/review/executor.ts b/src/review/executor.ts new file mode 100644 index 00000000..2eef9011 --- /dev/null +++ b/src/review/executor.ts @@ -0,0 +1,53 @@ +import type { ReviewModelCall, ReviewRequest, ReviewResult } from './types'; + +/** + * Above this target byte size (roughly the existing chunking threshold in + * bytes), `auto` model-call selection prefers the agent executor so the + * reviewer can page through target content for context management. + */ +export const AGENT_MODEL_CALL_BYTE_THRESHOLD = 600_000; + +/** + * The single source of truth for reviewer model-call strategies. Kept in sync + * with the {@link ReviewModelCall} union via `satisfies`. + */ +export const REVIEW_MODEL_CALLS = ['single', 'agent', 'auto'] as const satisfies readonly ReviewModelCall[]; + +/** + * The stable domain-level interface every executor implements. Single and + * agent executors are implementations behind this contract (audit Finding #2). + * A headless/autonomous workspace executor is explicitly NOT part of this + * contract. + */ +export interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +/** + * Agent-call capability: a bounded way to page through target content for + * context management. NOT a workspace tool (audit Product Decision). An agent + * executor receives this capability scoped to the target only. + */ +export interface ReviewTargetReadCapability { + /** Read a 1-based [startLine, endLine] window of the target content. */ + readTargetSection( + startLine: number, + endLine: number, + ): Promise<{ startLine: number; endLine: number; content: string }>; +} + +/** + * Resolves 'auto' to 'single' or 'agent'. Single for normal-sized inputs; + * agent for large inputs or multi-rule runs that benefit from paging. + */ +export function chooseModelCall( + modelCall: ReviewModelCall, + signal: { targetBytes: number; rules: number }, +): 'single' | 'agent' { + if (modelCall === 'single') return 'single'; + if (modelCall === 'agent') return 'agent'; + if (signal.targetBytes > AGENT_MODEL_CALL_BYTE_THRESHOLD || signal.rules > 5) { + return 'agent'; + } + return 'single'; +} diff --git a/src/review/index.ts b/src/review/index.ts new file mode 100644 index 00000000..c44dbe66 --- /dev/null +++ b/src/review/index.ts @@ -0,0 +1,15 @@ +/** + * Neutral review-domain contract for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. The CLI, headless adapters, + * and external callers all build a ReviewRequest and receive a ReviewResult. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + */ +export * from './types'; +export * from './schemas'; +export * from './budget'; +export * from './boundary'; +export * from './executor'; +export * from './request-builder'; diff --git a/src/review/request-builder.ts b/src/review/request-builder.ts new file mode 100644 index 00000000..72500596 --- /dev/null +++ b/src/review/request-builder.ts @@ -0,0 +1,88 @@ +import type { PromptFile } from '../schemas/prompt-schemas'; +import type { + ReviewBudget, + ReviewContext, + ReviewModelCall, + ReviewOutputPolicy, + ReviewRequest, + ReviewRule, + ReviewTarget, +} from './types'; +import { DEFAULT_REVIEW_BUDGET } from './budget'; +import { ValidationError } from '../errors'; + +/** Default output policy: include usage metadata, never record payloads. */ +export const DEFAULT_REVIEW_OUTPUT_POLICY: Readonly = Object.freeze({ + includeUsage: true, + recordPayloadTelemetry: false, +}); + +/** Optional overrides applied by {@link buildReviewRequest}. */ +export interface ReviewRequestBuilderConfig { + budget?: ReviewBudget; + outputPolicy?: ReviewOutputPolicy; + modelCall?: ReviewModelCall; +} + +export interface BuildReviewRequestParams { + target: ReviewTarget; + /** Existing prompt files to convert into source-backed ReviewRules. */ + prompts: readonly PromptFile[]; + /** Caller-supplied, in-scope context, passed through unchanged. */ + context?: ReviewContext[]; + config?: ReviewRequestBuilderConfig; +} + +/** Builds the stable ReviewRule id formatted Pack.Rule (mirrors src/agent/rule-id.ts). */ +function toReviewRuleId(prompt: PromptFile): string { + const pack = prompt.pack || 'Default'; + return `${pack}.${prompt.meta.id}`; +} + +/** + * Converts a single PromptFile into a source-backed ReviewRule, mapping only + * fields that belong in the neutral contract. Legacy `meta.type`, `evaluator`, + * `criteria`, and judge/rubric fields are intentionally NOT copied. + */ +function toReviewRule(prompt: PromptFile): ReviewRule { + const rule: ReviewRule = { + id: toReviewRuleId(prompt), + source: prompt.fullPath, + body: prompt.body, + }; + if (prompt.meta.name !== undefined) { + rule.name = prompt.meta.name; + } + // Severity is a string enum whose values ('error' | 'warning') match + // ReviewSeverity exactly, so it maps cleanly without transformation. + if (prompt.meta.severity !== undefined) { + rule.severity = prompt.meta.severity; + } + return rule; +} + +/** + * Builds a {@link ReviewRequest} from existing {@link PromptFile}s plus a + * target. This is the conservative bridge Phase 4 uses to convert the current + * prompt pipeline into the review contract without changing how prompts load. + * Throws a {@link ValidationError} when no prompts are supplied. + */ +export function buildReviewRequest(params: BuildReviewRequestParams): ReviewRequest { + const { target, prompts, context } = params; + if (prompts.length === 0) { + throw new ValidationError('buildReviewRequest requires at least one prompt.'); + } + + const config = params.config; + const request: ReviewRequest = { + target, + rules: prompts.map(toReviewRule), + budget: config?.budget ?? DEFAULT_REVIEW_BUDGET, + outputPolicy: config?.outputPolicy ?? DEFAULT_REVIEW_OUTPUT_POLICY, + modelCall: config?.modelCall ?? 'auto', + }; + if (context !== undefined) { + request.context = context; + } + return request; +} diff --git a/src/review/schemas.ts b/src/review/schemas.ts new file mode 100644 index 00000000..a1132cd3 --- /dev/null +++ b/src/review/schemas.ts @@ -0,0 +1,135 @@ +import { z } from 'zod'; +import { REVIEW_BUDGET_SCHEMA } from './budget'; + +/** + * Zod schemas mirroring src/review/types.ts (boundary validation). + * + * Every external review-domain shape has a paired schema here so callers and + * external adapters can validate untrusted input at the system boundary + * (audit Finding #3). Schemas are intentionally strict: unknown keys are + * rejected so legacy scoring-mode, rubric, and model-authored rule-override + * fields cannot leak into the new contract. + */ + +export const REVIEW_SEVERITY_SCHEMA = z.enum(['error', 'warning']); + +export const REVIEW_VIOLATION_CONDITION_SCHEMA = z + .object({ + id: z.string().min(1), + description: z.string().min(1), + }) + .strict(); + +export const REVIEW_TARGET_SCHEMA = z + .object({ + uri: z.string().min(1), + content: z.string(), + contentType: z.string().min(1), + byteLength: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RULE_SCHEMA = z + .object({ + id: z.string().min(1), + source: z.string().min(1), + body: z.string(), + name: z.string().optional(), + severity: REVIEW_SEVERITY_SCHEMA.default('warning'), + violationConditions: z.array(REVIEW_VIOLATION_CONDITION_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_CONTEXT_SCHEMA = z + .object({ + label: z.string().min(1), + content: z.string(), + relation: z.string().optional(), + uri: z.string().optional(), + }) + .strict(); + +export const REVIEW_OUTPUT_POLICY_SCHEMA = z + .object({ + includeUsage: z.boolean(), + recordPayloadTelemetry: z.boolean(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_LEVEL_SCHEMA = z.enum(['info', 'warn', 'error']); + +export const REVIEW_SCORE_COMPONENT_SCHEMA = z + .object({ + id: z.string().min(1), + scoreText: z.string(), + score: z.number(), + weight: z.number().optional(), + }) + .strict(); + +export const REVIEW_SCORE_SCHEMA = z + .object({ + ruleId: z.string().min(1), + score: z.number(), + scoreText: z.string(), + severity: REVIEW_SEVERITY_SCHEMA, + findingCount: z.number().int().nonnegative().optional(), + components: z.array(REVIEW_SCORE_COMPONENT_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_FINDING_SCHEMA = z + .object({ + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + severity: REVIEW_SEVERITY_SCHEMA, + message: z.string(), + line: z.number().int().positive(), + column: z.number().int().positive(), + match: z.string(), + analysis: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_SCHEMA = z + .object({ + level: REVIEW_DIAGNOSTIC_LEVEL_SCHEMA, + code: z.string().min(1), + message: z.string(), + ruleId: z.string().optional(), + context: z.record(z.string(), z.unknown()).optional(), + }) + .strict(); + +export const REVIEW_USAGE_SCHEMA = z + .object({ + inputTokens: z.number().int().nonnegative().optional(), + outputTokens: z.number().int().nonnegative().optional(), + modelCalls: z.number().int().nonnegative(), + costUsd: z.number().nonnegative().optional(), + wallClockMs: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RESULT_SCHEMA = z + .object({ + findings: z.array(REVIEW_FINDING_SCHEMA), + scores: z.array(REVIEW_SCORE_SCHEMA), + diagnostics: z.array(REVIEW_DIAGNOSTIC_SCHEMA), + usage: REVIEW_USAGE_SCHEMA.optional(), + hadOperationalErrors: z.boolean().optional(), + }) + .strict(); + +export const REVIEW_REQUEST_SCHEMA = z + .object({ + target: REVIEW_TARGET_SCHEMA, + rules: z.array(REVIEW_RULE_SCHEMA).min(1), + context: z.array(REVIEW_CONTEXT_SCHEMA).optional(), + budget: REVIEW_BUDGET_SCHEMA, + outputPolicy: REVIEW_OUTPUT_POLICY_SCHEMA, + modelCall: z.enum(['single', 'agent', 'auto']), + }) + .strict(); diff --git a/src/review/types.ts b/src/review/types.ts new file mode 100644 index 00000000..97e22039 --- /dev/null +++ b/src/review/types.ts @@ -0,0 +1,192 @@ +/** + * Neutral review-domain contract types for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. Callers build a + * {@link ReviewRequest} and executors return a {@link ReviewResult}. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + * + * This module is implementation-neutral: it deliberately exposes no legacy + * scoring-mode, rubric, or model-authored rule-override surface. `modelCall` + * selects how the reviewer model is invoked, not how rules are scored. + */ + +/** Finding/Rule severity. */ +export type ReviewSeverity = 'error' | 'warning'; + +/** Objective Via Negativa condition that counts as a violation when present. */ +export interface ReviewViolationCondition { + id: string; + description: string; +} + +/** + * Target content under review. The on-page boundary (audit Finding #5) is + * enforced against this: executors may only read sections of `content`. + */ +export interface ReviewTarget { + /** Stable absolute URI (file:// or virtual scheme for in-memory content). */ + uri: string; + /** Full target content. May be paged by an agent executor. */ + content: string; + /** MIME-ish content type, e.g. 'text/markdown'. */ + contentType: string; + /** Optional byte length hint; computed from content if absent. */ + byteLength?: number; +} + +/** + * A source-backed, caller-authored rule. Model-authored rule overrides are + * explicitly disallowed (audit Finding #1, #5). + */ +export interface ReviewRule { + /** Stable id, formatted Pack.Rule[.Criterion] in output. */ + id: string; + /** Canonical source path (where the rule body came from). */ + source: string; + /** The rule prompt body. Source-backed; never model-authored. */ + body: string; + /** Human-readable name. */ + name?: string; + /** 'error' | 'warning'. Defaults to 'warning'. */ + severity?: ReviewSeverity; + /** Optional structured Via Negativa violation conditions. */ + violationConditions?: ReviewViolationCondition[]; +} + +/** + * Caller-supplied context. Explicitly in scope (audit boundary rule): + * VectorLint does NOT discover workspace files as context on its own; the + * caller owns exploration and context gathering. + */ +export interface ReviewContext { + /** Short label used in diagnostics/tracing. */ + label: string; + /** The context content itself. */ + content: string; + /** How this context relates to the target, e.g. 'reference' | 'glossary'. */ + relation?: string; + /** Optional source URI for provenance. */ + uri?: string; +} + +/** + * The on-page boundary scope: the normalized URIs an executor may read. + * Built by {@link buildScope} (see boundary.ts) and checked by + * {@link isInScope}. + */ +export interface ReviewScope { + /** Normalized absolute URIs that are in scope (target + caller context). */ + readonly allowedUris: ReadonlySet; +} + +/** + * Hard bounds on a single review (audit Finding #7). These limit work, not + * output. Executors MUST check these and surface a ReviewDiagnostic (or fail + * the run) when exceeded. + */ +export interface ReviewBudget { + maxTargetBytes: number; + maxCallerContextBytes: number; + maxChunksPerRule: number; + maxModelCallsPerReview: number; + /** Maximum elapsed review time in milliseconds. This is the run timeout. */ + maxWallClockMs: number; +} + +/** + * Controls optional output/telemetry behavior. Diagnostics are always part of + * ReviewResult; this policy does not hide operational warnings. Audit + * Finding #9: payload telemetry must be a separate opt-in from metadata + * telemetry. + */ +export interface ReviewOutputPolicy { + /** Include usage/cost in the result. */ + includeUsage: boolean; + /** Opt-in: record prompt/content payloads to telemetry. Default false. */ + recordPayloadTelemetry: boolean; +} + +/** A verified finding anchored in the target content. */ +export interface ReviewFinding { + ruleId: string; + ruleSource: string; + severity: ReviewSeverity; + message: string; + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. Unverified finding evidence is a diagnostic. */ + match: string; + analysis?: string; + suggestion?: string; + fix?: string; +} + +/** A per-rule score produced through shared finding processing. */ +export interface ReviewScore { + ruleId: string; + score: number; + /** Human-readable score, e.g. "8.0/10". */ + scoreText: string; + severity: ReviewSeverity; + /** Count of verified findings that contributed to this score, if applicable. */ + findingCount?: number; + components?: ReviewScoreComponent[]; +} + +export interface ReviewScoreComponent { + id: string; + scoreText: string; + score: number; + weight?: number; +} + +export type ReviewDiagnosticLevel = 'info' | 'warn' | 'error'; + +/** An operational or finding-processing note. Always part of ReviewResult. */ +export interface ReviewDiagnostic { + level: ReviewDiagnosticLevel; + /** Stable machine code, e.g. 'finding-evidence-not-locatable'. */ + code: string; + message: string; + ruleId?: string; + /** Extra machine-readable detail (model-call count, evidence preview, ...). */ + context?: Record; +} + +/** Aggregated resource usage for a review run. */ +export interface ReviewUsage { + inputTokens?: number; + outputTokens?: number; + modelCalls: number; + costUsd?: number; + wallClockMs?: number; +} + +/** The output from any executor, produced through shared finding processing. */ +export interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; + /** True if the run hit an operational error but still returned partial results. */ + hadOperationalErrors?: boolean; +} + +/** How to call the reviewer model. 'auto' resolves via chooseModelCall(). */ +export type ReviewModelCall = 'single' | 'agent' | 'auto'; + +/** The complete input to any executor. */ +export interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + /** Caller-supplied, in-scope context (never workspace-discovered). */ + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; + /** Reviewer model call shape. 'auto' resolves via chooseModelCall(). */ + modelCall: ReviewModelCall; +} diff --git a/tests/review/boundary.test.ts b/tests/review/boundary.test.ts new file mode 100644 index 00000000..5ff92dad --- /dev/null +++ b/tests/review/boundary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { buildScope, isInScope, normalizeReviewUri } from '../../src/review'; + +describe('review boundary', () => { + const scope = buildScope({ + targetUri: 'file:///repo/docs/guide.md', + contextUris: ['file:///repo/docs/glossary.md'], + }); + + it('target is always in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/guide.md')).toBe(true); + }); + + it('caller-supplied context uri is in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/glossary.md')).toBe(true); + }); + + it('arbitrary workspace file is NOT in scope', () => { + expect(isInScope(scope, 'file:///repo/src/index.ts')).toBe(false); + }); + + it('rejects path traversal that escapes the target', () => { + expect(isInScope(scope, 'file:///repo/docs/../src/index.ts')).toBe(false); + }); + + it('normalizes traversal that resolves back onto an in-scope uri', () => { + expect(isInScope(scope, 'file:///repo/docs/sub/../guide.md')).toBe(true); + }); +}); + +describe('normalizeReviewUri', () => { + it('resolves dot segments in file uris', () => { + expect(normalizeReviewUri('file:///repo/a/../b/./c.md')).toBe('file:///repo/b/c.md'); + }); + + it('preserves authority', () => { + expect(normalizeReviewUri('file://host/x/../y.md')).toBe('file://host/y.md'); + }); +}); diff --git a/tests/review/budget.test.ts b/tests/review/budget.test.ts new file mode 100644 index 00000000..a63bcc24 --- /dev/null +++ b/tests/review/budget.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { + BudgetExceededError, + DEFAULT_REVIEW_BUDGET, + REVIEW_BUDGET_SCHEMA, + enforceBudget, +} from '../../src/review'; +import { VectorlintError } from '../../src/errors'; + +describe('review budget defaults', () => { + it('provides sensible positive defaults', () => { + expect(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxWallClockMs).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxTargetBytes).toBeGreaterThan(0); + }); + + it('does not cap findings or headless retries', () => { + expect('maxFindingsPerRule' in DEFAULT_REVIEW_BUDGET).toBe(false); + expect('maxHeadlessRetries' in DEFAULT_REVIEW_BUDGET).toBe(false); + }); +}); + +describe('REVIEW_BUDGET_SCHEMA', () => { + it('applies defaults for omitted fields', () => { + const parsed = REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 5 }); + expect(parsed.maxWallClockMs).toBe(DEFAULT_REVIEW_BUDGET.maxWallClockMs); + expect(parsed.maxModelCallsPerReview).toBe(5); + }); + + it('rejects non-positive limits', () => { + expect(() => REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 0 })).toThrow(); + }); +}); + +describe('enforceBudget', () => { + it('is silent within limits', () => { + expect(() => + enforceBudget(DEFAULT_REVIEW_BUDGET, { modelCalls: 1, elapsedMs: 1000 }), + ).not.toThrow(); + }); + + it('throws BudgetExceededError when model calls exceed the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 3 }, + { modelCalls: 4, elapsedMs: 0 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('throws BudgetExceededError when wall clock exceeds the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxWallClockMs: 100 }, + { modelCalls: 0, elapsedMs: 101 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('extends the repository error base', () => { + const err = new BudgetExceededError('boom', 'maxModelCallsPerReview', 9); + expect(err).toBeInstanceOf(VectorlintError); + expect(err.limit).toBe('maxModelCallsPerReview'); + expect(err.actual).toBe(9); + expect(err.code).toBe('BUDGET_EXCEEDED'); + }); +}); diff --git a/tests/review/executor.test.ts b/tests/review/executor.test.ts new file mode 100644 index 00000000..c15e670b --- /dev/null +++ b/tests/review/executor.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_MODEL_CALLS, + chooseModelCall, + type ReviewExecutor, + type ReviewRequest, +} from '../../src/review'; + +describe('REVIEW_MODEL_CALLS', () => { + it('exposes single | agent | auto', () => { + expect(REVIEW_MODEL_CALLS).toEqual(['single', 'agent', 'auto']); + }); +}); + +describe('chooseModelCall', () => { + it('respects an explicit single call', () => { + expect(chooseModelCall('single', { targetBytes: 2_000_000, rules: 10 })).toBe('single'); + }); + + it('respects an explicit agent call', () => { + expect(chooseModelCall('agent', { targetBytes: 10, rules: 1 })).toBe('agent'); + }); + + it('returns single for small inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 10_000, rules: 1 })).toBe('single'); + }); + + it('returns agent for large inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 2_000_000, rules: 1 })).toBe('agent'); + }); + + it('returns agent for many rules under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 1_000, rules: 6 })).toBe('agent'); + }); +}); + +describe('ReviewExecutor contract', () => { + it('can be implemented and run() returns a ReviewResult', async () => { + const fake: ReviewExecutor = { + run: () => Promise.resolve({ findings: [], scores: [], diagnostics: [] }), + }; + const result = await fake.run({} as unknown as ReviewRequest); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + }); +}); diff --git a/tests/review/module-surface.test.ts b/tests/review/module-surface.test.ts new file mode 100644 index 00000000..6c03ad30 --- /dev/null +++ b/tests/review/module-surface.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import * as review from '../../src/review'; + +describe('src/review public surface', () => { + it('exports the core contract values and functions', () => { + // Runtime presence checks for values; types are checked by tsc. + expect(review.REVIEW_MODEL_CALLS).toBeDefined(); + expect(review.DEFAULT_REVIEW_BUDGET).toBeDefined(); + expect(typeof review.chooseModelCall).toBe('function'); + expect(typeof review.isInScope).toBe('function'); + expect(typeof review.buildReviewRequest).toBe('function'); + }); + + it('exposes the boundary, budget, and error helpers', () => { + expect(typeof review.buildScope).toBe('function'); + expect(typeof review.enforceBudget).toBe('function'); + expect(typeof review.normalizeReviewUri).toBe('function'); + expect(review.BudgetExceededError).toBeDefined(); + }); + + it('exposes the Zod schemas for every external shape', () => { + expect(review.REVIEW_TARGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_RULE_SCHEMA).toBeDefined(); + expect(review.REVIEW_CONTEXT_SCHEMA).toBeDefined(); + expect(review.REVIEW_BUDGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_OUTPUT_POLICY_SCHEMA).toBeDefined(); + expect(review.REVIEW_FINDING_SCHEMA).toBeDefined(); + expect(review.REVIEW_SCORE_SCHEMA).toBeDefined(); + expect(review.REVIEW_DIAGNOSTIC_SCHEMA).toBeDefined(); + expect(review.REVIEW_USAGE_SCHEMA).toBeDefined(); + expect(review.REVIEW_RESULT_SCHEMA).toBeDefined(); + expect(review.REVIEW_REQUEST_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts new file mode 100644 index 00000000..618efa03 --- /dev/null +++ b/tests/review/request-builder.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_REVIEW_BUDGET, + REVIEW_RULE_SCHEMA, + buildReviewRequest, +} from '../../src/review'; +import { ValidationError } from '../../src/errors'; +import { Severity } from '../../src/evaluators/types'; +import type { PromptFile } from '../../src/schemas/prompt-schemas'; +import type { ReviewContext, ReviewTarget } from '../../src/review'; + +function makePrompt(overrides: Partial = {}): PromptFile { + return { + id: 'pseudo-advice', + filename: 'pseudo-advice.md', + fullPath: '/repo/presets/VectorLint/pseudo-advice.md', + meta: { + id: 'PseudoAdvice', + name: 'Pseudo Advice', + severity: Severity.WARNING, + type: 'check', + criteria: [{ id: 'Vague', name: 'Vague advice' }], + }, + body: 'Check for pseudo advice.', + pack: 'VectorLint', + ...overrides, + }; +} + +describe('buildReviewRequest', () => { + const target: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: '# Guide', + contentType: 'text/markdown', + }; + + it('maps one PromptFile to one ReviewRule with default budget and auto modelCall', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + expect(request.modelCall).toBe('auto'); + expect(request.budget).toEqual(DEFAULT_REVIEW_BUDGET); + expect(request.rules).toHaveLength(1); + + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule?.id).toBe('VectorLint.PseudoAdvice'); + expect(rule?.source).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(rule?.body).toBe('Check for pseudo advice.'); + expect(rule?.severity).toBe('warning'); + expect(rule?.name).toBe('Pseudo Advice'); + // Strict parse proves no legacy evaluator/criteria/type fields leaked. + expect(() => REVIEW_RULE_SCHEMA.parse(rule)).not.toThrow(); + }); + + it('does not copy legacy meta.type, evaluator, or criteria fields', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'type' in rule).toBe(false); + expect(rule && 'evaluator' in rule).toBe(false); + expect(rule && 'criteria' in rule).toBe(false); + }); + + it('throws a ValidationError when no prompts are supplied', () => { + expect(() => buildReviewRequest({ target, prompts: [] })).toThrow(ValidationError); + }); + + it('passes caller-supplied context through unchanged', () => { + const context: ReviewContext[] = [ + { label: 'glossary', content: 'term a means ...', relation: 'reference' }, + ]; + const request = buildReviewRequest({ target, prompts: [makePrompt()], context }); + expect(request.context).toBe(context); + }); + + it('honors an explicit modelCall override', () => { + const request = buildReviewRequest({ + target, + prompts: [makePrompt()], + config: { modelCall: 'agent' }, + }); + expect(request.modelCall).toBe('agent'); + }); + + it('omits severity on the rule when the prompt has none', () => { + const request = buildReviewRequest({ + target, + prompts: [ + makePrompt({ + meta: { id: 'PseudoAdvice', name: 'Pseudo Advice' }, + }), + ], + }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'severity' in rule).toBe(false); + }); +}); diff --git a/tests/review/result.test.ts b/tests/review/result.test.ts new file mode 100644 index 00000000..3be3767a --- /dev/null +++ b/tests/review/result.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_DIAGNOSTIC_SCHEMA, + REVIEW_FINDING_SCHEMA, + REVIEW_RESULT_SCHEMA, +} from '../../src/review'; + +describe('ReviewFinding schema', () => { + it('parses a complete finding', () => { + const finding = REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'VectorLint.Consistency', + ruleSource: '/repo/presets/VectorLint/consistency.md', + severity: 'warning', + message: 'Vague advice.', + line: 5, + column: 1, + match: 'consider leveraging', + analysis: 'too vague', + suggestion: 'be specific', + fix: 'consider using concrete terms', + }); + expect(finding.line).toBe(5); + expect(finding.match).toBe('consider leveraging'); + expect(finding.severity).toBe('warning'); + }); + + it('rejects a finding with an unknown severity', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'critical', + message: 'x', + line: 1, + column: 1, + match: 'x', + }), + ).toThrow(); + }); + + it('rejects a finding missing required anchored match evidence', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'warning', + message: 'x', + line: 1, + column: 1, + }), + ).toThrow(); + }); +}); + +describe('ReviewDiagnostic schema', () => { + it('parses a diagnostic with level/code/message', () => { + const diag = REVIEW_DIAGNOSTIC_SCHEMA.parse({ + level: 'warn', + code: 'finding-evidence-not-locatable', + message: 'could not anchor the finding', + }); + expect(diag.level).toBe('warn'); + }); + + it('rejects an unknown level', () => { + expect(() => + REVIEW_DIAGNOSTIC_SCHEMA.parse({ level: 'fatal', code: 'x', message: 'y' }), + ).toThrow(); + }); +}); + +describe('ReviewResult schema', () => { + it('accepts findings/scores/diagnostics and defaults usage to undefined', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [], diagnostics: [] }); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + expect(result.usage).toBeUndefined(); + }); + + it('keeps diagnostics mandatory', () => { + expect(() => + REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [] }), + ).toThrow(); + }); + + it('accepts usage when provided', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ + findings: [], + scores: [], + diagnostics: [], + usage: { modelCalls: 2, inputTokens: 10, outputTokens: 5 }, + }); + expect(result.usage?.modelCalls).toBe(2); + }); +}); diff --git a/tests/review/types.test.ts b/tests/review/types.test.ts new file mode 100644 index 00000000..d3bee632 --- /dev/null +++ b/tests/review/types.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { REVIEW_CONTEXT_SCHEMA, REVIEW_RULE_SCHEMA, REVIEW_TARGET_SCHEMA } from '../../src/review'; + +describe('ReviewTarget schema', () => { + it('parses a valid target', () => { + const target = REVIEW_TARGET_SCHEMA.parse({ + uri: 'file:///repo/docs/guide.md', + content: '# Guide\n\nBody text.', + contentType: 'text/markdown', + }); + expect(target.uri).toBe('file:///repo/docs/guide.md'); + }); + + it('allows empty content (streaming target)', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ uri: 'file:///x', content: '', contentType: 'text/plain' }), + ).not.toThrow(); + }); + + it('rejects a target missing a uri', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ content: 'x', contentType: 'text/plain' }), + ).toThrow(); + }); +}); + +describe('ReviewRule schema', () => { + it('parses a rule with required fields and defaults severity to warning', () => { + const rule = REVIEW_RULE_SCHEMA.parse({ + id: 'Consistency', + source: 'VectorLint/consistency.md', + body: 'Check internal consistency.', + violationConditions: [ + { id: 'Contradiction', description: 'The target makes mutually inconsistent claims.' }, + ], + }); + expect(rule.severity).toBe('warning'); + expect(rule.violationConditions?.[0]?.id).toBe('Contradiction'); + }); + + it('rejects legacy evaluator and judge criteria fields', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'Judge whether the tone is good.', + evaluator: 'judge', + criteria: [{ id: 'Tone', name: 'Tone quality' }], + }), + ).toThrow(); + }); + + it('rejects an unknown severity', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'x', + severity: 'critical', + }), + ).toThrow(); + }); +}); + +describe('ReviewContext schema', () => { + it('parses caller-supplied scoped content', () => { + const ctx = REVIEW_CONTEXT_SCHEMA.parse({ + label: 'related-glossary', + content: 'Term A means ...', + relation: 'reference', + }); + expect(ctx.label).toBe('related-glossary'); + }); +}); From e4259e7865a1604a4091d6432bcc04c516b9d2ee Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:11:06 +0100 Subject: [PATCH 10/27] Extract single finding evidence verifier - Add src/findings/finding-evidence-verifier.ts wrapping locateQuotedText from src/output/location.ts as the single source of truth (audit #6). - Anchored quotes return verified line/column/match; unanchored quotes return a finding-evidence-not-locatable warn diagnostic and no finding. - Never falls back to a model-provided line number the way the old agent path did. - Cover anchored, unanchored, no-line-hint, and truncation cases. --- src/findings/finding-evidence-verifier.ts | 84 +++++++++++++++++++ .../finding-evidence-verifier.test.ts | 69 +++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/findings/finding-evidence-verifier.ts create mode 100644 tests/findings/finding-evidence-verifier.test.ts diff --git a/src/findings/finding-evidence-verifier.ts b/src/findings/finding-evidence-verifier.ts new file mode 100644 index 00000000..7a6ffb6a --- /dev/null +++ b/src/findings/finding-evidence-verifier.ts @@ -0,0 +1,84 @@ +import { locateQuotedText, type LocationWithMatch } from '../output/location'; + +/** Stable machine code emitted when finding evidence cannot be anchored. */ +export const FINDING_EVIDENCE_NOT_LOCATABLE = 'finding-evidence-not-locatable' as const; + +/** Evidence markers a model returns for a candidate finding. */ +export interface FindingEvidenceInput { + quoted_text: string; + context_before?: string; + context_after?: string; + /** Optional 1-based line hint from the model; never used as a fallback. */ + line?: number; +} + +/** Located coordinates for a verified finding, anchored in target content. */ +export interface VerifiedFindingCoords { + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. */ + match: string; +} + +/** Operational note produced when evidence cannot be verified. */ +export interface FindingEvidenceDiagnostic { + code: string; + level: 'warn' | 'error'; + message: string; +} + +/** The outcome of verifying a single candidate finding's evidence. */ +export interface FindingEvidenceVerification { + verified: boolean; + finding?: VerifiedFindingCoords; + diagnostic?: FindingEvidenceDiagnostic; +} + +/** + * The single source of truth for finding evidence verification + * (audit Finding #6). + * + * Wraps the existing `locateQuotedText` fuzzy matcher; it never reimplements + * matching. On a successful anchor it returns a verified finding with the + * located line/column/match. When the quoted text cannot be anchored it + * returns a `finding-evidence-not-locatable` warn diagnostic and NO finding — + * it never falls back to a model-provided line number the way the old agent + * path did. + */ +export function verifyFindingEvidence( + content: string, + findingEvidence: FindingEvidenceInput, +): FindingEvidenceVerification { + const location: LocationWithMatch | null = locateQuotedText( + content, + { + quoted_text: findingEvidence.quoted_text || '', + context_before: findingEvidence.context_before || '', + context_after: findingEvidence.context_after || '', + }, + 80, + findingEvidence.line, + ); + + if (!location) { + return { + verified: false, + diagnostic: { + code: FINDING_EVIDENCE_NOT_LOCATABLE, + level: 'warn', + message: `Could not locate quoted text "${(findingEvidence.quoted_text || '').slice(0, 60)}" in target content.`, + }, + }; + } + + return { + verified: true, + finding: { + line: location.line, + column: location.column, + match: location.match, + }, + }; +} diff --git a/tests/findings/finding-evidence-verifier.test.ts b/tests/findings/finding-evidence-verifier.test.ts new file mode 100644 index 00000000..64d38209 --- /dev/null +++ b/tests/findings/finding-evidence-verifier.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, +} from '../../src/findings/finding-evidence-verifier'; + +const CONTENT = 'line one\nline two has the quote\nline three'; + +describe('verifyFindingEvidence', () => { + it('locates an exact quote and returns a verified finding', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'the quote', + context_before: '', + context_after: '', + line: 2, + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(2); + expect(result.finding?.match).toBe('the quote'); + expect(result.diagnostic).toBeUndefined(); + }); + + it('returns a warn diagnostic (not a fallback finding) when evidence cannot be located', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'this does not exist anywhere', + context_before: '', + context_after: '', + line: 99, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + expect(result.diagnostic?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostic?.level).toBe('warn'); + }); + + it('never returns a finding whose line came from the model when the quote did not anchor', () => { + // The old agent path fell back to the model-provided line (here 1) and + // surfaced the raw quoted_text as a verified match. Assert we do not. + // (Use a lexically distant quote that locateQuotedText cannot anchor.) + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'definitely not present xyzzy', + context_before: '', + context_after: '', + line: 1, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + }); + + it('still anchors a quote when no line hint is provided', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'line three', + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(3); + expect(result.finding?.column).toBe(1); + }); + + it('produces a diagnostic that mentions the (truncated) quoted text', () => { + const longQuote = 'x'.repeat(120); + const result = verifyFindingEvidence(CONTENT, { + quoted_text: longQuote, + }); + expect(result.verified).toBe(false); + expect(result.diagnostic?.message).toContain('x'.repeat(60)); + // Truncated, not the full 120 chars. + expect(result.diagnostic?.message).not.toContain('x'.repeat(61)); + }); +}); From 25ec49aa43ec112c56785ef41a20207ebb4c9c88 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:13:50 +0100 Subject: [PATCH 11/27] Add shared finding severity and scoring helpers - Add src/findings/types.ts with the FindingProcessingInput contract and strict Zod schemas; no modelCall/mode/agent/evaluator/judge/rubric fields, so legacy judge/evaluator-shaped input is rejected. - Add src/findings/severity.ts: resolveSeverity derives severity from the calculateCheckScore result (one path, no agent stamping); buildRuleId matches Pack.Rule[.Criterion] naming without importing src/agent/. - Add src/findings/scorer.ts as a thin wrapper over calculateCheckScore; the verified finding count drives the score. - Cover schema parse/reject, severity, rule-id, criterion resolution, and score numerics against calculateCheckScore. --- src/findings/scorer.ts | 60 ++++++++++++ src/findings/severity.ts | 52 +++++++++++ src/findings/types.ts | 160 ++++++++++++++++++++++++++++++++ tests/findings/scorer.test.ts | 94 +++++++++++++++++++ tests/findings/severity.test.ts | 73 +++++++++++++++ tests/findings/types.test.ts | 145 +++++++++++++++++++++++++++++ 6 files changed, 584 insertions(+) create mode 100644 src/findings/scorer.ts create mode 100644 src/findings/severity.ts create mode 100644 src/findings/types.ts create mode 100644 tests/findings/scorer.test.ts create mode 100644 tests/findings/severity.test.ts create mode 100644 tests/findings/types.test.ts diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts new file mode 100644 index 00000000..b5fb93fa --- /dev/null +++ b/src/findings/scorer.ts @@ -0,0 +1,60 @@ +import { calculateCheckScore, type CheckScoringOptions } from '../scoring'; +import type { CheckResult } from '../prompts/schema'; +import { resolveSeverity } from './severity'; +import type { RawViolation, RuleSeverity } from './types'; + +/** Optional scoring knobs mirroring {@link CheckScoringOptions}. */ +export interface ScoreCheckOptions { + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +} + +/** + * The count/density score result for a rule's verified findings. Mirrors the + * fields the processor needs to assemble a `ReviewScore`. The numeric values + * come straight from {@link calculateCheckScore}; this layer never + * reimplements the scoring math (audit Finding #4). + */ +export interface ScoredCheck { + score: number; + scoreText: string; + severity: RuleSeverity; + /** Count of verified findings that drove the score. */ + findingCount: number; + /** The raw check result, used to resolve severity. */ + scored: CheckResult; +} + +/** + * Scores a rule's verified findings using the existing check density formula. + * + * The score is driven by the **verified** finding count, not the raw candidate + * count (audit Finding #6): callers must pass only findings whose evidence has + * been anchored. This is a thin adapter over {@link calculateCheckScore}; it + * only normalizes the result into the shared score shape. + */ +export function scoreCheck(params: { + verifiedViolations: RawViolation[]; + wordCount: number; + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +}): ScoredCheck { + const scored = calculateCheckScore( + params.verifiedViolations, + params.wordCount, + { + ...(params.strictness !== undefined ? { strictness: params.strictness } : {}), + ...(params.promptSeverity !== undefined + ? { promptSeverity: params.promptSeverity } + : {}), + }, + ); + + return { + score: scored.final_score, + scoreText: `${scored.final_score.toFixed(1)}/10`, + severity: resolveSeverity({ scored }), + findingCount: scored.violation_count, + scored, + }; +} diff --git a/src/findings/severity.ts b/src/findings/severity.ts new file mode 100644 index 00000000..cca534f0 --- /dev/null +++ b/src/findings/severity.ts @@ -0,0 +1,52 @@ +import type { CheckResult } from '../prompts/schema'; +import type { FindingsCriterion, RuleSeverity } from './types'; + +/** Input to {@link resolveSeverity}. */ +export interface SeverityInput { + scored: CheckResult; +} + +/** + * The one severity resolution path for objective violation checks + * (audit Finding #4). Severity is derived from the existing count/density + * score result (`CheckResult.severity`); there is no agent-specific or + * mode-specific severity stamping here. + */ +export function resolveSeverity(input: SeverityInput): RuleSeverity { + return input.scored.severity; +} + +/** + * Builds the hierarchical output rule id matching the current standard + * orchestrator naming (`buildRuleName`): `Pack.Rule` or, when a violation is + * attributed to a criterion, `Pack.Rule.Criterion`. This is a faithful port of + * that behavior; it does not import from `src/agent/`. + */ +export function buildRuleId( + pack: string, + ruleId: string, + criterionId?: string, +): string { + const parts = [pack, ruleId]; + if (criterionId) { + parts.push(criterionId); + } + return parts.join('.'); +} + +/** + * Resolves a criterion id from the rule's declared criteria by matching a + * `criterionName` a violation carries to a declared criterion's `name`. + * Returns `undefined` when the violation is not attributed to a declared + * criterion (the common case for objective check violations). + */ +export function resolveCriterionId( + criteria: readonly FindingsCriterion[] | undefined, + criterionName: string | undefined, +): string | undefined { + if (!criterionName || !criteria) { + return undefined; + } + const match = criteria.find((c) => c.name === criterionName); + return match?.id; +} diff --git a/src/findings/types.ts b/src/findings/types.ts new file mode 100644 index 00000000..9a6360c6 --- /dev/null +++ b/src/findings/types.ts @@ -0,0 +1,160 @@ +import { z } from 'zod'; +import { Severity } from '../evaluators/types'; + +/** + * Input contract for shared finding processing (Phase 3, audit Finding #6). + * + * This module is the boundary between "what a model returned" (raw candidate + * violations) and "what the formatters receive" (`ReviewResult`). It is + * intentionally independent of model call (`single` | `agent`): there is no + * `modelCall`, `mode`, `agent`, `evaluator`, `judge`, or rubric field here. + * + * Following the `src/review/` pattern, the domain types below are explicit + * interfaces (so their optional properties stay assignable under + * `exactOptionalPropertyTypes` to `CheckItem`/`GateViolationLike`), and the + * Zod schemas mirror them for boundary validation. The schemas are strict so + * legacy `evaluator`/`judge`/`modelCall`/`mode`/`agent`/rubric payloads are + * rejected. + */ + +/** + * The two severity levels objective violation checks can resolve to. Mirrors + * `ReviewSeverity`; values are identical (`'error'` | `'warning'`). + */ +export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; + +/** Optional strictness knob for check density scoring. */ +export type Strictness = number | 'lenient' | 'standard' | 'strict'; + +/** + * The six boolean evidence gates a model returns per violation. Structurally + * identical to `GateChecks` in `src/prompts/schema.ts`. + */ +export interface GateChecks { + rule_supports_claim: boolean; + evidence_exact: boolean; + context_supports_violation: boolean; + plausible_non_violation: boolean; + fix_is_drop_in: boolean; + fix_preserves_meaning: boolean; +} + +/** + * A raw candidate finding returned by a reviewer model, before evidence + * verification or filtering. `analysis` is the only required descriptive + * field; everything else is optional to tolerate partial model output while + * still failing closed at the filter and verifier stages. + */ +export interface RawViolation { + line?: number; + quoted_text?: string; + context_before?: string; + context_after?: string; + description?: string; + /** Optional criterion attribution used to build `Pack.Rule.Criterion` ids. */ + criterionName?: string; + analysis: string; + message?: string; + suggestion?: string; + fix?: string; + rule_quote?: string; + confidence?: number; + checks?: GateChecks; +} + +/** + * Criterion metadata used ONLY to name output rule ids + * (`Pack.Rule.Criterion`). Weight and target are judge/rubric concerns and are + * deliberately absent so the finding-processing contract cannot leak them. + */ +export interface FindingsCriterion { + id: string; + name: string; +} + +/** + * The narrow slice of prompt metadata that finding processing needs: severity + * for scoring, strictness for density, and criteria for rule-id naming. It + * excludes `evaluator`, `type`, `evaluateAs`, `target`, and other legacy + * fields that do not affect objective violation-check processing. + */ +export interface PromptMetaForFindings { + severity?: RuleSeverity; + strictness?: Strictness; + criteria?: FindingsCriterion[]; +} + +/** + * The full finding-processing input. There is deliberately no `outputFormat`: + * the processor emits a format-agnostic `ReviewResult`; the caller routes it + * to a formatter. + */ +export interface FindingProcessingInput { + pack: string; + ruleId: string; + ruleSource: string; + candidateFindings: RawViolation[]; + wordCount: number; + promptMeta: PromptMetaForFindings; + targetContent: string; +} + +// --- Boundary validation schemas (mirror the interfaces above) --------------- + +export const GATE_CHECKS_SCHEMA = z + .object({ + rule_supports_claim: z.boolean(), + evidence_exact: z.boolean(), + context_supports_violation: z.boolean(), + plausible_non_violation: z.boolean(), + fix_is_drop_in: z.boolean(), + fix_preserves_meaning: z.boolean(), + }) + .strict(); + +export const RAW_VIOLATION_SCHEMA = z + .object({ + line: z.number().optional(), + quoted_text: z.string().optional(), + context_before: z.string().optional(), + context_after: z.string().optional(), + description: z.string().optional(), + criterionName: z.string().optional(), + analysis: z.string(), + message: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + rule_quote: z.string().optional(), + confidence: z.number().optional(), + checks: GATE_CHECKS_SCHEMA.optional(), + }) + .strict(); + +export const FINDINGS_CRITERION_SCHEMA = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + }) + .strict(); + +export const PROMPT_META_FOR_FINDINGS_SCHEMA = z + .object({ + severity: z.nativeEnum(Severity).optional(), + strictness: z + .union([z.number().positive(), z.enum(['lenient', 'standard', 'strict'])]) + .optional(), + criteria: z.array(FINDINGS_CRITERION_SCHEMA).optional(), + }) + .strict(); + +export const FINDING_PROCESSING_INPUT_SCHEMA = z + .object({ + pack: z.string().min(1), + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + candidateFindings: z.array(RAW_VIOLATION_SCHEMA), + wordCount: z.number().nonnegative(), + promptMeta: PROMPT_META_FOR_FINDINGS_SCHEMA, + targetContent: z.string(), + }) + .strict(); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts new file mode 100644 index 00000000..7e8147b0 --- /dev/null +++ b/tests/findings/scorer.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { calculateCheckScore } from '../../src/scoring'; +import { Severity } from '../../src/evaluators/types'; +import { scoreCheck } from '../../src/findings/scorer'; +import type { RawViolation } from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + analysis: 'Issue', + message: 'Issue message', + fix: 'Fix', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +describe('scoreCheck', () => { + it('returns the same numeric result as calculateCheckScore for the same density', () => { + const verified = [violation(), violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + const direct = calculateCheckScore(verified, 100, { + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + expect(result.score).toBe(direct.final_score); + expect(result.scoreText).toBe(`${direct.final_score.toFixed(1)}/10`); + expect(result.severity).toBe(direct.severity); + expect(result.findingCount).toBe(verified.length); + expect(result.findingCount).toBe(direct.violation_count); + }); + + it('is driven by the verified finding count, not a raw candidate count', () => { + // 1 verified violation over 100 words at standard strictness: + // density 1 -> 100 - 1*10 = 90 -> 9.0/10 + const one = scoreCheck({ + verifiedViolations: [violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(one.score).toBe(9.0); + expect(one.scoreText).toBe('9.0/10'); + + // 2 verified violations -> density 2 -> 100 - 2*10 = 80 -> 8.0/10 + const two = scoreCheck({ + verifiedViolations: [violation(), violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(two.score).toBe(8.0); + expect(two.findingCount).toBe(2); + }); + + it('resolves error severity from the density score when score is low and prompt severity is error', () => { + // 20 violations over 100 words at standard strictness: + // density 20 -> 100 - 20*10 = -100 -> clamp 0 -> 0.0/10 -> prompt severity + const result = scoreCheck({ + verifiedViolations: Array.from({ length: 20 }, () => violation()), + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.ERROR, + }); + expect(result.score).toBe(0); + expect(result.severity).toBe(Severity.ERROR); + }); + + it('forwards strictness and prompt severity options to calculateCheckScore', () => { + const verified = [violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'strict', + }); + const direct = calculateCheckScore(verified, 100, { strictness: 'strict' }); + expect(result.score).toBe(direct.final_score); + }); +}); diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts new file mode 100644 index 00000000..46e0c93a --- /dev/null +++ b/tests/findings/severity.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from 'vitest'; +import { + buildRuleId, + resolveCriterionId, + resolveSeverity, +} from '../../src/findings/severity'; +import { EvaluationType, Severity } from '../../src/evaluators/types'; +import type { CheckResult } from '../../src/prompts/schema'; + +function makeCheckResult(severity: Severity): CheckResult { + return { + type: EvaluationType.CHECK, + final_score: 5, + percentage: 50, + violation_count: 1, + items: [], + severity, + message: 'Found 1 issue', + violations: [], + }; +} + +describe('resolveSeverity', () => { + it('returns the density-derived severity from the check score result', () => { + expect(resolveSeverity({ scored: makeCheckResult(Severity.WARNING) })).toBe( + Severity.WARNING, + ); + expect(resolveSeverity({ scored: makeCheckResult(Severity.ERROR) })).toBe( + Severity.ERROR, + ); + }); +}); + +describe('buildRuleId', () => { + it('builds Pack.Rule when no criterion is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern')).toBe('VectorLint.AiPattern'); + }); + + it('builds Pack.Rule.Criterion when a criterion id is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern', 'Hedging')).toBe( + 'VectorLint.AiPattern.Hedging', + ); + }); + + it('ignores an empty criterion id', () => { + expect(buildRuleId('VectorLint', 'AiPattern', '')).toBe( + 'VectorLint.AiPattern', + ); + }); +}); + +describe('resolveCriterionId', () => { + const criteria = [ + { id: 'Hedging', name: 'Hedge words' }, + { id: 'Filler', name: 'Filler phrases' }, + ]; + + it('maps a criterion name to its declared id', () => { + expect(resolveCriterionId(criteria, 'Hedge words')).toBe('Hedging'); + }); + + it('returns undefined when the name does not match a criterion', () => { + expect(resolveCriterionId(criteria, 'Unknown')).toBeUndefined(); + }); + + it('returns undefined when no criterion name is provided', () => { + expect(resolveCriterionId(criteria, undefined)).toBeUndefined(); + }); + + it('returns undefined when no criteria are declared', () => { + expect(resolveCriterionId(undefined, 'Hedge words')).toBeUndefined(); + }); +}); diff --git a/tests/findings/types.test.ts b/tests/findings/types.test.ts new file mode 100644 index 00000000..4309ead6 --- /dev/null +++ b/tests/findings/types.test.ts @@ -0,0 +1,145 @@ +import { describe, it, expect } from 'vitest'; +import { + FINDING_PROCESSING_INPUT_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function validInput() { + return { + pack: 'VectorLint', + ruleId: 'AiPattern', + ruleSource: '/repo/presets/VectorLint/ai-pattern.md', + candidateFindings: [ + { + line: 1, + quoted_text: 'leverage synergies', + context_before: '', + context_after: '', + description: 'cliché phrase', + analysis: 'vague corporate cliché', + message: 'Avoid cliché phrase', + suggestion: 'Use concrete verbs', + fix: 'work together', + rule_quote: 'Avoid clichés', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + }, + ], + wordCount: 100, + promptMeta: { + severity: 'warning' as const, + strictness: 'standard' as const, + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + targetContent: 'leverage synergies everywhere', + }; +} + +describe('RAW_VIOLATION_SCHEMA', () => { + it('parses a representative candidate finding', () => { + const parsed = RAW_VIOLATION_SCHEMA.parse(validInput().candidateFindings[0]); + expect(parsed.analysis).toBe('vague corporate cliché'); + expect(parsed.checks?.evidence_exact).toBe(true); + }); + + it('requires analysis', () => { + expect(() => + RAW_VIOLATION_SCHEMA.parse({ message: 'no analysis field' }), + ).toThrow(); + }); +}); + +describe('PROMPT_META_FOR_FINDINGS_SCHEMA', () => { + it('rejects legacy evaluator/type/evaluateAs/target fields', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + severity: 'warning', + evaluator: 'base', + }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ type: 'judge' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ evaluateAs: 'document' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ target: { regex: 'x' } }), + ).toThrow(); + }); + + it('rejects rubric weight on criteria', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + criteria: [{ id: 'Hedging', name: 'Hedge words', weight: 2 }], + }), + ).toThrow(); + }); +}); + +describe('FINDING_PROCESSING_INPUT_SCHEMA', () => { + it('parses a representative supported finding-processing input', () => { + const parsed = FINDING_PROCESSING_INPUT_SCHEMA.parse(validInput()); + expect(parsed.pack).toBe('VectorLint'); + expect(parsed.candidateFindings).toHaveLength(1); + expect(parsed.promptMeta.criteria?.[0]?.id).toBe('Hedging'); + }); + + it('has no modelCall/mode/agent field and rejects them', () => { + for (const legacyKey of ['modelCall', 'mode', 'agent'] as const) { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), [legacyKey]: 'agent' }), + ).toThrow(); + } + }); + + it('rejects legacy judge/evaluator/rubric-shaped top-level input', () => { + // Legacy judge result shape: criteria array with scores at the top level. + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + pack: 'VectorLint', + ruleId: 'Tone', + ruleSource: '/x.md', + wordCount: 100, + targetContent: 'x', + promptMeta: {}, + evaluator: 'judge', + criteria: [{ name: 'Tone', score: 2, violations: [] }], + }), + ).toThrow(); + }); + + it('rejects a legacy evaluator field nested under promptMeta', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + promptMeta: { ...validInput().promptMeta, evaluator: 'base' }, + }), + ).toThrow(); + }); + + it('rejects malformed candidate findings', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + candidateFindings: [{ message: 'no analysis field' }], + }), + ).toThrow(); + }); + + it('rejects an unknown top-level key', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), judge: true }), + ).toThrow(); + }); +}); From 20d42ff7d9441f3269d6fdea53f527ead80342b8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:15:37 +0100 Subject: [PATCH 12/27] Implement shared finding processor contract - Add src/findings/processor.ts: processFindings runs the single pipeline (filter -> verify -> dedupe -> score -> severity -> diagnostics) and returns the Phase 2 ReviewResult from src/review/types.ts. - Verified finding count, not raw candidate count, drives the score; unanchored evidence becomes a warn diagnostic and emits no finding; hadOperationalErrors is true only for error-level diagnostics. - Add src/findings/index.ts barrel export. - Golden processor test asserts findings/score match the standard check path; cover diagnostics, filtering, verified-count fix, dedupe, rule-id attribution, and ReviewResult contract validation. --- src/findings/index.ts | 48 +++++ src/findings/processor.ts | 167 ++++++++++++++++ tests/findings/module-surface.test.ts | 27 +++ tests/findings/processor.test.ts | 278 ++++++++++++++++++++++++++ 4 files changed, 520 insertions(+) create mode 100644 src/findings/index.ts create mode 100644 src/findings/processor.ts create mode 100644 tests/findings/module-surface.test.ts create mode 100644 tests/findings/processor.test.ts diff --git a/src/findings/index.ts b/src/findings/index.ts new file mode 100644 index 00000000..095b878c --- /dev/null +++ b/src/findings/index.ts @@ -0,0 +1,48 @@ +/** + * Shared finding-processing layer (Phase 3). + * + * Transforms raw candidate violation findings into a formatter-ready + * `ReviewResult` through one pipeline: evidence verification, filtering, + * count/density scoring, severity resolution, and diagnostics. + * + * This module is independent of model call (`single` | `agent`) and contains + * no legacy `judge`, `evaluator`, rubric, or autonomous-loop output + * compatibility code. See + * docs/plans/2026-07-10-phase-3-share-result-projection.md. + */ +export { + FINDING_EVIDENCE_NOT_LOCATABLE, + verifyFindingEvidence, + type FindingEvidenceDiagnostic, + type FindingEvidenceInput, + type FindingEvidenceVerification, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; + +export { + buildRuleId, + resolveCriterionId, + resolveSeverity, + type SeverityInput, +} from './severity'; + +export { scoreCheck, type ScoreCheckOptions, type ScoredCheck } from './scorer'; + +export { + processFindings, +} from './processor'; + +export { + FINDING_PROCESSING_INPUT_SCHEMA, + FINDINGS_CRITERION_SCHEMA, + GATE_CHECKS_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, + type FindingsCriterion, + type FindingProcessingInput, + type GateChecks, + type PromptMetaForFindings, + type RawViolation, + type RuleSeverity, + type Strictness, +} from './types'; diff --git a/src/findings/processor.ts b/src/findings/processor.ts new file mode 100644 index 00000000..45a1c7ae --- /dev/null +++ b/src/findings/processor.ts @@ -0,0 +1,167 @@ +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewResult, + ReviewScore, +} from '../review/types'; +import { computeFilterDecision } from '../evaluators/violation-filter'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; +import { scoreCheck } from './scorer'; +import { buildRuleId, resolveCriterionId } from './severity'; +import type { FindingProcessingInput, RawViolation } from './types'; + +interface VerifiedEntry { + raw: RawViolation; + coords: VerifiedFindingCoords; +} + +/** + * The single finding-processing pipeline for objective Via Negativa violation + * checks (Phase 3). Transforms raw candidate findings into a formatter-ready + * {@link ReviewResult}. + * + * Order (audit lines 167-174): + * 1. filter candidates by the evidence gate (`computeFilterDecision`); + * 2. verify each surfaced finding's evidence (anchored quote or diagnostic); + * 3. deduplicate verified findings by `(quoted_text, line)`; + * 4. score by verified finding count/density; + * 5. resolve severity from the density score; + * 6. assemble findings, score, and diagnostics. + * + * The verified finding count — not the raw candidate count — drives the score + * (audit Finding #6). Unanchored evidence becomes a + * `finding-evidence-not-locatable` warn diagnostic and emits no finding. + * `hadOperationalErrors` is `true` only when a diagnostic is `error` level. + */ +export function processFindings(input: FindingProcessingInput): ReviewResult { + const surfaced = filterCandidates(input.candidateFindings); + const { verified, evidenceDiagnostics } = verifyAndDedupe( + surfaced, + input.targetContent, + ); + + const scored = scoreCheck({ + verifiedViolations: verified.map((entry) => entry.raw), + wordCount: input.wordCount, + ...(input.promptMeta.strictness !== undefined + ? { strictness: input.promptMeta.strictness } + : {}), + ...(input.promptMeta.severity !== undefined + ? { promptSeverity: input.promptMeta.severity } + : {}), + }); + + const severity = scored.severity; + const scoreRuleId = buildRuleId(input.pack, input.ruleId); + + const findings: ReviewFinding[] = verified.map(({ raw, coords }) => { + const criterionId = resolveCriterionId( + input.promptMeta.criteria, + raw.criterionName, + ); + const ruleId = buildRuleId(input.pack, input.ruleId, criterionId); + const message = (raw.message || raw.analysis || '').trim(); + return { + ruleId, + ruleSource: input.ruleSource, + severity, + message, + line: coords.line, + column: coords.column, + match: coords.match, + ...(raw.analysis ? { analysis: raw.analysis } : {}), + ...(raw.suggestion ? { suggestion: raw.suggestion } : {}), + ...(raw.fix ? { fix: raw.fix } : {}), + }; + }); + + const scores: ReviewScore[] = [ + { + ruleId: scoreRuleId, + score: scored.score, + scoreText: scored.scoreText, + severity, + findingCount: verified.length, + }, + ]; + + const diagnostics: ReviewDiagnostic[] = [...evidenceDiagnostics]; + const hadOperationalErrors = diagnostics.some( + (diagnostic) => diagnostic.level === 'error', + ); + + return { findings, scores, diagnostics, hadOperationalErrors }; +} + +/** + * Drops candidate findings that fail the evidence gate. Filtered candidates + * contribute neither findings nor diagnostics, matching the current standard + * orchestrator behavior. + */ +function filterCandidates(candidates: readonly RawViolation[]): RawViolation[] { + const surfaced: RawViolation[] = []; + for (const candidate of candidates) { + const decision = computeFilterDecision(candidate); + if (decision.surface) { + surfaced.push(candidate); + } + } + return surfaced; +} + +/** + * Verifies each surfaced finding's evidence and deduplicates verified findings + * by `(quoted_text, line)`. Unanchored findings become warn diagnostics and are + * not emitted. + */ +function verifyAndDedupe( + surfaced: readonly RawViolation[], + targetContent: string, +): { verified: VerifiedEntry[]; evidenceDiagnostics: ReviewDiagnostic[] } { + const verified: VerifiedEntry[] = []; + const evidenceDiagnostics: ReviewDiagnostic[] = []; + const seen = new Set(); + + for (const candidate of surfaced) { + const verification = verifyFindingEvidence(targetContent, { + quoted_text: candidate.quoted_text || '', + ...(candidate.context_before !== undefined + ? { context_before: candidate.context_before } + : {}), + ...(candidate.context_after !== undefined + ? { context_after: candidate.context_after } + : {}), + ...(candidate.line !== undefined ? { line: candidate.line } : {}), + }); + + if (!verification.verified || !verification.finding) { + evidenceDiagnostics.push({ + level: 'warn', + code: FINDING_EVIDENCE_NOT_LOCATABLE, + message: + verification.diagnostic?.message ?? + 'Could not locate finding evidence in target content.', + }); + continue; + } + + const coords = verification.finding; + const dedupeKey = candidate.quoted_text + ? `${candidate.quoted_text}:${coords.line}` + : null; + if (dedupeKey) { + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + } + + verified.push({ raw: candidate, coords }); + } + + return { verified, evidenceDiagnostics }; +} diff --git a/tests/findings/module-surface.test.ts b/tests/findings/module-surface.test.ts new file mode 100644 index 00000000..a0e5c481 --- /dev/null +++ b/tests/findings/module-surface.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import * as findings from '../../src/findings'; + +describe('src/findings public surface', () => { + it('exports the processing entry point and helpers', () => { + expect(typeof findings.processFindings).toBe('function'); + expect(typeof findings.verifyFindingEvidence).toBe('function'); + expect(typeof findings.resolveSeverity).toBe('function'); + expect(typeof findings.buildRuleId).toBe('function'); + expect(typeof findings.resolveCriterionId).toBe('function'); + expect(typeof findings.scoreCheck).toBe('function'); + }); + + it('exports the stable diagnostic code constant', () => { + expect(findings.FINDING_EVIDENCE_NOT_LOCATABLE).toBe( + 'finding-evidence-not-locatable', + ); + }); + + it('exports the Zod boundary schemas', () => { + expect(findings.FINDING_PROCESSING_INPUT_SCHEMA).toBeDefined(); + expect(findings.RAW_VIOLATION_SCHEMA).toBeDefined(); + expect(findings.GATE_CHECKS_SCHEMA).toBeDefined(); + expect(findings.FINDINGS_CRITERION_SCHEMA).toBeDefined(); + expect(findings.PROMPT_META_FOR_FINDINGS_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/findings/processor.test.ts b/tests/findings/processor.test.ts new file mode 100644 index 00000000..90f9641d --- /dev/null +++ b/tests/findings/processor.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { processFindings } from '../../src/findings/processor'; +import { FINDING_EVIDENCE_NOT_LOCATABLE } from '../../src/findings/finding-evidence-verifier'; +import { REVIEW_RESULT_SCHEMA } from '../../src/review'; +import type { + FindingProcessingInput, + RawViolation, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + line: 1, + quoted_text: 'Alpha text', + context_before: '', + context_after: '', + analysis: 'Issue 1', + message: 'Issue 1', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +function baseInput( + candidateFindings: RawViolation[], + overrides: Partial = {}, +): FindingProcessingInput { + return { + pack: 'TestPack', + ruleId: 'CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + candidateFindings, + wordCount: 100, + promptMeta: { severity: 'warning', strictness: 'standard' }, + targetContent: 'Alpha text\nBeta text\n', + ...overrides, + }; +} + +describe('processFindings', () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + // Default threshold (0.75) unless a test opts in. + delete process.env.CONFIDENCE_THRESHOLD; + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + }); + + describe('golden objective-check output', () => { + beforeEach(() => { + // Surface every gate-passing candidate regardless of confidence. + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('produces byte-for-byte findings/score matching the standard orchestrator path', () => { + const input = baseInput([ + violation(), + violation({ + line: 2, + quoted_text: 'Beta text', + analysis: 'Issue 2', + message: 'Issue 2', + suggestion: 'Suggestion 2', + fix: 'Fix 2', + }), + ]); + + const result = processFindings(input); + + // Score: density 2/100 at standard strictness (10) -> 100 - 20 -> 8.0/10. + expect(result.scores).toEqual([ + { + ruleId: 'TestPack.CheckPrompt', + score: 8.0, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 2, + }, + ]); + + // Findings: Pack.Rule (check violations carry no criterion name), + // anchored lines/columns/match. + expect(result.findings).toHaveLength(2); + expect(result.findings[0]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + severity: 'warning', + message: 'Issue 1', + line: 1, + column: 1, + match: 'Alpha text', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + }); + expect(result.findings[1]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + severity: 'warning', + message: 'Issue 2', + line: 2, + column: 1, + match: 'Beta text', + }); + + expect(result.diagnostics).toEqual([]); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('returns a ReviewResult that validates against the review contract', () => { + const result = processFindings(baseInput([violation()])); + expect(() => REVIEW_RESULT_SCHEMA.parse(result)).not.toThrow(); + }); + }); + + describe('evidence verification (audit Finding #6)', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('turns unanchored evidence into a warn diagnostic and emits no finding', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'this quote is not in the content' }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostics[0]?.level).toBe('warn'); + // Score reflects 0 verified findings -> perfect 10.0/10. + expect(result.scores[0]?.score).toBe(10.0); + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('counts only verified findings toward the score, not raw candidate count', () => { + // One anchored + one unanchored candidate. Verified count = 1. + // Density 1/100 -> 100 - 10 -> 9.0/10 (the intentional fix). + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text' }), + violation({ quoted_text: 'nowhere to be found' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.score).toBe(9.0); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.diagnostics).toHaveLength(1); + }); + + it('does not set hadOperationalErrors for warn-level evidence diagnostics', () => { + const result = processFindings( + baseInput([violation({ quoted_text: 'missing' })]), + ); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('deduplicates verified findings by quoted_text and line', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', line: 1 }), + violation({ quoted_text: 'Alpha text', line: 1, message: 'dup' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + }); + }); + + describe('candidate filtering', () => { + it('drops candidates that fail computeFilterDecision without a diagnostic', () => { + // Default threshold 0.75: confidence 0.2 is filtered out. + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.2 }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + // No verified findings -> 10.0/10. + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('filtered candidates do not contribute to the score count', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.9 }), + violation({ quoted_text: 'Beta text', line: 2, confidence: 0.2 }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.scores[0]?.score).toBe(9.0); + }); + }); + + describe('rule id building', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('attributes findings to Pack.Rule when violations carry no criterion name', () => { + const result = processFindings(baseInput([violation({ quoted_text: 'Alpha text' })])); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt'); + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + + it('attributes findings to Pack.Rule.Criterion when a violation names a declared criterion', () => { + const result = processFindings( + baseInput( + [ + violation({ + quoted_text: 'Alpha text', + criterionName: 'Hedge words', + }), + ], + { + promptMeta: { + severity: 'warning', + strictness: 'standard', + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + }, + ), + ); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt.Hedging'); + // The score stays at Pack.Rule (no criterion), matching the orchestrator. + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + }); + + describe('severity resolution', () => { + it('resolves error severity from the density score and stamps it on findings', () => { + // 20 verified violations over 100 words -> 0.0/10 -> prompt severity error. + const result = processFindings( + baseInput( + Array.from({ length: 20 }, (_, i) => + violation({ + quoted_text: `Alpha text${i}`, + analysis: `Issue ${i}`, + message: `Issue ${i}`, + }), + ), + { + targetContent: Array.from({ length: 20 }, (_, i) => `Alpha text${i}`).join('\n') + '\n', + promptMeta: { severity: 'error', strictness: 'standard' }, + }, + ), + ); + + expect(result.scores[0]?.severity).toBe('error'); + expect(result.findings.every((f) => f.severity === 'error')).toBe(true); + }); + }); +}); From 671da9cd5e1cfae9863970274e338c541fc63893 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 03:24:02 +0100 Subject: [PATCH 13/27] Route standard check evaluation through shared processor - Build FindingProcessingInput and call processFindings in the standard objective-check branch of routePromptResult - Feed returned findings/scores/diagnostics through the existing line/json/rdjson/vale sinks and quality-score output - Remove inline filter/score/severity/quote-location logic from the check path; computeFilterDecision kept only for the debug-run artifact - Verified finding count now drives counts and score (audit Finding #6); unanchored quotes become warn diagnostics and no longer flag operational errors - Add orchestrator regression tests covering finding/score output, JSON/Vale shape, and exit behavior for supported objective-check reviews --- src/cli/orchestrator.ts | 120 ++++--- tests/orchestrator-check-processor.test.ts | 356 +++++++++++++++++++++ 2 files changed, 413 insertions(+), 63 deletions(-) create mode 100644 tests/orchestrator-check-processor.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f8df367a..a8e24436 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -12,6 +12,7 @@ import { printFileHeader, printIssueRow, printEvaluationSummaries, type Evaluati import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; +import { processFindings } from '../findings'; import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; @@ -616,79 +617,72 @@ function routePromptResult( let promptErrors = 0; let promptWarnings = 0; - // Handle Check Result + // Handle Check Result — routed through the shared finding processor + // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, + // filters, deduplicates, scores, and resolves severity; the orchestrator + // only feeds the returned ReviewResult to the existing formatter sinks. if (!isJudgeResult(result)) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - - // Score calculated from surfaced violations only — matches what user sees - const scored = calculateCheckScore( - surfacedViolations, - result.word_count, - { - strictness: promptFile.meta.strictness, - promptSeverity: promptFile.meta.severity, - } - ); - const severity = scored.severity; - // Group violations by criterionName - const violationsByCriterion = new Map< - string | undefined, - typeof surfacedViolations - >(); - for (const v of surfacedViolations) { - const criterionName = v.criterionName; - if (!violationsByCriterion.has(criterionName)) { - violationsByCriterion.set(criterionName, []); - } - violationsByCriterion.get(criterionName)!.push(v); - } - - // Report violations grouped by criterion - let totalErrors = 0; - let totalWarnings = 0; + const reviewResult = processFindings({ + pack: promptFile.pack, + ruleId: promptId, + ruleSource: promptFile.fullPath, + candidateFindings: result.violations, + wordCount: result.word_count, + promptMeta: { + ...(meta.severity !== undefined ? { severity: meta.severity } : {}), + ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), + ...(meta.criteria ? { criteria: meta.criteria } : {}), + }, + targetContent: content, + }); - for (const [criterionName, violations] of violationsByCriterion) { - // Find criterion ID from meta - let criterionId: string | undefined; - if (criterionName && meta.criteria) { - const criterion = meta.criteria.find(c => c.name === criterionName); - criterionId = criterion?.id; - } + // processFindings always returns exactly one score entry (processor contract). + const ruleScore = reviewResult.scores[0]!; + const severity = + ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - const ruleName = buildRuleName(promptFile.pack, promptId, criterionId); - - if (violations.length > 0) { - const violationResult = locateAndReportViolations({ - violations, - content, - relFile, - severity, - ruleName, - scoreText: '', - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; + // Report only verified findings through the existing line/json/rdjson/vale sink. + for (const finding of reviewResult.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity, + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); + } - if (severity === Severity.ERROR) { - totalErrors += violations.length; - } else { - totalWarnings += violations.length; - } + // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, + // consistent with prior operational reporting. They are warn-level and do not + // flag the run as operationally failed (audit Finding #6). + if (verbose) { + for (const diagnostic of reviewResult.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } } - // Create scoreEntry for Quality Scores display + // Verified finding count drives the counts (audit Finding #6). + const findingCount = reviewResult.findings.length; + const totalErrors = severity === Severity.ERROR ? findingCount : 0; + const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; + const scoreEntry: EvaluationSummary = { - id: buildRuleName(promptFile.pack, promptId, undefined), - scoreText: `${scored.final_score.toFixed(1)}/10`, - score: scored.final_score, + id: ruleScore.ruleId, + scoreText: ruleScore.scoreText, + score: ruleScore.score, }; if (debugJson) { + const { decisions, surfacedViolations } = getViolationFilterResults( + result.violations + ); const runId = randomUUID(); const model = getModelInfoFromEnv(); @@ -721,7 +715,7 @@ function routePromptResult( return { errors: totalErrors, warnings: totalWarnings, - hadOperationalErrors, + hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, scoreEntries: [scoreEntry], }; diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts new file mode 100644 index 00000000..eaaa1529 --- /dev/null +++ b/tests/orchestrator-check-processor.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { Result } from "../src/output/json-formatter"; +import type { ValeOutput } from "../src/schemas/vale-responses"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; + +const { EVALUATE_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), +})); + +type CheckViolation = RawCheckResult["violations"][number]; + +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +} as const; + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +/** + * Builds a check violation that passes the confidence gate by default. Real + * model output always carries `message`; the helper sets it explicitly so the + * test reflects production behavior rather than the analysis fallback. + */ +function makeCheckViolation( + overrides: Partial = {} +): CheckViolation { + return { + line: 1, + analysis: "Issue 1", + message: "Issue 1", + suggestion: "Suggestion 1", + fix: "Fix 1", + quoted_text: "Alpha text", + context_before: "", + context_after: "", + rule_quote: "Rule quote", + checks: FULLY_SUPPORTED_CHECKS, + confidence: 0.9, + ...overrides, + }; +} + +function makeCheckResult(params: { + violations: CheckViolation[]; + wordCount?: number; +}): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: params.violations, + word_count: params.wordCount ?? 100, + }; +} + +describe("standard check evaluation via the shared finding processor", () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + delete process.env.CONFIDENCE_THRESHOLD; + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + vi.restoreAllMocks(); + }); + + it("reports fully locatable findings, score, and counts unchanged", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckPrompt", + name: "Check Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First issue", analysis: "First issue" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second issue", + analysis: "Second issue", + suggestion: "Suggestion 2", + fix: "Fix 2", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. + expect(run.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("8.0/10"); + // The line summary prints the last segment of the Pack.Rule score id. + expect(scoreLine).toContain("CheckPrompt"); + }); + + it("counts only verified findings and excludes unanchored quotes from the score", async () => { + // One anchored + one unanchored quote. Both pass the confidence gate, so + // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CountingFixPrompt", + name: "Counting Fix Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "this quote is not anywhere in the content", + message: "Ghost issue", + analysis: "Ghost issue", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a + // finding; it contributes neither to the count nor the score. + expect(run.totalWarnings).toBe(1); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("9.0/10"); + }); + + it("does not flag severity errors when no finding can be verified", async () => { + // severity=error, but the only quote cannot anchor -> 0 verified findings -> + // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorNoVerifiedPrompt", + name: "Error No Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ + line: 9, + quoted_text: "nowhere to be found", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it("flags severity errors when verified error-severity findings exist", async () => { + // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> + // prompt severity error applies. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorVerifiedPrompt", + name: "Error Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [makeCheckViolation({ quoted_text: "Alpha text" })], + wordCount: 10, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it("emits verified findings with anchored location through the JSON sink", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckJsonPrompt", + name: "Check JSON Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First", analysis: "First" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second", + analysis: "Second", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.Json, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as Result; + const issues = Object.values(parsed.files).flatMap((file) => file.issues); + + expect(issues).toHaveLength(2); + expect(issues[0]).toMatchObject({ + line: 1, + column: 1, + severity: Severity.WARNING, + message: "First", + rule: "TestPack.CheckJsonPrompt", + match: "Alpha text", + }); + expect(issues[1]).toMatchObject({ + line: 2, + message: "Second", + match: "Beta text", + }); + }); + + it("omits unanchored quotes from the Vale JSON output", async () => { + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CheckValePrompt", + name: "Check Vale Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "missing quote", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.ValeJson, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as ValeOutput; + const issues = Object.values(parsed).flat(); + + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + Check: "TestPack.CheckValePrompt", + Line: 1, + Match: "Alpha text", + Severity: "warning", + }); + }); +}); From 9fe6440c7cc768ec0da72cb2a7fd2d73330b51d5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:36 +0100 Subject: [PATCH 14/27] Reject judge prompt type at the loader boundary - PROMPT_META_SCHEMA now rejects `type: judge` and its deprecated alias `subjective` via a superRefine, so judge-typed prompts fail to load with a descriptive warning instead of reaching evaluation. - superRefine (not refine) keeps the inferred union including "judge" so the legacy BaseEvaluator type-checks unchanged until Phase 4 deletes that path. - Add loader tests covering judge/subjective rejection and that check still loads; subjective and semi-objective are documented as deprecated aliases. --- src/schemas/prompt-schemas.ts | 26 ++++++++++--- tests/prompt-loader-validation.test.ts | 51 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index f8689645..2dadd85e 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -27,12 +27,15 @@ export const PROMPT_CRITERION_SCHEMA = z.object({ * - 'technical-accuracy': specialized evaluator with claim extraction + search * * Evaluation type: - * - 'judge': 1-4 scores per criterion, normalized to 1-10 * - 'check': density-based scoring (errors per 100 words) + * - 'semi-objective': deprecated alias for 'check' * - * Deprecated aliases (still supported): - * - 'subjective' → 'judge' - * - 'semi-objective' → 'check' + * 'judge' and its deprecated alias 'subjective' are no longer supported + * (Phase 3): subjective rubric scoring is not a future-facing review type. + * The enum still admits them so the Zod error is descriptive and the inferred + * type stays compatible with the legacy BaseEvaluator until Phase 4 deletes + * that path; a superRefine rejects them at this boundary so judge-typed prompts + * fail to load. * * Strictness factor for check scoring: * - Determines penalty weight per 1% error density. @@ -44,11 +47,24 @@ export const PROMPT_META_SCHEMA = z.object({ type: z .enum(["judge", "check", "subjective", "semi-objective"]) .transform((val) => { - // Map deprecated values to new canonical values + // Map deprecated values to their canonical forms. if (val === "subjective") return "judge" as const; if (val === "semi-objective") return "check" as const; return val; }) + .superRefine((val, ctx) => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). + // superRefine (not refine) so the inferred union still includes 'judge' + // and stays compatible with the legacy BaseEvaluator until Phase 4 deletes + // that path; the value is rejected at this boundary regardless. + if (val === "judge") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "judge evaluation is no longer supported; use check (Via Negativa) rules", + }); + } + }) .optional(), id: z.string(), name: z.string(), diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index 094ad7cb..fc34cc1d 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -130,4 +130,55 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); }); + + describe('Judge boundary rejection (Phase 3)', () => { + it('rejects judge-typed prompts at the loader boundary', () => { + createPrompt('judge.md', `--- +evaluator: base +id: Judge +name: Judge Prompt +type: judge +criteria: + - name: Clarity + id: Clarity +--- +Judge content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('judge.md'); + expect(warnings[0]).toMatch(/judge/i); + }); + + it('rejects subjective-typed prompts (deprecated judge alias)', () => { + createPrompt('subjective.md', `--- +evaluator: base +id: Subjective +name: Subjective Prompt +type: subjective +--- +Subjective content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('subjective.md'); + }); + + it('still loads check-typed prompts', () => { + createPrompt('check.md', `--- +evaluator: base +id: Check +name: Check Prompt +type: check +--- +Check content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(warnings).toHaveLength(0); + expect(prompts).toHaveLength(1); + expect(prompts[0].meta.type).toBe('check'); + }); + }); }); From d36d06dd139439667ac765b4adfe1e1162a76e42 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:54 +0100 Subject: [PATCH 15/27] Remove legacy judge review path - Drop the judge branch of routePromptResult and the now-dead criterion extraction/reporting helpers (extractAndReportCriterion, validateCriteriaCompleteness, validateScores, locateAndReportViolations, buildRuleName); subjective rubric scoring is not a future-facing review type. - A JudgeResult that still reaches the orchestrator is refused explicitly (operational error, no findings) rather than misprojected as a check result; no adapter is added for old judge metadata. - Standard objective-check orchestration still routes through processFindings unchanged. - Replace the judge-filtering orchestrator test with a judge-rejection test. --- src/cli/orchestrator.ts | 545 +-------------------------- tests/orchestrator-filtering.test.ts | 41 +- 2 files changed, 28 insertions(+), 558 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index a8e24436..4d09e917 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -6,10 +6,9 @@ import * as os from 'os'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; -import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { processFindings } from '../findings'; @@ -21,8 +20,7 @@ import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '. import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; import type { EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessViolationsParams, - ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, + ReportIssueParams, ProcessPromptResultParams, RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, RunPromptEvaluationResultSuccess } from './types'; @@ -33,7 +31,6 @@ import { import { calculateCheckScore } from '../scoring'; import { countWords } from '../chunking/utils'; import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { locateQuotedText } from "../output/location"; import { computeFilterDecision, type FilterDecision, @@ -64,23 +61,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } -/* - * Constructs a hierarchical rule name following the pattern: - * - With criterion: PackName.RuleId.CriterionId - * - Without criterion: PackName.RuleId - */ -function buildRuleName( - packName: string, - ruleId: string, - criterionId: string | undefined -): string { - const parts = [packName, ruleId]; - if (criterionId) { - parts.push(criterionId); - } - return parts.join('.'); -} - /* * Generic concurrency runner that executes workers in parallel up to a specified limit. * Preserves result order matching input order. @@ -174,119 +154,6 @@ function reportIssue(params: ReportIssueParams): void { } } -/* - * Locates and reports each violation using pre/post evidence markers. - * If location matching fails (missing markers, content mismatch), logs warning - * and continues processing. Returns hadOperationalErrors=true if any violations - * couldn't be located, signaling text matching issues vs. content quality issues. - */ -function locateAndReportViolations(params: ProcessViolationsParams): { - hadOperationalErrors: boolean; -} { - const { - violations, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose, - } = params; - - let hadOperationalErrors = false; - - // Locate all violations and filter out those that can't be verified - // Then de-duplicate by (quoted_text, line) - const seen = new Set(); - const verifiedViolations: Array<{ - v: (typeof violations)[0]; - line: number; - column: number; - matchedText: string; - rowSummary: string; - }> = []; - - for (const v of violations) { - if (!v) continue; - - const rowSummary = (v.message || "").trim(); - - try { - const locWithMatch = locateQuotedText( - content, - { - quoted_text: v.quoted_text || "", - context_before: v.context_before || "", - context_after: v.context_after || "", - }, - 80, - v.line - ); - - if (!locWithMatch) { - // Can't verify this quote exists - skip it entirely - if (verbose) { - console.warn( - `[vectorlint] Skipping unverifiable quote: "${v.quoted_text}"` - ); - } - hadOperationalErrors = true; - continue; - } - - const line = locWithMatch.line; - const column = locWithMatch.column; - const matchedText = locWithMatch.match || ""; - - // De-duplicate by (quoted_text, line) - skip if quoted_text is empty - const dedupeKey = v.quoted_text ? `${v.quoted_text}:${line}` : null; - if (dedupeKey && seen.has(dedupeKey)) { - continue; // Skip duplicate - } - if (dedupeKey) { - seen.add(dedupeKey); - } - - verifiedViolations.push({ v, line, column, matchedText, rowSummary }); - } catch (e: unknown) { - const err = handleUnknownError(e, "Locating evidence"); - if (verbose) { - console.warn(`[vectorlint] Error locating evidence: ${err.message}`); - } - hadOperationalErrors = true; - } - } - - // Report only verified, unique violations - for (const { - v, - line, - column, - matchedText, - rowSummary, - } of verifiedViolations) { - reportIssue({ - file: relFile, - line, - column, - severity, - summary: rowSummary, - ruleName, - outputFormat, - jsonFormatter, - ...(v.analysis !== undefined && { analysis: v.analysis }), - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), - ...(v.fix !== undefined && { fix: v.fix }), - scoreText, - match: matchedText, - }); - } - - return { hadOperationalErrors }; -} - function getViolationFilterResults< TViolation extends Parameters[0] >( @@ -304,297 +171,10 @@ function getViolationFilterResults< } /* - * Extracts pre-calculated scores from a subjective evaluation criterion and reports surfaced violations. - * Violations that do not pass computeFilterDecision are not reported. - * Returns error/warning counts, score entry for Quality Scores, and score components for JSON. - */ -function extractAndReportCriterion( - params: ProcessCriterionParams -): ProcessCriterionResult { - const { - exp, - result, - content, - relFile, - packName, - promptId, - meta, - outputFormat, - jsonFormatter, - verbose, - } = params; - let hadOperationalErrors = false; - let hadSeverityErrors = false; - - const nameKey = String(exp.name || exp.id || ""); - const criterionId = exp.id - ? String(exp.id) - : exp.name - ? String(exp.name) - .replace(/[^A-Za-z0-9]+/g, " ") - .split(" ") - .filter(Boolean) - .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) - .join("") - : ""; - const ruleName = buildRuleName(packName, promptId, criterionId); - - const weightNum = exp.weight || 1; - const maxScore = weightNum; - - // Target gating (deterministic precheck) - const metaTargetSpec = meta.target; - const expTargetSpec = exp.target; - const targetCheck = checkTarget(content, metaTargetSpec, expTargetSpec); - const missingTarget = targetCheck.missing; - - if (missingTarget) { - hadSeverityErrors = true; - const summary = "target not found"; - const suggestion = - targetCheck.suggestion || - expTargetSpec?.suggestion || - metaTargetSpec?.suggestion || - "Add the required target section."; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity: Severity.ERROR, - summary, - ruleName, - outputFormat, - jsonFormatter, - suggestion, - scoreText: "nil", - match: "", - }); - return { - errors: 1, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "0.0/10", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const got = result.criteria.find( - (c) => c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) { - return { - errors: 0, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "-", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const score = Number(got.score); - - // Use pre-calculated values from evaluator - const rawWeighted = got.weighted_points; - const normalizedScore = got.normalized_score; - const userScore = rawWeighted; - const violations = got.violations; - const { surfacedViolations } = getViolationFilterResults(violations); - - // Display normalized score (1-10) in CLI output - const scoreText = `${normalizedScore.toFixed(1)}/10`; - - // Determine severity based on violations - // If there are violations, use evaluator's scoring to determine severity - // Score <= 1 = error, score = 2 = warning, score > 2 = no severity needed (but we still create scoreEntry) - let errors = 0; - let warnings = 0; - let severity: Severity | undefined; - - if (surfacedViolations.length > 0) { - // Determine severity from score for violations - if (score <= 1) { - severity = Severity.ERROR; - hadSeverityErrors = true; - errors = surfacedViolations.length; - } else if (score === 2) { - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } else { - // Score > 2 but has violations - this is informational - // Use WARNING as default for informational violations - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } - - // Report surfaced violations only - const violationResult = locateAndReportViolations({ - violations: surfacedViolations as Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - analysis?: string; - suggestion?: string; - }>, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = - hadOperationalErrors || violationResult.hadOperationalErrors; - } else if (score <= 2) { - // No violations but low score - report with summary - severity = score <= 1 ? Severity.ERROR : Severity.WARNING; - if (severity === Severity.ERROR) { - hadSeverityErrors = true; - errors = 1; - } else { - warnings = 1; - } - - const sum = got.summary.trim(); - const words = sum.split(/\s+/).filter(Boolean); - const limited = words.slice(0, 15).join(" "); - const summaryText = limited || "No findings"; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: summaryText, - ruleName, - outputFormat, - jsonFormatter, - scoreText, - match: "", - }); - } - - return { - errors, - warnings, - userScore, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText, score: normalizedScore }, - scoreComponent: { - criterion: nameKey, - rawScore: score, - maxScore: 4, - weightedScore: rawWeighted, - weightedMaxScore: weightNum, - normalizedScore: normalizedScore, - normalizedMaxScore: 10, - }, - }; -} - -/* - * Validates that all expected criteria are present in the result. - */ -function validateCriteriaCompleteness(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - const expectedNames = new Set( - (meta.criteria || []).map((c) => String(c.name || c.id || "")) - ); - const returnedNames = new Set( - result.criteria.map((c: { name: string }) => c.name) - ); - - // Create normalized maps for case-insensitive lookup - const expectedNormalized = new Set(); - const expectedOriginalMap = new Map(); - for (const name of expectedNames) { - const norm = name.toLowerCase(); - expectedNormalized.add(norm); - expectedOriginalMap.set(norm, name); - } - - const returnedNormalized = new Set(); - for (const name of returnedNames) { - returnedNormalized.add(name.toLowerCase()); - } - - for (const norm of expectedNormalized) { - if (!returnedNormalized.has(norm)) { - console.error( - `Missing criterion in model output: ${expectedOriginalMap.get(norm)}` - ); - hadErrors = true; - } - } - - for (const name of returnedNames) { - if (!expectedNormalized.has(name.toLowerCase())) { - console.warn( - `[vectorlint] Extra criterion returned by model (ignored): ${name}` - ); - } - } - - return hadErrors; -} - -/* - * Validates that all criterion scores are within valid range. - */ -function validateScores(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - for (const exp of meta.criteria || []) { - const nameKey = String(exp.name || exp.id || ""); - const got = result.criteria.find( - (c) => - c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) continue; - - const score = Number(got.score); - if (!Number.isFinite(score) || score < 0 || score > 4) { - console.error(`Invalid score for ${nameKey}: ${score}`); - hadErrors = true; - } - } - - return hadErrors; -} - -/* - * Routes evaluation results through check or judge processing paths. - * Check: Reports violations, creates scoreEntry using final_score. - * Judge: Iterates through criteria, validates scores, creates scoreEntry per criterion. - * Both paths generate scoreEntries for Quality Scores display. + * Routes an evaluation result through the shared finding processor. + * Check results are verified, filtered, scored, and reported; judge/rubric + * results are rejected (Phase 3) because subjective scoring is not a + * future-facing review type. */ function routePromptResult( params: ProcessPromptResultParams @@ -612,11 +192,6 @@ function routePromptResult( const meta = promptFile.meta; const promptId = (meta.id || "").toString(); - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let promptErrors = 0; - let promptWarnings = 0; - // Handle Check Result — routed through the shared finding processor // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, // filters, deduplicates, scores, and resolves severity; the orchestrator @@ -721,105 +296,21 @@ function routePromptResult( }; } - // Handle Judge Result - // Validate criterion completeness and scores - hadOperationalErrors = - validateCriteriaCompleteness({ meta, result }) || hadOperationalErrors; - hadOperationalErrors = - validateScores({ meta, result }) || hadOperationalErrors; - - // Reset promptErrors and promptWarnings for subjective results - promptErrors = 0; - promptWarnings = 0; - const criterionScores: EvaluationSummary[] = []; - const scoreComponents: ScoreComponent[] = []; - - // Iterate through each criterion - for (const exp of meta.criteria || []) { - const criterionResult = extractAndReportCriterion({ - exp, - result, - content, - relFile, - packName: promptFile.pack, - promptId, - promptFilename: promptFile.filename, - meta, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - - promptErrors += criterionResult.errors; - promptWarnings += criterionResult.warnings; - hadOperationalErrors = - hadOperationalErrors || criterionResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || criterionResult.hadSeverityErrors; - criterionScores.push(criterionResult.scoreEntry); - - if (criterionResult.scoreComponent) { - scoreComponents.push(criterionResult.scoreComponent); - } - } - - if (outputFormat === OutputFormat.Json && scoreComponents.length > 0) { - (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore( - relFile, - { - id: promptId || promptFile.filename.replace(/\.md$/, ""), - scores: scoreComponents, - } + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a + // caller bypassed that boundary. Refuse it explicitly rather than + // misprojecting it as a check result. No adapter is added for judge metadata. + if (verbose) { + console.warn( + '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' ); } - - if (debugJson) { - const runId = randomUUID(); - const flat = result.criteria.flatMap((c) => - (c.violations || []).map((v, i) => ({ - criterion: c.name, - index: i, - violation: v, - decision: computeFilterDecision(v), - })) - ); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "judge", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: flat.map((x) => ({ - criterion: x.criterion, - index: x.index, - surface: x.decision.surface, - reasons: x.decision.reasons, - })), - surfaced_violations: flat.filter((x) => x.decision.surface).map((x) => ({ - criterion: x.criterion, - violation: x.violation, - })), - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - return { - errors: promptErrors, - warnings: promptWarnings, - hadOperationalErrors, - hadSeverityErrors, - scoreEntries: criterionScores, + errors: 0, + warnings: 0, + hadOperationalErrors: true, + hadSeverityErrors: false, + scoreEntries: [], }; } diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index 73e56119..b4d2cf89 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -307,7 +307,11 @@ describe("CLI violation filtering", () => { expect(scoreLine).not.toContain("8.0/10"); }); - it("filters low-confidence judge violations from CLI counts by default", async () => { + it("rejects judge/rubric results instead of projecting them as check findings", async () => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // orchestrator refuses a JudgeResult rather than running the old criterion + // extraction/reporting path: no findings are projected and the run is + // flagged with an operational error. const targetFile = createTempFile("Alpha text\nBeta text\n"); const prompt = createPrompt({ id: "JudgePrompt", @@ -323,42 +327,17 @@ describe("CLI violation filtering", () => { makeJudgeViolation({ line: 2, quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", confidence: 0.2, }), ]) ); - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - confidence: 0.2, - }), - ]) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.totalWarnings).toBe(0); + expect(run.hadOperationalErrors).toBe(true); + expect(run.hadSeverityErrors).toBe(false); }); it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { From d4ea9a7764f682e1288a09303f7fa94d0c20c599 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:20 +0100 Subject: [PATCH 16/27] Remove orphaned orchestrator param types from CLI contract - Drop ProcessViolationsParams, ProcessCriterionParams, ProcessCriterionResult, and ValidationParams from src/cli/types.ts. They were orphaned when the orchestrator's inline violation/criterion processing moved to src/findings (Tasks 2-3) and the judge path was removed, and are referenced nowhere else. - Remove the now-unused imports PromptMeta, PromptCriterionSpec, ScoreComponent, and JudgeResult. Severity and PromptEvaluationResult are retained (still used by ReportIssueParams / ProcessPromptResultParams). --- src/cli/types.ts | 42 ++---------------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/src/cli/types.ts b/src/cli/types.ts index 7f95192a..a083f2e2 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,13 +1,12 @@ import type { PromptFile } from '../prompts/prompt-loader'; import type { LLMProvider } from '../providers/llm-provider'; import type { SearchProvider } from '../providers/search-provider'; -import type { PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; -import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; +import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; @@ -95,43 +94,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessViolationsParams extends EvaluationContext { - violations: Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - message?: string; - analysis?: string; - suggestion?: string; - fix?: string; - }>; - severity: Severity; - ruleName: string; - scoreText: string; -} - -export interface ProcessCriterionParams extends EvaluationContext { - exp: PromptCriterionSpec; - result: JudgeResult; - packName: string; - promptId: string; - promptFilename: string; - meta: PromptMeta; -} - -export interface ProcessCriterionResult extends ErrorTrackingResult { - userScore: number; - maxScore: number; - scoreEntry: { id: string; scoreText: string; score?: number }; - scoreComponent?: ScoreComponent; -} - -export interface ValidationParams { - meta: PromptMeta; - result: JudgeResult; -} - export interface ProcessPromptResultParams extends EvaluationContext { promptFile: PromptFile; result: PromptEvaluationResult; From 15ec7a3e146df0820d39b9680e06b0369d5dd33e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:33 +0100 Subject: [PATCH 17/27] Document finding-processing behavior and clean stale judge docs - Add tests/findings/README.md mapping the behavior moved into src/findings (filtering, evidence verification, density scoring, rule-id/severity, formatter routing) and the intentionally removed judge/rubric and agent behavior, with agent tests marked out of scope for Phase 3 (Phase 3 Task 8). - Stop describing type: judge/subjective as a valid future-facing prompt type in CREATING_RULES.md, docs/frontmatter-fields.mdx, and docs/style-guide.mdx; note that judge/subjective are now rejected at load. - Drop the stale rubric-based scoring feature blurb from README.md. --- CREATING_RULES.md | 197 +++--------------------------------- README.md | 3 +- docs/frontmatter-fields.mdx | 80 ++------------- docs/style-guide.mdx | 113 ++------------------- tests/findings/README.md | 83 +++++++++++++++ 5 files changed, 117 insertions(+), 359 deletions(-) create mode 100644 tests/findings/README.md diff --git a/CREATING_RULES.md b/CREATING_RULES.md index ec90954f..7b800c9d 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -8,7 +8,6 @@ A comprehensive guide to creating powerful, reusable content evaluations using V - [Rule Anatomy](#rule-anatomy) - [Evaluation Modes](#evaluation-modes) - [Check Rules](#check-rules) -- [Judge Rules](#judge-rules) - [Target Specification](#target-specification) - [Configuration Reference](#configuration-reference) - [Best Practices](#best-practices) @@ -25,7 +24,7 @@ VectorLint rules are Markdown files with YAML frontmatter that define how your c - **Rule = Prompt file** (`.md` file organized in rule packs) - **Pack** = Subdirectory containing related rules (typically named after a company/style guide) - **Criteria** = Individual quality checks within a rule -- **Score** = LLM-assigned rating (1-4 scale for judge, density-based for check) +- **Score** = Density-based quality score derived from violation count - **Severity** = How failures are reported (`error` or `warning`) @@ -74,27 +73,20 @@ project/ ## Evaluation Modes -VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in two distinct modes, determined by the `type` field: +VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in one mode, determined by the `type` field: | Mode | `type` | Use Case | Scoring | Output | | ---------- | ------- | ------------------------------------- | --------------------------- | ----------------------- | | **Check** | `check` | Pass/fail checks, counting violations | 10 points - 1 per violation | List of specific issues | -| **Judge** | `judge` | Multi-dimensional quality scoring | 0-4 scale per criterion | Weighted average score | -### When to Use Each +The deprecated `type: judge` (alias `subjective`) is no longer supported; a rule using it is rejected at load. -**Use Check when:** +### When to Use Check - You need to find specific errors (e.g., "Find all grammar mistakes") - The check is binary (Pass/Fail) for each item - You want a list of specific violations to fix -**Use Judge when:** - -- You're measuring quality on a spectrum (e.g., "How engaging is this?") -- You have multiple dimensions (Clarity, Tone, Depth) -- You need weighted importance (some criteria matter more) - --- ## Check Rules @@ -141,71 +133,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes --- -## Judge Rules - -Judge rules use weighted criteria and a 1-4 rubric for sophisticated quality measurement. - -### Structure - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator - -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator... [Your detailed instructions] - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit - -### Good - -Clear benefit but less specific impact - -... -``` - -### The 1-4 Scoring Scale - -VectorLint uses a **1-4 scale** for all judge criteria, which is then normalized to a 1-10 scale: - -| LLM Rating | Meaning | Normalized Score | -| :--------- | :-------- | :--------------- | -| **4** | Excellent | **10.0** | -| **3** | Good | **7.0** | -| **2** | Fair | **4.0** | -| **1** | Poor | **1.0** | - -### Score Calculation - -1. **Normalization**: We map the 1-4 rating to a 1-10 score using the formula: `1 + ((Rating - 1) / 3) * 9`. -2. **Weighted Average**: The final score is the weighted average of all normalized criterion scores. - -**Example:** - -- Criterion: "Value Communication" (weight=12) -- Rating: 3 (Good) -> Normalized: 7.0 -- Weighted Points: 7.0 \* 12 = 84 points - ---- - ## Target Specification The `target` field allows you to: @@ -243,27 +170,17 @@ target: ### Frontmatter Fields -| Field | Type | Required | Description | -| ------------- | ------------- | -------- | ------------------------------------------------------------------ | -| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | -| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | -| `type` | string | No | Mode: `judge` or `check` (default: `check`) | -| `id` | string | **Yes** | Unique identifier (used in error reporting) | -| `name` | string | **Yes** | Human-readable name | +| Field | Type | Required | Description | +| ------------- | ------------- | -------- | -------------------------------------------------------------- | +| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | +| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | +| `type` | string | No | Mode: `check` (default: `check`). `judge`/`subjective` rejected at load | +| `id` | string | **Yes** | Unique identifier (used in error reporting) | +| `name` | string | **Yes** | Human-readable name | | `severity` | string | No | `error` or `warning` (default: `warning`) | | `evaluateAs` | string | No | `document` or `chunk` - whether to evaluate content as a whole or in chunks (default: `chunk`) | | `target` | object | No | Content matching specification | -| `criteria` | array | **Yes\*** | List of evaluation criteria (\*required for judge) | - -### Criterion Fields - -| Field | Type | Required | Description | -| -------- | ------ | -------- | ------------------------------------------ | -| `name` | string | **Yes** | Human-readable criterion name | -| `id` | string | **Yes** | Unique identifier (PascalCase recommended) | -| `weight` | number | No | Importance weight (default: 1) | -| `target` | object | No | Criterion-specific content matching | --- @@ -288,25 +205,10 @@ You are a headline evaluator for developer blog posts. Assess whether the headli 2. Uses natural, conversational language (avoid buzzwords) 3. Creates curiosity without being clickbait -For each criterion, provide a score (0-4) and specific examples from the text. -``` - -### 2. **Use Meaningful Weights (Subjective)** - -Scale weights to reflect real-world importance: - -```yaml -criteria: - # Technical accuracy is critical - - name: Technical Accuracy - weight: 40 - - # Readability is important - - name: Readability - weight: 30 +For each violation, quote the exact text and suggest a concrete fix. ``` -### 3. **Provide Context in Prompts** +### 2. **Provide Context in Prompts** Help the LLM understand your domain: @@ -338,79 +240,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes Report any errors found with specific examples. ``` -### Example 2: Headline Evaluator (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: Headline -name: Headline Evaluator - -severity: error -target: - regex: '^#\s+(.+)$' - flags: "mu" - group: 1 - required: true - suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 10 - - name: Language Authenticity - id: LanguageAuthenticity - weight: 5 ---- - -You are a headline evaluator. Assess the H1 headline for: - -1. **Value Communication** (10 points): Does it clearly state what the reader gains? -2. **Language Authenticity** (5 points): Does it use natural, conversational language? - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit clearly stated - -... -``` - -### Example 3: AI Pattern Detector (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: AIPatterns -name: AI Pattern Detector - -severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 ---- - -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Buzzwords**: leverage, synergy, elevate -2. **Formulaic transitions**: Moreover, Furthermore - ... -``` - ## Resources - [VectorLint README](./README.md) - Installation and basic usage diff --git a/README.md b/README.md index f6511e6a..662ac7f0 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,9 @@ If you can write a prompt for it, you can lint it with VectorLint. ## Quality Scores -VectorLint scores your content using error density and a rubric-based system, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. +VectorLint scores your content using error density, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. - **Density-Based Scoring:** For errors that can be counted, scores are calculated based on **error density (errors per 100 words)**, making quality assessment fair across documents of any length. -- **Rubric-Based Scoring:** For more nuanced quality standards, like flow and completeness, scores are graded on a 1-4 rubric system and then normalized to a **1-10 scale**. ## How VectorLint Reduces False Positives diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index 82e7d93c..d840d7b9 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -27,14 +27,12 @@ Your LLM prompt goes here. | Field | Default | Description | |---|---|---| -| `specVersion` | _(none)_ | Rule spec version. Set to `1.0.0` for rules using the full judge schema. | +| `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | | `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | -| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` scores content against weighted criteria on a 1–4 rubric. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations and scores them by error density. `judge` (and its deprecated `subjective` alias) is no longer supported — a rule with `type: judge` or `type: subjective` is rejected at load. | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | | `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | | `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | -| `criteria` | _(none)_ | Required when `type` is `judge`. Defines the scoring dimensions and their weights. See [Criteria fields](#criteria-fields) below. | - ## Target fields The `target` field narrows evaluation to a specific part of the document. It is an object with the following sub-fields: @@ -58,40 +56,16 @@ target: suggestion: Add an H1 headline for the article. ``` -## Criteria fields - -The `criteria` field is an array of criterion objects used with `type: judge`. Each criterion defines one scoring dimension. - -| Field | Type | Required | Description | -|---|---|---|---| -| `name` | string | Yes | Human-readable criterion name shown in CLI output. | -| `id` | string | Yes | Unique identifier for this criterion. PascalCase recommended (e.g., `TechnicalAccuracy`). Referenced in the Markdown body rubric as `# CriterionName `. | -| `weight` | number | No | Relative importance of this criterion in the weighted average. Higher values increase the criterion's influence on the final score. Defaults to `1`. | -| `target` | object | No | Criterion-specific content matching. Uses the same sub-fields as the top-level `target`. Overrides the rule-level target for this criterion only. | - -**Example — judge rule with two weighted criteria:** - -```yaml -criteria: - - name: Technical Accuracy - id: TechnicalAccuracy - weight: 40 - - name: Readability - id: Readability - weight: 30 -``` - ## Complete example -The following rule uses all commonly-used fields. It targets the H1 headline, judges it against two criteria, and requires the headline to exist before evaluation runs. +The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before evaluation runs. ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator +type: check +id: HeadlineQuality +name: Headline Quality severity: error evaluateAs: document target: @@ -100,44 +74,10 @@ target: group: 1 required: true suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 --- -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. +You are a headline reviewer for developer blog posts. Evaluate the H1 headline +and report specific, fixable violations — for example: a vague benefit, a +missing subject, or passive voice. For each violation, quote the exact text and +suggest a concrete rewrite. ``` diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index 1fcaccb1..3dfe0c20 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -79,21 +79,19 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| | `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | -| `type` | `check` | Evaluation mode: `check` or `judge`. See below. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` (and its `subjective` alias) is no longer supported and is rejected at load. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | | `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | -| `criteria` | _(none)_ | Required for `judge` rules. Defines the scoring dimensions. | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | ## Evaluation Modes -VectorLint supports two evaluation modes, chosen with the `type` field. +VectorLint uses a single evaluation mode, set with the `type` field. | Mode | `type` value | Best for | |------|-------------|----------| | Check | `check` | Finding specific, countable violations (grammar errors, banned terms) | -| Judge | `judge` | Measuring quality on a spectrum (clarity, tone, completeness) | ### Check rules @@ -122,69 +120,6 @@ Report any errors found with specific examples. | `standard` (4–7) | ~10 points | | `strict` (8–10) | ~20 points | -### Judge rules - -The LLM scores content against multiple weighted criteria using a 1–4 rubric. VectorLint normalizes each score to a 1–10 scale and computes a weighted average. - -| LLM rating | Meaning | Normalized | -|------------|---------|------------| -| 4 | Excellent | 10.0 | -| 3 | Good | 7.0 | -| 2 | Fair | 4.0 | -| 1 | Poor | 1.0 | - -Define criteria in the frontmatter and expand each into a rubric section in the Markdown body: - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. -``` - ## Targeting Specific Content The `target` field lets you evaluate a specific portion of a document — for example, only the H1 headline — rather than the full content. @@ -227,51 +162,23 @@ RunRules=Acme ## Examples -### AI Pattern Detector (Judge) +### AI Pattern Detector (Check) ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge +type: check id: AIPatterns name: AI Pattern Detector severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 --- -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Overused buzzwords**: leverage, synergy, elevate, unlock, empower -2. **Formulaic transitions**: Moreover, Furthermore, In conclusion -3. **Hollow openings**: "In today's fast-paced world..." -4. **Excessive hedging**: "It's worth noting that...", "It's important to mention..." - -## RUBRIC - -# Language Authenticity - -### Excellent -Natural, human voice throughout. No AI patterns detected. - -### Good -Mostly natural. One or two minor patterns present. - -### Fair -Several AI patterns reduce authenticity. - -### Poor -Pervasive AI patterns throughout. +Detect AI-generated writing patterns in this content. Report each occurrence as a +violation: overused buzzwords (leverage, synergy, elevate, unlock, empower), +formulaic transitions (Moreover, Furthermore, In conclusion), hollow openings +("In today's fast-paced world..."), and excessive hedging ("It's worth noting +that..."). For each violation, quote the exact text and suggest a concrete, +human-sounding rewrite. ``` ## Next Steps diff --git a/tests/findings/README.md b/tests/findings/README.md new file mode 100644 index 00000000..f48cc664 --- /dev/null +++ b/tests/findings/README.md @@ -0,0 +1,83 @@ +# Finding Processing — Test Coverage & Behavior Mapping + +This directory owns the regression coverage for the **shared objective +violation-check finding-processing pipeline** (`src/findings/`). It documents +which behaviors moved out of `src/cli/orchestrator.ts` (and adjacent output +tests) into `src/findings/`, which behaviors were **intentionally removed**, and +which behaviors are **out of scope** for this phase. + +The pipeline (`processFindings`) composes a single path: + +```text +candidate findings ──▶ verify evidence ──▶ filter ──▶ dedupe ──▶ score ──▶ severity ──▶ ReviewResult +``` + +It is independent of model call (`single` / `agent` / `auto`) and carries no +`judge`, `evaluator`, rubric, or autonomous-agent compatibility path. + +--- + +## Behavior-preservation mapping + +| Behavior (origin) | Preserved by | Notes | +|---|---|---| +| Violation filtering / surfacing (`orchestrator.ts` `computeFilterDecision`) | `processor.test.ts` — *drops candidates that fail computeFilterDecision without a diagnostic*; *filtered candidates do not contribute to the score count* | Filter logic itself stays in `src/debug/violation-filter.ts`; the processor is a caller. `orchestrator-filtering.test.ts` still guards the end-to-end CLI surfacing. | +| Fuzzy finding-evidence location (`locateQuotedText`) | `finding-evidence-verifier.test.ts` — exact-quote location, warn diagnostic on miss, **no model-line fallback**, anchors without a line hint, diagnostic names the quoted text | One verifier (audit Finding #6). The agent path's model-line fallback is **not** preserved (see Removed). | +| Verified-finding counting & density scoring (`calculateCheckScore`) | `scorer.test.ts` — same numeric result as `calculateCheckScore`; score driven by verified count, not raw candidate count; resolves error severity; forwards strictness/severity | `scorer.ts` is a thin wrapper; no reimplemented math. | +| Severity + rule-id attribution | `severity.test.ts` — `resolveSeverity`, `buildRuleId` (`Pack.Rule` / `Pack.Rule.Criterion`), `resolveCriterionId`; `processor.test.ts` — stamps severity, attributes to pack rule / criterion | One severity path; one rule-id builder (audit Finding #4). | +| Golden check output (byte-for-byte) | `processor.test.ts` — *produces byte-for-byte findings/score matching the standard orchestrator path* | Proves no regression before the orchestrator was rewired (Task 2). | +| Unanchored-evidence diagnostics & exit behavior | `processor.test.ts` — *turns unanchored evidence into a warn diagnostic and emits no finding*; *counts only verified findings toward the score*; *does not set hadOperationalErrors for warn-level evidence diagnostics* | Intentional Phase 3 fix: unanchored quotes no longer flag the run as operationally failed. | +| Deduplication | `processor.test.ts` — *deduplicates verified findings by quoted_text and line* | | +| Contract / input boundary | `types.test.ts` — `FINDING_PROCESSING_INPUT_SCHEMA`, `RAW_VIOLATION_SCHEMA`, `PROMPT_META_FOR_FINDINGS_SCHEMA` parse supported input and reject legacy `evaluator`/`judge`/`rubric weight`/`agent`/`modelCall`/`mode`; `module-surface.test.ts` — public surface + diagnostic code constant | Strict Zod at the boundary; no `any`. | +| Formatter routing (line / JSON / RDJSON / Vale) | `orchestrator-check-processor.test.ts` — locatable findings + score + counts unchanged; verified findings through the JSON sink; unanchored quotes omitted from Vale JSON. `orchestrator-filtering.test.ts` — no dummy JSON/Vale issues when nothing is surfaced | End-to-end coverage of the rewired check path feeding existing formatters from `ReviewResult`. | + +--- + +## Intentionally removed behavior (no replacement fixture) + +These were legacy paths that the Phase 3 contract no longer supports. They are +covered by **rejection tests**, not behavior-parity tests. + +| Removed behavior | Rejection coverage | +|---|---| +| Subjective `judge` / rubric criterion evaluation | `prompt-loader-validation.test.ts` — *Judge boundary rejection (Phase 3)* rejects `type: judge` and the `subjective` alias at the loader boundary; `orchestrator-filtering.test.ts` — *rejects judge/rubric results instead of projecting them as check findings* | +| Legacy `evaluator` / `judge` / rubric criteria in the review contract | `review/types.test.ts` — *rejects legacy evaluator and judge criteria fields*; `findings/types.test.ts` — input contract rejects judge/evaluator/rubric shapes | +| Agent finding-evidence model-line fallback | `finding-evidence-verifier.test.ts` — *never returns a finding whose line came from the model when the quote did not anchor* (the removed behavior is asserted as absent) | + +Judge criterion output behavior (1–4 rubric, weighted average) has **no** +replacement fixture in this phase. Objective rule coverage belongs to the +objective-check test inventory, not a ported rubric. + +--- + +## Out of scope for Phase 3 (Phase 4 territory) + +These are **not** behavior-preservation targets for this phase and are +intentionally left untouched: + +- `tests/agent/*`, the orchestrator test covering legacy agent output, and + `tests/providers/*agent-loop*.test.ts` — the old autonomous agent surface. + Phase 4 deletes or replaces these with agent-model-call executor tests. +- `BaseEvaluator`'s internal judge mode (`runJudgeEvaluation`), + `buildJudgeLLMSchema` (`src/prompts/schema.ts`), the judge scoring helpers in + `src/scoring/`, `tests/scoring-types.test.ts`, and the judge cases in + `tests/prompt-schema.test.ts` — evaluator internals that are now **unreachable + from the CLI** (judge prompts fail to load) but are scheduled for deletion in + Phase 4 (executors / provider interface / agent deletion). + +Phase 3 added **no** compatibility code for the legacy agent surface — no +agent tool-loop wiring, no review-instruction handler, and no agent result +projection — to `src/findings/`. + +--- + +## Files + +| File | Covers | +|---|---| +| `finding-evidence-verifier.test.ts` | Single evidence verifier; no model-line fallback. | +| `severity.test.ts` | `resolveSeverity`, `buildRuleId`, `resolveCriterionId`. | +| `scorer.test.ts` | `scoreCheck` numerics vs `calculateCheckScore`; verified-count scoring. | +| `types.test.ts` | Zod input contract: parses supported input, rejects legacy shapes. | +| `processor.test.ts` | `processFindings` golden output, diagnostics, counting fix, dedupe, rule-id attribution, `ReviewResult` validation. | +| `module-surface.test.ts` | Public barrel surface + stable diagnostic code constant. | From 4fa07011105f3f2b84df3ae22d41fd774e27749e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:50:55 +0100 Subject: [PATCH 18/27] Sanitize prompt criteria at the findings boundary - Map meta.criteria to { id, name } before processFindings so legacy PromptCriterionSpec rubric fields (weight, target) cannot cross the standard check orchestration -> findings contract - Add tests/orchestrator-finding-criteria.test.ts mocking processFindings to assert weight/target are stripped on the production path - prompt-loader criteria support is unchanged; criterion id/name still drive output rule-id attribution --- src/cli/orchestrator.ts | 6 +- tests/orchestrator-finding-criteria.test.ts | 126 ++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 tests/orchestrator-finding-criteria.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 4d09e917..1e61a47d 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -206,7 +206,11 @@ function routePromptResult( promptMeta: { ...(meta.severity !== undefined ? { severity: meta.severity } : {}), ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - ...(meta.criteria ? { criteria: meta.criteria } : {}), + // Sanitize to the findings contract ({ id, name }): meta.criteria is + // PromptCriterionSpec[], which can carry legacy rubric weight/target. + ...(meta.criteria + ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } + : {}), }, targetContent: content, }); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts new file mode 100644 index 00000000..dc0b1ad8 --- /dev/null +++ b/tests/orchestrator-finding-criteria.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; +import type { ReviewResult } from "../src/review/types"; + +const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), + PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), +})); + +// createEvaluator returns a controllable evaluator so evaluateFiles reaches +// routePromptResult -> processFindings without a real model call. +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +// Replace only processFindings (the findings boundary); keep every other +// findings export real so unrelated imports stay intact. +vi.mock("../src/findings", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; +}); + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +function makeCheckResult(): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: [], + word_count: 100, + }; +} + +describe("standard check orchestration sanitizes criteria at the findings boundary", () => { + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + PROCESS_FINDINGS_MOCK.mockReset(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("drops rubric weight/target from criteria before reaching processFindings", async () => { + const targetFile = createTempFile("Alpha text\n"); + // PromptCriterionSpec admits legacy rubric fields (weight, target). The + // orchestrator must strip them so only { id, name } crosses the boundary. + const prompt = createPrompt({ + id: "SanitizePrompt", + name: "Sanitize Prompt", + type: "check", + severity: Severity.WARNING, + criteria: [ + { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, + ], + }); + + EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); + PROCESS_FINDINGS_MOCK.mockReturnValue({ + findings: [], + scores: [ + { + ruleId: "TestPack.SanitizePrompt", + score: 10, + scoreText: "10.0/10", + severity: "warning", + }, + ], + diagnostics: [], + }); + + await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); + const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { + promptMeta: { criteria?: Array> }; + }; + + // toEqual fails if any extra (weight/target) key survives the mapping. + expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); + for (const criterion of input.promptMeta.criteria ?? []) { + expect(criterion).not.toHaveProperty("weight"); + expect(criterion).not.toHaveProperty("target"); + } + }); +}); From 699f3557e31ec86c0ac62d64ce8270e15f31221f Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 09:23:27 +0100 Subject: [PATCH 19/27] Define structured and tool-calling model clients - Add StructuredModelClient (runPromptStructured + LLMResult/TokenUsage) as the permanent single structured-output capability - Add ToolCallingModelClient: bounded, executor-owned tool transport (runWithTools) that names no workspace/agent-finding concepts - Make LLMProvider extend StructuredModelClient; keep runAgentToolLoop only as a temporary compile bridge for src/agent/ (deleted in a later task) - VercelAIProvider implements both capabilities via a bounded runWithTools transport (caller-owned tools + caller-set step budget), no product loop - Extend AIExecutionContext.operation with a transport-shaped 'tool-calling' label - Add provider contract tests asserting the new surfaces and the absence of legacy workspace-agent concepts Refs: docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md (Finding #2; Product Decision) --- src/observability/ai-observability.ts | 2 +- src/providers/index.ts | 8 +- src/providers/llm-provider.ts | 23 ++-- src/providers/structured-model-client.ts | 28 +++++ src/providers/tool-calling-model-client.ts | 53 ++++++++ src/providers/vercel-ai-provider.ts | 119 +++++++++++++++++- .../providers/structured-model-client.test.ts | 67 ++++++++++ .../tool-calling-model-client.test.ts | 95 ++++++++++++++ 8 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 src/providers/structured-model-client.ts create mode 100644 src/providers/tool-calling-model-client.ts create mode 100644 tests/providers/structured-model-client.test.ts create mode 100644 tests/providers/tool-calling-model-client.test.ts diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 5369c639..e16a3c9d 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,5 +1,5 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'agent-tool-loop'; + operation: 'structured-eval' | 'tool-calling' | 'agent-tool-loop'; provider: string; model: string; evaluator?: string; diff --git a/src/providers/index.ts b/src/providers/index.ts index 4bdb538c..2c02f6e0 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,4 +1,10 @@ -export { LLMProvider, type LLMResult } from './llm-provider'; +export { LLMProvider } from './llm-provider'; +export { type StructuredModelClient, type LLMResult } from './structured-model-client'; +export { + type ToolCallingModelClient, + type ToolCallDefinition, + type ToolCallRunOptions, +} from './tool-calling-model-client'; export { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; export { SearchProvider } from './search-provider'; export { PerplexitySearchProvider, type PerplexitySearchConfig } from './perplexity-provider'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index f87304dd..635d5ecd 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,10 +1,12 @@ import type { TokenUsage } from './token-usage'; -import type { EvalContext } from './request-builder'; +import type { StructuredModelClient } from './structured-model-client'; -export interface LLMResult { - data: T; - usage?: TokenUsage; -} +/** + * `LLMResult` now lives on the permanent structured-output capability. It is + * re-exported here so existing deep imports (`from './llm-provider'`) keep + * compiling until the legacy provider surface is removed in a later task. + */ +export type { LLMResult } from './structured-model-client'; export interface AgentToolDefinition { description: string; @@ -25,7 +27,14 @@ export interface AgentToolLoopResult { usage?: TokenUsage; } -export interface LLMProvider { - runPromptStructured(content: string, promptText: string, schema: { name: string; schema: Record }, context?: EvalContext): Promise>; +/** + * Temporary compile bridge, preserved only until later tasks remove the + * autonomous agent surface. It extends the permanent + * {@link StructuredModelClient} (single structured output) with the legacy + * `runAgentToolLoop` method still consumed by `src/agent/`. Do not expand this + * surface; `runAgentToolLoop` and the `Agent*` types above are deleted once the + * agent tree is removed (audit Finding #2; Product Decision). + */ +export interface LLMProvider extends StructuredModelClient { runAgentToolLoop(params: AgentToolLoopParams): Promise; } diff --git a/src/providers/structured-model-client.ts b/src/providers/structured-model-client.ts new file mode 100644 index 00000000..99b1a1e0 --- /dev/null +++ b/src/providers/structured-model-client.ts @@ -0,0 +1,28 @@ +import type { TokenUsage } from './token-usage'; +import type { EvalContext } from './request-builder'; + +/** + * Result of a structured model call: validated output plus optional usage. + */ +export interface LLMResult { + data: T; + usage?: TokenUsage; +} + +/** + * Permanent structured-output model capability (audit Finding #2). + * + * A {@link StructuredModelClient} makes a single structured model call and + * returns validated output. It owns no tool surface and no autonomous agent + * loop; that product-level autonomy is removed from the provider surface + * (audit Product Decision). The single structured-call executor builds on this + * capability. + */ +export interface StructuredModelClient { + runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record }, + context?: EvalContext, + ): Promise>; +} diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts new file mode 100644 index 00000000..2139c8f2 --- /dev/null +++ b/src/providers/tool-calling-model-client.ts @@ -0,0 +1,53 @@ +import type { ZodType } from 'zod'; +import type { LLMResult } from './structured-model-client'; + +/** + * An executor-owned tool definition supplied to the bounded tool-calling + * transport. The provider is transport only: it does not define or name + * product tools. Descriptions and parameter schemas come from the caller (the + * agent executor), keyed by the caller-supplied tool name, never from the + * provider (audit Finding #2). The object key in the tools map is the tool + * name, matching the existing provider tool-mapping convention. + */ +export interface ToolCallDefinition { + description: string; + parameters: ZodType; + execute: (input: unknown) => Promise; +} + +/** + * Bounded execution metadata for a single tool-calling run. The caller + * (executor) sets the bounds from the review budget (audit Finding #7); the + * transport honors them. It does not invent its own product-level loop. + */ +export interface ToolCallRunOptions { + /** Maximum number of model steps (tool-call rounds) before the run stops. */ + maxSteps?: number; + /** Provider retry budget for transient failures. */ + maxRetries?: number; + /** Maximum concurrent tool executions per model step. Defaults to 1. */ + maxParallelToolCalls?: number; + /** Optional abort signal for cooperative cancellation. */ + signal?: AbortSignal; +} + +/** + * Permanent bounded tool-calling model capability (audit Finding #2). + * + * A {@link ToolCallingModelClient} performs a single bounded generation that + * may execute caller-supplied tools and then emits structured output. The tool + * map is executor-owned and transport-shaped: this interface deliberately + * names no product, workspace, or agent-finding concept (audit Product + * Decision; Finding #1). The autonomous product loop is removed; bounded + * orchestration lives in the executor, which supplies the only tool(s) it + * permits (for example a target-scoped reader). + */ +export interface ToolCallingModelClient { + runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise>; +} diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 5cd833fd..ed314001 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -3,11 +3,20 @@ import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from '../observability/ai-observability'; import { handleUnknownError } from '../errors'; +/** + * Conservative default step cap for a single bounded tool-calling run when the + * caller does not supply one. Executors should pass an explicit `maxSteps` + * derived from the review budget (audit Finding #7) rather than relying on + * this default. + */ +const DEFAULT_TOOL_CALLING_MAX_STEPS = 10; + export interface VercelAIConfig { model: LanguageModel; providerName?: string; @@ -21,7 +30,7 @@ export interface VercelAIConfig { observability?: AIObservability; } -export class VercelAIProvider implements LLMProvider { +export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; @@ -135,6 +144,114 @@ export class VercelAIProvider implements LLMProvider { } } + /** + * Bounded tool-calling transport (audit Finding #2). Performs a single + * bounded generation that may execute caller-supplied tools and then emits + * structured output. The tool map, step budget, and schema are all + * executor-owned; the provider defines no product tools and runs no + * autonomous product loop (audit Product Decision). + */ + async runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise> { + const options = params.options ?? {}; + const maxParallel = options.maxParallelToolCalls ?? 1; + const limit = pLimit(maxParallel); + + const zodSchema = this.jsonSchemaToZod(params.schema); + + const mappedTools = Object.fromEntries( + Object.entries(params.tools).map(([name, definition]) => [ + name, + tool({ + description: definition.description, + inputSchema: definition.parameters, + execute: (input: unknown) => limit(() => definition.execute(input)), + }), + ]), + ); + + if (this.config.debug) { + this.logger.debug('[vectorlint] Sending tool-calling request via Vercel AI SDK', { + model: this.config.model, + toolNames: Object.keys(params.tools), + maxSteps: options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS, + }); + } + + try { + const result = await generateText({ + model: this.config.model, + system: params.systemPrompt, + prompt: params.prompt, + ...(this.config.temperature !== undefined && { temperature: this.config.temperature }), + ...(this.config.maxTokens !== undefined && { maxTokens: this.config.maxTokens }), + ...(options.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + ...(options.signal !== undefined ? { abortSignal: options.signal } : {}), + ...this.getObservabilityOptions({ + operation: 'tool-calling', + provider: this.config.providerName ?? 'unknown', + model: this.config.modelName ?? 'unknown', + }), + stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), + providerOptions: { + openai: { + parallelToolCalls: maxParallel > 1, + }, + }, + tools: mappedTools, + output: Output.object({ + schema: zodSchema, + }), + }); + + if (this.config.debug && result.usage) { + this.logger.debug('[vectorlint] tool-calling LLM response meta', { + usage: { + input_tokens: result.usage.inputTokens, + output_tokens: result.usage.outputTokens, + total_tokens: result.usage.totalTokens, + }, + finish_reason: result.finishReason, + steps: result.steps.length, + }); + } + + const usage = result.usage + ? { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + } + : undefined; + + const output: unknown = result.output; + if (output === undefined || output === null) { + throw new Error( + `LLM returned no structured output. Raw text: ${result.text?.slice(0, 500) ?? '(empty)'}` + ); + } + + const llmResult: LLMResult = { data: output as T }; + if (usage) { + llmResult.usage = usage; + } + return llmResult; + } catch (e: unknown) { + if (NoObjectGeneratedError.isInstance(e)) { + const rawText = e instanceof Error && 'text' in e ? String(e.text) : 'unknown'; + throw new Error( + `LLM failed to generate valid structured output. Raw text: ${rawText}` + ); + } + const err = e instanceof Error ? e : new Error(String(e)); + throw new Error(`Vercel AI SDK tool-calling call failed: ${err.message}`); + } + } + runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { const maxParallel = params.maxParallelToolCalls ?? 1; const limit = pLimit(maxParallel); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts new file mode 100644 index 00000000..7fa6b61d --- /dev/null +++ b/tests/providers/structured-model-client.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import type { + LLMResult, + StructuredModelClient, +} from '../../src/providers/structured-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Schema', schema: { type: 'object' } }; + +describe('StructuredModelClient contract', () => { + it('exposes runPromptStructured and returns an LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: { ok: true } }), + }; + + expect(typeof client.runPromptStructured).toBe('function'); + + const result = await client.runPromptStructured('content', 'prompt', SCHEMA); + expect(result.data).toEqual({ ok: true }); + expect(result.usage).toBeUndefined(); + }); + + it('carries optional token usage on LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => + Promise.resolve({ data: 1, usage: { inputTokens: 3, outputTokens: 4 } }), + }; + + const result: LLMResult = await client.runPromptStructured( + 'content', + 'prompt', + SCHEMA, + ); + + expect(result.data).toBe(1); + expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 4 }); + }); + + it('does not declare an autonomous agent-loop method', () => { + // Compile-time guard: the capability surface is exactly runPromptStructured. + // If a second member were added, this conditional type would resolve to + // `false` and the assignment below would fail tsc. + type StructuredKeys = keyof StructuredModelClient; + const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; + expect(onlyRunPromptStructured).toBe(true); + + // Runtime guard: a minimal implementation exposes only runPromptStructured. + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: null }), + }; + expect( + (client as unknown as Record).runAgentToolLoop, + ).toBeUndefined(); + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy StructuredModelClient). + const client: StructuredModelClient = provider; + expect(typeof client.runPromptStructured).toBe('function'); + }); +}); diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts new file mode 100644 index 00000000..ccf84f68 --- /dev/null +++ b/tests/providers/tool-calling-model-client.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import { z } from 'zod'; +import type { + ToolCallDefinition, + ToolCallRunOptions, + ToolCallingModelClient, +} from '../../src/providers/tool-calling-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Findings', schema: { type: 'object' } }; + +describe('ToolCallingModelClient contract', () => { + it('exposes a single bounded runWithTools method', () => { + // Compile-time guard: the capability surface is exactly runWithTools. + type ToolCallingKeys = keyof ToolCallingModelClient; + const onlyRunWithTools: ToolCallingKeys extends 'runWithTools' ? true : false = true; + expect(onlyRunWithTools).toBe(true); + + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: { findings: [] } }), + }; + expect(typeof client.runWithTools).toBe('function'); + }); + + it('returns structured output plus optional usage', async () => { + const client: ToolCallingModelClient = { + runWithTools: () => + Promise.resolve({ + data: { findings: [{ line: 1 }] }, + usage: { inputTokens: 5, outputTokens: 6 }, + }), + }; + + const result = await client.runWithTools({ + systemPrompt: 'system', + prompt: 'prompt', + tools: { read: makeTool() }, + schema: SCHEMA, + options: { maxSteps: 3, maxParallelToolCalls: 1 } satisfies ToolCallRunOptions, + }); + + expect(result.data).toEqual({ findings: [{ line: 1 }] }); + expect(result.usage).toEqual({ inputTokens: 5, outputTokens: 6 }); + }); + + it('accepts executor-owned tool definitions with typed parameters', () => { + const parameters = z.object({ startLine: z.number().int().positive() }); + const definition: ToolCallDefinition = { + description: 'paged read', + parameters, + execute: (input) => Promise.resolve(input), + }; + + expect(definition.parameters).toBe(parameters); + expect(typeof definition.execute).toBe('function'); + }); + + it('does not name workspace-agent or product-finding concepts on the capability surface', () => { + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: {} }), + }; + const keys = Object.keys(client as Record); + + const forbidden = [ + 'read_file', + 'search_content', + 'report_finding', + 'reviewInstruction', + 'runAgentToolLoop', + ]; + for (const concept of forbidden) { + expect(keys).not.toContain(concept); + } + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy ToolCallingModelClient). + const client: ToolCallingModelClient = provider; + expect(typeof client.runWithTools).toBe('function'); + }); +}); + +function makeTool(): ToolCallDefinition { + return { + description: 'a bounded executor-owned tool', + parameters: z.object({ value: z.number().int().positive() }), + execute: () => Promise.resolve({ ok: true }), + }; +} From 74957bcb6528d35988c11f66705d4875ade1f503 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 09:54:05 +0100 Subject: [PATCH 20/27] Implement SingleModelCallExecutor for single modelCall - Add SingleModelCallExecutor behind the Phase 2 ReviewExecutor contract, reviewing source-backed rules with one structured model call per (rule, chunk) through an injected StructuredModelClient. - Build rule prompts verbatim from request.rules[i].body; no tool surface and no reviewInstruction. - Reuse buildCheckLLMSchema, RecursiveChunker, mergeViolations, prependLineNumbers, and processFindings instead of parallel projection. - Enforce ReviewBudget.maxModelCallsPerReview via enforceBudget before each call; on exhaustion record an error diagnostic + hadOperationalErrors and return partial results rather than throwing past the contract. - Record usage (modelCalls, wallClockMs always; input/output tokens gated on outputPolicy.includeUsage) consistent with ReviewResult.usage. - Add focused tests: one call per rule, projected findings/scores/diagnostics, unanchored-evidence diagnostic, budget exhaustion, no tool calls, and large-target chunking/merging. --- src/executors/single-model-call-executor.ts | 227 ++++++++++++++++ .../single-model-call-executor.test.ts | 248 ++++++++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 src/executors/single-model-call-executor.ts create mode 100644 tests/executors/single-model-call-executor.test.ts diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts new file mode 100644 index 00000000..de3957cb --- /dev/null +++ b/src/executors/single-model-call-executor.ts @@ -0,0 +1,227 @@ +import path from 'path'; + +import type { ReviewExecutor } from '../review/executor'; +import { BudgetExceededError, enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, + ReviewUsage, +} from '../review/types'; +import type { EvalContext } from '../providers/request-builder'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import { Severity } from '../evaluators/types'; +import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; +import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; +import { prependLineNumbers } from '../output/line-numbering'; +import { processFindings } from '../findings'; + +/** + * Word-count threshold above which the single-call executor chunks the target + * before reviewing it. Mirrors the check evaluator's chunking threshold so the + * single-call path preserves the existing chunk/merge behavior for large + * documents. + */ +const CHUNKING_WORD_THRESHOLD = 600; +const MAX_CHUNK_WORDS = 500; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules and chunks. + */ +interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * The single modelCall {@link ReviewExecutor} (audit Finding #2). + * + * Reviews target content against source-backed rules with one structured model + * call per (rule, chunk) through an injected {@link StructuredModelClient}. It + * owns no tool surface and no autonomous loop: rule prompts come verbatim from + * {@link ReviewRule.body}, candidate findings flow through the shared + * {@link processFindings} pipeline, and the model-call budget is enforced via + * the review budget module before every call. + * + * This is the bounded, transport-only review strategy selected by + * `--model-call single` (and by `auto` for normal-sized inputs). + */ +export class SingleModelCallExecutor implements ReviewExecutor { + constructor(private readonly client: StructuredModelClient) {} + + async run(request: ReviewRequest): Promise { + const schema = buildCheckLLMSchema(); + const context = this.buildContext(request.target.uri); + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const ruleResult = await this.reviewRule( + request, + rule, + schema, + context, + counters, + elapsedMs, + ); + findings.push(...ruleResult.findings); + scores.push(...ruleResult.scores); + diagnostics.push(...ruleResult.diagnostics); + if (ruleResult.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + if (error instanceof BudgetExceededError) { + // Surface the existing budget error pattern as an operational failure: + // the run returns partial results plus an error diagnostic rather than + // throwing past the ReviewExecutor contract. + hadOperationalErrors = true; + diagnostics.push({ + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: this.buildUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: chunks the line-numbered target, makes one + * structured model call per chunk, merges violations across chunks, and + * projects the merged candidates through {@link processFindings}. + */ + private async reviewRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + context: EvalContext, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + const numberedContent = prependLineNumbers(request.target.content); + const wordCount = countWords(request.target.content) || 1; + const chunks = this.chunkTarget(numberedContent, wordCount, request.budget.maxChunksPerRule); + + const chunkViolations: CheckLLMResult['violations'][] = []; + for (const chunk of chunks) { + // Enforce the model-call budget before committing to another call. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the call that would push the run over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runPromptStructured( + chunk.content, + rule.body, + schema, + context, + ); + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + chunkViolations.push(data.violations); + } + + const { pack, ruleId } = splitRuleId(rule.id); + const mergedViolations = mergeViolations(chunkViolations); + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: mergedViolations, + wordCount, + // Map the review contract's plain severity union onto the finding + // processor's Severity enum at this boundary. + promptMeta: { + ...(rule.severity !== undefined + ? { severity: rule.severity === 'error' ? Severity.ERROR : Severity.WARNING } + : {}), + }, + targetContent: request.target.content, + }); + } + + /** + * Chunks line-numbered target content for context management. Small targets + * review as a single chunk; larger targets are split recursively and capped + * at the review budget's per-rule chunk limit. + */ + private chunkTarget( + numberedContent: string, + wordCount: number, + maxChunks: number, + ): Chunk[] { + if (wordCount <= CHUNKING_WORD_THRESHOLD) { + return [{ content: numberedContent, index: 0 }]; + } + const chunker = new RecursiveChunker(); + return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); + } + + private buildContext(uri: string): EvalContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; + } + + private buildUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, + ): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; + } +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} diff --git a/tests/executors/single-model-call-executor.test.ts b/tests/executors/single-model-call-executor.test.ts new file mode 100644 index 00000000..5172f9b0 --- /dev/null +++ b/tests/executors/single-model-call-executor.test.ts @@ -0,0 +1,248 @@ +import { describe, expect, it } from 'vitest'; + +import { SingleModelCallExecutor } from '../../src/executors/single-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { + ReviewRequest, + ReviewRule, + ReviewTarget, +} from '../../src/review'; +import type { LLMResult, StructuredModelClient } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { CheckLLMResult } from '../../src/prompts/schema'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = CheckLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +type FakeStructuredClient = StructuredModelClient & { + runWithTools: () => Promise; + structuredCalls: number; + toolCalls: number; + lastPromptText?: string; +}; + +function makeFakeClient( + respond: (content: string, promptText: string) => { data: CheckLLMResult; usage?: TokenUsage }, +): FakeStructuredClient { + const client = { + structuredCalls: 0, + toolCalls: 0, + lastPromptText: undefined as string | undefined, + runPromptStructured: ( + content: string, + promptText: string, + ): Promise> => { + client.structuredCalls += 1; + client.lastPromptText = promptText; + const { data, usage } = respond(content, promptText); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + runWithTools: (): Promise => { + client.toolCalls += 1; + return Promise.reject( + new Error('SingleModelCallExecutor must not call runWithTools'), + ); + }, + }; + return client; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'single', + ...overrides, + }; +} + +describe('SingleModelCallExecutor', () => { + it('makes one structured call per rule and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + // One structured call, with the rule body as the source-backed prompt. + expect(client.structuredCalls).toBe(1); + expect(client.lastPromptText).toBe('Flag vague advice.'); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.line).toBeGreaterThanOrEqual(1); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('reviews each rule independently and aggregates usage across calls', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.structuredCalls).toBe(2); + expect(result.scores).toHaveLength(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's single call fits within maxModelCallsPerReview = 1. + expect(client.structuredCalls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('never invokes the tool-calling surface', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + await executor.run(makeRequest([makeRule()])); + + expect(client.toolCalls).toBe(0); + expect(client.structuredCalls).toBe(1); + }); + + it('chunks large targets into multiple model calls and merges per-chunk violations', async () => { + // > 600 words triggers recursive chunking. + const targetContent = Array.from({ length: 200 }, (_, i) => `vague text line ${i}`).join('\n'); + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text line 0' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([makeRule()], { + target: { uri: 'file:///repo/docs/big.md', content: targetContent, contentType: 'text/markdown' }, + }), + ); + + expect(client.structuredCalls).toBeGreaterThan(1); + expect(result.usage?.modelCalls).toBe(client.structuredCalls); + expect(result.scores).toHaveLength(1); + // Identical violations across chunks dedupe to a single anchored finding. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + }); +}); From 8bf8e86c4012b41680bc24be3d7b98e0887ad166 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 11:10:34 +0100 Subject: [PATCH 21/27] Implement AgentModelCallExecutor with target-only read tool - Add AgentModelCallExecutor implementing ReviewExecutor via one bounded ToolCallingModelClient.runWithTools call per rule - Expose exactly one executor-owned tool, read_target_section, through TargetReadCapability bound to in-memory request.target.content (no filesystem or non-target URI access) - Map out-of-range/invalid ranges to model-visible error results so the bounded run continues instead of aborting - Enforce ReviewBudget.maxModelCallsPerReview via enforceBudget before each run, surfacing exhaustion as an error diagnostic with partial results - Pass request.rules[i].body verbatim as the source-backed prompt; no reviewInstruction is accepted or introduced - Project results through src/findings/processor.ts so anchored findings, scores, and diagnostics match the shared pipeline - Extract shared executor helpers (splitRuleId, buildEvalContext, buildReviewUsage, toFindingSeverity, budgetExceededDiagnostic, RunCounters) into src/executors/shared.ts and reuse from the single executor to remove duplication - Add focused tests for the agent executor and target-read adapter --- src/executors/agent-model-call-executor.ts | 172 +++++++++++ src/executors/shared.ts | 96 ++++++ src/executors/single-model-call-executor.ts | 87 ++---- .../target-read-capability-adapter.ts | 162 ++++++++++ .../agent-model-call-executor.test.ts | 277 ++++++++++++++++++ .../target-read-capability-adapter.test.ts | 108 +++++++ 6 files changed, 833 insertions(+), 69 deletions(-) create mode 100644 src/executors/agent-model-call-executor.ts create mode 100644 src/executors/shared.ts create mode 100644 src/executors/target-read-capability-adapter.ts create mode 100644 tests/executors/agent-model-call-executor.test.ts create mode 100644 tests/executors/target-read-capability-adapter.test.ts diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts new file mode 100644 index 00000000..1b09f388 --- /dev/null +++ b/src/executors/agent-model-call-executor.ts @@ -0,0 +1,172 @@ +import type { ToolCallDefinition, ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { EvalContext, RequestBuilder } from '../providers/request-builder'; +import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; +import { countWords } from '../chunking'; +import { processFindings } from '../findings'; +import { enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, +} from '../review/types'; +import type { ReviewExecutor } from '../review/executor'; +import { TargetReadCapability, buildReadTargetSectionTool } from './target-read-capability-adapter'; +import { + budgetExceededDiagnostic, + buildEvalContext, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; + +/** + * The agent `modelCall` {@link ReviewExecutor} (audit Product Decision; Finding #2). + * + * Reviews target content against source-backed rules through a single bounded + * tool-calling run per rule via an injected {@link ToolCallingModelClient}. The + * only executor-owned tool exposed to the model is `read_target_section`, + * which pages through the in-memory `request.target.content` (the on-page + * boundary, audit Finding #5). Rule prompts come verbatim from + * {@link ReviewRule.body} — no model-supplied rule override is introduced — + * candidate findings flow through the shared {@link processFindings} pipeline, + * and the model-call budget is enforced via the review budget module before + * every call. + * + * This is the bounded, target-only review strategy selected by + * `--model-call agent` (and by `auto` for large inputs). + */ +export class AgentModelCallExecutor implements ReviewExecutor { + constructor( + private readonly client: ToolCallingModelClient, + private readonly builder: RequestBuilder, + ) {} + + async run(request: ReviewRequest): Promise { + const schema = buildCheckLLMSchema(); + const capability = new TargetReadCapability(request.target.content); + // Exactly one executor-owned tool is exposed: target-section paging. + const tools = buildReadTargetSectionTool(capability); + const context = buildEvalContext(request.target.uri); + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const ruleResult = await this.reviewRule( + request, + rule, + schema, + tools, + context, + capability.lineCount, + counters, + elapsedMs, + ); + findings.push(...ruleResult.findings); + scores.push(...ruleResult.scores); + diagnostics.push(...ruleResult.diagnostics); + if (ruleResult.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { + hadOperationalErrors = true; + diagnostics.push(diagnostic); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: buildReviewUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: makes one bounded tool-calling run that lets the + * model page through the target via `read_target_section`, then projects the + * returned violations through {@link processFindings}. + */ + private async reviewRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + tools: Record, + context: EvalContext, + targetLineCount: number, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + // Enforce the model-call budget before committing to the run. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the run that would push the review over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runWithTools({ + // The source-backed rule body, wrapped with the directive/user + // instructions exactly as the single-call path does. No model-supplied + // rule override is introduced. + systemPrompt: this.builder.buildPromptBodyForStructured(rule.body, context), + prompt: this.buildTargetPrompt(request.target.uri, targetLineCount), + tools, + schema, + options: { + // Each run is bounded by the budget's per-rule section limit: the + // model may page through at most maxChunksPerRule sections before + // emitting its structured finding set. + maxSteps: request.budget.maxChunksPerRule, + maxParallelToolCalls: 1, + }, + }); + + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + + const { pack, ruleId } = splitRuleId(rule.id); + const wordCount = countWords(request.target.content) || 1; + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: data.violations, + wordCount, + promptMeta: { + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), + }, + targetContent: request.target.content, + }); + } + + private buildTargetPrompt(uri: string, lineCount: number): string { + return [ + `Target URI: ${uri}`, + `Target lines: ${lineCount}`, + '', + 'Page through the target with the read_target_section tool, then report verified violations in the structured output.', + ].join('\n'); + } +} diff --git a/src/executors/shared.ts b/src/executors/shared.ts new file mode 100644 index 00000000..e9e24ef9 --- /dev/null +++ b/src/executors/shared.ts @@ -0,0 +1,96 @@ +import path from 'path'; + +import { Severity } from '../evaluators/types'; +import type { EvalContext } from '../providers/request-builder'; +import { BudgetExceededError } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewRequest, + ReviewSeverity, + ReviewUsage, +} from '../review/types'; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +export const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules (and chunks/sections) by both + * the single and agent model-call executors. + */ +export interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +export function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} + +/** + * Builds the provider {@link EvalContext} (file-type hint) for a target URI. + * Shared by both executors so structured and tool-calling calls receive the + * same file-type context for directive substitution. + */ +export function buildEvalContext(uri: string): EvalContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; +} + +/** + * Maps the review contract's plain {@link ReviewSeverity} union onto the + * finding processor's {@link Severity} enum at the executor boundary (no + * unsafe cast). + */ +export function toFindingSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; +} + +/** + * Aggregates run-wide counters into a {@link ReviewUsage}, including token + * counts only when the output policy opts in to usage reporting. + */ +export function buildReviewUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, +): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; +} + +/** + * Returns a `review-budget-exceeded` error diagnostic when `error` is a + * {@link BudgetExceededError}; otherwise returns `undefined` so the caller can + * rethrow non-budget errors unchanged. Shared by both executors' run loops so + * budget exhaustion surfaces as a partial-result operational failure instead + * of throwing past the {@link ReviewExecutor} contract. + */ +export function budgetExceededDiagnostic(error: unknown): ReviewDiagnostic | undefined { + if (!(error instanceof BudgetExceededError)) { + return undefined; + } + return { + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }; +} diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index de3957cb..3809cbb3 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -1,7 +1,5 @@ -import path from 'path'; - import type { ReviewExecutor } from '../review/executor'; -import { BudgetExceededError, enforceBudget } from '../review/budget'; +import { enforceBudget } from '../review/budget'; import type { ReviewDiagnostic, ReviewFinding, @@ -9,15 +7,21 @@ import type { ReviewResult, ReviewRule, ReviewScore, - ReviewUsage, } from '../review/types'; import type { EvalContext } from '../providers/request-builder'; import type { StructuredModelClient } from '../providers/structured-model-client'; -import { Severity } from '../evaluators/types'; import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; import { prependLineNumbers } from '../output/line-numbering'; import { processFindings } from '../findings'; +import { + budgetExceededDiagnostic, + buildEvalContext, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; /** * Word-count threshold above which the single-call executor chunks the target @@ -28,21 +32,6 @@ import { processFindings } from '../findings'; const CHUNKING_WORD_THRESHOLD = 600; const MAX_CHUNK_WORDS = 500; -/** - * Stable diagnostic code recorded when a run stops because the model-call - * budget was exhausted before every rule could be reviewed. - */ -const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; - -/** - * Mutable run-wide counters shared across rules and chunks. - */ -interface RunCounters { - modelCalls: number; - inputTokens: number; - outputTokens: number; -} - /** * The single modelCall {@link ReviewExecutor} (audit Finding #2). * @@ -61,7 +50,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { async run(request: ReviewRequest): Promise { const schema = buildCheckLLMSchema(); - const context = this.buildContext(request.target.uri); + const context = buildEvalContext(request.target.uri); const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; @@ -90,17 +79,13 @@ export class SingleModelCallExecutor implements ReviewExecutor { } } } catch (error: unknown) { - if (error instanceof BudgetExceededError) { - // Surface the existing budget error pattern as an operational failure: - // the run returns partial results plus an error diagnostic rather than - // throwing past the ReviewExecutor contract. + // Surface budget exhaustion as an operational failure: the run returns + // partial results plus an error diagnostic rather than throwing past the + // ReviewExecutor contract. Non-budget errors propagate unchanged. + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { hadOperationalErrors = true; - diagnostics.push({ - level: 'error', - code: REVIEW_BUDGET_EXCEEDED_CODE, - message: error.message, - context: { limit: error.limit, actual: error.actual }, - }); + diagnostics.push(diagnostic); } else { throw error; } @@ -111,7 +96,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { scores, diagnostics, hadOperationalErrors, - usage: this.buildUsage(request, counters, elapsedMs()), + usage: buildReviewUsage(request, counters, elapsedMs()), }; } @@ -165,12 +150,8 @@ export class SingleModelCallExecutor implements ReviewExecutor { ruleSource: rule.source, candidateFindings: mergedViolations, wordCount, - // Map the review contract's plain severity union onto the finding - // processor's Severity enum at this boundary. promptMeta: { - ...(rule.severity !== undefined - ? { severity: rule.severity === 'error' ? Severity.ERROR : Severity.WARNING } - : {}), + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), }, targetContent: request.target.content, }); @@ -192,36 +173,4 @@ export class SingleModelCallExecutor implements ReviewExecutor { const chunker = new RecursiveChunker(); return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); } - - private buildContext(uri: string): EvalContext { - const ext = path.extname(uri); - return ext ? { fileType: ext } : {}; - } - - private buildUsage( - request: ReviewRequest, - counters: RunCounters, - wallClockMs: number, - ): ReviewUsage { - const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; - if (request.outputPolicy.includeUsage) { - usage.inputTokens = counters.inputTokens; - usage.outputTokens = counters.outputTokens; - } - return usage; - } -} - -/** - * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. - * The review contract carries the composite id, while {@link processFindings} - * rebuilds the same id from the parts via `buildRuleId`. Splits on the first - * dot; pack names are single path segments and rule ids are PascalCase. - */ -function splitRuleId(id: string): { pack: string; ruleId: string } { - const dot = id.indexOf('.'); - if (dot === -1) { - return { pack: id, ruleId: id }; - } - return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; } diff --git a/src/executors/target-read-capability-adapter.ts b/src/executors/target-read-capability-adapter.ts new file mode 100644 index 00000000..041c286a --- /dev/null +++ b/src/executors/target-read-capability-adapter.ts @@ -0,0 +1,162 @@ +import { z } from 'zod'; + +import { VectorlintError } from '../errors'; +import type { ToolCallDefinition } from '../providers/tool-calling-model-client'; +import type { ReviewTargetReadCapability } from '../review/executor'; + +/** + * The single executor-owned tool name exposed to the bounded tool-calling + * transport (audit Product Decision): page through the target under review by + * 1-based line range. This is the only agent-like capability that survives the + * Phase 4 agent removal. + */ +export const READ_TARGET_SECTION_TOOL_NAME = 'read_target_section'; + +/** + * Zod schema for {@link TargetReadCapability.readTargetSection} arguments: + * 1-based positive integers. The tool adapter parses model input against this + * before slicing, so malformed arguments become model-visible errors. + */ +export const READ_TARGET_SECTION_PARAMETERS = z + .object({ + startLine: z.number().int().positive(), + endLine: z.number().int().positive(), + }) + .strict(); + +const READ_TARGET_SECTION_DESCRIPTION = [ + 'Read a 1-based inclusive [startLine, endLine] window of the target content under review.', + 'Returns the window with original line numbers prepended so findings can cite exact lines.', + 'A window outside [1, targetLineCount], or with startLine > endLine, returns an error result', + 'describing the valid range instead of aborting the review.', +].join(' '); + +/** + * Thrown by {@link TargetReadCapability.readTargetSection} when the requested + * window is outside the target's valid line range. The tool adapter translates + * this into a model-visible error result so the bounded run continues. + */ +export class TargetSectionRangeError extends VectorlintError { + constructor( + message: string, + public readonly lineCount: number, + ) { + super(message, 'TARGET_SECTION_RANGE'); + this.name = 'TargetSectionRangeError'; + } +} + +/** Success result of a {@link TargetReadCapability.readTargetSection} call. */ +export interface TargetSectionResult { + startLine: number; + endLine: number; + /** The requested window, each line prefixed with its 1-based line number. */ + content: string; +} + +/** Model-visible error result returned in place of a window for invalid calls. */ +export interface TargetSectionErrorResult { + error: string; + /** Total lines in the target, so the model can retry with a valid range. */ + lineCount: number; +} + +/** + * The on-page boundary adapter (audit Finding #5): a target-only + * {@link ReviewTargetReadCapability} bound to the in-memory + * `ReviewTarget.content`. It performs NO filesystem access and reads no URI + * other than the target it was constructed from. The agent executor exposes + * this single capability to the model via {@link buildReadTargetSectionTool}. + */ +export class TargetReadCapability implements ReviewTargetReadCapability { + private readonly lines: readonly string[]; + + constructor(content: string) { + // Mirrors prependLineNumbers: a trailing newline yields a final empty line. + this.lines = content.split('\n'); + } + + /** Number of addressable lines in the target. */ + get lineCount(): number { + return this.lines.length; + } + + readTargetSection(startLine: number, endLine: number): Promise { + this.assertValidRange(startLine, endLine); + const slice = this.lines.slice(startLine - 1, endLine); + const content = slice.map((line, index) => `${startLine + index}\t${line}`).join('\n'); + return Promise.resolve({ startLine, endLine, content }); + } + + private assertValidRange(startLine: number, endLine: number): void { + const lineCount = this.lineCount; + if (!Number.isInteger(startLine) || !Number.isInteger(endLine)) { + throw new TargetSectionRangeError( + `read_target_section requires integer line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine < 1 || endLine < 1) { + throw new TargetSectionRangeError( + `read_target_section requires 1-based positive line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > endLine) { + throw new TargetSectionRangeError( + `read_target_section requires startLine <= endLine; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section startLine=${startLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (endLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section endLine=${endLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + } +} + +/** + * Builds the single executor-owned tool map exposed to the + * {@link ToolCallingModelClient}: exactly one entry, `read_target_section`, + * bound to the target-only {@link TargetReadCapability}. Argument-parse and + * range failures are returned as model-visible error results (never thrown + * past the tool boundary) so the bounded review run continues. + */ +export function buildReadTargetSectionTool( + capability: TargetReadCapability, +): Record { + return { + [READ_TARGET_SECTION_TOOL_NAME]: { + description: READ_TARGET_SECTION_DESCRIPTION, + parameters: READ_TARGET_SECTION_PARAMETERS, + execute: async ( + input: unknown, + ): Promise => { + const parsed = READ_TARGET_SECTION_PARAMETERS.safeParse(input); + if (!parsed.success) { + return { + error: `Invalid read_target_section arguments: ${parsed.error.message}.`, + lineCount: capability.lineCount, + }; + } + const { startLine, endLine } = parsed.data; + try { + return await capability.readTargetSection(startLine, endLine); + } catch (error: unknown) { + if (error instanceof TargetSectionRangeError) { + return { error: error.message, lineCount: error.lineCount }; + } + throw error; + } + }, + }, + }; +} diff --git a/tests/executors/agent-model-call-executor.test.ts b/tests/executors/agent-model-call-executor.test.ts new file mode 100644 index 00000000..9ade0f1a --- /dev/null +++ b/tests/executors/agent-model-call-executor.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from 'vitest'; + +import { AgentModelCallExecutor } from '../../src/executors/agent-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { ReviewRequest, ReviewRule, ReviewTarget } from '../../src/review'; +import { DefaultRequestBuilder } from '../../src/providers/request-builder'; +import type { ToolCallDefinition, ToolCallingModelClient, ToolCallRunOptions } from '../../src/providers/tool-calling-model-client'; +import type { LLMResult } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { CheckLLMResult } from '../../src/prompts/schema'; +import type { TargetSectionErrorResult, TargetSectionResult } from '../../src/executors/target-read-capability-adapter'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = CheckLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +interface CapturedCall { + systemPrompt: string; + prompt: string; + tools: Record; + options?: ToolCallRunOptions; +} + +interface FakeToolClient extends ToolCallingModelClient { + calls: number; + captured: CapturedCall[]; +} + +function makeFakeClient( + respond: () => { data: CheckLLMResult; usage?: TokenUsage }, +): FakeToolClient { + const fake = { + calls: 0, + captured: [] as CapturedCall[], + runWithTools: ( + params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }, + ): Promise> => { + fake.calls += 1; + fake.captured.push({ + systemPrompt: params.systemPrompt, + prompt: params.prompt, + tools: params.tools, + ...(params.options !== undefined ? { options: params.options } : {}), + }); + const { data, usage } = respond(); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + }; + return fake; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\nthird line\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'agent', + ...overrides, + }; +} + +describe('AgentModelCallExecutor', () => { + it('exposes exactly one tool (read_target_section) and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + // Exactly one bounded model run, exposing exactly one tool. + expect(client.calls).toBe(1); + expect(Object.keys(client.captured[0]!.tools)).toEqual(['read_target_section']); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('passes the source-backed rule body verbatim as the system prompt (no model-supplied rule override)', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule({ body: 'Flag vague advice.' })])); + + // With an empty directive/user-instructions builder, the system prompt is + // the rule body verbatim — no model-supplied rule override is introduced. + expect(client.captured[0]!.systemPrompt).toBe('Flag vague advice.'); + // The agent instruction names the only tool and the target. + expect(client.captured[0]!.prompt).toContain('read_target_section'); + expect(client.captured[0]!.prompt).toContain('file:///repo/docs/guide.md'); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's run fits within maxModelCallsPerReview = 1. + expect(client.calls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('reviews each rule independently and aggregates usage across runs', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.calls).toBe(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('exposes a read_target_section that slices only request.target.content', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule()])); + + const tool = client.captured[0]!.tools['read_target_section']!; + const result = (await tool.execute({ startLine: 2, endLine: 3 })) as TargetSectionResult; + + // The window matches the in-memory target lines with their real line numbers. + expect(result).toEqual({ + startLine: 2, + endLine: 3, + content: '2\tsecond line is fine\n3\tthird line', + }); + }); + + it('returns a model-visible error for out-of-range windows without aborting the run', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + // The run completes and returns a normal ReviewResult despite the model + // (simulated below) requesting an out-of-range section. + const result = await executor.run(makeRequest([makeRule()])); + expect(result.hadOperationalErrors).toBe(false); + + const tool = client.captured[0]!.tools['read_target_section']!; + const errorResult = (await tool.execute({ startLine: 1, endLine: 999 })) as TargetSectionErrorResult; + + expect(errorResult.error).toContain('999'); + expect(errorResult.lineCount).toBe(4); + }); + + it('bounds each run by the budget maxChunksPerRule step limit', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run( + makeRequest([makeRule()], { budget: { ...DEFAULT_REVIEW_BUDGET, maxChunksPerRule: 7 } }), + ); + + expect(client.captured[0]!.options?.maxSteps).toBe(7); + expect(client.captured[0]!.options?.maxParallelToolCalls).toBe(1); + }); +}); diff --git a/tests/executors/target-read-capability-adapter.test.ts b/tests/executors/target-read-capability-adapter.test.ts new file mode 100644 index 00000000..c46016af --- /dev/null +++ b/tests/executors/target-read-capability-adapter.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildReadTargetSectionTool, + READ_TARGET_SECTION_TOOL_NAME, + TargetReadCapability, + type TargetSectionErrorResult, + type TargetSectionResult, +} from '../../src/executors/target-read-capability-adapter'; + +const CONTENT = 'alpha line\nbeta line\ngamma line\n'; + +describe('TargetReadCapability', () => { + it('counts lines including the trailing empty line (mirrors prependLineNumbers)', () => { + expect(new TargetReadCapability(CONTENT).lineCount).toBe(4); + expect(new TargetReadCapability('only one').lineCount).toBe(1); + }); + + it('slices only the in-memory target content with original line numbers', async () => { + const capability = new TargetReadCapability(CONTENT); + + const first = await capability.readTargetSection(1, 2); + expect(first).toEqual({ + startLine: 1, + endLine: 2, + content: '1\talpha line\n2\tbeta line', + }); + + const tail = await capability.readTargetSection(3, 4); + expect(tail).toEqual({ + startLine: 3, + endLine: 4, + content: '3\tgamma line\n4\t', + }); + }); +}); + +describe('buildReadTargetSectionTool', () => { + function toolFor(content: string) { + return buildReadTargetSectionTool(new TargetReadCapability(content)); + } + + it('exposes exactly one tool named read_target_section', () => { + const tools = toolFor(CONTENT); + expect(Object.keys(tools)).toEqual([READ_TARGET_SECTION_TOOL_NAME]); + }); + + it('returns numbered target windows for valid ranges (target-only slicing)', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 3, + })) as TargetSectionResult; + + expect(result).toEqual({ + startLine: 1, + endLine: 3, + content: '1\talpha line\n2\tbeta line\n3\tgamma line', + }); + }); + + it('returns a model-visible error (never throws) when endLine exceeds the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 99, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('99'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error when startLine is past the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 10, + endLine: 12, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('10'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error for an inverted range', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 3, + endLine: 1, + })) as TargetSectionErrorResult; + + expect(result.error).toMatch(/startLine <= endLine/); + }); + + it('returns a model-visible error for malformed arguments', async () => { + const tools = toolFor(CONTENT); + const execute = tools[READ_TARGET_SECTION_TOOL_NAME]!.execute; + + const missing = (await execute({ startLine: 1 })) as TargetSectionErrorResult; + expect(missing.error).toMatch(/Invalid read_target_section arguments/); + expect(missing.lineCount).toBe(4); + + const negative = (await execute({ startLine: 0, endLine: 2 })) as TargetSectionErrorResult; + expect(negative.error).toMatch(/Invalid read_target_section arguments/); + + const floats = (await execute({ startLine: 1.5, endLine: 2 })) as TargetSectionErrorResult; + expect(floats.error).toMatch(/Invalid read_target_section arguments/); + }); +}); From 8bfb47de8264fd7e75440347d64e3c87e2b2f4fb Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:18:00 +0100 Subject: [PATCH 22/27] Wire --model-call executors into the CLI and remove the agent surface (core) --- src/agent/executor.ts | 843 ------------------ src/agent/path-utils.ts | 73 -- src/agent/progress.ts | 282 ------ src/agent/prompt-builder.ts | 57 -- src/agent/review-session-store.ts | 94 -- src/agent/rule-id.ts | 11 - src/agent/tools-registry.ts | 80 -- src/agent/types.ts | 134 --- src/cli/commands.ts | 22 +- src/cli/orchestrator.ts | 772 ++++------------ src/cli/types.ts | 65 +- src/executors/index.ts | 38 + src/providers/llm-provider.ts | 42 +- src/providers/provider-factory.ts | 5 +- src/providers/vercel-ai-provider.ts | 67 +- src/schemas/cli-schemas.ts | 5 +- tests/agent/agent-executor.test.ts | 841 ----------------- tests/agent/progress.test.ts | 90 -- tests/agent/prompt-builder.test.ts | 51 -- tests/agent/review-session-store.test.ts | 142 --- tests/agent/types-contract.test.ts | 83 -- tests/main-command-observability.test.ts | 6 +- tests/orchestrator-agent-output.test.ts | 165 ---- tests/orchestrator-check-processor.test.ts | 356 -------- tests/orchestrator-filtering.test.ts | 412 --------- tests/orchestrator-finding-criteria.test.ts | 126 --- tests/providers/llm-provider-contract.test.ts | 14 - .../providers/structured-model-client.test.ts | 6 +- .../tool-calling-model-client.test.ts | 2 - .../vercel-ai-provider-agent-loop.test.ts | 314 ------- 30 files changed, 263 insertions(+), 4935 deletions(-) delete mode 100644 src/agent/executor.ts delete mode 100644 src/agent/path-utils.ts delete mode 100644 src/agent/progress.ts delete mode 100644 src/agent/prompt-builder.ts delete mode 100644 src/agent/review-session-store.ts delete mode 100644 src/agent/rule-id.ts delete mode 100644 src/agent/tools-registry.ts delete mode 100644 src/agent/types.ts create mode 100644 src/executors/index.ts delete mode 100644 tests/agent/agent-executor.test.ts delete mode 100644 tests/agent/progress.test.ts delete mode 100644 tests/agent/prompt-builder.test.ts delete mode 100644 tests/agent/review-session-store.test.ts delete mode 100644 tests/agent/types-contract.test.ts delete mode 100644 tests/orchestrator-agent-output.test.ts delete mode 100644 tests/orchestrator-check-processor.test.ts delete mode 100644 tests/orchestrator-filtering.test.ts delete mode 100644 tests/orchestrator-finding-criteria.test.ts delete mode 100644 tests/providers/llm-provider-contract.test.ts delete mode 100644 tests/providers/vercel-ai-provider-agent-loop.test.ts diff --git a/src/agent/executor.ts b/src/agent/executor.ts deleted file mode 100644 index 749a5685..00000000 --- a/src/agent/executor.ts +++ /dev/null @@ -1,843 +0,0 @@ -import { readdir, readFile } from 'fs/promises'; -import * as os from 'os'; -import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; -import type { PromptFile } from '../prompts/prompt-loader'; -import { Type, Severity } from '../evaluators/types'; -import { computeFilterDecision } from '../evaluators/violation-filter'; -import { locateQuotedText } from '../output/location'; -import type { AgentToolLoopResult, LLMProvider } from '../providers/llm-provider'; -import type { TokenUsage } from '../providers/token-usage'; -import type { OutputFormat } from '../cli/types'; -import { createEvaluator } from '../evaluators'; -import { createReviewSessionStore } from './review-session-store'; -import { buildAgentSystemPrompt } from './prompt-builder'; -import { AgentToolError } from '../errors'; -import { ScanPathResolver } from '../boundaries/scan-path-resolver'; -import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import { - createAgentTools, - listAvailableTools, - type AgentToolHandler, - type AgentToolName, -} from './tools-registry'; -import { - LINT_TOOL_INPUT_SCHEMA, - FINALIZE_REVIEW_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, - SESSION_EVENT_TYPE, - type SessionEvent, -} from './types'; -import { resolveGlobPatternWithinRoot, resolveWithinRoot, toRelativePathFromRoot } from './path-utils'; -import type { AgentProgressReporter, VisibleToolName, VisibleToolProgress } from './progress'; -import { buildRuleId, normalizeRuleSource } from './rule-id'; -export interface AgentFinding { - file: string; - line: number; - column: number; - severity: Severity; - message: string; - ruleId: string; - ruleSource: string; - analysis?: string; - suggestion?: string; - fix?: string; - match?: string; -} - -export interface RunAgentExecutorParams { - targets: string[]; - prompts: PromptFile[]; - provider: LLMProvider; - workspaceRoot: string; - scanPaths: FilePatternConfig[]; - outputFormat: OutputFormat; - printMode: boolean; - sessionHomeDir?: string; - progressReporter?: AgentProgressReporter; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; - userInstructions?: string; -} - -export interface AgentExecutorResult { - findings: AgentFinding[]; - events: SessionEvent[]; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - requestFailures: number; - hadOperationalErrors: boolean; - errorMessage?: string; - usage?: AgentToolLoopResult['usage']; -} - -const MAX_SEARCH_FILE_RESULTS = 500; -const MAX_CONTENT_MATCH_RESULTS = 200; -const VISIBLE_TOOL_NAMES = new Set(['read_file', 'list_directory', 'lint']); - -function mergeTokenUsage(left?: TokenUsage, right?: TokenUsage): TokenUsage | undefined { - if (!left && !right) { - return undefined; - } - - return { - inputTokens: (left?.inputTokens ?? 0) + (right?.inputTokens ?? 0), - outputTokens: (left?.outputTokens ?? 0) + (right?.outputTokens ?? 0), - }; -} - -function buildUnknownRuleSourceError(ruleSource: string, validSources: string[]): Error { - const validHint = validSources.length > 0 ? validSources.join(', ') : '(none)'; - return new AgentToolError( - `Unknown ruleSource "${ruleSource}". Valid sources: ${validHint}`, - 'UNKNOWN_RULE_SOURCE' - ); -} - -function fallbackMessage(result: PromptEvaluationResult): string { - if (!isJudgeResult(result) && result.reasoning) { - return result.reasoning; - } - return 'Potential issue detected'; -} - -function severityFromPrompt(prompt: PromptFile): Severity { - return prompt.meta.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING; -} - -function resolvePromptBySource( - ruleSource: string, - promptBySource: Map -): PromptFile | undefined { - const normalized = normalizeRuleSource(ruleSource); - return promptBySource.get(normalized); -} - -function resolveTargetForTopLevel( - workspaceRoot: string, - targets: string[], - file?: string -): string { - if (file && file.trim().length > 0) { - const resolved = resolveWithinRoot(workspaceRoot, file); - return toRelativePathFromRoot(workspaceRoot, resolved); - } - if (targets.length > 0) { - return toRelativePathFromRoot(workspaceRoot, targets[0]!); - } - return '.'; -} - -function withReviewInstructionOverride(prompt: PromptFile, reviewInstruction?: string): PromptFile { - const instruction = reviewInstruction?.trim(); - if (!instruction) { - return prompt; - } - return { - ...prompt, - body: instruction, - }; -} - -function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { - const findings: AgentFinding[] = []; - - for (const event of events) { - if ( - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedInline && - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedTopLevel - ) { - continue; - } - - const payload = event.payload; - const file = payload.file ?? '.'; - const line = payload.line ?? 1; - findings.push({ - file, - line, - column: payload.column ?? 1, - severity: payload.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, - message: payload.message, - ruleId: payload.ruleId ?? payload.ruleSource, - ruleSource: payload.ruleSource, - ...('analysis' in payload && payload.analysis ? { analysis: payload.analysis } : {}), - ...('suggestion' in payload && payload.suggestion ? { suggestion: payload.suggestion } : {}), - ...('fix' in payload && payload.fix ? { fix: payload.fix } : {}), - ...('match' in payload && payload.match ? { match: payload.match } : {}), - }); - } - - return findings; -} - -// Build the concrete matched file-to-rule pairs that the agent should review for this run. -function buildFileRuleMatches( - relativeTargets: string[], - prompts: PromptFile[], - scanPaths: FilePatternConfig[] -): Array<{ file: string; ruleSource: string }> { - const resolver = new ScanPathResolver(); - const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); - const matches: Array<{ file: string; ruleSource: string }> = []; - - for (const relFile of relativeTargets) { - const resolution = - scanPaths.length > 0 - ? resolver.resolveConfiguration(relFile, scanPaths, availablePacks) - : { packs: availablePacks, overrides: {} }; - const matchedPrompts = prompts.filter((prompt) => { - if (prompt.pack === '') return true; - if (!prompt.pack) return false; - if (scanPaths.length > 0 && !resolution.packs.includes(prompt.pack)) return false; - if (!prompt.meta?.id) return true; - const disableKey = `${prompt.pack}.${prompt.meta.id}`; - const overrideValue = resolution.overrides[disableKey]; - return typeof overrideValue !== 'string' || overrideValue.toLowerCase() !== 'disabled'; - }); - - for (const prompt of matchedPrompts) { - matches.push({ file: relFile, ruleSource: normalizeRuleSource(prompt.fullPath) }); - } - } - - return matches; -} - -function summarizeToolOutput(toolName: AgentToolName, output: unknown): unknown { - if (typeof output !== 'object' || output === null) { - return output; - } - - const candidate = output as Record; - - switch (toolName) { - case 'read_file': { - const content = typeof candidate.content === 'string' ? candidate.content : ''; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - contentLength: content.length, - }; - } - case 'search_content': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'search_files': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'list_directory': { - const entries = Array.isArray(candidate.entries) ? candidate.entries : []; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - entryCount: entries.length, - }; - } - default: - return output; - } -} - -interface VisibleToolContext extends VisibleToolProgress { - signature: string; - progressFile?: string; -} - -function countLines(content: string): number { - if (content.length === 0) { - return 0; - } - return content.split('\n').filter((_, index, lines) => { - return !(index === lines.length - 1 && lines[index] === ''); - }).length; -} - -function tryResolveRelativePath(workspaceRoot: string, rawPath: string): string | undefined { - try { - return toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, rawPath)); - } catch { - return undefined; - } -} - -function resolveVisibleToolContext(params: { - toolName: AgentToolName; - input: unknown; - workspaceRoot: string; - promptBySource: Map; - targetFiles: Set; - currentProgressFile?: string; - defaultProgressFile?: string; -}): VisibleToolContext | undefined { - const { - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - currentProgressFile, - defaultProgressFile, - } = params; - - if (!VISIBLE_TOOL_NAMES.has(toolName as VisibleToolName)) { - return undefined; - } - - switch (toolName) { - case 'read_file': { - const parsed = READ_FILE_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'read_file', - path, - ...(progressFile ? { progressFile } : {}), - signature: `read_file:${path}`, - }; - } - case 'list_directory': { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = currentProgressFile ?? defaultProgressFile; - return { - toolName: 'list_directory', - path, - ...(progressFile ? { progressFile } : {}), - signature: `list_directory:${path}`, - }; - } - case 'lint': { - const parsed = LINT_TOOL_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const prompt = resolvePromptBySource(parsed.data.ruleSource, promptBySource); - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); - const path = resolvedPath ?? parsed.data.file; - const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'lint', - path, - ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), - ruleText, - ...(progressFile ? { progressFile } : {}), - signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, - }; - } - } -} - -function buildVisibleToolSuccessState( - context: VisibleToolContext, - output: unknown -): VisibleToolProgress { - switch (context.toolName) { - case 'read_file': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const content = typeof record.content === 'string' ? record.content : ''; - return { - ...context, - lineCount: countLines(content), - }; - } - case 'list_directory': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const entries = Array.isArray(record.entries) ? record.entries : []; - return { - ...context, - entryCount: entries.length, - }; - } - case 'lint': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - return { - ...context, - findingsCount: - typeof record.findingsRecorded === 'number' ? Math.max(0, Math.trunc(record.findingsRecorded)) : 0, - }; - } - } -} - -type FindingLikeViolation = { - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - description?: string; - analysis?: string; - message?: string; - suggestion?: string; - fix?: string; - confidence?: number; - checks?: GateChecks; -}; - -async function appendInlineFinding(params: { - violation: FindingLikeViolation; - result: PromptEvaluationResult; - content: string; - relFile: string; - prompt: PromptFile; - ruleSource: string; - store: Awaited>; -}): Promise { - const { violation, result, content, relFile, prompt, ruleSource, store } = params; - const filterDecision = computeFilterDecision(violation); - if (!filterDecision.surface) { - return false; - } - - const location = locateQuotedText( - content, - { - quoted_text: violation.quoted_text || '', - context_before: violation.context_before || '', - context_after: violation.context_after || '', - }, - 80, - violation.line - ); - - const line = location?.line ?? Math.max(1, Math.trunc(violation.line ?? 1)); - const column = location?.column ?? 1; - const match = location?.match ?? violation.quoted_text ?? ''; - const message = (violation.message || violation.description || fallbackMessage(result)).trim(); - - const finding: AgentFinding = { - file: relFile, - line, - column, - severity: severityFromPrompt(prompt), - message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(ruleSource), - ...(violation.analysis ? { analysis: violation.analysis } : {}), - ...(violation.suggestion ? { suggestion: violation.suggestion } : {}), - ...(violation.fix ? { fix: violation.fix } : {}), - ...(match ? { match } : {}), - }; - - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), - }, - }); - - return true; -} - -export async function runAgentExecutor(params: RunAgentExecutorParams): Promise { - const { - targets, - prompts, - provider, - workspaceRoot, - scanPaths, - sessionHomeDir = os.homedir(), - progressReporter, - maxSteps, - maxRetries, - maxParallelToolCalls, - userInstructions, - } = params; - - const promptBySource = new Map(); - for (const prompt of prompts) { - promptBySource.set(normalizeRuleSource(prompt.fullPath), prompt); - } - const validSources = Array.from(promptBySource.keys()).sort(); - - const store = await createReviewSessionStore({ homeDir: sessionHomeDir }); - let findingsCount = 0; - const relativeTargets = targets.map((target) => - toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, target)) - ); - const fileRuleMatches = buildFileRuleMatches(relativeTargets, prompts, scanPaths); - const defaultRuleName = String(prompts[0]?.meta.name || prompts[0]?.meta.id || 'Rule'); - const targetFiles = new Set(relativeTargets); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { - cwd: workspaceRoot, - targets: relativeTargets, - }, - }); - - let finalized = false; - let currentProgressFile: string | undefined; - const failedVisibleToolSignatures = new Set(); - let nestedToolUsage: TokenUsage | undefined; - - async function runTool( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ): Promise { - const visibleToolContext = resolveVisibleToolContext({ - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - ...(currentProgressFile ? { currentProgressFile } : {}), - ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), - }); - - if (visibleToolContext?.progressFile) { - const nextRuleName = visibleToolContext.ruleName ?? defaultRuleName; - if (currentProgressFile !== visibleToolContext.progressFile) { - progressReporter?.startFile(visibleToolContext.progressFile, nextRuleName); - currentProgressFile = visibleToolContext.progressFile; - } else if (visibleToolContext.ruleName) { - progressReporter?.updateRule(visibleToolContext.ruleName); - } - } - - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName, input }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolStart({ - ...visibleToolContext, - retrying: failedVisibleToolSignatures.has(visibleToolContext.signature), - }); - } - - try { - const output = await handler(input); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: true, - output: summarizeToolOutput(toolName, output), - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolSuccess(buildVisibleToolSuccessState(visibleToolContext, output)); - failedVisibleToolSignatures.delete(visibleToolContext.signature); - } - return output; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: false, - error: message, - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolError(visibleToolContext); - failedVisibleToolSignatures.add(visibleToolContext.signature); - } - throw error; - } - } - - async function lintToolHandler(input: unknown): Promise { - const parsed = LINT_TOOL_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - const promptForCall = withReviewInstructionOverride(prompt, parsed.reviewInstruction); - - const absoluteFile = resolveWithinRoot(workspaceRoot, parsed.file); - const relFile = toRelativePathFromRoot(workspaceRoot, absoluteFile); - const content = await readFile(absoluteFile, 'utf8'); - - const evaluator = createEvaluator( - promptForCall.meta.evaluator || Type.BASE, - provider, - promptForCall - ); - const result = await evaluator.evaluate(relFile, content); - nestedToolUsage = mergeTokenUsage(nestedToolUsage, result.usage); - - let findingsRecorded = 0; - const violations = isJudgeResult(result) - ? result.criteria.flatMap((criterion) => criterion.violations) - : result.violations; - for (const violation of violations) { - const wasRecorded = await appendInlineFinding({ - violation, - result, - content, - relFile, - prompt, - ruleSource: parsed.ruleSource, - store, - }); - if (wasRecorded) { - findingsCount += 1; - findingsRecorded += 1; - } - } - - return { - ok: true, - findingsRecorded, - schema: buildCheckLLMSchema().name, - }; - } - - async function reportFindingToolHandler(input: unknown): Promise { - const parsed = TOP_LEVEL_REPORT_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - - const references = parsed.references && parsed.references.length > 0 - ? parsed.references - : [{ file: resolveTargetForTopLevel(workspaceRoot, targets), startLine: 1, endLine: 1 }]; - - let findingsRecorded = 0; - for (const reference of references) { - const relFile = resolveTargetForTopLevel(workspaceRoot, targets, reference.file); - const finding: AgentFinding = { - file: relFile, - line: Math.max(1, Math.trunc(reference.startLine)), - column: 1, - severity: severityFromPrompt(prompt), - message: parsed.message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(parsed.ruleSource), - ...(parsed.suggestion ? { suggestion: parsed.suggestion } : {}), - }; - - findingsCount += 1; - findingsRecorded += 1; - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(parsed.references ? { references: parsed.references } : {}), - }, - }); - } - - return { ok: true, findingsRecorded }; - } - - async function readFileToolHandler(input: unknown): Promise { - const parsed = READ_FILE_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const content = await readFile(absolutePath, 'utf8'); - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - content, - }; - } - - async function searchFilesToolHandler(input: unknown): Promise { - const parsed = SEARCH_FILES_INPUT_SCHEMA.parse(input); - const scope = resolveGlobPatternWithinRoot(workspaceRoot, parsed.pattern); - const allMatches = await fg(scope.pattern, { - cwd: scope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const truncated = allMatches.length > MAX_SEARCH_FILE_RESULTS; - const matches = allMatches - .slice(0, MAX_SEARCH_FILE_RESULTS) - .map((match) => toRelativePathFromRoot(workspaceRoot, match)) - .sort((a, b) => a.localeCompare(b)); - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function listDirectoryToolHandler(input: unknown): Promise { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const entries = await readdir(absolutePath, { withFileTypes: true }); - - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - entries: entries - .map((entry) => ({ - name: entry.name, - type: entry.isDirectory() ? 'directory' : 'file', - })) - .sort((left, right) => left.name.localeCompare(right.name)), - }; - } - - async function searchContentToolHandler(input: unknown): Promise { - const parsed = SEARCH_CONTENT_INPUT_SCHEMA.parse(input); - const absoluteSearchRoot = resolveWithinRoot(workspaceRoot, parsed.path || '.'); - const globScope = resolveGlobPatternWithinRoot(absoluteSearchRoot, parsed.glob || '**/*'); - const files = await fg(globScope.pattern, { - cwd: globScope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const matches: Array<{ file: string; line: number; text: string }> = []; - let truncated = false; - - outer: for (const filePath of files) { - let content = ''; - try { - content = await readFile(filePath, 'utf8'); - } catch { - continue; - } - - const lines = content.split('\n'); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!; - if (line.includes(parsed.pattern)) { - if (matches.length >= MAX_CONTENT_MATCH_RESULTS) { - truncated = true; - break outer; - } - matches.push({ - file: toRelativePathFromRoot(workspaceRoot, filePath), - line: index + 1, - text: line, - }); - } - } - } - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function finalizeReviewToolHandler(input: unknown): Promise { - const parsed = FINALIZE_REVIEW_INPUT_SCHEMA.parse(input); - if (finalized) { - throw new AgentToolError( - 'finalize_review can only be called once per session.', - 'FINALIZE_REVIEW_ALREADY_CALLED' - ); - } - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { - totalFindings: findingsCount, - ...(parsed.summary ? { summary: parsed.summary } : {}), - }, - }); - finalized = true; - return { ok: true }; - } - - const tools = createAgentTools({ - runTool, - handlers: { - lint: lintToolHandler, - report_finding: reportFindingToolHandler, - read_file: readFileToolHandler, - search_files: searchFilesToolHandler, - list_directory: listDirectoryToolHandler, - search_content: searchContentToolHandler, - finalize_review: finalizeReviewToolHandler, - }, - }); - const availableTools = listAvailableTools(tools); - - let usage: AgentToolLoopResult['usage'] | undefined; - let requestFailures = 0; - let hadOperationalErrors = false; - let errorMessage: string | undefined; - - try { - const result = await provider.runAgentToolLoop({ - systemPrompt: buildAgentSystemPrompt({ - workspaceRoot, - fileRuleMatches, - availableTools, - ...(userInstructions ? { userInstructions } : {}), - }), - prompt: [ - `Workspace root: ${workspaceRoot}`, - `Targets: ${relativeTargets.join(', ')}`, - `Available ruleSources: ${validSources.join(', ')}`, - ].join('\n'), - tools, - ...(maxSteps !== undefined ? { maxSteps } : {}), - ...(maxRetries !== undefined ? { maxRetries } : {}), - ...(maxParallelToolCalls !== undefined ? { maxParallelToolCalls } : {}), - }); - usage = result.usage; - } catch (error) { - requestFailures += 1; - hadOperationalErrors = true; - errorMessage = error instanceof Error ? error.message : String(error); - } - - const events = await store.replay(); - const hasFinalizedEvent = events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - - if (!hasFinalizedEvent) { - hadOperationalErrors = true; - if (!errorMessage) { - errorMessage = 'Agent run ended without finalize_review.'; - } - } - - const findings = findingsFromEvents(events); - const aggregatedUsage = mergeTokenUsage(usage, nestedToolUsage); - progressReporter?.finishRun(hadOperationalErrors ? 'failed' : 'completed'); - - return { - findings, - events, - fileRuleMatches, - requestFailures, - hadOperationalErrors, - ...(errorMessage ? { errorMessage } : {}), - ...(aggregatedUsage ? { usage: aggregatedUsage } : {}), - }; -} diff --git a/src/agent/path-utils.ts b/src/agent/path-utils.ts deleted file mode 100644 index 55378366..00000000 --- a/src/agent/path-utils.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { realpathSync } from 'fs'; -import * as path from 'path'; -import { AgentToolError } from '../errors'; - -function isGlobSegment(segment: string): boolean { - return /[*?[\]{}()!+@]/.test(segment); -} - -export function resolveWithinRoot(root: string, inputPath: string): string { - const rootResolved = path.resolve(root); - const targetResolved = path.resolve(rootResolved, inputPath); - - // Lexical check: fast reject for obvious traversal (e.g. ../../etc) - const relative = path.relative(rootResolved, targetResolved); - const outsideRoot = relative.startsWith('..') || path.isAbsolute(relative); - if (outsideRoot) { - throw new AgentToolError( - `Path "${inputPath}" is outside workspace root bounds.`, - 'PATH_OUTSIDE_WORKSPACE_ROOT' - ); - } - - // Symlink check: resolve symlinks on both sides before comparing, so that - // a symlink inside the repo pointing to an external path is caught. - try { - const realRoot = realpathSync(rootResolved); - const realTarget = realpathSync(targetResolved); - const realRelative = path.relative(realRoot, realTarget); - if (realRelative.startsWith('..') || path.isAbsolute(realRelative)) { - throw new AgentToolError( - `Path "${inputPath}" escapes workspace root via symlink.`, - 'PATH_ESCAPES_WORKSPACE_ROOT_SYMLINK' - ); - } - } catch (err) { - // ENOENT means the path does not yet exist — no symlinks to follow, - // so the lexical check above is sufficient. Re-throw any other error. - if (!(err instanceof Error && 'code' in err && (err as { code: string }).code === 'ENOENT')) { - throw err; - } - } - - return targetResolved; -} - -export function resolveGlobPatternWithinRoot(root: string, pattern: string): { cwd: string; pattern: string } { - const normalizedPattern = pattern.replace(/\\/g, '/').trim(); - const segments = normalizedPattern.split('/').filter((segment) => segment.length > 0); - const firstGlobIndex = segments.findIndex((segment) => isGlobSegment(segment)); - - if (firstGlobIndex === -1) { - const literalPath = resolveWithinRoot(root, normalizedPattern || '.'); - return { - cwd: path.dirname(literalPath), - pattern: path.basename(literalPath), - }; - } - - const baseSegments = segments.slice(0, firstGlobIndex); - const globSegments = segments.slice(firstGlobIndex); - const basePath = baseSegments.length > 0 ? baseSegments.join('/') : '.'; - - return { - cwd: resolveWithinRoot(root, basePath), - pattern: globSegments.join('/'), - }; -} - -export function toRelativePathFromRoot(root: string, absolutePath: string): string { - const rootResolved = path.resolve(root); - const absoluteResolved = path.resolve(absolutePath); - return path.relative(rootResolved, absoluteResolved).replace(/\\/g, '/'); -} diff --git a/src/agent/progress.ts b/src/agent/progress.ts deleted file mode 100644 index dff0f64d..00000000 --- a/src/agent/progress.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { OutputFormat } from '../cli/types'; -import * as readline from 'node:readline'; -import ora, { type Ora } from 'ora'; - -const TOOL_PREFIX = ' └ '; -const MAX_TOOL_LINE_LENGTH = 140; - -export type VisibleToolName = 'read_file' | 'list_directory' | 'lint'; - -export interface VisibleToolProgress { - toolName: VisibleToolName; - path?: string; - ruleName?: string; - ruleText?: string; - lineCount?: number; - entryCount?: number; - findingsCount?: number; -} - -type RunStatus = 'completed' | 'failed'; - -export class AgentProgressReporter { - private readonly enabled: boolean; - private readonly spinner: Ora | undefined; - private readonly runStartedAt = Date.now(); - private activeFile: string | undefined; - private activeRuleName: string | undefined; - private activeLine = TOOL_PREFIX; - - constructor(enabled: boolean) { - this.enabled = enabled; - if (enabled) { - this.spinner = ora({ - spinner: 'dots', - stream: createOraStream(process.stderr), - isEnabled: true, - discardStdin: false, - color: false, - }); - } - } - - private writeLine(message: string): void { - if (!this.enabled) { - return; - } - process.stderr.write(`${message}\n`); - } - - private currentBlockText(): string { - return `Reviewing ${this.activeFile ?? ''} for ${this.activeRuleName ?? 'Rule'}\n${this.activeLine}`; - } - - private renderCurrent(): void { - if (!this.enabled || !this.activeFile || !this.spinner) { - return; - } - - const nextText = this.currentBlockText(); - if (!this.spinner.isSpinning) { - this.spinner.start(nextText); - return; - } - this.spinner.text = nextText; - this.spinner.render(); - } - - startFile(file: string, ruleName: string): void { - if (!this.enabled || !this.spinner) { - return; - } - - if (this.spinner.isSpinning && this.activeFile && this.activeFile !== file) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - - this.activeFile = file; - this.activeRuleName = ruleName; - this.activeLine = TOOL_PREFIX; - this.renderCurrent(); - } - - updateRule(ruleName: string): void { - if (!this.enabled || !this.activeFile || this.activeRuleName === ruleName) { - return; - } - - this.activeRuleName = ruleName; - this.renderCurrent(); - } - - showVisibleToolStart(params: VisibleToolProgress & { retrying?: boolean }): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = params.retrying - ? formatRetryingLine(params) - : formatInvocationLine(params); - this.renderCurrent(); - } - - showVisibleToolSuccess(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatSuccessLine(params); - this.renderCurrent(); - } - - showVisibleToolError(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatErrorLine(params); - this.renderCurrent(); - } - - finishRun(status: RunStatus = 'completed'): void { - if (!this.enabled) { - return; - } - - if (this.spinner?.isSpinning && this.activeFile) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - this.writeLine(formatRunFooter(status, this.runStartedAt)); - this.activeFile = undefined; - this.activeRuleName = undefined; - this.activeLine = TOOL_PREFIX; - } -} - -function formatInvocationLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Read(${sanitizePath(params.path)})`); - case 'list_directory': - return formatToolLine(`List(${sanitizePath(params.path)})`); - case 'lint': - return formatToolLine(`Lint("${formatRuleSnippet(params.ruleText)}")`); - } -} - -function formatRetryingLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Retrying Read(${sanitizePath(params.path)})...`); - case 'list_directory': - return formatToolLine(`Retrying List(${sanitizePath(params.path)})...`); - case 'lint': - return formatToolLine(`Retrying Lint("${formatRuleSnippet(params.ruleText)}")...`); - } -} - -function formatSuccessLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine( - `Read ${formatCount(params.lineCount ?? 0, 'line')} from ${sanitizePath(params.path)}` - ); - case 'list_directory': - return formatToolLine( - `Listed ${formatCount(params.entryCount ?? 0, 'entry')} in ${sanitizePath(params.path)}` - ); - case 'lint': - if ((params.findingsCount ?? 0) === 0) { - return formatToolLine(`Found no issues in ${sanitizePath(params.path)}`); - } - return formatToolLine( - `Found ${formatCount(params.findingsCount ?? 0, 'issue')} in ${sanitizePath(params.path)}` - ); - } -} - -function formatErrorLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Error reading ${sanitizePath(params.path)}`); - case 'list_directory': - return formatToolLine(`Error listing ${sanitizePath(params.path)}`); - case 'lint': - return formatToolLine(`Error linting ${sanitizePath(params.path)}`); - } -} - -function formatToolLine(message: string): string { - return truncate(`${TOOL_PREFIX}${message}`, MAX_TOOL_LINE_LENGTH); -} - -function formatRuleSnippet(ruleText: string | undefined): string { - const sanitized = sanitizeInline(ruleText || ''); - if (sanitized.length === 0) { - return '...'; - } - return truncate(`${sanitized}...`, 52); -} - -function sanitizePath(path: string | undefined): string { - const value = sanitizeInline(path || '.'); - return value.length > 0 ? value : '.'; -} - -function formatCount(count: number, noun: string): string { - return `${count} ${noun}${count === 1 ? '' : 's'}`; -} - -function sanitizeInline(value: string): string { - return value.replace(/\s+/g, ' ').trim(); -} - -function truncate(value: string, maxLength: number): string { - if (value.length <= maxLength) { - return value; - } - return `${value.slice(0, Math.max(0, maxLength - 3))}...`; -} - -function formatElapsed(startedAt: number): string { - const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); - const hours = Math.floor(elapsedSeconds / 3600); - const minutes = Math.floor((elapsedSeconds % 3600) / 60); - const seconds = elapsedSeconds % 60; - - if (hours > 0) { - return `${hours}h ${minutes}m ${seconds}s`; - } - if (minutes > 0) { - return `${minutes}m ${seconds}s`; - } - return `${seconds}s`; -} - -function formatRunFooter(status: RunStatus, startedAt: number): string { - const elapsed = formatElapsed(startedAt); - return status === 'failed' - ? `Review failed after ${elapsed}.` - : `Completed review in ${elapsed}.`; -} - -function createOraStream(stream: NodeJS.WriteStream): NodeJS.WriteStream { - return { - ...stream, - isTTY: stream.isTTY === true, - columns: stream.columns, - write: stream.write.bind(stream), - cursorTo: (x: number) => readline.cursorTo(stream, x), - clearLine: (dir: -1 | 0 | 1) => readline.clearLine(stream, dir), - moveCursor: (dx: number, dy: number) => readline.moveCursor(stream, dx, dy), - } as NodeJS.WriteStream; -} - -export function shouldEmitAgentProgress(params: { outputFormat: OutputFormat; printMode: boolean }): boolean { - const { outputFormat, printMode } = params; - if (printMode) { - return false; - } - if (outputFormat !== OutputFormat.Line) { - return false; - } - return process.stderr.isTTY === true; -} diff --git a/src/agent/prompt-builder.ts b/src/agent/prompt-builder.ts deleted file mode 100644 index 0d16870a..00000000 --- a/src/agent/prompt-builder.ts +++ /dev/null @@ -1,57 +0,0 @@ -export interface BuildAgentSystemPromptParams { - workspaceRoot: string; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - availableTools: Array<{ name: string; description: string }>; - userInstructions?: string; -} - -function formatBulletedList(values: string[]): string { - if (values.length === 0) { - return '- (none)'; - } - return values.map((value) => `- ${value}`).join('\n'); -} - -function formatFileRuleMatches( - matches: Array<{ file: string; ruleSource: string }> -): string { - if (matches.length === 0) { - return '- (none)'; - } - - const rulesByFile = new Map(); - for (const { file, ruleSource } of matches) { - const rules = rulesByFile.get(file) ?? []; - rules.push(ruleSource); - rulesByFile.set(file, rules); - } - - return Array.from(rulesByFile.entries()) - .map(([file, rules]) => `- ${file}\n${rules.map((rule) => ` - ${rule}`).join('\n')}`) - .join('\n'); -} - -export function buildAgentSystemPrompt(params: BuildAgentSystemPromptParams): string { - const date = new Date().toISOString().slice(0, 10); - const userInstructions = params.userInstructions?.trim(); - const fileRuleMatches = formatFileRuleMatches(params.fileRuleMatches); - - return `You are a senior technical writer. You review documentation files against source-backed rules to identify quality issues, inconsistencies, and violations. - -Your goal is to produce a thorough, complete review of every file against every matched rule. - -Workflow: -1. You are given matched file-rule pairs. Work through each file one at a time — complete every matched rule for a file before moving to the next. -2. For each file-rule pair, review the file against the rule. -3. After reviewing the file, read the rule. If the rule contains top-level review instructions — such as checking for documentation drift, verifying that certain files exist, or any other workspace-level check — carry them out and report any findings. -4. When every file has been reviewed against all of its matched rules, you MUST call the finalize_review tool. - -Available tools: -${formatBulletedList(params.availableTools.map((toolDef) => `${toolDef.name}: ${toolDef.description}`))} - -Review files and matched rules: -${fileRuleMatches}${userInstructions ? `\n\nUser Instructions (from VECTORLINT.md):\n${userInstructions}` : ''} - -Current date: ${date} -Workspace root: ${params.workspaceRoot}`; -} diff --git a/src/agent/review-session-store.ts b/src/agent/review-session-store.ts deleted file mode 100644 index d5fb7262..00000000 --- a/src/agent/review-session-store.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { randomUUID } from 'crypto'; -import { appendFile, mkdir, open, readFile } from 'fs/promises'; -import * as path from 'path'; -import { SESSION_EVENT_SCHEMA, SESSION_EVENT_TYPE, type SessionEvent } from './types'; - -type AppendableSessionEvent = Omit; - -export interface ReviewSessionStore { - sessionId: string; - sessionFilePath: string; - append: (event: AppendableSessionEvent) => Promise; - replay: () => Promise; - hasFinalizedEvent: () => Promise; -} - -function buildSessionFilePath(reviewsDir: string, sessionId: string): string { - return path.join(reviewsDir, `${sessionId}.jsonl`); -} - -async function createUniqueSessionFile(reviewsDir: string): Promise<{ sessionId: string; sessionFilePath: string }> { - while (true) { - const sessionId = randomUUID(); - const sessionFilePath = buildSessionFilePath(reviewsDir, sessionId); - - try { - const handle = await open(sessionFilePath, 'wx'); - await handle.close(); - return { sessionId, sessionFilePath }; - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'EEXIST') { - continue; - } - throw error; - } - } -} - -export async function createReviewSessionStore({ homeDir }: { homeDir: string }): Promise { - const reviewsDir = path.join(homeDir, '.vectorlint', 'reviews'); - await mkdir(reviewsDir, { recursive: true }); - - const { sessionId, sessionFilePath } = await createUniqueSessionFile(reviewsDir); - - async function append(event: AppendableSessionEvent): Promise { - const parsed = SESSION_EVENT_SCHEMA.parse({ - sessionId, - timestamp: new Date().toISOString(), - ...event, - }); - await appendFile(sessionFilePath, `${JSON.stringify(parsed)}\n`, 'utf8'); - } - - async function replay(): Promise { - let raw = ''; - try { - raw = await readFile(sessionFilePath, 'utf8'); - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'ENOENT') { - return []; - } - throw error; - } - - const events: SessionEvent[] = []; - const lines = raw.split('\n').filter((line) => line.trim().length > 0); - - for (const line of lines) { - try { - const candidate = JSON.parse(line) as unknown; - const parsed = SESSION_EVENT_SCHEMA.safeParse(candidate); - if (parsed.success) { - events.push(parsed.data); - } - } catch { - // Ignore malformed JSONL lines so replay can recover valid events. - } - } - - return events; - } - - async function hasFinalizedEvent(): Promise { - const events = await replay(); - return events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - } - - return { - sessionId, - sessionFilePath, - append, - replay, - hasFinalizedEvent, - }; -} diff --git a/src/agent/rule-id.ts b/src/agent/rule-id.ts deleted file mode 100644 index 2ee68813..00000000 --- a/src/agent/rule-id.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { PromptFile } from '../prompts/prompt-loader'; - -export function buildRuleId(prompt: PromptFile): string { - const pack = prompt.pack || 'Default'; - const rule = String(prompt.meta.id || prompt.filename || 'Rule'); - return `${pack}.${rule}`; -} - -export function normalizeRuleSource(ruleSource: string): string { - return ruleSource.replace(/\\/g, '/').replace(/^\.\//, ''); -} diff --git a/src/agent/tools-registry.ts b/src/agent/tools-registry.ts deleted file mode 100644 index 4bc49dea..00000000 --- a/src/agent/tools-registry.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { AgentToolDefinition } from '../providers/llm-provider'; -import { - FINALIZE_REVIEW_INPUT_SCHEMA, - LINT_TOOL_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, -} from './types'; - -export type AgentToolName = - | 'lint' - | 'report_finding' - | 'read_file' - | 'search_files' - | 'list_directory' - | 'search_content' - | 'finalize_review'; - -export type AgentToolHandler = (input: unknown) => Promise; -export type AgentToolHandlers = Record; - -export function createAgentTools(params: { - runTool: ( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ) => Promise; - handlers: AgentToolHandlers; -}): Record { - const { runTool, handlers } = params; - - return { - lint: { - description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.', - inputSchema: LINT_TOOL_INPUT_SCHEMA, - execute: (input) => runTool('lint', input, handlers.lint), - }, - report_finding: { - description: 'Record a top-level finding for the report.', - inputSchema: TOP_LEVEL_REPORT_INPUT_SCHEMA, - execute: (input) => runTool('report_finding', input, handlers.report_finding), - }, - read_file: { - description: 'Read a file inside the workspace root.', - inputSchema: READ_FILE_INPUT_SCHEMA, - execute: (input) => runTool('read_file', input, handlers.read_file), - }, - search_files: { - description: 'Find files in the workspace by glob pattern.', - inputSchema: SEARCH_FILES_INPUT_SCHEMA, - execute: (input) => runTool('search_files', input, handlers.search_files), - }, - list_directory: { - description: 'List files and directories inside a path in the workspace.', - inputSchema: LIST_DIRECTORY_INPUT_SCHEMA, - execute: (input) => runTool('list_directory', input, handlers.list_directory), - }, - search_content: { - description: 'Search workspace text content by substring and optional glob.', - inputSchema: SEARCH_CONTENT_INPUT_SCHEMA, - execute: (input) => runTool('search_content', input, handlers.search_content), - }, - finalize_review: { - description: 'REQUIRED: Call this tool when all matched file-rule pairs have been reviewed.', - inputSchema: FINALIZE_REVIEW_INPUT_SCHEMA, - execute: (input) => runTool('finalize_review', input, handlers.finalize_review), - }, - }; -} - -export function listAvailableTools( - tools: Record -): Array<{ name: string; description: string }> { - return Object.entries(tools).map(([name, definition]) => ({ - name, - description: definition.description, - })); -} diff --git a/src/agent/types.ts b/src/agent/types.ts deleted file mode 100644 index e1f2407d..00000000 --- a/src/agent/types.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { z } from 'zod'; - -export const SESSION_EVENT_TYPE = { - SessionStarted: 'session_started', - ToolCallStarted: 'tool_call_started', - ToolCallFinished: 'tool_call_finished', - FindingRecordedInline: 'finding_recorded_inline', - FindingRecordedTopLevel: 'finding_recorded_top_level', - SessionFinalized: 'session_finalized', -} as const; - -export const LINT_TOOL_INPUT_SCHEMA = z.object({ - file: z.string().min(1), - ruleSource: z.string().min(1), - reviewInstruction: z.string().min(1).optional(), -}); - -export const TOP_LEVEL_REFERENCE_SCHEMA = z.object({ - file: z.string().min(1), - startLine: z.number().int().positive(), - endLine: z.number().int().positive(), -}); - -export const TOP_LEVEL_REPORT_INPUT_SCHEMA = z.object({ - kind: z.literal('top-level'), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), -}); - -export const READ_FILE_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_FILES_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), -}); - -export const LIST_DIRECTORY_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_CONTENT_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), - path: z.string().optional(), - glob: z.string().optional(), -}); - -export const FINALIZE_REVIEW_INPUT_SCHEMA = z.object({ - summary: z.string().optional(), -}); - -const SESSION_EVENT_BASE_SCHEMA = z.object({ - sessionId: z.string().min(1), - timestamp: z.string().min(1), -}); - -const SESSION_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionStarted), - payload: z.object({ - cwd: z.string().min(1), - targets: z.array(z.string().min(1)), - }), -}); - -const TOOL_CALL_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallStarted), - payload: z.object({ - toolName: z.string().min(1), - input: z.unknown(), - }), -}); - -const TOOL_CALL_FINISHED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallFinished), - payload: z.object({ - toolName: z.string().min(1), - ok: z.boolean(), - output: z.unknown().optional(), - error: z.string().optional(), - }), -}); - -const FINDING_RECORDED_INLINE_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedInline), - payload: z.object({ - file: z.string().min(1), - line: z.number().int().positive(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - analysis: z.string().optional(), - suggestion: z.string().optional(), - fix: z.string().optional(), - match: z.string().optional(), - }), -}); - -const FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedTopLevel), - payload: z.object({ - file: z.string().min(1).optional(), - line: z.number().int().positive().optional(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), - }), -}); - -const SESSION_FINALIZED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionFinalized), - payload: z.object({ - totalFindings: z.number().int().nonnegative(), - summary: z.string().optional(), - }), -}); - -export const SESSION_EVENT_SCHEMA = z.discriminatedUnion('eventType', [ - SESSION_STARTED_EVENT_SCHEMA, - TOOL_CALL_STARTED_EVENT_SCHEMA, - TOOL_CALL_FINISHED_EVENT_SCHEMA, - FINDING_RECORDED_INLINE_EVENT_SCHEMA, - FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA, - SESSION_FINALIZED_EVENT_SCHEMA, -]); - -export type SessionEvent = z.infer; diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 50dbffc8..b2e7f0b7 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -18,7 +18,7 @@ import { resolveTargets } from '../scan/file-resolver'; import { parseCliOptions, parseEnvironment } from '../boundaries/index'; import { handleUnknownError } from '../errors/index'; import { evaluateFiles } from './orchestrator'; -import { DEFAULT_REVIEW_MODE, OUTPUT_FORMATS, OutputFormat } from './types'; +import { DEFAULT_REVIEW_MODEL_CALL, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; import { createNoopLogger } from '../logging/logger'; @@ -82,8 +82,8 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) - .option('-p, --print', 'Suppress interactive progress output in agent mode') + .option('--model-call ', 'Model call strategy: single (one structured call per rule), agent (bounded target-only paging), or auto (default)', DEFAULT_REVIEW_MODEL_CALL) + .option('--mode ', 'Removed: use --model-call instead') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') .action(async (paths: string[] = []) => { @@ -94,6 +94,15 @@ export function registerMainCommand(program: Command): void { return; } + // --mode is removed (audit Product Decision): the autonomous workspace-agent + // surface is gone. Fail clearly and point users at the replacement rather + // than silently mapping it onto --model-call. + const rawCliOpts = program.opts() as { mode?: unknown }; + if (rawCliOpts.mode !== undefined) { + console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); + process.exit(1); + } + // Parse and validate CLI options const cliOptions = await runOrExit('Parsing CLI options', () => parseCliOptions(program.opts())); @@ -192,6 +201,7 @@ export function registerMainCommand(program: Command): void { try { observability = await initializeObservability(env, runtimeLogger); + const requestBuilder = new DefaultRequestBuilder(directive, userInstructions.content || undefined); const provider = createProvider( env, { @@ -201,7 +211,7 @@ export function registerMainCommand(program: Command): void { logger: runtimeLogger, observability, }, - new DefaultRequestBuilder(directive, userInstructions.content || undefined) + requestBuilder ); // Create search provider if API key is available @@ -214,14 +224,14 @@ export function registerMainCommand(program: Command): void { prompts, rulesPath, provider, + requestBuilder, ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, - mode: cliOptions.mode, - printMode: cliOptions.print, + modelCall: cliOptions.modelCall, scanPaths: config.scanPaths, pricing: { inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 1e61a47d..205a2a80 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,41 +1,29 @@ import { readFileSync } from 'fs'; -import { readFile } from 'fs/promises'; import { randomUUID } from 'crypto'; import * as path from 'path'; -import * as os from 'os'; +import { pathToFileURL } from 'url'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { isJudgeResult } from '../prompts/schema'; -import { handleUnknownError, MissingDependencyError } from '../errors/index'; -import { processFindings } from '../findings'; -import { createEvaluator } from '../evaluators/index'; -import { Type, Severity } from '../evaluators/types'; +import { handleUnknownError } from '../errors/index'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from './types'; -import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '../agent/executor'; -import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; +import { OutputFormat } from './types'; +import { executorFor } from '../executors'; +import { chooseModelCall } from '../review/executor'; +import { buildReviewRequest } from '../review/request-builder'; +import type { ReviewResult, ReviewSeverity, ReviewTarget } from '../review/types'; import type { - EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessPromptResultParams, - RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, - RunPromptEvaluationResultSuccess + EvaluationOptions, EvaluationResult, ReportIssueParams, EvaluateFileResult, } from './types'; import { calculateCost, TokenUsageStats } from '../providers/token-usage'; -import { calculateCheckScore } from '../scoring'; -import { countWords } from '../chunking/utils'; -import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { - computeFilterDecision, - type FilterDecision, -} from "../evaluators/violation-filter"; -import { writeDebugRunArtifact } from "../debug/run-artifact"; +import { writeDebugRunArtifact } from '../debug/run-artifact'; +import { Severity } from '../evaluators/types'; function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } { const provider = process.env.LLM_PROVIDER; @@ -60,34 +48,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string return { ...(provider && { provider }), ...(name && { name }), ...(tag && { tag }) }; } - -/* - * Generic concurrency runner that executes workers in parallel up to a specified limit. - * Preserves result order matching input order. - */ -async function runWithConcurrency( - items: T[], - limit: number, - worker: (item: T, index: number) => Promise -): Promise { - const results: R[] = new Array(items.length); - let i = 0; - const workers = new Array(Math.min(limit, items.length)) - .fill(0) - .map(async () => { - while (true) { - const idx = i++; - if (idx >= items.length) break; - const item = items[idx]; - if (item !== undefined) { - results[idx] = await worker(item, idx); - } - } - }); - await Promise.all(workers); - return results; -} - /* * Reports an issue in either line or JSON format. */ @@ -154,259 +114,62 @@ function reportIssue(params: ReportIssueParams): void { } } -function getViolationFilterResults< - TViolation extends Parameters[0] ->( - violations: TViolation[] -): { - decisions: FilterDecision[]; - surfacedViolations: TViolation[]; -} { - const decisions = violations.map((v) => computeFilterDecision(v)); - const surfacedViolations = violations.filter( - (_v, i) => decisions[i]?.surface === true - ); - - return { decisions, surfacedViolations }; +function toSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; } -/* - * Routes an evaluation result through the shared finding processor. - * Check results are verified, filtered, scored, and reported; judge/rubric - * results are rejected (Phase 3) because subjective scoring is not a - * future-facing review type. - */ -function routePromptResult( - params: ProcessPromptResultParams -): ErrorTrackingResult { - const { - promptFile, - result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - debugJson, - } = params; - const meta = promptFile.meta; - const promptId = (meta.id || "").toString(); - - // Handle Check Result — routed through the shared finding processor - // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, - // filters, deduplicates, scores, and resolves severity; the orchestrator - // only feeds the returned ReviewResult to the existing formatter sinks. - if (!isJudgeResult(result)) { - const reviewResult = processFindings({ - pack: promptFile.pack, - ruleId: promptId, - ruleSource: promptFile.fullPath, - candidateFindings: result.violations, - wordCount: result.word_count, - promptMeta: { - ...(meta.severity !== undefined ? { severity: meta.severity } : {}), - ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - // Sanitize to the findings contract ({ id, name }): meta.criteria is - // PromptCriterionSpec[], which can carry legacy rubric weight/target. - ...(meta.criteria - ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } - : {}), - }, - targetContent: content, - }); - - // processFindings always returns exactly one score entry (processor contract). - const ruleScore = reviewResult.scores[0]!; - const severity = - ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - - // Report only verified findings through the existing line/json/rdjson/vale sink. - for (const finding of reviewResult.findings) { - reportIssue({ - file: relFile, - line: finding.line, - column: finding.column, - severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), - ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), - ...(finding.fix !== undefined ? { fix: finding.fix } : {}), - match: finding.match, - }); - } - - // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, - // consistent with prior operational reporting. They are warn-level and do not - // flag the run as operationally failed (audit Finding #6). - if (verbose) { - for (const diagnostic of reviewResult.diagnostics) { - console.warn(`[vectorlint] ${diagnostic.message}`); - } - } - - // Verified finding count drives the counts (audit Finding #6). - const findingCount = reviewResult.findings.length; - const totalErrors = severity === Severity.ERROR ? findingCount : 0; - const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; - - const scoreEntry: EvaluationSummary = { - id: ruleScore.ruleId, - scoreText: ruleScore.scoreText, - score: ruleScore.score, - }; - - if (debugJson) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - const runId = randomUUID(); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "check", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: decisions.map((d, i) => ({ - index: i, - surface: d.surface, - reasons: d.reasons, - })), - surfaced_violations: surfacedViolations, - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - - return { - errors: totalErrors, - warnings: totalWarnings, - hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, - hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, - scoreEntries: [scoreEntry], - }; - } +function contentTypeFor(file: string): string { + return path.extname(file).toLowerCase() === '.md' ? 'text/markdown' : 'text/plain'; +} - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a - // caller bypassed that boundary. Refuse it explicitly rather than - // misprojecting it as a check result. No adapter is added for judge metadata. - if (verbose) { - console.warn( - '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' - ); - } - return { - errors: 0, - warnings: 0, - hadOperationalErrors: true, - hadSeverityErrors: false, - scoreEntries: [], - }; +function emptyTokenUsage(): TokenUsageStats { + return { totalInputTokens: 0, totalOutputTokens: 0 }; } /* - * Runs a single prompt evaluation. - * BaseEvaluator auto-detects mode from criteria presence: - * - criteria defined → scored mode - * - no criteria → basic mode + * Writes a debug JSON artifact for a review run. The executor path reviews + * through the shared finding processor, so raw model output and per-candidate + * filter decisions are no longer reachable here; the artifact records the + * verified findings that surfaced and the review metadata instead. */ -async function runPromptEvaluation( - params: RunPromptEvaluationParams -): Promise { - const { promptFile, relFile, content, provider, searchProvider } = params; - +function writeReviewDebugArtifact(relFile: string, result: ReviewResult): void { + const runId = randomUUID(); + const model = getModelInfoFromEnv(); try { - const meta = promptFile.meta; - - const evaluatorType = String(meta.evaluator || Type.BASE); - const baseEvaluatorType = String(Type.BASE); - - // Specialized evaluators (e.g., technical-accuracy) require criteria - // BaseEvaluator handles both modes: scored (with criteria) and basic (without) - if (evaluatorType !== baseEvaluatorType) { - if ( - !meta || - !Array.isArray(meta.criteria) || - meta.criteria.length === 0 - ) { - throw new Error( - `Prompt ${promptFile.filename} has no criteria in frontmatter` - ); - } - } - const evaluator = createEvaluator( - evaluatorType, - provider, - promptFile, - searchProvider - ); - const result = await evaluator.evaluate(relFile, content); - - - const resultObj: RunPromptEvaluationResultSuccess = { ok: true, result }; - - return resultObj; - } catch (e: unknown) { - const err = handleUnknownError(e, `Running prompt ${promptFile.filename}`); - return { ok: false, error: err }; + const filePath = writeDebugRunArtifact(process.cwd(), runId, { + file: relFile, + ...(Object.keys(model).length > 0 ? { model } : {}), + ...(model.tag !== undefined ? { subdir: model.tag } : {}), + prompt: { + evaluation_type: 'check', + }, + raw_model_output: null, + filter_decisions: [], + surfaced_violations: result.findings, + }); + console.warn(`[vectorlint] Debug JSON written: ${filePath}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[vectorlint] Debug JSON write failed: ${message}`); } } /* - * Evaluates a single file with all applicable prompts. + * Determines the source-backed prompts that apply to a file, honoring the + * configured scan paths. When no rules match but a VECTORLINT.md user + * instruction guide exists, a synthetic rule is added so the reviewer model + * evaluates against the user instructions in the system prompt. */ -async function evaluateFile( - params: EvaluateFileParams -): Promise { - const { file, options, jsonFormatter } = params; - const { - prompts, - provider, - searchProvider, - concurrency, - scanPaths, - outputFormat = OutputFormat.Line, - verbose, - debugJson, - } = options; - - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let totalErrors = 0; - let totalWarnings = 0; - let requestFailures = 0; - let totalInputTokens = 0; - let totalOutputTokens = 0; - - const allScores = new Map(); - - const content = readFileSync(file, "utf-8"); - const relFile = path.relative(process.cwd(), file) || file; - - if (outputFormat === OutputFormat.Line) { - printFileHeader(relFile); - } - - // Determine applicable prompts for this file +function resolveApplicablePrompts( + relFile: string, + prompts: PromptFile[], + scanPaths: EvaluationOptions['scanPaths'], + userInstructionContent: string | undefined, +): PromptFile[] { const toRun: PromptFile[] = []; if (scanPaths && scanPaths.length > 0) { const resolver = new ScanPathResolver(); - // Extract available packs from loaded prompts const availablePacks = Array.from( new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p)) ); @@ -417,9 +180,7 @@ async function evaluateFile( availablePacks ); - // Filter prompts by active packs - only runs if explicitly in RunRules const activePrompts = prompts.filter((p) => { - // Prompts with empty pack (style guide only) always run if (p.pack === '') return true; if (!p.pack || !resolution.packs.includes(p.pack)) return false; if (!p.meta?.id) return true; @@ -433,13 +194,10 @@ async function evaluateFile( toRun.push(...activePrompts); } else { - // Fallback: When no scanPaths configured, run all prompts. toRun.push(...prompts); } - // If no rules matched but VECTORLINT.md exists, run an evaluation using it. - // The LLM will use the VECTORLINT.md content from the system prompt. - if (options.userInstructionContent) { + if (userInstructionContent) { toRun.push({ id: USER_INSTRUCTION_FILENAME, filename: USER_INSTRUCTION_FILENAME, @@ -454,346 +212,165 @@ async function evaluateFile( }); } - const results = await runWithConcurrency( - toRun, - concurrency, - async (prompt) => { - return runPromptEvaluation({ - promptFile: prompt, - relFile, - content, - provider, - ...(searchProvider !== undefined && { searchProvider }), - }); - } - ); + return toRun; +} - // Aggregate results from each prompt - for (let idx = 0; idx < toRun.length; idx++) { - const p = toRun[idx]; - const r = results[idx]; - if (!p || !r) continue; - - if (r.ok !== true) { - // Check if this is a missing dependency error - if so, skip gracefully - if (r.error instanceof MissingDependencyError) { - console.warn(`[vectorlint] Skipping ${p.filename}: ${r.error.message}`); - if (r.error.hint) { - console.warn(`[vectorlint] Hint: ${r.error.hint}`); - } - // Skip this evaluation entirely - don't count it as a failure - continue; - } +/* + * Evaluates a single file: builds a ReviewRequest from the applicable + * source-backed prompts, resolves the model-call strategy, dispatches through + * the selected ReviewExecutor, and routes the resulting ReviewResult to the + * existing line/json/vale output sinks. + */ +async function evaluateFile( + file: string, + options: EvaluationOptions, + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter, +): Promise { + const { + prompts, + scanPaths, + outputFormat = OutputFormat.Line, + verbose, + debugJson, + } = options; - // Other errors are actual failures - console.error(` Prompt failed: ${p.filename}`); - console.error(r.error); - hadOperationalErrors = true; - requestFailures += 1; - continue; - } + const allScores = new Map(); - // Accumulate token usage - if (r.result.usage) { - totalInputTokens += r.result.usage.inputTokens; - totalOutputTokens += r.result.usage.outputTokens; - } + const content = readFileSync(file, "utf-8"); + const relFile = path.relative(process.cwd(), file) || file; - const promptResult = routePromptResult({ - promptFile: p, - result: r.result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - ...(debugJson !== undefined ? { debugJson } : {}), - }); - totalErrors += promptResult.errors; - totalWarnings += promptResult.warnings; - hadOperationalErrors = - hadOperationalErrors || promptResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || promptResult.hadSeverityErrors; - - if (promptResult.scoreEntries && promptResult.scoreEntries.length > 0) { - const ruleName = (p.meta.id || p.filename).toString(); - allScores.set(ruleName, promptResult.scoreEntries); - } + if (outputFormat === OutputFormat.Line) { + printFileHeader(relFile); } - const tokenUsageStats: TokenUsageStats = { - totalInputTokens, - totalOutputTokens, - }; + const toRun = resolveApplicablePrompts( + relFile, + prompts, + scanPaths, + options.userInstructionContent, + ); - if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); - console.log(""); + // No applicable rules: nothing to review for this file. + if (toRun.length === 0) { + if (outputFormat === OutputFormat.Line) { + printEvaluationSummaries(allScores); + console.log(""); + } + return { + errors: 0, + warnings: 0, + requestFailures: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } - return { - errors: totalErrors, - warnings: totalWarnings, - requestFailures, - hadOperationalErrors, - hadSeverityErrors, - tokenUsage: tokenUsageStats + // Build the review request and dispatch through the executor selected by the + // resolved model-call strategy (audit Product Decision; Finding #2). + const target: ReviewTarget = { + uri: pathToFileURL(path.resolve(file)).href, + content, + contentType: contentTypeFor(file), + byteLength: Buffer.byteLength(content), }; -} - -function reportAgentFinding(params: { - finding: AgentFinding; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -}): void { - const { finding, outputFormat, jsonFormatter } = params; - - reportIssue({ - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), + const request = buildReviewRequest({ + target, + prompts: toRun, + config: { modelCall: options.modelCall }, + }); + const resolvedModelCall = chooseModelCall(request.modelCall, { + targetBytes: request.target.byteLength ?? Buffer.byteLength(content), + rules: request.rules.length, + }); + const executor = executorFor(resolvedModelCall, { + structuredModelClient: options.provider, + toolCallingModelClient: options.provider, + builder: options.requestBuilder, }); -} - -type AgentRuleScore = { - ruleId: string; - score: number; - scoreText: string; -}; - -async function getAgentFileWordCount( - file: string, - workspaceRoot: string, - cache: Map -): Promise { - const workspaceRelative = path.relative(workspaceRoot, path.resolve(workspaceRoot, file)) || file; - if (cache.has(workspaceRelative)) { - return cache.get(workspaceRelative)!; - } - const absolutePath = path.resolve(workspaceRoot, workspaceRelative); + let result: ReviewResult; try { - const content = await readFile(absolutePath, 'utf-8'); - const words = Math.max(1, countWords(content) || 1); - cache.set(workspaceRelative, words); - return words; - } catch { - cache.set(workspaceRelative, 1); - return 1; + result = await executor.run(request); + } catch (e: unknown) { + const err = handleUnknownError(e, `Reviewing ${relFile}`); + console.error(`Error reviewing file ${relFile}: ${err.message}`); + return { + errors: 0, + warnings: 0, + requestFailures: 1, + hadOperationalErrors: true, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } -} -async function buildAgentRuleScores( - findings: AgentFinding[], - prompts: PromptFile[], - fileRuleMatches: Array<{ file: string; ruleSource: string }>, - workspaceRoot: string -): Promise { - const fileWordCountCache = new Map(); - const findingsByRule = new Map(); - const filesByRuleSource = new Map>(); - - for (const finding of findings) { - const existing = findingsByRule.get(finding.ruleId) ?? []; - existing.push(finding); - findingsByRule.set(finding.ruleId, existing); - } - for (const match of fileRuleMatches) { - const files = filesByRuleSource.get(match.ruleSource) ?? new Set(); - files.add(match.file); - filesByRuleSource.set(match.ruleSource, files); + // Route the ReviewResult's verified findings through the existing sinks. + for (const finding of result.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity: toSeverity(finding.severity), + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); } - const results: AgentRuleScore[] = []; - for (const prompt of prompts) { - const ruleId = buildRuleId(prompt); - const ruleFindings = findingsByRule.get(ruleId) ?? []; - const matchedFiles = filesByRuleSource.get(normalizeRuleSource(prompt.fullPath)) ?? new Set(); - - if (matchedFiles.size === 0) { - results.push({ - ruleId, - score: 10, - scoreText: '10.0/10', - }); - continue; - } + for (const score of result.scores) { + allScores.set(score.ruleId, [ + { id: score.ruleId, scoreText: score.scoreText, score: score.score }, + ]); + } - let totalWords = 0; - for (const file of matchedFiles) { - totalWords += await getAgentFileWordCount(file, workspaceRoot, fileWordCountCache); + if (verbose) { + for (const diagnostic of result.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } - - const syntheticViolations = Array.from({ length: ruleFindings.length }, (_, index) => ({ - line: index + 1, - description: 'Agent finding', - analysis: 'Agent finding', - })); - - const scored = calculateCheckScore( - syntheticViolations, - Math.max(1, totalWords), - { - strictness: prompt.meta.strictness, - promptSeverity: prompt.meta.severity, - } - ); - - results.push({ - ruleId, - score: scored.final_score, - scoreText: `${scored.final_score.toFixed(1)}/10`, - }); } - return results; -} -// Retained in quarantine: unreachable from the CLI after --mode agent was -// deprecated (now falls back to standard evaluation). Removed in Phase 4. -// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine -async function evaluateFilesInAgentMode( - targets: string[], - options: EvaluationOptions, - outputFormat: OutputFormat, - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter -): Promise { - const workspaceRoot = inferAgentWorkspaceRoot(targets); - const progressReporter = new AgentProgressReporter( - shouldEmitAgentProgress({ - outputFormat, - printMode: options.printMode ?? false, - }) - ); + const totalErrors = result.findings.filter((f) => f.severity === 'error').length; + const totalWarnings = result.findings.filter((f) => f.severity === 'warning').length; - const agentResult: AgentExecutorResult = await runAgentExecutor({ - targets, - prompts: options.prompts, - provider: options.provider, - workspaceRoot, - scanPaths: options.scanPaths, - outputFormat, - printMode: options.printMode ?? false, - sessionHomeDir: os.homedir(), - progressReporter, - maxParallelToolCalls: 3, - maxRetries: options.agentMaxRetries ?? 10, - ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), - }); + const tokenUsage: TokenUsageStats = { + totalInputTokens: result.usage?.inputTokens ?? 0, + totalOutputTokens: result.usage?.outputTokens ?? 0, + }; - let totalErrors = 0; - let totalWarnings = 0; - const printedFileHeaders = new Set(); - for (const finding of agentResult.findings) { - if (outputFormat === OutputFormat.Line && !printedFileHeaders.has(finding.file)) { - printFileHeader(finding.file); - printedFileHeaders.add(finding.file); - } - reportAgentFinding({ finding, outputFormat, jsonFormatter }); - if (finding.severity === Severity.ERROR) { - totalErrors += 1; - } else { - totalWarnings += 1; - } + if (debugJson) { + writeReviewDebugArtifact(relFile, result); } if (outputFormat === OutputFormat.Line) { - const ruleScores = await buildAgentRuleScores( - agentResult.findings, - options.prompts, - agentResult.fileRuleMatches, - workspaceRoot - ); - const scoreSummary = new Map( - ruleScores.map((entry) => [ - entry.ruleId, - [{ id: 'overall', scoreText: entry.scoreText, score: entry.score }], - ]) - ); - printEvaluationSummaries(scoreSummary); - - if (agentResult.hadOperationalErrors) { - const message = agentResult.errorMessage ?? 'Agent run encountered an operational error.'; - console.error(`\n[agent] ${message}`); - } - } - - if ( - outputFormat === OutputFormat.Json || - outputFormat === OutputFormat.ValeJson || - outputFormat === OutputFormat.RdJson - ) { - console.log(jsonFormatter.toJson()); + printEvaluationSummaries(allScores); + console.log(""); } - const tokenUsage = { - totalInputTokens: agentResult.usage?.inputTokens ?? 0, - totalOutputTokens: agentResult.usage?.outputTokens ?? 0, - }; - return { - totalFiles: targets.length, - totalErrors, - totalWarnings, - requestFailures: agentResult.requestFailures, - hadOperationalErrors: agentResult.hadOperationalErrors, + errors: totalErrors, + warnings: totalWarnings, + requestFailures: 0, + hadOperationalErrors: result.hadOperationalErrors ?? false, hadSeverityErrors: totalErrors > 0, tokenUsage, }; } -function inferAgentWorkspaceRoot(targets: string[]): string { - if (targets.length === 0) { - return process.cwd(); - } - - const directories = targets.map((target) => path.dirname(path.resolve(target))); - let root = directories[0]!; - - for (const directory of directories.slice(1)) { - root = commonPathPrefix(root, directory); - } - - return root; -} - -function commonPathPrefix(left: string, right: string): string { - let candidate = path.resolve(left); - const target = path.resolve(right); - - while (true) { - const relative = path.relative(candidate, target); - const insideCandidate = !relative.startsWith('..') && !path.isAbsolute(relative); - if (insideCandidate) { - return candidate; - } - - const parent = path.dirname(candidate); - if (parent === candidate) { - return candidate; - } - candidate = parent; - } -} - /* - * Runs evaluations across all target files with configurable concurrency. - * Coordinates prompt-to-file mapping, evaluation execution, and result aggregation. - * Returns aggregated results for reporting. + * Runs reviews across all target files, dispatching each through the executor + * selected by the model-call strategy, and aggregates the results. */ export async function evaluateFiles( targets: string[], options: EvaluationOptions ): Promise { - const { outputFormat = OutputFormat.Line, mode = DEFAULT_REVIEW_MODE } = options; + const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -813,18 +390,10 @@ export async function evaluateFiles( jsonFormatter = new ValeJsonFormatter(); } - if (mode === AGENT_REVIEW_MODE) { - options.logger?.warn( - '--mode agent is deprecated and now falls back to standard mode. ' + - 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', - ); - // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. - } - for (const file of targets) { try { totalFiles += 1; - const fileResult = await evaluateFile({ file, options, jsonFormatter }); + const fileResult = await evaluateFile(file, options, jsonFormatter); totalErrors += fileResult.errors; totalWarnings += fileResult.warnings; requestFailures += fileResult.requestFailures; @@ -832,7 +401,6 @@ export async function evaluateFiles( hadOperationalErrors || fileResult.hadOperationalErrors; hadSeverityErrors = hadSeverityErrors || fileResult.hadSeverityErrors; - // Aggregate token usage if (fileResult.tokenUsage) { totalInputTokens += fileResult.tokenUsage.totalInputTokens; totalOutputTokens += fileResult.tokenUsage.totalOutputTokens; diff --git a/src/cli/types.ts b/src/cli/types.ts index a083f2e2..900b1037 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,15 +1,20 @@ import type { PromptFile } from '../prompts/prompt-loader'; -import type { LLMProvider } from '../providers/llm-provider'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; import type { SearchProvider } from '../providers/search-provider'; +import type { RequestBuilder } from '../providers/request-builder'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; +import type { ReviewModelCall } from '../review/types'; + +export { REVIEW_MODEL_CALLS } from '../review/executor'; +export type { ReviewModelCall } from '../review/types'; export enum OutputFormat { Line = "line", @@ -27,25 +32,26 @@ export const OUTPUT_FORMATS = [ export const DEFAULT_OUTPUT_FORMAT = OUTPUT_FORMATS[0]; -export const REVIEW_MODES = ['standard', 'agent'] as const; -export const DEFAULT_REVIEW_MODE = REVIEW_MODES[0]; -export const AGENT_REVIEW_MODE = REVIEW_MODES[1]; - -export type ReviewMode = (typeof REVIEW_MODES)[number]; +/** + * How the reviewer model is invoked for a review (audit Product Decision; + * Finding #2). `single` is one structured call per rule/chunk; `agent` is a + * bounded target-only paging run; `auto` resolves via `chooseModelCall`. + */ +export const DEFAULT_REVIEW_MODEL_CALL: ReviewModelCall = 'auto'; export interface EvaluationOptions { prompts: PromptFile[]; rulesPath: string | undefined; - provider: LLMProvider; + provider: StructuredModelClient & ToolCallingModelClient; + /** Retained for the search-provider capability; the executor path reviews structured output. */ searchProvider?: SearchProvider; + requestBuilder: RequestBuilder; concurrency: number; verbose: boolean; debugJson?: boolean; scanPaths: FilePatternConfig[]; outputFormat?: OutputFormat; - mode?: ReviewMode; - printMode?: boolean; - agentMaxRetries?: number; + modelCall: ReviewModelCall; pricing?: PricingConfig; userInstructionContent?: string; logger?: Logger; @@ -69,15 +75,6 @@ export interface ErrorTrackingResult { scoreEntries?: EvaluationSummary[]; } -export interface EvaluationContext { - content: string; - relFile: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - verbose?: boolean; - debugJson?: boolean; -} - export interface ReportIssueParams { file: string; line: number; @@ -94,34 +91,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessPromptResultParams extends EvaluationContext { - promptFile: PromptFile; - result: PromptEvaluationResult; -} - -export interface RunPromptEvaluationParams { - promptFile: PromptFile; - relFile: string; - content: string; - provider: LLMProvider; - searchProvider?: SearchProvider; -} - -export interface RunPromptEvaluationResultSuccess { - ok: true; - result: PromptEvaluationResult; -} - -export type RunPromptEvaluationResult = - | RunPromptEvaluationResultSuccess - | { ok: false; error: Error }; - -export interface EvaluateFileParams { - file: string; - options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -} - export interface EvaluateFileResult extends ErrorTrackingResult { requestFailures: number; tokenUsage?: TokenUsageStats; diff --git a/src/executors/index.ts b/src/executors/index.ts new file mode 100644 index 00000000..2b565a29 --- /dev/null +++ b/src/executors/index.ts @@ -0,0 +1,38 @@ +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { RequestBuilder } from '../providers/request-builder'; +import type { ReviewExecutor } from '../review/executor'; +import { SingleModelCallExecutor } from './single-model-call-executor'; +import { AgentModelCallExecutor } from './agent-model-call-executor'; + +/** + * The model-client dependencies every executor composes. Both capabilities are + * supplied because the resolved {@link ModelCall} is not known until + * {@link chooseModelCall} runs at review time (audit Finding #2). + */ +export interface ExecutorDeps { + structuredModelClient: StructuredModelClient; + toolCallingModelClient: ToolCallingModelClient; + builder: RequestBuilder; +} + +/** A resolved reviewer model-call strategy (after `chooseModelCall`). */ +export type ModelCall = 'single' | 'agent'; + +/** + * Selects the {@link ReviewExecutor} for a resolved model call. `single` maps + * to {@link SingleModelCallExecutor}; `agent` maps to + * {@link AgentModelCallExecutor}. Callers resolve `auto` via + * {@link chooseModelCall} before calling this factory. + */ +export function executorFor(modelCall: ModelCall, deps: ExecutorDeps): ReviewExecutor { + if (modelCall === 'agent') { + return new AgentModelCallExecutor(deps.toolCallingModelClient, deps.builder); + } + return new SingleModelCallExecutor(deps.structuredModelClient); +} + +export { SingleModelCallExecutor } from './single-model-call-executor'; +export { AgentModelCallExecutor } from './agent-model-call-executor'; +export { TargetReadCapability } from './target-read-capability-adapter'; +export { REVIEW_BUDGET_EXCEEDED_CODE, splitRuleId } from './shared'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index 635d5ecd..50b0a21a 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,40 +1,20 @@ -import type { TokenUsage } from './token-usage'; import type { StructuredModelClient } from './structured-model-client'; /** - * `LLMResult` now lives on the permanent structured-output capability. It is + * `LLMResult` lives on the permanent structured-output capability. It is * re-exported here so existing deep imports (`from './llm-provider'`) keep - * compiling until the legacy provider surface is removed in a later task. + * compiling. */ export type { LLMResult } from './structured-model-client'; -export interface AgentToolDefinition { - description: string; - inputSchema: unknown; - execute: (input: unknown) => Promise; -} - -export interface AgentToolLoopParams { - systemPrompt: string; - prompt: string; - tools: Record; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; -} - -export interface AgentToolLoopResult { - usage?: TokenUsage; -} - /** - * Temporary compile bridge, preserved only until later tasks remove the - * autonomous agent surface. It extends the permanent - * {@link StructuredModelClient} (single structured output) with the legacy - * `runAgentToolLoop` method still consumed by `src/agent/`. Do not expand this - * surface; `runAgentToolLoop` and the `Agent*` types above are deleted once the - * agent tree is removed (audit Finding #2; Product Decision). + * The provider capability historically named `LLMProvider`. The autonomous + * agent loop and the `Agent*` types that used to live here are removed (audit + * Finding #2; Product Decision): the provider surface is now the permanent + * {@link StructuredModelClient} structured-output capability, plus the bounded + * {@link ToolCallingModelClient} transport implemented by concrete providers. + * + * Kept as an alias so existing evaluator and factory imports continue to + * resolve the structured-output capability. */ -export interface LLMProvider extends StructuredModelClient { - runAgentToolLoop(params: AgentToolLoopParams): Promise; -} +export type LLMProvider = StructuredModelClient; diff --git a/src/providers/provider-factory.ts b/src/providers/provider-factory.ts index 3234087d..cbef40ef 100644 --- a/src/providers/provider-factory.ts +++ b/src/providers/provider-factory.ts @@ -4,7 +4,8 @@ import { createAnthropic } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; import type { LanguageModel } from 'ai'; -import { LLMProvider } from './llm-provider'; +import type { StructuredModelClient } from './structured-model-client'; +import type { ToolCallingModelClient } from './tool-calling-model-client'; import { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; import { RequestBuilder } from './request-builder'; import type { EnvConfig } from '../schemas/env-schemas'; @@ -38,7 +39,7 @@ export function createProvider( envConfig: EnvConfig, options: ProviderOptions = {}, builder?: RequestBuilder -): LLMProvider { +): StructuredModelClient & ToolCallingModelClient { let model: LanguageModel; let modelName: string; let temperature = 0.2; diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index ed314001..dba87369 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -2,7 +2,8 @@ import { generateText, Output, NoObjectGeneratedError, stepCountIs, tool } from import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; -import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { LLMProvider } from './llm-provider'; +import type { LLMResult } from './structured-model-client'; import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; @@ -252,70 +253,6 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { } } - runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { - const maxParallel = params.maxParallelToolCalls ?? 1; - const limit = pLimit(maxParallel); - - const mappedTools = Object.fromEntries( - Object.entries(params.tools).map(([name, definition]) => [ - name, - tool({ - description: definition.description, - inputSchema: definition.inputSchema as z.ZodType, - execute: (input: unknown) => limit(() => definition.execute(input)), - }), - ]) - ); - - const result = await generateText({ - model: this.config.model, - system: params.systemPrompt, - prompt: params.prompt, - ...(params.maxRetries !== undefined ? { maxRetries: params.maxRetries } : {}), - ...this.getObservabilityOptions({ - operation: 'agent-tool-loop', - provider: this.config.providerName ?? 'unknown', - model: this.config.modelName ?? 'unknown', - }), - stopWhen: stepCountIs(params.maxSteps ?? 1000), - providerOptions: { - openai: { - parallelToolCalls: maxParallel > 1, - }, - }, - tools: mappedTools, - }); - - if (this.config.debug) { - for (const [i, step] of result.steps.entries()) { - const toolNames = step.toolCalls.map((c) => c.toolName).join(', ') || '(none)'; - this.logger.debug( - `[agent] step ${i + 1}: finishReason=${step.finishReason} tools=[${toolNames}]` - ); - if (step.text) { - this.logger.debug( - `[agent] step ${i + 1} text: ${step.text.slice(0, 500)}${step.text.length > 500 ? '...' : ''}` - ); - } - } - this.logger.debug(`[agent] final finishReason=${result.finishReason} steps=${result.steps.length}`); - if (result.text) { - this.logger.debug( - `[agent] final text: ${result.text.slice(0, 500)}${result.text.length > 500 ? '...' : ''}` - ); - } - } - - return result.usage - ? { - usage: { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, - }, - } - : {}; - } - private getObservabilityOptions(context: AIExecutionContext): Record { if (!this.observability) { return {}; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 654f41df..468a0aca 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODE, OutputFormat, REVIEW_MODES } from '../cli/types'; +import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODEL_CALL, OutputFormat, REVIEW_MODEL_CALLS } from '../cli/types'; // CLI options schema for command line argument validation export const CLI_OPTIONS_SCHEMA = z.object({ @@ -8,8 +8,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.nativeEnum(OutputFormat).default(DEFAULT_OUTPUT_FORMAT), - mode: z.enum(REVIEW_MODES).default(DEFAULT_REVIEW_MODE), - print: z.boolean().default(false), + modelCall: z.enum(REVIEW_MODEL_CALLS).default(DEFAULT_REVIEW_MODEL_CALL), prompts: z.string().optional(), config: z.string().optional(), }); diff --git a/tests/agent/agent-executor.test.ts b/tests/agent/agent-executor.test.ts deleted file mode 100644 index 2f3d1ec5..00000000 --- a/tests/agent/agent-executor.test.ts +++ /dev/null @@ -1,841 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, describe, expect, it } from 'vitest'; -import type { PromptFile } from '../../src/prompts/prompt-loader'; -import type { LLMProvider } from '../../src/providers/llm-provider'; -import { Severity } from '../../src/evaluators/types'; -import { OutputFormat } from '../../src/cli/types'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; -import { AgentToolError } from '../../src/errors'; - -function makePrompt(): PromptFile { - return { - id: 'consistency', - filename: 'consistency.md', - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording.', - meta: { - id: 'Consistency', - name: 'Consistency', - type: 'check', - severity: Severity.WARNING, - }, - }; -} - -function makeProvider( - script: (params: Record) => Promise<{ usage?: { inputTokens: number; outputTokens: number } }> -): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: script as unknown as never, - } as unknown as LLMProvider; -} - -describe('agent executor', () => { - const tempDirs: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-agent-')); - tempDirs.push(repo); - return repo; - } - - afterEach(() => { - for (const dir of tempDirs.splice(0, tempDirs.length)) { - rmSync(dir, { recursive: true, force: true }); - } - }); - - it('exposes only non-mutating analysis tools plus finalize_review', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = (params.tools ?? {}) as Record; - const names = Object.keys(tools).sort(); - expect(names).toEqual([ - 'finalize_review', - 'lint', - 'list_directory', - 'read_file', - 'report_finding', - 'search_content', - 'search_files', - ]); - - const finalize = tools.finalize_review as { execute: (input: unknown) => Promise }; - await finalize.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('returns explicit tool error for unknown ruleSource with valid-source hints', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - let errorMessage = ''; - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - expect(error).toBeInstanceOf(AgentToolError); - errorMessage = error instanceof Error ? error.message : String(error); - } - expect(errorMessage).toContain('Unknown ruleSource'); - expect(errorMessage).toContain('Valid sources'); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('returns explicit tool error for unknown ruleSource in report_finding', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/does-not-exist.md', - message: 'Unknown source', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('Unknown ruleSource'); - expect(result.errorMessage).toContain('Valid sources'); - }); - - it('returns findings reconstructed from persisted inline and top-level session events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({ summary: 'done' }); - return { usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - const inlineEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedInline - ).length; - const topLevelEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedTopLevel - ).length; - const replayableFindingEvents = inlineEventCount + topLevelEventCount; - - expect(replayableFindingEvents).toBeGreaterThanOrEqual(2); - expect(result.findings.length).toBe(replayableFindingEvents); - expect(result.findings.some((finding: { line: number }) => finding.line > 1)).toBe(false); - expect( - result.findings.some( - (finding: { ruleId: string; ruleSource: string }) => - finding.ruleId === 'Default.Consistency' && - finding.ruleSource === 'packs/default/consistency.md' - ) - ).toBe(true); - expect(result.events.some((event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized)).toBe(true); - }); - - it('aggregates nested lint usage with agent loop usage', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [], - }, - usage: { - inputTokens: 7, - outputTokens: 3, - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - }; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.usage).toEqual({ - inputTokens: 18, - outputTokens: 8, - }); - }); - - it('falls back to matching all prompts when scanPaths is empty', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('records the required session event stream and preserves lifecycle ordering', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const eventTypes = result.events.map((event: { eventType: string }) => event.eventType); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallFinished); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionFinalized); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.SessionStarted)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.FindingRecordedInline)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - }); - - it('returns an operational error when finalize_review is called more than once', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.finalize_review.execute({ summary: 'first' }); - await tools.finalize_review.execute({ summary: 'second' }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - const finalizedCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized - ).length; - expect(finalizedCount).toBe(1); - }); - - it.each([ - { - toolName: 'read_file', - input: { path: '../outside.md' }, - }, - { - toolName: 'search_files', - input: { pattern: '../*.md' }, - }, - { - toolName: 'list_directory', - input: { path: '../' }, - }, - { - toolName: 'search_content', - input: { pattern: 'bad phrase', path: '../', glob: '**/*.md' }, - }, - ])( - 'returns explicit tool errors when $toolName is asked to access outside repository bounds', - async ({ toolName, input }) => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - let toolErrorValue: unknown; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools[toolName]!.execute(input); - } catch (error) { - toolErrorValue = error; - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(toolErrorValue).toBeInstanceOf(AgentToolError); - expect(toolError).toBeTruthy(); - expect(toolError).toMatch(/outside|workspace|root|bounds/i); - } - ); - - it('continues producing findings after a recoverable tool error and reports the tool diagnostic', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 2, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(toolError).toContain('Unknown ruleSource'); - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings.length).toBeGreaterThan(0); - expect( - result.events.some( - (event: { eventType: string; payload?: { ok?: boolean } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.ok === false - ) - ).toBe(true); - }); - - it('allows read-only search_content tool usage without requiring mutation capabilities', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\nanother line\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - const output = await tools.search_content.execute({ - pattern: 'bad phrase', - path: '.', - glob: '**/*.md', - }); - - expect(output).toBeTruthy(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('marks the run as operationally failed but preserves findings when finalize_review is missing', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - expect(result.findings.length).toBeGreaterThan(0); - }); - - it('uses reviewInstruction to override the prompt body for that lint invocation', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for wording consistency using the evidence you gathered.', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe( - 'Review this file for wording consistency using the evidence you gathered.' - ); - }); - - it('keeps lint prompt body unchanged when reviewInstruction is not provided', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe('Find inconsistent wording.'); - }); - - it('records judge-style violations as inline findings in agent mode', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const basePrompt = makePrompt(); - const judgePrompt: PromptFile = { - ...basePrompt, - body: 'Judge the document for clarity.', - meta: { - ...basePrompt.meta, - type: 'judge', - criteria: [{ id: 'Clarity', name: 'Clarity', weight: 1 }], - }, - }; - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - criteria: [ - { - name: 'Clarity', - score: 2, - summary: 'Needs work', - reasoning: 'The wording is unclear.', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Unclear wording', - analysis: 'This phrase is vague.', - message: 'Use clearer wording', - suggestion: 'Replace the vague phrase', - fix: 'better phrase', - rule_quote: 'Prefer precise language', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [judgePrompt], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - file: 'doc.md', - ruleId: 'Default.Consistency', - message: 'Use clearer wording', - }), - ]) - ); - }); - - it('redacts raw read_file content from persisted tool_call_finished events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'secret text\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const readFileEvent = result.events.find( - (event: { - eventType: string; - payload?: { toolName?: string; output?: { path?: string; contentLength?: number } }; - }) => event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.toolName === 'read_file' - ); - - expect(readFileEvent?.payload?.output).toEqual({ - path: 'doc.md', - contentLength: 'secret text\n'.length, - }); - }); - - it('records started and failed events for visible-tool path errors', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const started = result.events.find( - (event: { eventType: string; payload?: { toolName?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallStarted && event.payload?.toolName === 'read_file' - ); - const failed = result.events.find( - (event: { eventType: string; payload?: { toolName?: string; ok?: boolean; error?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished - && event.payload?.toolName === 'read_file' - && event.payload?.ok === false - ); - - expect(started).toBeDefined(); - expect(failed?.payload?.error).toContain('outside workspace root'); - }); -}); diff --git a/tests/agent/progress.test.ts b/tests/agent/progress.test.ts deleted file mode 100644 index df213ab4..00000000 --- a/tests/agent/progress.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { AgentProgressReporter } from '../../src/agent/progress'; - -describe('agent progress reporter', () => { - afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - it('animates spinner frames independently of content changes while active', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - reporter.showVisibleToolStart({ - toolName: 'lint', - path: 'README.md', - ruleName: 'Repetition', - ruleText: '# Repetition Flag any instance where the same wording repeats', - }); - - const initialWrites = writeSpy.mock.calls.length; - vi.advanceTimersByTime(250); - - expect(writeSpy.mock.calls.length).toBeGreaterThan(initialWrites); - - reporter.finishRun(); - }); - - it('is a no-op when disabled', () => { - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(false); - reporter.startFile('README.md', 'Repetition'); - reporter.finishRun(); - - expect(writeSpy).not.toHaveBeenCalled(); - }); - - it.each([ - { elapsedMs: 0, expected: 'Completed review in 0s.' }, - { elapsedMs: 60_000, expected: 'Completed review in 1m 0s.' }, - { elapsedMs: 3_600_000, expected: 'Completed review in 1h 0m 0s.' }, - ])('formats elapsed time boundaries for $expected', ({ elapsedMs, expected }) => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(elapsedMs); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain(expected); - }); - - it('formats the completion footer with elapsed time', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Completed review in 1m 25s.'); - }); - - it('formats a failed footer when the run ends with errors', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun('failed'); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Review failed after 1m 25s.'); - }); -}); diff --git a/tests/agent/prompt-builder.test.ts b/tests/agent/prompt-builder.test.ts deleted file mode 100644 index fe31074c..00000000 --- a/tests/agent/prompt-builder.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('agent prompt builder', () => { - it('builds a non-empty system prompt for valid inputs', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [ - { file: 'README.md', ruleSource: 'packs/default/ai-pattern.md' }, - { file: 'README.md', ruleSource: 'packs/default/consistency.md' }, - ], - availableTools: [ - { name: 'read_file', description: 'Read a file inside the workspace root.' }, - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - { name: 'finalize_review', description: 'Finalize review output and close the session.' }, - ], - }); - - expect(typeof prompt).toBe('string'); - expect(prompt.length).toBeGreaterThan(0); - expect(prompt).toContain('Workspace root: /workspace'); - expect(prompt).toContain('matched rules'); - }); - - it('supports optional user instructions without breaking prompt generation', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const withoutUserInstructions = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - }); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - userInstructions: 'Always enforce concise phrasing.', - }); - - expect(typeof withoutUserInstructions).toBe('string'); - expect(typeof prompt).toBe('string'); - expect(withoutUserInstructions.length).toBeGreaterThan(0); - expect(prompt.length).toBeGreaterThan(withoutUserInstructions.length); - }); -}); diff --git a/tests/agent/review-session-store.test.ts b/tests/agent/review-session-store.test.ts deleted file mode 100644 index 0258b561..00000000 --- a/tests/agent/review-session-store.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { appendFileSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -beforeEach(() => { - vi.resetModules(); -}); - -afterEach(() => { - vi.doUnmock('crypto'); - vi.resetModules(); -}); - -describe('review session store', () => { - it('creates session files in the VectorLint reviews directory', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(store.sessionFilePath).toContain( - `${path.sep}.vectorlint${path.sep}reviews${path.sep}` - ); - expect(path.basename(store.sessionFilePath)).toMatch(/\.jsonl$/); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); - - it('persists each appended event as a valid JSONL record', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const raw = readFileSync(store.sessionFilePath, 'utf8'); - const lines = raw.trim().split('\n'); - expect(lines).toHaveLength(2); - - const parsed = lines.map((line) => JSON.parse(line) as { eventType?: string }); - expect(parsed.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('reads persisted session events in order and reports completion state', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(await store.hasFinalizedEvent()).toBe(false); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - expect(await store.hasFinalizedEvent()).toBe(true); - }); - - it('recovers valid events when an existing session file contains malformed JSONL lines', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - - appendFileSync(store.sessionFilePath, 'not-json\n', 'utf8'); - appendFileSync(store.sessionFilePath, '{"eventType":"unknown"}\n', 'utf8'); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0 }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('creates a unique session file when an initial generated session id collides', async () => { - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const reviewsDir = path.join(home, '.vectorlint', 'reviews'); - mkdirSync(reviewsDir, { recursive: true }); - writeFileSync(path.join(reviewsDir, 'collision-id.jsonl'), '', 'utf8'); - - const randomUUIDMock = vi - .fn() - .mockReturnValueOnce('collision-id') - .mockReturnValueOnce('fresh-id'); - - vi.doMock('crypto', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - randomUUID: randomUUIDMock, - }; - }); - - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const store = await createReviewSessionStore({ homeDir: home }); - expect(path.basename(store.sessionFilePath)).toBe('fresh-id.jsonl'); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); -}); diff --git a/tests/agent/types-contract.test.ts b/tests/agent/types-contract.test.ts deleted file mode 100644 index 6363587a..00000000 --- a/tests/agent/types-contract.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -describe('agent contracts', () => { - it('accepts ruleSource-based inputs for lint and top-level findings', async () => { - const contracts = await import('../../src/agent/types'); - - const lintInput = contracts.LINT_TOOL_INPUT_SCHEMA.parse({ - file: 'docs/guide.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for consistency.', - }); - expect(lintInput.ruleSource).toBe('packs/default/consistency.md'); - - const topLevel = contracts.TOP_LEVEL_REPORT_INPUT_SCHEMA.parse({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file inconsistency', - references: [{ file: 'docs/guide.md', startLine: 2, endLine: 4 }], - }); - expect(topLevel.kind).toBe('top-level'); - expect(topLevel.ruleSource).toBe('packs/default/consistency.md'); - }); - - it('accepts finalized session events with completion metadata', async () => { - const contracts = await import('../../src/agent/types'); - - const event = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 1, summary: 'done' }, - }); - - expect(event.eventType).toBe(SESSION_EVENT_TYPE.SessionFinalized); - expect(event.payload.totalFindings).toBe(1); - }); - - it('accepts required tool and finding session event variants for deterministic replay', async () => { - const contracts = await import('../../src/agent/types'); - - const started = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName: 'lint', input: { file: 'docs/guide.md' } }, - }); - - const finished = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:01.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { toolName: 'lint', ok: true }, - }); - - const inlineFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:02.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: 'docs/guide.md', - line: 2, - message: 'Inconsistent term', - ruleSource: 'packs/default/consistency.md', - }, - }); - - const topLevelFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:03.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - message: 'Cross-file mismatch', - ruleSource: 'packs/default/consistency.md', - }, - }); - - expect(started.eventType).toBe(SESSION_EVENT_TYPE.ToolCallStarted); - expect(finished.eventType).toBe(SESSION_EVENT_TYPE.ToolCallFinished); - expect(inlineFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(topLevelFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - }); -}); diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 359da161..7d652c46 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -97,8 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'auto', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -142,8 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'auto', config: undefined, }); const observability = { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts deleted file mode 100644 index 0b61ea63..00000000 --- a/tests/orchestrator-agent-output.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; -import type { PromptFile } from '../src/prompts/prompt-loader'; -import type { LLMProvider } from '../src/providers/llm-provider'; -import type { Logger } from '../src/logging/logger'; -import { Severity } from '../src/evaluators/types'; - -function makePrompt(): PromptFile { - const id = 'consistency'; - const name = 'Consistency'; - return { - id, - filename: `${id}.md`, - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording', - meta: { - id: name, - name, - type: 'check', - severity: Severity.WARNING, - }, - }; -} - -type LoggerSpy = Logger & { warn: ReturnType }; - -function makeLogger(): LoggerSpy { - return { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - } as unknown as LoggerSpy; -} - -interface StandardProviderSpies { - provider: LLMProvider; - runPromptStructured: ReturnType; - runAgentToolLoop: ReturnType; -} - -function makeStandardProvider(): StandardProviderSpies { - const runPromptStructured = vi.fn().mockResolvedValue({ - data: { reasoning: 'ok', violations: [] }, - }); - const runAgentToolLoop = vi.fn().mockResolvedValue({ - usage: { inputTokens: 0, outputTokens: 0 }, - }); - const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; - return { provider, runPromptStructured, runAgentToolLoop }; -} - -// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no -// longer reaches the autonomous agent executor: it warns through the injected -// logger and falls back to standard evaluation. The retained agent executor -// code is covered directly by tests/agent/* and removed in Phase 4. -describe('agent mode deprecation', () => { - const tempRepos: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(process.cwd(), 'tmp-agent-orch-')); - tempRepos.push(repo); - return repo; - } - - beforeEach(() => { - vi.spyOn(console, 'log').mockImplementation(() => undefined); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); - vi.spyOn(console, 'error').mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - for (const repo of tempRepos.splice(0, tempRepos.length)) { - rmSync(repo, { recursive: true, force: true }); - } - }); - - it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('deprecated'), - ); - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), - ); - // Standard evaluation ran; the autonomous executor did not. - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); - - it('does not invoke the agent executor in line output mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - }); - - it('stays silent in standard mode without a logger', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runAgentToolLoop } = makeStandardProvider(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: DEFAULT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - }); - - // No deprecation warning and no executor invocation in standard mode. - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); -}); diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts deleted file mode 100644 index eaaa1529..00000000 --- a/tests/orchestrator-check-processor.test.ts +++ /dev/null @@ -1,356 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type CheckViolation = RawCheckResult["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -/** - * Builds a check violation that passes the confidence gate by default. Real - * model output always carries `message`; the helper sets it explicitly so the - * test reflects production behavior rather than the analysis fallback. - */ -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { - return { - line: 1, - analysis: "Issue 1", - message: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; - wordCount?: number; -}): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -describe("standard check evaluation via the shared finding processor", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("reports fully locatable findings, score, and counts unchanged", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ message: "First issue", analysis: "First issue" }), - makeCheckViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second issue", - analysis: "Second issue", - suggestion: "Suggestion 2", - fix: "Fix 2", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. - expect(run.totalWarnings).toBe(2); - expect(run.totalErrors).toBe(0); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("8.0/10"); - // The line summary prints the last segment of the Pack.Rule score id. - expect(scoreLine).toContain("CheckPrompt"); - }); - - it("counts only verified findings and excludes unanchored quotes from the score", async () => { - // One anchored + one unanchored quote. Both pass the confidence gate, so - // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CountingFixPrompt", - name: "Counting Fix Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ - line: 9, - quoted_text: "this quote is not anywhere in the content", - message: "Ghost issue", - analysis: "Ghost issue", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a - // finding; it contributes neither to the count nor the score. - expect(run.totalWarnings).toBe(1); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - }); - - it("does not flag severity errors when no finding can be verified", async () => { - // severity=error, but the only quote cannot anchor -> 0 verified findings -> - // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorNoVerifiedPrompt", - name: "Error No Verified Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - line: 9, - quoted_text: "nowhere to be found", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.hadSeverityErrors).toBe(false); - expect(run.hadOperationalErrors).toBe(false); - }); - - it("flags severity errors when verified error-severity findings exist", async () => { - // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> - // prompt severity error applies. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorVerifiedPrompt", - name: "Error Verified Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [makeCheckViolation({ quoted_text: "Alpha text" })], - wordCount: 10, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(1); - expect(run.hadSeverityErrors).toBe(true); - }); - - it("emits verified findings with anchored location through the JSON sink", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ message: "First", analysis: "First" }), - makeCheckViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second", - analysis: "Second", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const issues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(issues).toHaveLength(2); - expect(issues[0]).toMatchObject({ - line: 1, - column: 1, - severity: Severity.WARNING, - message: "First", - rule: "TestPack.CheckJsonPrompt", - match: "Alpha text", - }); - expect(issues[1]).toMatchObject({ - line: 2, - message: "Second", - match: "Beta text", - }); - }); - - it("omits unanchored quotes from the Vale JSON output", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValePrompt", - name: "Check Vale Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ - line: 9, - quoted_text: "missing quote", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const issues = Object.values(parsed).flat(); - - expect(issues).toHaveLength(1); - expect(issues[0]).toMatchObject({ - Check: "TestPack.CheckValePrompt", - Line: 1, - Match: "Alpha text", - Severity: "warning", - }); - }); -}); diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts deleted file mode 100644 index b4d2cf89..00000000 --- a/tests/orchestrator-filtering.test.ts +++ /dev/null @@ -1,412 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { JudgeResult, RawCheckResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type CheckViolation = RawCheckResult["violations"][number]; -type JudgeViolation = JudgeResult["criteria"][number]["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-filtering-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -const EMPTY_CHECK_NOTES: JudgeViolation["check_notes"] = { - rule_supports_claim: "", - evidence_exact: "", - context_supports_violation: "", - plausible_non_violation: "", - fix_is_drop_in: "", - fix_preserves_meaning: "", -}; - -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { - return { - line: 1, - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeJudgeViolation( - overrides: Partial = {} -): JudgeViolation { - const { check_notes: checkNotesOverrides, ...rest } = overrides; - return { - line: 1, - quoted_text: "Alpha text", - context_before: "", - context_after: "", - description: "Issue 1", - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: { - ...EMPTY_CHECK_NOTES, - ...(checkNotesOverrides ?? {}), - }, - confidence: 0.9, - ...rest, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; - wordCount?: number; -}): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -function makeJudgeResult(violations: JudgeViolation[]): JudgeResult { - return { - type: EvaluationType.JUDGE, - final_score: 5, - criteria: [ - { - name: "Clarity", - weight: 1, - score: 2, - normalized_score: 5, - weighted_points: 1, - summary: "Needs work", - reasoning: "Reasoning", - violations, - }, - ], - }; -} - -describe("CLI violation filtering", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("filters low-confidence check violations from CLI counts by default", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation(), - makeCheckViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation(), - makeCheckViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); - }); - - it("does not mark severity error when no check violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckErrorPrompt", - name: "Check Error Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalErrors).toBe(0); - expect(defaultRun.hadSeverityErrors).toBe(false); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalErrors).toBe(1); - expect(zeroThresholdRun.hadSeverityErrors).toBe(true); - }); - - it("score reflects only surfaced violations, not filtered-out ones", async () => { - // 100-word file: 2 violations from model, 1 fails confidence gate - // With default threshold, only 1 violation surfaces - // Density: 1/100 * 100 * 10 = 10 penalty → score = 9.0 - // If bug were present (scoring all 2): 2/100 * 100 * 10 = 20 penalty → score = 8.0 - - const content = new Array(100).fill("word").join(" ") + "\n"; - const targetFile = createTempFile(content); - - const prompt = createPrompt({ - id: "ScorePrompt", - name: "Score Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: content.split(" ")[0] ?? "word" }), - makeCheckViolation({ - quoted_text: content.split(" ")[1] ?? "word", - confidence: 0.2, // fails confidence gate — should NOT affect score - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Score should reflect 1 surfaced violation, not 2 - const scoreLine = logCalls.find(l => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - expect(scoreLine).not.toContain("8.0/10"); - }); - - it("rejects judge/rubric results instead of projecting them as check findings", async () => { - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // orchestrator refuses a JudgeResult rather than running the old criterion - // extraction/reporting path: no findings are projected and the run is - // flagged with an operational error. - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "JudgePrompt", - name: "Judge Prompt", - type: "judge", - criteria: [{ id: "Clarity", name: "Clarity", weight: 1 }], - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - confidence: 0.2, - }), - ]) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.totalWarnings).toBe(0); - expect(run.hadOperationalErrors).toBe(true); - expect(run.hadSeverityErrors).toBe(false); - }); - - it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); - - it("does not emit dummy issues in Vale JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValeJsonPrompt", - name: "Check Vale JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const allIssues = Object.values(parsed).flat(); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); -}); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts deleted file mode 100644 index dc0b1ad8..00000000 --- a/tests/orchestrator-finding-criteria.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; -import type { ReviewResult } from "../src/review/types"; - -const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), - PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), -})); - -// createEvaluator returns a controllable evaluator so evaluateFiles reaches -// routePromptResult -> processFindings without a real model call. -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -// Replace only processFindings (the findings boundary); keep every other -// findings export real so unrelated imports stay intact. -vi.mock("../src/findings", async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; -}); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -function makeCheckResult(): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: [], - word_count: 100, - }; -} - -describe("standard check orchestration sanitizes criteria at the findings boundary", () => { - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - PROCESS_FINDINGS_MOCK.mockReset(); - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("drops rubric weight/target from criteria before reaching processFindings", async () => { - const targetFile = createTempFile("Alpha text\n"); - // PromptCriterionSpec admits legacy rubric fields (weight, target). The - // orchestrator must strip them so only { id, name } crosses the boundary. - const prompt = createPrompt({ - id: "SanitizePrompt", - name: "Sanitize Prompt", - type: "check", - severity: Severity.WARNING, - criteria: [ - { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, - ], - }); - - EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); - PROCESS_FINDINGS_MOCK.mockReturnValue({ - findings: [], - scores: [ - { - ruleId: "TestPack.SanitizePrompt", - score: 10, - scoreText: "10.0/10", - severity: "warning", - }, - ], - diagnostics: [], - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); - const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { - promptMeta: { criteria?: Array> }; - }; - - // toEqual fails if any extra (weight/target) key survives the mapping. - expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); - for (const criterion of input.promptMeta.criteria ?? []) { - expect(criterion).not.toHaveProperty("weight"); - expect(criterion).not.toHaveProperty("target"); - } - }); -}); diff --git a/tests/providers/llm-provider-contract.test.ts b/tests/providers/llm-provider-contract.test.ts deleted file mode 100644 index 249854ca..00000000 --- a/tests/providers/llm-provider-contract.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { LanguageModel } from 'ai'; -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('LLMProvider agent contract', () => { - it('supports agent-mode execution via a provider loop interface', () => { - const provider = new VercelAIProvider({ - model: {} as unknown as LanguageModel, - }); - - const runAgentToolLoop = (provider as { runAgentToolLoop?: unknown }).runAgentToolLoop; - expect(typeof runAgentToolLoop).toBe('function'); - }); -}); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts index 7fa6b61d..45db05cc 100644 --- a/tests/providers/structured-model-client.test.ts +++ b/tests/providers/structured-model-client.test.ts @@ -45,13 +45,11 @@ describe('StructuredModelClient contract', () => { const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; expect(onlyRunPromptStructured).toBe(true); - // Runtime guard: a minimal implementation exposes only runPromptStructured. + // Runtime guard: a minimal implementation exposes exactly one capability. const client: StructuredModelClient = { runPromptStructured: () => Promise.resolve({ data: null }), }; - expect( - (client as unknown as Record).runAgentToolLoop, - ).toBeUndefined(); + expect(Object.keys(client)).toEqual(['runPromptStructured']); }); it('is satisfied by VercelAIProvider', () => { diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts index ccf84f68..a2239552 100644 --- a/tests/providers/tool-calling-model-client.test.ts +++ b/tests/providers/tool-calling-model-client.test.ts @@ -66,8 +66,6 @@ describe('ToolCallingModelClient contract', () => { 'read_file', 'search_content', 'report_finding', - 'reviewInstruction', - 'runAgentToolLoop', ]; for (const concept of forbidden) { expect(keys).not.toContain(concept); diff --git a/tests/providers/vercel-ai-provider-agent-loop.test.ts b/tests/providers/vercel-ai-provider-agent-loop.test.ts deleted file mode 100644 index b1bca660..00000000 --- a/tests/providers/vercel-ai-provider-agent-loop.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { z } from 'zod'; -import type { LanguageModel } from 'ai'; -import { createMockLogger } from '../utils'; - -const MOCK_GENERATE_TEXT = vi.hoisted(() => vi.fn()); -const MOCK_STEP_COUNT_IS = vi.hoisted(() => vi.fn(() => ({ type: 'stepCount' }))); -const MOCK_TOOL = vi.hoisted(() => - vi.fn((definition: Record) => definition) -); - -const ERROR_CLASSES = vi.hoisted(() => { - class NoObjectGeneratedError extends Error { - text: string; - - constructor(message: string, text: string) { - super(message); - this.name = 'NoObjectGeneratedError'; - this.text = text; - } - - static isInstance(error: unknown): error is NoObjectGeneratedError { - return error instanceof NoObjectGeneratedError; - } - } - - return { NoObjectGeneratedError }; -}); - -vi.mock('ai', () => { - const { NoObjectGeneratedError } = ERROR_CLASSES; - return { - generateText: MOCK_GENERATE_TEXT, - stepCountIs: MOCK_STEP_COUNT_IS, - tool: MOCK_TOOL, - Output: { - object: vi.fn((schema: unknown) => ({ - _outputType: 'object', - schema, - })), - }, - NoObjectGeneratedError, - }; -}); - -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('VercelAIProvider agent loop', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('applies configured retry and tool-concurrency limits during agent execution', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - const runAgentToolLoop = ( - provider as unknown as { - runAgentToolLoop?: (params: Record) => Promise; - } - ).runAgentToolLoop; - - expect(typeof runAgentToolLoop).toBe('function'); - if (typeof runAgentToolLoop !== 'function') { - throw new Error('runAgentToolLoop is not implemented'); - } - - await runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxRetries: 4, - maxSteps: 42, - maxParallelToolCalls: 1, - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - maxRetries?: number; - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - - expect(call.maxRetries).toBe(4); - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: false }, - }); - }); - - it('adds observability options to agent-loop generateText calls', async () => { - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => ({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.agent-tool-loop' }, - })), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(observability.decorateCall).toHaveBeenCalledWith({ - operation: 'agent-tool-loop', - provider: 'openai', - model: 'gpt-4o-mini', - }); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).toHaveProperty('experimental_telemetry'); - }); - - it('continues agent-loop AI calls when observability decoration fails', async () => { - const logger = createMockLogger(); - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => { - throw new Error('telemetry failed'); - }), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - logger, - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).not.toHaveProperty('experimental_telemetry'); - expect(logger.warn).toHaveBeenCalledWith( - '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', - expect.objectContaining({ error: 'telemetry failed', operation: 'agent-tool-loop' }) - ); - }); - - it('limits concurrent tool executes to maxParallelToolCalls', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxParallelToolCalls: 2, - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - tool_d: { description: 'd', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBeLessThanOrEqual(2); - expect(maxConcurrent).toBeGreaterThan(1); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: true }, - }); - }); - - it('defaults tool concurrency to 1 when maxParallelToolCalls is not supplied', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBe(1); - }); - - it('emits agent-loop debug output through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'final summary', - finishReason: 'stop', - steps: [ - { - finishReason: 'tool-calls', - text: 'step text', - toolCalls: [{ toolName: 'lint' }], - }, - ], - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const logger = createMockLogger(); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - debug: true, - logger, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(logger.debug).toHaveBeenCalled(); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] step 1: finishReason=tool-calls tools=[lint]') - ) - ).toBe(true); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] final finishReason=stop steps=1') - ) - ).toBe(true); - }); -}); From 6e2a04feda87a83ec9feb4c2c8259d12470c9fc8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:28:56 +0100 Subject: [PATCH 23/27] refactor(cli): reject legacy mode flag - Reject the removed --mode option without silently remapping it.\n- Cover single, agent, and auto executor dispatch plus ReviewResult routing.\n- Preserve the breaking CLI migration to --model-call single|agent|auto. --- src/cli/commands.ts | 3 +- tests/cli-mode-rejection.test.ts | 37 +++ tests/orchestrator-executor-dispatch.test.ts | 229 +++++++++++++++++++ 3 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 tests/cli-mode-rejection.test.ts create mode 100644 tests/orchestrator-executor-dispatch.test.ts diff --git a/src/cli/commands.ts b/src/cli/commands.ts index b2e7f0b7..9c0e362a 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -97,8 +97,7 @@ export function registerMainCommand(program: Command): void { // --mode is removed (audit Product Decision): the autonomous workspace-agent // surface is gone. Fail clearly and point users at the replacement rather // than silently mapping it onto --model-call. - const rawCliOpts = program.opts() as { mode?: unknown }; - if (rawCliOpts.mode !== undefined) { + if (program.opts().mode !== undefined) { console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); process.exit(1); } diff --git a/tests/cli-mode-rejection.test.ts b/tests/cli-mode-rejection.test.ts new file mode 100644 index 00000000..38c0f3db --- /dev/null +++ b/tests/cli-mode-rejection.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; + +describe('CLI --mode rejection', () => { + let exitSpy: ReturnType; + + beforeEach(() => { + vi.spyOn(console, 'error').mockImplementation(() => undefined); + exitSpy = vi.spyOn(process, 'exit').mockImplementation( + ((code?: string | number | null) => { + throw new Error(`exit:${code ?? ''}`); + }) as never, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('rejects --mode and points users at --model-call instead of mapping it', async () => { + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect( + program.parseAsync(['node', 'test', 'README.md', '--mode', 'agent']), + ).rejects.toThrow('exit:1'); + + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--mode is no longer supported'), + ); + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--model-call'), + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); +}); diff --git a/tests/orchestrator-executor-dispatch.test.ts b/tests/orchestrator-executor-dispatch.test.ts new file mode 100644 index 00000000..b4a1d093 --- /dev/null +++ b/tests/orchestrator-executor-dispatch.test.ts @@ -0,0 +1,229 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import * as path from 'path'; +import { evaluateFiles } from '../src/cli/orchestrator'; +import { OutputFormat, type EvaluationOptions } from '../src/cli/types'; +import { Severity } from '../src/evaluators/types'; +import type { PromptFile } from '../src/prompts/prompt-loader'; +import type { ReviewRequest, ReviewResult } from '../src/review/types'; + +const { EXECUTOR_FOR_MOCK, FAKE_EXECUTOR_RUN } = vi.hoisted(() => ({ + EXECUTOR_FOR_MOCK: vi.fn(), + FAKE_EXECUTOR_RUN: vi.fn(), +})); + +// Replace executorFor with a spy that returns a controllable executor so the +// orchestrator's wiring (ReviewRequest build -> chooseModelCall -> dispatch -> +// ReviewResult routing) is exercised without a real model call. +vi.mock('../src/executors', () => ({ + executorFor: EXECUTOR_FOR_MOCK, +})); + +function makePrompt(id: string): PromptFile { + return { + id, + filename: `${id}.md`, + fullPath: path.join(process.cwd(), 'prompts', `${id}.md`), + pack: 'TestPack', + body: `Rule body for ${id}`, + meta: { id, name: id, type: 'check', severity: Severity.WARNING }, + }; +} + +function makeOptions(prompts: PromptFile[], overrides: Partial = {}): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + requestBuilder: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Json, + modelCall: 'auto', + ...overrides, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), 'vectorlint-dispatch-')); + const filePath = path.join(dir, 'input.md'); + writeFileSync(filePath, content); + return filePath; +} + +function makeReviewResult(overrides: Partial = {}): ReviewResult { + return { + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: path.join(process.cwd(), 'prompts', 'CheckPrompt.md'), + severity: 'warning', + message: 'Vague advice found', + line: 1, + column: 1, + match: 'vague text', + suggestion: 'Be specific.', + }, + ], + scores: [ + { + ruleId: 'TestPack.CheckPrompt', + score: 8, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 1, + }, + ], + diagnostics: [], + hadOperationalErrors: false, + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5 }, + ...overrides, + }; +} + +describe('orchestrator executor dispatch', () => { + beforeEach(() => { + EXECUTOR_FOR_MOCK.mockReset(); + FAKE_EXECUTOR_RUN.mockReset(); + EXECUTOR_FOR_MOCK.mockReturnValue({ run: FAKE_EXECUTOR_RUN }); + FAKE_EXECUTOR_RUN.mockResolvedValue(makeReviewResult()); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('resolves auto to single for a normal-sized target and routes findings to JSON output', async () => { + const file = createTempFile('vague text here\n'); + const run = await evaluateFiles([file], makeOptions([makePrompt('CheckPrompt')])); + + // auto + small target + one rule resolves to the single executor. + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + + // The verified finding reaches the JSON sink. + const parsed = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) as { + files: Record> }>; + }; + const issues = Object.values(parsed.files).flatMap((f) => f.issues); + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + line: 1, + severity: Severity.WARNING, + message: 'Vague advice found', + rule: 'TestPack.CheckPrompt', + match: 'vague text', + }); + + expect(run.totalWarnings).toBe(1); + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it('resolves auto to agent for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await evaluateFiles([file], makeOptions([makePrompt('BigPrompt')])); + + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('honors an explicit single modelCall even for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await evaluateFiles([file], makeOptions([makePrompt('ForcedSingle')], { modelCall: 'single' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + }); + + it('honors an explicit agent modelCall even for a small target', async () => { + const file = createTempFile('small\n'); + await evaluateFiles([file], makeOptions([makePrompt('ForcedAgent')], { modelCall: 'agent' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('forwards the built ReviewRequest (target + rules + modelCall) to the executor', async () => { + const file = createTempFile('target content line one\n'); + await evaluateFiles([file], makeOptions([makePrompt('FwdPrompt')], { modelCall: 'agent' })); + + expect(FAKE_EXECUTOR_RUN).toHaveBeenCalledTimes(1); + const request = FAKE_EXECUTOR_RUN.mock.calls[0]![0] as ReviewRequest; + expect(request.target.content).toBe('target content line one\n'); + expect(request.rules).toHaveLength(1); + expect(request.rules[0]?.body).toBe('Rule body for FwdPrompt'); + expect(request.modelCall).toBe('agent'); + }); + + it('aggregates error-severity findings and flags hadSeverityErrors', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: 'src', + severity: 'error', + message: 'Severe issue', + line: 2, + column: 1, + match: 'vague text', + }, + ], + scores: [ + { ruleId: 'TestPack.CheckPrompt', score: 0, scoreText: '0.0/10', severity: 'error', findingCount: 1 }, + ], + }), + ); + const file = createTempFile('vague text\n'); + + const run = await evaluateFiles([file], makeOptions([makePrompt('ErrorPrompt')])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it('routes ReviewResult diagnostics to verbose console output', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [], + diagnostics: [{ level: 'warn', code: 'finding-evidence-not-locatable', message: 'could not anchor quote' }], + }), + ); + const file = createTempFile('vague text\n'); + + await evaluateFiles([file], makeOptions([makePrompt('DiagPrompt')], { verbose: true })); + + expect(vi.mocked(console.warn)).toHaveBeenCalledWith( + expect.stringContaining('could not anchor quote'), + ); + }); + + it('aggregates token usage from the ReviewResult', async () => { + const file = createTempFile('vague text\n'); + const run = await evaluateFiles([file], makeOptions([makePrompt('UsagePrompt')])); + + expect(run.tokenUsage?.totalInputTokens).toBe(10); + expect(run.tokenUsage?.totalOutputTokens).toBe(5); + }); + + it('skips the executor when no prompts apply', async () => { + const file = createTempFile('content\n'); + // A scan-path config that runs a pack none of the prompts belong to. + const run = await evaluateFiles( + [file], + makeOptions([makePrompt('OrphanPrompt')], { + scanPaths: [{ pattern: '**/*.md', runRules: ['OtherPack'], overrides: {} }], + }), + ); + + expect(EXECUTOR_FOR_MOCK).not.toHaveBeenCalled(); + expect(run.totalFiles).toBe(1); + expect(run.totalWarnings).toBe(0); + }); +}); From 64e434d88350204c20c2eb671d9fa4181059d7d6 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:39:52 +0100 Subject: [PATCH 24/27] refactor(cli): harden config and payload telemetry - create global config resources with restrictive permissions\n- propagate opt-in payload telemetry through both executors\n- cover CLI, package entrypoints, observability, and provider logging --- src/config/global-config.ts | 4 ++-- src/executors/agent-model-call-executor.ts | 6 +++++- src/executors/single-model-call-executor.ts | 5 ++++- src/observability/ai-observability.ts | 1 + src/observability/langfuse-observability.ts | 4 ++-- src/providers/request-builder.ts | 1 + src/providers/tool-calling-model-client.ts | 2 ++ src/providers/vercel-ai-provider.ts | 6 ++++++ tests/global-config.test.ts | 17 ++++++++++++++- tests/main-command-observability.test.ts | 4 ++-- .../langfuse-observability.test.ts | 21 ++++++++++++++++++- tests/package-entrypoint.test.ts | 21 +++++++++++++++++++ tests/vercel-ai-provider.test.ts | 1 + 13 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 tests/package-entrypoint.test.ts diff --git a/src/config/global-config.ts b/src/config/global-config.ts index 98b1a7fc..6a694122 100644 --- a/src/config/global-config.ts +++ b/src/config/global-config.ts @@ -74,11 +74,11 @@ export function ensureGlobalConfig(): string { const configDir = path.dirname(configPath); if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); + mkdirSync(configDir, { recursive: true, mode: 0o700 }); } if (!existsSync(configPath)) { - writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, 'utf-8'); + writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, { encoding: 'utf-8', mode: 0o600 }); } return configPath; diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts index 1b09f388..1679fecb 100644 --- a/src/executors/agent-model-call-executor.ts +++ b/src/executors/agent-model-call-executor.ts @@ -50,7 +50,10 @@ export class AgentModelCallExecutor implements ReviewExecutor { const capability = new TargetReadCapability(request.target.content); // Exactly one executor-owned tool is exposed: target-section paging. const tools = buildReadTargetSectionTool(capability); - const context = buildEvalContext(request.target.uri); + const context = { + ...buildEvalContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; @@ -136,6 +139,7 @@ export class AgentModelCallExecutor implements ReviewExecutor { // emitting its structured finding set. maxSteps: request.budget.maxChunksPerRule, maxParallelToolCalls: 1, + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, }, }); diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index 3809cbb3..2bb87ec4 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -50,7 +50,10 @@ export class SingleModelCallExecutor implements ReviewExecutor { async run(request: ReviewRequest): Promise { const schema = buildCheckLLMSchema(); - const context = buildEvalContext(request.target.uri); + const context = { + ...buildEvalContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index e16a3c9d..88a76f26 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -4,6 +4,7 @@ export interface AIExecutionContext { model: string; evaluator?: string; rule?: string; + recordPayloadTelemetry?: boolean; } export interface AIObservability { diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index 25f4d32c..bae2d555 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -63,8 +63,8 @@ export class LangfuseObservability implements AIObservability { ...(context.evaluator ? { evaluator: context.evaluator } : {}), ...(context.rule ? { rule: context.rule } : {}), }, - recordInputs: true, - recordOutputs: true, + recordInputs: context.recordPayloadTelemetry === true, + recordOutputs: context.recordPayloadTelemetry === true, }, }; } diff --git a/src/providers/request-builder.ts b/src/providers/request-builder.ts index 806c53e8..9e0b239c 100644 --- a/src/providers/request-builder.ts +++ b/src/providers/request-builder.ts @@ -2,6 +2,7 @@ export interface EvalContext { fileType?: string; + recordPayloadTelemetry?: boolean; } export interface RequestBuilder { diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts index 2139c8f2..a038e7e5 100644 --- a/src/providers/tool-calling-model-client.ts +++ b/src/providers/tool-calling-model-client.ts @@ -21,6 +21,8 @@ export interface ToolCallDefinition { * transport honors them. It does not invent its own product-level loop. */ export interface ToolCallRunOptions { + /** Opt in to recording prompt and generated payloads in telemetry. */ + recordPayloadTelemetry?: boolean; /** Maximum number of model steps (tool-call rounds) before the run stops. */ maxSteps?: number; /** Provider retry budget for transient failures. */ diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index dba87369..634e08f8 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -91,6 +91,9 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { model: this.config.modelName ?? 'unknown', ...(evaluator ? { evaluator } : {}), ...(rule ? { rule } : {}), + ...(context?.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: context.recordPayloadTelemetry } + : {}), }); const result = await generateText({ @@ -197,6 +200,9 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { operation: 'tool-calling', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', + ...(options.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: options.recordPayloadTelemetry } + : {}), }), stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), providerOptions: { diff --git a/tests/global-config.test.ts b/tests/global-config.test.ts index 2a8d8c7b..acee276c 100644 --- a/tests/global-config.test.ts +++ b/tests/global-config.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import { loadGlobalConfig } from '../src/config/global-config'; +import { ensureGlobalConfig, loadGlobalConfig } from '../src/config/global-config'; import { GLOBAL_CONFIG_DIR, GLOBAL_CONFIG_FILE } from '../src/config/constants'; // Mock fs and os @@ -49,6 +49,21 @@ BOOL_KEY = true expect(fs.readFileSync).toHaveBeenCalledWith(mockConfigPath, 'utf-8'); }); + it('creates the global config directory and file with restrictive modes', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + + expect(ensureGlobalConfig()).toBe(mockConfigPath); + expect(fs.mkdirSync).toHaveBeenCalledWith( + path.dirname(mockConfigPath), + { recursive: true, mode: 0o700 }, + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + mockConfigPath, + expect.any(String), + { encoding: 'utf-8', mode: 0o600 }, + ); + }); + it('should NOT overwrite existing env vars', () => { const mockToml = ` [env] diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 7d652c46..d1619ad9 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -97,7 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - modelCall: 'auto', + modelCall: 'single', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -141,7 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - modelCall: 'auto', + modelCall: 'single', config: undefined, }); const observability = { diff --git a/tests/observability/langfuse-observability.test.ts b/tests/observability/langfuse-observability.test.ts index 56448489..ca0f73c4 100644 --- a/tests/observability/langfuse-observability.test.ts +++ b/tests/observability/langfuse-observability.test.ts @@ -26,7 +26,7 @@ describe('LangfuseObservability', () => { SHUTDOWN_MOCK.mockResolvedValue(undefined); }); - it('returns AI SDK telemetry options with full payload recording enabled', () => { + it('does not record payloads by default', () => { const subject = new LangfuseObservability({ publicKey: 'pk-lf-test', secretKey: 'sk-lf-test', @@ -49,6 +49,25 @@ describe('LangfuseObservability', () => { evaluator: 'clarity', rule: 'no-fluff', }, + recordInputs: false, + recordOutputs: false, + }, + }); + }); + + it('records payloads only when the review output policy opts in', () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + }); + + expect(subject.decorateCall({ + operation: 'structured-eval', + provider: 'openai', + model: 'gpt-4o', + recordPayloadTelemetry: true, + })).toMatchObject({ + experimental_telemetry: { recordInputs: true, recordOutputs: true, }, diff --git a/tests/package-entrypoint.test.ts b/tests/package-entrypoint.test.ts new file mode 100644 index 00000000..ff99ac77 --- /dev/null +++ b/tests/package-entrypoint.test.ts @@ -0,0 +1,21 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const PACKAGE_PATH = resolve(dirname(fileURLToPath(import.meta.url)), '../package.json'); + +describe('package entrypoints', () => { + it('keeps both published bin aliases pointed at the built CLI', () => { + const packageJson = JSON.parse(readFileSync(PACKAGE_PATH, 'utf8')) as { + main?: string; + bin?: Record; + }; + + expect(packageJson.main).toBe('dist/index.js'); + expect(packageJson.bin).toEqual({ + vectorlint: './dist/index.js', + veclint: './dist/index.js', + }); + }); +}); diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index ab258dd4..80bbaf5f 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -391,6 +391,7 @@ describe('VercelAIProvider', () => { }) as Record, }) ); + expect(logger.debug).not.toHaveBeenCalledWith('Test content'); }); it('does not log when debug is disabled', async () => { From 0c18c2dfbb58cb03854bfad19d40cc3f69a79c26 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:01:26 +0100 Subject: [PATCH 25/27] refactor(observability): remove legacy agent telemetry operation - Keep observability operation types aligned with the executor-only review surface --- src/observability/ai-observability.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 88a76f26..7ed80e03 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,5 +1,5 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'tool-calling' | 'agent-tool-loop'; + operation: 'structured-eval' | 'tool-calling'; provider: string; model: string; evaluator?: string; From 9c122f81ff2f60bbae4ad02dd861c705b0b82e17 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:34:45 +0100 Subject: [PATCH 26/27] refactor(errors): drop orphaned AgentToolError class - AgentToolError's only consumers were src/agent/executor.ts, src/agent/path-utils.ts, and tests/agent/agent-executor.test.ts, all deleted with the autonomous workspace-agent surface - Remove the now-unused class so no agent-tied symbol lingers in the error hierarchy Refs: docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md (Product Decision; Finding #2) --- src/errors/index.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/errors/index.ts b/src/errors/index.ts index 86e257ba..ca196d95 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -30,14 +30,6 @@ export class ProcessingError extends VectorlintError { } } -// Agent tool/runtime error for agent-mode operational failures -export class AgentToolError extends VectorlintError { - constructor(message: string, code = 'AGENT_TOOL_ERROR') { - super(message, code); - this.name = 'AgentToolError'; - } -} - // Missing dependency error for when required dependencies are not available export class MissingDependencyError extends VectorlintError { constructor( From 84776cf6eb23c2f3bba881e488cd1e5a34049e50 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Fri, 24 Jul 2026 00:05:17 +0100 Subject: [PATCH 27/27] refactor(review): remove search-backed evaluator architecture - Remove technical-accuracy, Perplexity, and evaluator-registry surfaces so callers provide review context directly. - Rename runtime contracts, telemetry, output fields, and fixtures from evaluation terminology to the review domain. - Review target content with semantically accurate executor methods and include caller-supplied context in both model-call paths. --- AGENTS.md | 31 +- CONFIGURATION.md | 29 +- CREATING_RULES.md | 27 +- README.md | 4 +- docs/VECTORLINT.md | 4 +- docs/best-practices.mdx | 10 +- docs/ci-integration.mdx | 2 +- docs/cli-reference.mdx | 4 +- docs/configuration-schema.mdx | 12 +- docs/configuration.mdx | 10 +- docs/customize-style-rules.mdx | 5 +- docs/env-variables.mdx | 25 +- docs/false-positive-tuning.mdx | 10 +- docs/frontmatter-fields.mdx | 16 +- docs/how-it-works.mdx | 14 +- docs/initial-style-check.mdx | 6 +- docs/installation.mdx | 4 +- docs/introduction.mdx | 6 +- docs/llm-providers.mdx | 23 +- docs/presets.mdx | 2 +- docs/project-config.mdx | 20 +- docs/quickstart.mdx | 8 +- docs/style-guide.mdx | 35 +-- docs/troubleshooting.mdx | 20 +- docs/use-cases.mdx | 16 +- package-lock.json | 53 +--- package.json | 1 - presets/VectorLint/ai-pattern.md | 2 - presets/VectorLint/capitalization.md | 2 - presets/VectorLint/consistency.md | 2 - presets/VectorLint/inclusivity.md | 2 - presets/VectorLint/passive-voice.md | 2 - presets/VectorLint/repetition.md | 2 - presets/VectorLint/unsupported-claims.md | 2 - presets/VectorLint/wordiness.md | 2 - src/boundaries/rule-pack-loader.ts | 6 +- src/chunking/merger.ts | 6 +- src/cli/commands.ts | 21 +- src/cli/init-command.ts | 4 +- src/cli/orchestrator.ts | 32 +- src/cli/types.ts | 15 +- src/cli/validate-command.ts | 2 +- src/config/global-config.ts | 7 - src/debug/violation-filter.ts | 2 +- src/evaluators/accuracy-evaluator.ts | 254 ---------------- src/evaluators/base-evaluator.ts | 128 -------- src/evaluators/evaluator-registry.ts | 81 ----- src/evaluators/evaluator.ts | 9 - src/evaluators/index.ts | 31 -- src/evaluators/prompt-loader.ts | 29 -- src/evaluators/prompts.json | 3 - src/evaluators/types.ts | 12 - src/executors/agent-model-call-executor.ts | 36 ++- src/executors/shared.ts | 39 ++- src/executors/single-model-call-executor.ts | 41 +-- .../filter-decision.ts} | 0 src/findings/processor.ts | 2 +- src/findings/scorer.ts | 4 +- src/findings/severity.ts | 6 +- src/findings/types.ts | 2 +- src/index.ts | 3 - src/observability/ai-observability.ts | 4 +- src/observability/langfuse-observability.ts | 2 +- src/output/json-formatter.ts | 14 +- src/output/rdjson-formatter.ts | 14 +- src/output/reporter.ts | 12 +- src/prompts/directive-loader.ts | 10 +- src/prompts/prompt-loader.ts | 2 +- src/prompts/prompt-validator.ts | 2 +- src/prompts/schema.ts | 24 +- src/providers/index.ts | 2 - src/providers/perplexity-provider.ts | 87 ------ src/providers/request-builder.ts | 6 +- src/providers/search-provider.ts | 7 - src/providers/structured-model-client.ts | 4 +- src/providers/vercel-ai-provider.ts | 10 +- src/review/severity.ts | 5 + src/schemas/index.ts | 1 - src/schemas/perplexity-responses.ts | 14 - src/schemas/prompt-schemas.ts | 4 +- src/scoring/scorer.ts | 8 +- src/types/external.d.ts | 12 - tests/config-loader-integration.test.ts | 6 +- tests/evaluations/README.md | 67 ---- tests/evaluator.test.ts | 53 ---- .../agent-model-call-executor.test.ts | 25 +- .../single-model-call-executor.test.ts | 26 +- tests/file-sections.test.ts | 10 +- tests/findings/scorer.test.ts | 2 +- tests/findings/severity.test.ts | 12 +- .../capitalization-rules/capitalization.md | 2 - .../rules/consistency-rules/consistency.md | 2 - .../rules/inclusivity-rules/inclusivity.md | 2 - .../passive-voice-rules/passive-voice.md | 2 - .../rules/repetition-rules/repetition.md | 2 - tests/fixtures/technical-accuracy/test.md | 31 -- .../unsupported-claims.md | 2 - .../rules/wordiness-rules/wordiness.md | 2 - tests/fixtures/wordiness/test.md | 2 +- tests/main-command-observability.test.ts | 6 +- .../langfuse-observability.test.ts | 10 +- .../observability/noop-observability.test.ts | 2 +- tests/orchestrator-executor-dispatch.test.ts | 26 +- tests/perplexity-provider.test.ts | 286 ------------------ tests/prompt-loader-validation.test.ts | 32 +- tests/prompt-schema.test.ts | 6 +- tests/rdjson-formatter.test.ts | 2 +- tests/review/request-builder.test.ts | 2 +- .../{evaluations => reviews}/.vectorlint.ini | 0 tests/{evaluations => reviews}/TEST_FILE.md | 0 .../test-rules/Test/clarity.md | 2 - .../test-rules/Test/consistency.md | 2 - .../test-rules/Test/passive-voice.md | 2 - .../test-rules/Test/readability.md | 4 +- .../test-rules/Test/wordiness.md | 2 - tests/rule-pack-e2e.test.ts | 26 +- tests/rule-pack-loader.test.ts | 14 +- tests/schemas/mock-schemas.ts | 35 --- tests/scoring-types.test.ts | 162 ---------- tests/vercel-ai-provider.test.ts | 12 +- tests/violation-filter.test.ts | 2 +- 121 files changed, 424 insertions(+), 1891 deletions(-) delete mode 100644 src/evaluators/accuracy-evaluator.ts delete mode 100644 src/evaluators/base-evaluator.ts delete mode 100644 src/evaluators/evaluator-registry.ts delete mode 100644 src/evaluators/evaluator.ts delete mode 100644 src/evaluators/index.ts delete mode 100644 src/evaluators/prompt-loader.ts delete mode 100644 src/evaluators/prompts.json delete mode 100644 src/evaluators/types.ts rename src/{evaluators/violation-filter.ts => findings/filter-decision.ts} (100%) delete mode 100644 src/providers/perplexity-provider.ts delete mode 100644 src/providers/search-provider.ts create mode 100644 src/review/severity.ts delete mode 100644 src/schemas/perplexity-responses.ts delete mode 100644 src/types/external.d.ts delete mode 100644 tests/evaluations/README.md delete mode 100644 tests/evaluator.test.ts delete mode 100644 tests/fixtures/technical-accuracy/test.md delete mode 100644 tests/perplexity-provider.test.ts rename tests/{evaluations => reviews}/.vectorlint.ini (100%) rename tests/{evaluations => reviews}/TEST_FILE.md (100%) rename tests/{evaluations => reviews}/test-rules/Test/clarity.md (89%) rename tests/{evaluations => reviews}/test-rules/Test/consistency.md (91%) rename tests/{evaluations => reviews}/test-rules/Test/passive-voice.md (90%) rename tests/{evaluations => reviews}/test-rules/Test/readability.md (75%) rename tests/{evaluations => reviews}/test-rules/Test/wordiness.md (90%) delete mode 100644 tests/scoring-types.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0373fad3..bf199f48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,26 +1,28 @@ # Repository Guidelines -This repository implements VectorLint — a prompt‑driven, structured‑output content evaluator. Use this guide to navigate the codebase, run it locally, and contribute safely. +This repository implements VectorLint — a prompt‑driven, structured‑output content reviewer. Use this guide to navigate the codebase, run it locally, and contribute safely. ## Project Structure & Module Organization - `src/` - - `index.ts` — CLI entry; orchestrates config, discovery, evaluation, reporting + - `index.ts` — CLI entry; orchestrates config, discovery, review, reporting - `boundaries/` — external data validation (config, CLI args, env vars, YAML, API responses) - `chunking/` — content chunking for large documents (recursive chunker, merger, utilities) - `cli/` — command definitions and CLI orchestration - `config/` — configuration loading and management - `errors/` — custom error types and validation errors - - `evaluators/` — evaluation logic (base evaluator, registry, specific evaluators) + - `executors/` — bounded single-call and target-paging review strategies + - `findings/` — finding verification, filtering, severity, and scoring - `output/` — TTY formatting (reporter, evidence location, line numbering) - `prompts/` — YAML frontmatter parsing, schema validation, directive loading - `providers/` — LLM abstractions (OpenAI, Anthropic, Azure, Gemini), request builder, provider factory - `scan/` — file discovery (fast‑glob) honoring config and exclusions + - `review/` — shared review contracts, budgets, boundaries, and request building - `schemas/` — Zod schemas for all external data (API responses, config, CLI, env) - `scoring/` — density-based score calculation - `types/` — TypeScript type definitions - `presets/` — bundled rule packs (e.g., `VectorLint/`) -- `tests/` — Vitest specs for config, scanning, evaluation, providers +- `tests/` — Vitest specs for config, scanning, review, providers ## Agent Behavior Guidelines @@ -112,7 +114,7 @@ Rules must be organized into subdirectories (packs) within `RulesPath`. ### Zero-Config Mode -If you just want to evaluate against a user instruction guide without specific rules: +If you just want to review against a user instruction guide without specific rules: 1. Create a `VECTORLINT.md` file with your user instruction content 2. Run `vectorlint doc.md` — VectorLint creates a synthetic rule from your user instructions @@ -131,19 +133,18 @@ If you just want to evaluate against a user instruction guide without specific r VectorLint assembles prompts in this order: -1. **Directive** (`src/prompts/directive-loader.ts`) — Role definition, task, and evaluation instructions +1. **Directive** (`src/prompts/directive-loader.ts`) — Role definition, task, and review instructions 2. **User Instructions** (`VECTORLINT.md`) — Optional global style context -3. **Rule** (the prompt body from the rule file) — Specific evaluation criteria +3. **Rule** (the prompt body from the rule file) — Specific review criteria -The content to evaluate is sent as a **user message** with line numbers prepended. +The content to review is sent as a **user message** with line numbers prepended. ### Chunking For documents >600 words, VectorLint automatically chunks content: - Uses recursive splitting (paragraphs → lines → sentences → words) -- Each chunk is evaluated separately +- Each chunk is reviewed separately - Results are merged and deduplicated -- Disable with `evaluateAs: document` in rule frontmatter ## Coding Style & Naming Conventions @@ -180,11 +181,11 @@ For documents >600 words, VectorLint automatically chunks content: - Boundary validation: all external data (files, CLI, env, APIs) validated at system boundaries using Zod schemas - Type safety: strict TypeScript with no `any`; use `unknown` + schema validation for external data -- Dependency inversion: depend on `LLMProvider` and `SearchProvider` interfaces; keep providers thin (transport only) +- Dependency inversion: depend on `StructuredModelClient` and `ToolCallingModelClient`; keep providers thin (transport only) - Dependency injection: inject `RequestBuilder` via provider constructor to avoid coupling -- Separation of concerns: rules define criteria; schemas enforce structure; CLI orchestrates; evaluators process; reporters format +- Separation of concerns: rules define criteria; schemas enforce structure; CLI orchestrates; executors review; finding processors verify; reporters format - Separation of concerns: when a file starts combining contracts, orchestration, and utility logic, extract shared helpers and types into focused modules -- Extensibility: add providers by implementing `LLMProvider` or `SearchProvider`; add evaluators via registry pattern +- Extensibility: add providers by implementing the model-client contracts; add review strategies behind `ReviewExecutor` - Shared domain constants: avoid magic strings for core runtime concepts; define shared constants, enums, or types and import them where needed - Naming: choose domain-accurate names that reflect the real abstraction level; avoid use-case-specific terminology in shared runtime code - Logging: route runtime logging through an injected logger interface; keep concrete logger implementations behind the abstraction @@ -206,7 +207,3 @@ VectorLint supports multiple output formats via the `--output` flag: - Anthropic: Claude models (Opus, Sonnet, Haiku) - Azure OpenAI: Azure-hosted OpenAI models - Google Gemini: Gemini Pro and other Gemini models - -### Search Providers - -- Perplexity: Sonar models with web search capabilities (used by technical-accuracy evaluator) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index ea7f8506..a6432375 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -15,7 +15,7 @@ VectorLint is configured via a `.vectorlint.ini` file in the root of your projec # Optional: Path to custom rules directory # If omitted, only preset rules (from RunRules) are used # RulesPath=.github/rules -# Number of concurrent evaluations (Default: 4) +# Number of concurrent reviews (Default: 4) Concurrency=4 # Default severity for violations (Default: warning) DefaultSeverity=warning @@ -34,8 +34,6 @@ GrammarChecker.strictness=7 RunRules=Acme # Higher strictness for docs GrammarChecker.strictness=9 -# Require a higher score for technical accuracy -TechnicalAccuracy.threshold=8.0 # Marketing content - run "TechCorp" pack [content/marketing/**/*.md] @@ -56,7 +54,7 @@ These settings control the application's core behavior. | Setting | Type | Default | Description | | ----------------- | ------- | ------------ | ------------------------------------------------------------------ | | `RulesPath` | string | (none) | Root directory for custom rule packs. If omitted, only presets are used. | -| `Concurrency` | integer | `4` | Number of concurrent evaluations to run. | +| `Concurrency` | integer | `4` | Number of concurrent reviews to run. | | `DefaultSeverity` | string | `warning` | Default severity level (`warning` or `error`) for reported issues. | --- @@ -69,36 +67,23 @@ You can place a `VECTORLINT.md` file in your project root to define global style If no `.vectorlint.ini` exists, VectorLint will automatically: 1. Detect `VECTORLINT.md` 2. Create a synthetic "Style Guide Compliance" rule -3. Evaluate your contents against it +3. Review your contents against it ### Combined Mode -If you have configured rules (via `.vectorlint.ini`), the content of `VECTORLINT.md` is **prepended** to the system prompt for every evaluation. This ensures your global style preferences (tone, terminology) are respected across all specific rules. +If you have configured rules (via `.vectorlint.ini`), the content of `VECTORLINT.md` is **prepended** to the system prompt for every review. This ensures your global style preferences (tone, terminology) are respected across all specific rules. > **Note:** Keep `VECTORLINT.md` concise. VectorLint will emit a warning if the file exceeds ~4,000 tokens, as very large contexts can degrade performance and increase costs. --- -## LLM & Search Providers +## LLM Providers -VectorLint relies on LLM and Search providers. These are configured globally in `~/.vectorlint/config.toml`, or project scope using a `.env` file (which takes precedence). +VectorLint relies on an LLM provider. Configure it globally in `~/.vectorlint/config.toml`, or at project scope using a `.env` file (which takes precedence). You can generate these files using the `vectorlint init` command. -### LLM Providers - VectorLint supports multiple LLM providers. Set `LLM_PROVIDER` to your desired provider (e.g., `openai`, `anthropic`, `gemini`) and provide the corresponding API key. -### Search Provider - -Some evaluators, such as **TechnicalAccuracy**, require access to current external knowledge to verify facts. VectorLint supports search providers to fetch this information. - -**Example configuration for Perplexity:** - -```bash -SEARCH_PROVIDER=perplexity -PERPLEXITY_API_KEY=pplx-... -``` - ### Observability VectorLint can optionally emit AI execution telemetry to Langfuse. The first implementation is scoped to the Vercel AI SDK calls made through `VercelAIProvider`. @@ -224,7 +209,7 @@ You can use named levels or direct numeric multipliers: [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=strict -TechnicalAccuracy.strictness=20 +Terminology.strictness=20 ``` --- diff --git a/CREATING_RULES.md b/CREATING_RULES.md index d9c60cdf..961f4883 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -1,6 +1,6 @@ # Creating Rules for VectorLint -A comprehensive guide to creating powerful, reusable content evaluations using VectorLint's prompt system. +A comprehensive guide to creating powerful, reusable content reviews using VectorLint's prompt system. ## Table of Contents @@ -36,9 +36,8 @@ Every rule is a Markdown file with two parts: ```markdown --- # YAML Frontmatter (Configuration) -id: MyEval -name: My Content Evaluator -evaluator: base +id: MyRule +name: My Content Reviewer severity: error --- @@ -57,9 +56,7 @@ project/ │ └── rules/ │ ├── Acme/ ← Company style guide pack │ │ ├── grammar-checker.md -│ │ ├── headline-evaluator.md -│ │ └── Technical/ ← Nested organization supported -│ │ └── technical-accuracy.md +│ │ └── headline-reviewer.md │ └── TechCorp/ ← Another company's pack │ └── brand-voice.md └── .vectorlint.ini @@ -78,7 +75,6 @@ score from the verified findings. ```markdown --- -evaluator: base id: GrammarChecker name: Grammar Checker severity: error @@ -117,7 +113,7 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes The `target` field allows you to: -1. **Specify which part** of content to evaluate (via regex) +1. **Specify which part** of content to review (via regex) 2. **Require certain content** to exist (e.g., "must have an H1 headline") 3. **Provide helpful suggestions** when content is missing @@ -136,13 +132,13 @@ target: **When `required: true`:** -- If content matches → Evaluation proceeds normally +- If content matches → Review proceeds normally - If no match → Immediate `error` with the suggestion message **When `required: false` or omitted:** -- If content matches → Evaluate the matched content -- If no match → Evaluate entire content +- If content matches → Review the matched content +- If no match → Review entire content --- @@ -153,12 +149,10 @@ target: | Field | Type | Required | Description | | ------------- | ------------- | -------- | -------------------------------------------------------------- | | `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | -| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | | `id` | string | **Yes** | Unique identifier (used in error reporting) | | `name` | string | **Yes** | Human-readable name | | `severity` | string | No | `error` or `warning` (default: `warning`) | | `strictness` | number/string | No | Density penalty: a positive number, `lenient`, `standard`, or `strict` | -| `evaluateAs` | string | No | `document` or `chunk` (default: `chunk`) | | `target` | object | No | Content matching specification | --- @@ -178,7 +172,7 @@ Check if the headline is good. ✅ **Good:** ```markdown -You are a headline evaluator for developer blog posts. Assess whether the headline: +You are a headline reviewer for developer blog posts. Assess whether the headline: 1. Clearly communicates a specific benefit 2. Uses natural, conversational language (avoid buzzwords) @@ -208,7 +202,6 @@ Help the LLM understand your domain: ```markdown --- -evaluator: base id: GrammarChecker name: Grammar Checker severity: error @@ -226,4 +219,4 @@ Report any errors found with specific examples. --- -**Happy evaluating! 🚀** +**Happy reviewing! 🚀** diff --git a/README.md b/README.md index bdbef88b..10016f7b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # VectorLint: Prompt it, Lint it! [![npm version](https://img.shields.io/npm/v/vectorlint.svg)](https://www.npmjs.com/package/vectorlint) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -VectorLint is a command-line tool that uses LLMs to find and score terminology, technical accuracy, and style issues that require contextual understanding. +VectorLint is a command-line tool that uses LLMs to find and score terminology, consistency, and style issues that require contextual understanding. ![VectorLint Screenshot](./assets/VectorLint_screenshot.jpeg) @@ -51,7 +51,7 @@ VectorLint scores your content using error density, enabling you to measure qual ## How VectorLint Reduces False Positives -VectorLint uses a PAT (Pay A Tax) evaluation approach: +VectorLint uses a PAT (Pay A Tax) review approach: 1. **Candidate generation:** the model returns all potential violations with required gate-check fields (rule support, exact evidence, context support, plausible non-violation, and fix quality). 2. **Deterministic surfacing:** VectorLint applies a strict filter and only surfaces violations that pass all required gates. diff --git a/docs/VECTORLINT.md b/docs/VECTORLINT.md index 9fe79f70..49a2a514 100644 --- a/docs/VECTORLINT.md +++ b/docs/VECTORLINT.md @@ -1,7 +1,7 @@ # VectorLint example styling rules -{/* This file is prepended to the system prompt for every VectorLint evaluation - This file defines global style instructions for all VectorLint evaluations. +{/* This file is prepended to the system prompt for every VectorLint review + This file defines global style instructions for all VectorLint reviews. VectorLint prepends its contents to the system prompt for every rule it runs. Keep this file under ~800 tokens to avoid performance and cost issues. Adapt the rules below to match your team's style guide. diff --git a/docs/best-practices.mdx b/docs/best-practices.mdx index 9eab45e4..ed807cde 100644 --- a/docs/best-practices.mdx +++ b/docs/best-practices.mdx @@ -13,7 +13,7 @@ After you run `VECTORLINT.md` against your content library and complete your fir ## Keep VECTORLINT.md under 800 tokens -VectorLint emits a warning at 4,000 tokens, but a practical target is *much* lower. Under 800 tokens leaves headroom for rule-specific prompts to add context without the combined system prompt becoming unwieldy. Long context degrades LLM precision and increases API costs on every evaluation. +VectorLint emits a warning at 4,000 tokens, but a practical target is *much* lower. Under 800 tokens leaves headroom for rule-specific prompts to add context without the combined system prompt becoming unwieldy. Long context degrades LLM precision and increases API costs on every review. If your `VECTORLINT.md` is growing, that's usually a sign that some rules are specific enough to belong in a dedicated rule pack file instead. @@ -28,7 +28,7 @@ Check if the writing is clear. Write: ``` -You are a clarity evaluator for developer documentation. Flag sentences that: +You are a clarity reviewer for developer documentation. Flag sentences that: 1. Exceed 25 words 2. Use passive voice where active voice is possible 3. Contain filler phrases: "it is important to note", "please be aware", "in order to" @@ -38,7 +38,7 @@ The second prompt gives the LLM a defined audience, measurable criteria, and spe ## Give rules domain context -LLMs evaluate content relative to an implied standard. Make that standard explicit with a context block in your rule prompt. Tell the model who the audience is, what they value, and what good looks like for your specific content type. +LLMs review content relative to an implied standard. Make that standard explicit with a context block in your rule prompt. Tell the model who the audience is, what they value, and what good looks like for your specific content type. ```markdown ## CONTEXT BANK @@ -85,7 +85,7 @@ The goal is a workflow where every finding is worth reading. That takes iteratio ## Set a higher confidence threshold in CI than locally -In CI, a false positive blocks a merge. Set `CONFIDENCE_THRESHOLD` higher in your CI environment than in local development so only the highest-confidence findings gate a merge. Lower-confidence candidates still surface locally where a writer can evaluate them in context. +In CI, a false positive blocks a merge. Set `CONFIDENCE_THRESHOLD` higher in your CI environment than in local development so only the highest-confidence findings gate a merge. Lower-confidence candidates still surface locally where a writer can review them in context. ```bash # Local development — catch more, review in context @@ -117,6 +117,6 @@ Skipping this step leads to rules that look correct on paper but produce noise i ## Next steps -- [Tuning evaluation precision](/false-positive-tuning) — detailed guidance on CONFIDENCE_THRESHOLD and strictness +- [Tuning review precision](/false-positive-tuning) — detailed guidance on CONFIDENCE_THRESHOLD and strictness - [CI Integration](/ci-integration) — set up content quality gates in your pipeline - [Customize style rules](/customize-style-rules) — write effective prompts for rule pack files diff --git a/docs/ci-integration.mdx b/docs/ci-integration.mdx index 0b34eec1..d02c481e 100644 --- a/docs/ci-integration.mdx +++ b/docs/ci-integration.mdx @@ -155,6 +155,6 @@ Store API keys in your CI system's secret or environment variable manager — ne ## Next steps -- [Tuning evaluation precision](/false-positive-tuning) — set `CONFIDENCE_THRESHOLD` and strictness for CI +- [Tuning review precision](/false-positive-tuning) — set `CONFIDENCE_THRESHOLD` and strictness for CI - [Project Configuration](/project-config) — configure which files CI checks - [CLI reference](/cli-reference) — full command and exit code reference diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index febe11cc..b140f0b8 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -89,8 +89,6 @@ VectorLint reads configuration from environment variables in addition to config | `ANTHROPIC_API_KEY` | — | API key for Anthropic | | `GEMINI_API_KEY` | — | API key for Google Gemini | | `AZURE_OPENAI_API_KEY` | — | API key for Azure OpenAI | -| `SEARCH_PROVIDER` | — | Search provider for `technical-accuracy` evaluator: `perplexity` | -| `PERPLEXITY_API_KEY` | — | API key for Perplexity search | | `CONFIDENCE_THRESHOLD` | `0.75` | PAT pipeline confidence gate. Lower surfaces more findings; higher surfaces fewer. | For the full environment variable reference see [Environment variables](/env-variables). @@ -99,5 +97,5 @@ For the full environment variable reference see [Environment variables](/env-var - [Installation](/installation) — install VectorLint globally or run with npx - [Configuration](/configuration) — configure VectorLint for your project -- [Tuning evaluation precision](/false-positive-tuning) — adjust `CONFIDENCE_THRESHOLD` and strictness +- [Tuning review precision](/false-positive-tuning) — adjust `CONFIDENCE_THRESHOLD` and strictness - [CI Integration](/ci-integration) — use exit codes to gate merges on content quality diff --git a/docs/configuration-schema.mdx b/docs/configuration-schema.mdx index 3ce62cb3..98fe19aa 100644 --- a/docs/configuration-schema.mdx +++ b/docs/configuration-schema.mdx @@ -14,7 +14,7 @@ These settings go at the top of `.vectorlint.ini`, outside any file pattern sect | Setting | Default | Description | |---|---|---| | `RulesPath` | _(none)_ | Path to your custom rule packs directory. Subdirectories inside this path become available pack names. If omitted, only the bundled `VectorLint` preset is available. | -| `Concurrency` | `4` | Number of rule evaluations to run in parallel. Reduce this if you hit API rate limits. | +| `Concurrency` | `4` | Number of rule reviews to run in parallel. Reduce this if you hit API rate limits. | | `DefaultSeverity` | `warning` | Default severity for reported violations: `warning` or `error`. Individual rules can override this with their own `severity` frontmatter field. | ```ini @@ -27,7 +27,7 @@ DefaultSeverity=warning ## File pattern sections -File pattern sections map glob patterns to rule packs. VectorLint evaluates a file only if at least one pattern matches it. +File pattern sections map glob patterns to rule packs. VectorLint reviews a file only if at least one pattern matches it. ```ini [**/*.md] @@ -91,7 +91,7 @@ Set strictness per rule, per pattern: [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.strictness=strict +Clarity.strictness=strict ``` ### Strictness levels @@ -111,9 +111,9 @@ GrammarChecker.strictness=strict ## Global style context (VECTORLINT.md) -Place a `VECTORLINT.md` file in your project root to define style instructions that apply to every evaluation. VectorLint prepends its contents to the system prompt for every rule, automatically applying your tone, terminology, and baseline standards across all checks. +Place a `VECTORLINT.md` file in your project root to define style instructions that apply to every review. VectorLint prepends its contents to the system prompt for every rule, automatically applying your tone, terminology, and baseline standards across all checks. -Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds 4,000 tokens, as large context blocks degrade evaluation performance and increase costs. +Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds 4,000 tokens, as large context blocks degrade review performance and increase costs. ### Zero-config mode @@ -170,7 +170,7 @@ GrammarChecker.strictness=7 [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.threshold=9 +Clarity.threshold=9 # Marketing content — brand voice rules [content/marketing/**/*.md] diff --git a/docs/configuration.mdx b/docs/configuration.mdx index 4567a758..6a610136 100644 --- a/docs/configuration.mdx +++ b/docs/configuration.mdx @@ -7,7 +7,7 @@ VectorLint is configured through four files. Running `vectorlint init` creates t | File | Location | Purpose | Required | |------|----------|---------|----------| -| `VECTORLINT.md` | Project root | Global style instructions in plain language — prepended to every evaluation | Required for zero-config mode | +| `VECTORLINT.md` | Project root | Global style instructions in plain language — prepended to every review | Required for zero-config mode | | `.vectorlint.ini` | Project root | File patterns, rule packs, strictness overrides | Required for rule packs and file-specific configuration | | `config.toml` | `~/.vectorlint/` | LLM provider API keys — applies globally across projects | Always required | | Rule pack files | `RulesPath` directory | Targeted LLM prompts for specific checks — for advanced content quality workflows | Optional | @@ -29,7 +29,7 @@ If you want to check content against a style guide with no additional setup: vectorlint init --quick ``` -This creates a `VECTORLINT.md` file in your project root where you paste your style guide. VectorLint automatically creates a "Style Guide Compliance" rule from it and evaluates your content against it. +This creates a `VECTORLINT.md` file in your project root where you paste your style guide. VectorLint automatically creates a "Style Guide Compliance" rule from it and reviews your content against it. Run a check: @@ -53,7 +53,7 @@ This creates both `.vectorlint.ini` and `~/.vectorlint/config.toml`. Edit them m Each file has its own reference page with complete field definitions, examples, and behavior details. -- [LLM Providers](/llm-providers) — configure your LLM provider, search provider, and false-positive filtering threshold +- [LLM Providers](/llm-providers) — configure your LLM provider and false-positive filtering threshold - [Project Configuration](/project-config) — map file patterns to rule packs, set strictness overrides, and understand cascading behavior - [Defining your style rules](/style-guide) — create `VECTORLINT.md` and write targeted rule pack files @@ -75,10 +75,10 @@ VectorLint ships with a built-in preset containing rules for AI pattern detectio - Configure the model and API keys that power your evaluations. + Configure the model and API keys that power your reviews. - Map files to rule packs and tune evaluation behavior. + Map files to rule packs and tune review behavior. Create VECTORLINT.md and write custom rule pack files. diff --git a/docs/customize-style-rules.mdx b/docs/customize-style-rules.mdx index e19760c4..75521561 100644 --- a/docs/customize-style-rules.mdx +++ b/docs/customize-style-rules.mdx @@ -27,7 +27,6 @@ Create a new file at `.github/rules/MyTeam/grammar-checker.md`: --- id: GrammarChecker name: Grammar Checker -evaluator: base severity: error --- @@ -74,7 +73,7 @@ See [Project Configuration](/project-config) for the full strictness reference. ## Writing effective prompts -The Markdown body of your rule file is the prompt sent to the LLM. Specificity here directly determines evaluation quality — a vague prompt produces vague findings. +The Markdown body of your rule file is the prompt sent to the LLM. Specificity here directly determines review quality — a vague prompt produces vague findings. **Be explicit about what you're looking for:** @@ -83,7 +82,7 @@ The Markdown body of your rule file is the prompt sent to the LLM. Specificity h Check if the headline is good. # ✅ Specific -You are a headline evaluator for developer blog posts. Assess whether the headline: +You are a headline reviewer for developer blog posts. Assess whether the headline: 1. Clearly communicates a specific benefit to the reader 2. Uses natural, conversational language (avoid buzzwords like "leverage" or "unlock") diff --git a/docs/env-variables.mdx b/docs/env-variables.mdx index 20aced06..cb7e8fa6 100644 --- a/docs/env-variables.mdx +++ b/docs/env-variables.mdx @@ -26,22 +26,13 @@ In CI, pass them directly as pipeline environment variables or secrets. See [CI | `AZURE_OPENAI_ENDPOINT` | If using Azure | Your Azure OpenAI resource endpoint URL. | | `AZURE_OPENAI_DEPLOYMENT` | If using Azure | Your Azure OpenAI deployment name. | -## Search provider - -Required only for rules that use the `technical-accuracy` evaluator. If not set, rules that depend on external lookup return reduced-confidence results. - -| Variable | Required | Description | -|----------|----------|-------------| -| `SEARCH_PROVIDER` | If using technical-accuracy | Search provider to use. Accepted value: `perplexity`. | -| `PERPLEXITY_API_KEY` | If using Perplexity | API key for Perplexity search. | - -## Evaluation behavior +## Review behavior | Variable | Default | Description | |----------|---------|-------------| | `CONFIDENCE_THRESHOLD` | `0.75` | PAT pipeline confidence gate. Controls how strictly raw model candidates are filtered before surfacing violations. Accepted range: `0`–`1`. Invalid values fall back to `0.75`. | -Lower values surface more findings (higher recall, more noise). Higher values surface fewer findings (higher precision, fewer false positives). See [Tuning evaluation precision](/false-positive-tuning) for guidance on when to adjust this. +Lower values surface more findings (higher recall, more noise). Higher values surface fewer findings (higher precision, fewer false positives). See [Tuning review precision](/false-positive-tuning) for guidance on when to adjust this. ## Example configurations @@ -60,16 +51,6 @@ LLM_PROVIDER=anthropic ANTHROPIC_API_KEY=sk-ant-... ``` -### With search provider - -```toml -[env] -LLM_PROVIDER = "openai" -OPENAI_API_KEY = "sk-..." -SEARCH_PROVIDER = "perplexity" -PERPLEXITY_API_KEY = "pplx-..." -``` - ### CI environment with higher confidence threshold ```bash @@ -90,4 +71,4 @@ When the same variable is set in multiple places, VectorLint resolves it in this - [LLM Providers](/llm-providers) — configure your provider with full setup examples - [CI Integration](/ci-integration) — pass environment variables securely in pipelines -- [Tuning evaluation precision](/false-positive-tuning) — when and how to adjust `CONFIDENCE_THRESHOLD` +- [Tuning review precision](/false-positive-tuning) — when and how to adjust `CONFIDENCE_THRESHOLD` diff --git a/docs/false-positive-tuning.mdx b/docs/false-positive-tuning.mdx index 50e00f0e..8c044a36 100644 --- a/docs/false-positive-tuning.mdx +++ b/docs/false-positive-tuning.mdx @@ -1,11 +1,11 @@ --- -title: Tuning evaluation precision +title: Tuning review precision description: Use `CONFIDENCE_THRESHOLD` and strictness overrides together to reduce noise and get accurate findings across different content types. --- -VectorLint gives you two complementary levers for controlling evaluation precision. Used together, they let you calibrate how aggressively VectorLint surfaces findings — globally across your project, and per content type. +VectorLint gives you two complementary levers for controlling review precision. Used together, they let you calibrate how aggressively VectorLint surfaces findings — globally across your project, and per content type. -- **`CONFIDENCE_THRESHOLD`** — controls how strictly the PAT pipeline filters raw model candidates before surfacing them. A global setting that applies to every evaluation. +- **`CONFIDENCE_THRESHOLD`** — controls how strictly the PAT pipeline filters raw model candidates before surfacing them. A global setting that applies to every review. - **Strictness overrides** — controls how harshly rules score error density for specific file patterns. Set per content type in `.vectorlint.ini`. Understanding when to reach for each one is the key to a low-noise, high-signal workflow. @@ -47,7 +47,7 @@ Different content types warrant different quality bars. A draft circulated inter [content/docs/**/*.md] RunRules=TechDocs GrammarChecker.strictness=strict -TechnicalAccuracy.strictness=9 +Clarity.strictness=9 # Blog posts — standard. Quality matters, but tone is flexible. [content/blog/**/*.md] @@ -75,7 +75,7 @@ RunRules= In CI, false positives block merges. A finding that a writer might reasonably dismiss becomes a pipeline failure that needs explaining. Two adjustments help: -**Raise `CONFIDENCE_THRESHOLD` in CI.** Set it higher in your CI environment's `.env` than in local development. This means only the highest-confidence findings block a merge — lower-confidence candidates still get caught locally where a writer can evaluate them in context. +**Raise `CONFIDENCE_THRESHOLD` in CI.** Set it higher in your CI environment's `.env` than in local development. This means only the highest-confidence findings block a merge — lower-confidence candidates still get caught locally where a writer can review them in context. ```bash # .env in CI environment diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index 31ea9267..a07a35fd 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -3,7 +3,7 @@ title: Frontmatter field reference description: Complete reference for YAML frontmatter fields in VectorLint rule files. --- -Every VectorLint rule file begins with a YAML frontmatter block that controls how VectorLint runs the evaluation, handles the result, and reports violations. The Markdown body that follows is the prompt sent to the LLM. +Every VectorLint rule file begins with a YAML frontmatter block that controls how VectorLint runs the review, handles the result, and reports violations. The Markdown body that follows is the prompt sent to the LLM. ```markdown --- @@ -27,20 +27,18 @@ Your LLM prompt goes here. | Field | Default | Description | |---|---|---| | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | -| `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | -| `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | -| `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | +| `target` | _(none)_ | Regex specification to extract and review a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | ## Target fields -The `target` field narrows evaluation to a specific part of the document. It is an object with the following sub-fields: +The `target` field narrows review to a specific part of the document. It is an object with the following sub-fields: | Field | Type | Required | Description | |---|---|---|---| | `regex` | string | Yes | Regular expression pattern to match the target content. | | `flags` | string | No | Regex flags (e.g., `"mu"` for multiline and Unicode). | | `group` | number | No | Capture group index to extract. If omitted, the full match is used. | -| `required` | boolean | No | When `true`, a missing match immediately reports an `error` with the `suggestion` text and skips LLM evaluation. When `false` or omitted, a missing match causes VectorLint to evaluate the full document instead. | +| `required` | boolean | No | When `true`, a missing match immediately reports an `error` with the `suggestion` text and skips LLM review. When `false` or omitted, a missing match causes VectorLint to review the full document instead. | | `suggestion` | string | No | Message shown when `required: true` and the pattern does not match. | **Example — targeting only the H1 headline:** @@ -56,15 +54,13 @@ target: ## Complete example -The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before evaluation runs. +The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before review runs. ```markdown --- -evaluator: base id: HeadlineQuality name: Headline Quality severity: error -evaluateAs: document target: regex: '^#\s+(.+)$' flags: "mu" @@ -73,7 +69,7 @@ target: suggestion: Add an H1 headline for the article. --- -You are a headline reviewer for developer blog posts. Evaluate the H1 headline +You are a headline reviewer for developer blog posts. Review the H1 headline and report specific, fixable violations — for example: a vague benefit, a missing subject, or passive voice. For each violation, quote the exact text and suggest a concrete rewrite. diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index dd1bc7c9..dbede090 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -1,23 +1,23 @@ --- title: How it works -description: How VectorLint evaluates content, filters findings, and calculates scores. +description: How VectorLint reviews content, filters findings, and calculates scores. --- VectorLint sends your content to a large language model (LLM) with a structured prompt that defines your quality criteria. The model returns candidate findings, which VectorLint filters through a deterministic pipeline before surfacing violations in the CLI. If you haven't run your first check yet, start with the [Quickstart](/quickstart). This page is for understanding the architecture behind what you're running. -## The evaluation pipeline +## The review pipeline Every time you run `vectorlint doc.md`, three things happen in sequence. ### 1. Rule resolution -VectorLint reads your `.vectorlint.ini` to determine which rule packs apply to the file. It loads those rule files, prepends the contents of `VECTORLINT.md` (if present) to each rule's system prompt, and assembles the evaluation context. +VectorLint reads your `.vectorlint.ini` to determine which rule packs apply to the file. It loads those rule files, prepends the contents of `VECTORLINT.md` (if present) to each rule's system prompt, and assembles the review context. -If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic "Style Guide Compliance" rule automatically. This is the zero-config path — the fastest way to get from nothing to a working evaluation without writing any rule pack files. +If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic "Style Guide Compliance" rule automatically. This is the zero-config path — the fastest way to get from nothing to a working review without writing any rule pack files. -### 2. LLM evaluation +### 2. LLM review VectorLint sends your content to the configured LLM provider with each rule's prompt. The model returns a list of specific violations with supporting evidence. @@ -44,7 +44,7 @@ See [Quality scoring](/quality-scoring) for the full scoring reference. VectorLint gives you two complementary tools for expressing your content standards: -- **`VECTORLINT.md`** — plain-language style instructions that apply globally to every evaluation. The fastest path to useful output: no rule files, no configuration syntax, just plain English instructions that the LLM uses as context for every check. +- **`VECTORLINT.md`** — plain-language style instructions that apply globally to every review. The fastest path to useful output: no rule files, no configuration syntax, just plain English instructions that the LLM uses as context for every check. - **Rule pack files** — structured LLM prompts for specific, measurable standards. Use these when you need reproducible, auditable results on a particular dimension of quality. The two work together: `VECTORLINT.md` sets the baseline context, and rule pack files enforce specific criteria on top of it. @@ -53,6 +53,6 @@ See [Defining your style rules](/style-guide) for how to create both. ## Next steps -- [Quickstart](/quickstart) — run your first evaluation in minutes +- [Quickstart](/quickstart) — run your first review in minutes - [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules - [Configuring a project](/project-config) — map files to rule packs diff --git a/docs/initial-style-check.mdx b/docs/initial-style-check.mdx index ededf0fe..14eb7b01 100644 --- a/docs/initial-style-check.mdx +++ b/docs/initial-style-check.mdx @@ -1,9 +1,9 @@ --- title: Running your first style check -description: Use VECTORLINT.md to run your first content evaluation without writing any rules. +description: Use VECTORLINT.md to run your first content review without writing any rules. --- -The fastest way to get value from VectorLint is to create a `VECTORLINT.md` file in your project root. This is the zero-config path — no rule pack files, no `.vectorlint.ini` required. VectorLint detects the file automatically, creates a "Style Guide Compliance" rule from its contents, and evaluates your content against it. +The fastest way to get value from VectorLint is to create a `VECTORLINT.md` file in your project root. This is the zero-config path — no rule pack files, no `.vectorlint.ini` required. VectorLint detects the file automatically, creates a "Style Guide Compliance" rule from its contents, and reviews your content against it. ## Before you start @@ -11,7 +11,7 @@ You'll need a working VectorLint installation with an LLM provider configured. I ## Step 1: Create your VECTORLINT.md -Create a file called `VECTORLINT.md` in your project root. Write your style standards as plain-language instructions — short imperative rules grouped under headings. VectorLint passes this content directly to the LLM as context for every evaluation. +Create a file called `VECTORLINT.md` in your project root. Write your style standards as plain-language instructions — short imperative rules grouped under headings. VectorLint passes this content directly to the LLM as context for every review. Here's a minimal example to start from: diff --git a/docs/installation.mdx b/docs/installation.mdx index 961bb1bf..2a92e77a 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -8,7 +8,7 @@ description: Install VectorLint globally or run it with npx. Covers Node.js requ VectorLint requires Node.js 18 or later. Node.js 22 LTS is recommended. If you need to install or update Node.js, use [nvm](https://github.com/nvm-sh/nvm) (recommended) or download directly from [nodejs.org](https://nodejs.org). -Installing VectorLint makes the CLI available, but the tool requires a configured LLM provider to evaluate content. If you run `vectorlint` against a file before completing provider configuration, you get no output and no error. After installing, complete [Configuring LLM providers](/llm-providers) before running your first check. +Installing VectorLint makes the CLI available, but the tool requires a configured LLM provider to review content. If you run `vectorlint` against a file before completing provider configuration, you get no output and no error. After installing, complete [Configuring LLM providers](/llm-providers) before running your first check. --- @@ -55,4 +55,4 @@ npx vectorlint@2.5.0 path/to/doc.md - [Quickstart](/quickstart) — configure your LLM provider and run your first check end to end - [Configuring LLM providers](/llm-providers) — set up your API key for OpenAI, Anthropic, Gemini, Azure OpenAI, or Amazon Bedrock -- [Using VECTORLINT.md](/initial-style-check) — start evaluating content without writing any rule pack files \ No newline at end of file +- [Using VECTORLINT.md](/initial-style-check) — start reviewing content without writing any rule pack files \ No newline at end of file diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 1041f912..505970d1 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -5,13 +5,13 @@ description: "VectorLint is a large language model (LLM)-based prose linter that ## What is VectorLint? -VectorLint is a command-line tool that evaluates and scores documentation using large language models (LLMs). Unlike regex patterns that catch only surface-level issues, VectorLint can find terminology misuse, technical inaccuracies, and style inconsistencies that require contextual understanding. +VectorLint is a command-line tool that reviews and scores documentation using large language models (LLMs). Unlike regex patterns that catch only surface-level issues, VectorLint can find terminology misuse, unsupported claims, and style inconsistencies that require contextual understanding. If you can write a prompt for it, you can lint it with VectorLint. ## Why VectorLint exists -Traditional prose linters like Vale work by matching text against fixed regex patterns and word lists. They catch what you explicitly tell them to catch — but they can't reason about meaning, context, or technical accuracy. +Traditional prose linters like Vale work by matching text against fixed regex patterns and word lists. They catch what you explicitly tell them to catch — but they can't reason about meaning, context, or internal consistency. VectorLint fills that gap. You define a rule once as a Markdown prompt, and the LLM applies it across your entire content library — scoring each document, surfacing specific violations, and explaining why each violation matters. @@ -45,7 +45,7 @@ VectorLint filters raw LLM candidates through a series of gate checks before sur 1. **Candidate generation** — the LLM returns all potential violations, each with required gate-check fields: rule support, exact evidence, context support, plausible non-violation, and fix quality. 2. **Deterministic filtering** — VectorLint applies a strict filter and only surfaces violations that pass all required gates. -VectorLint's output is intentionally stricter than raw model candidates; it reports only findings that pass all gates. You can tune how aggressively the pipeline filters findings to match your content workflow. See [Tuning evaluation precision](/false-positive-tuning). +VectorLint's output is intentionally stricter than raw model candidates; it reports only findings that pass all gates. You can tune how aggressively the pipeline filters findings to match your content workflow. See [Tuning review precision](/false-positive-tuning). ## Next steps diff --git a/docs/llm-providers.mdx b/docs/llm-providers.mdx index 90eecf0b..e2578572 100644 --- a/docs/llm-providers.mdx +++ b/docs/llm-providers.mdx @@ -3,7 +3,7 @@ title: Configuring LLM providers description: Connect VectorLint to OpenAI, Anthropic, Azure OpenAI, Google Gemini, or Amazon Bedrock. --- -VectorLint sends your content to an LLM (Large Language Model) for evaluation. Configure a provider and supply credentials before running your first check. +VectorLint sends your content to an LLM (Large Language Model) for review. Configure a provider and supply credentials before running your first check. ## Supported providers @@ -199,27 +199,6 @@ export AWS_SECRET_ACCESS_KEY=... If your environment already provides AWS credentials through an IAM role or `~/.aws/credentials`, you can omit AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. -## Search provider (optional) - -The `technical-accuracy` evaluator verifies factual claims against live web search. You need a separate search provider credential. VectorLint currently supports [Perplexity](https://www.perplexity.ai/) for this purpose. - -**Global config** (`~/.vectorlint/config.toml`) - -```toml -[env] -SEARCH_PROVIDER = "perplexity" -PERPLEXITY_API_KEY = "pplx-..." -``` - -**Project `.env` file** - -```bash -SEARCH_PROVIDER=perplexity -PERPLEXITY_API_KEY=pplx-... -``` - -You don't need a search provider for standard `base` evaluator rules. - ## Tracking LLM costs You can configure per-token pricing so VectorLint calculates and reports estimated costs after each run. Check your provider's pricing page for current values. diff --git a/docs/presets.mdx b/docs/presets.mdx index bb9b77d2..0fe880fa 100644 --- a/docs/presets.mdx +++ b/docs/presets.mdx @@ -1,6 +1,6 @@ --- title: Presets -description: Use VectorLint's built-in rule presets to start evaluating content without writing custom rules. +description: Use VectorLint's built-in rule presets to start reviewing content without writing custom rules. --- VectorLint ships with a built-in rule preset that activates automatically when you run `vectorlint init`. Presets give you immediate, useful output without requiring you to write any rules first. diff --git a/docs/project-config.mdx b/docs/project-config.mdx index e78d00b7..e42797d5 100644 --- a/docs/project-config.mdx +++ b/docs/project-config.mdx @@ -1,6 +1,6 @@ --- title: Configuring a project -description: Map files to rule packs and tune evaluation behavior with .vectorlint.ini. +description: Map files to rule packs and tune review behavior with .vectorlint.ini. --- `.vectorlint.ini` is VectorLint's project configuration file. It lives in your project root and controls three things: global behavior settings, where VectorLint looks for custom rules, and which rule packs run on which files. @@ -9,12 +9,12 @@ Run `vectorlint init` to generate a starter file. You can edit it manually at an ## Global Settings -These settings apply to all evaluations in the project. +These settings apply to all reviews in the project. | Setting | Type | Default | Description | |---------|------|---------|-------------| | `RulesPath` | string | _(none)_ | Root directory for custom rule packs. If omitted, only built-in presets are used. | -| `Concurrency` | integer | `4` | Number of evaluations to run in parallel. | +| `Concurrency` | integer | `4` | Number of reviews to run in parallel. | | `DefaultSeverity` | string | `warning` | Severity level for reported violations: `warning` or `error`. | ```ini @@ -36,7 +36,7 @@ RunRules=Acme [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.threshold=8.0 +Clarity.threshold=8.0 [content/marketing/**/*.md] RunRules=TechCorp @@ -116,7 +116,7 @@ You can use named levels (`lenient`, `standard`, `strict`) or direct numeric mul [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=strict -TechnicalAccuracy.strictness=20 +Clarity.strictness=20 ``` ## Complete Example @@ -137,7 +137,7 @@ GrammarChecker.strictness=5 [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.threshold=8.0 +Clarity.threshold=8.0 # Marketing — brand voice rules [content/marketing/**/*.md] @@ -153,15 +153,15 @@ RunRules= In addition to `.vectorlint.ini`, you can place a `VECTORLINT.md` file in your project root to define global style instructions in plain language. -**Zero-config mode:** If no `.vectorlint.ini` exists, VectorLint automatically detects `VECTORLINT.md`, creates a synthetic "Style Guide Compliance" rule, and evaluates content against it. +**Zero-config mode:** If no `.vectorlint.ini` exists, VectorLint automatically detects `VECTORLINT.md`, creates a synthetic "Style Guide Compliance" rule, and reviews content against it. -**Combined mode:** When `.vectorlint.ini` is present, the contents of `VECTORLINT.md` are prepended to the system prompt for every evaluation — making your global tone and terminology preferences active across all rules. +**Combined mode:** When `.vectorlint.ini` is present, the contents of `VECTORLINT.md` are prepended to the system prompt for every review — making your global tone and terminology preferences active across all rules. -Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds approximately 4,000 tokens, as very large contexts can degrade evaluation quality and increase API costs. +Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds approximately 4,000 tokens, as very large contexts can degrade review quality and increase API costs. ## Next Steps -- [LLM Providers](/llm-providers) — configure the model that runs evaluations +- [LLM Providers](/llm-providers) — configure the model that runs reviews - [Style Guide](/style-guide) — write custom rules for your content standards diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index c6775b23..bd77f8ac 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -3,13 +3,13 @@ title: Quickstart description: Install VectorLint, configure your LLM provider, and run your first content check. --- -Set up VectorLint and run your first content evaluation in under ten minutes. You install VectorLint, add your style instructions, connect an LLM (Large Language Model) provider, and see real output on a file you own. +Set up VectorLint and run your first content review in under ten minutes. You install VectorLint, add your style instructions, connect an LLM (Large Language Model) provider, and see real output on a file you own. ## Before you begin VectorLint requires Node.js 18 or later. Node.js 22 LTS is recommended. If you need to install or update Node.js, use [nvm](https://github.com/nvm-sh/nvm) or download directly from [nodejs.org](https://nodejs.org). -You will also need an API key for a supported LLM provider. VectorLint sends your content to the provider you configure for evaluation. Supported providers include: OpenAI, Anthropic, Google Gemini, Azure OpenAI, Amazon Bedrock. +You will also need an API key for a supported LLM provider. VectorLint sends your content to the provider you configure for review. Supported providers include: OpenAI, Anthropic, Google Gemini, Azure OpenAI, Amazon Bedrock. VectorLint requires a configured LLM provider to produce output. Installing the package alone is not enough — if you skip provider configuration, running `vectorlint` against a file will return no results and no error. Complete Step 2 before running a check. @@ -61,7 +61,7 @@ You can also set provider values in a `.env` file in your project root. Project- ## Step 3: Create your VECTORLINT.md -After you initialize VectorLint in your repository, the application generates a VECTORLINT.md file in your project root. After you copy your style standards as plain-language instructions into the file, VectorLint builds a "Style Guide Compliance" rule and applies it to your VectorLint evaluations. +After you initialize VectorLint in your repository, the application generates a VECTORLINT.md file in your project root. After you copy your style standards as plain-language instructions into the file, VectorLint builds a "Style Guide Compliance" rule and applies it to your VectorLint reviews. Run the quick-init command to create the file: @@ -103,7 +103,7 @@ Point VectorLint at any Markdown file in your project: vectorlint path/to/doc.md ``` -VectorLint reads your `VECTORLINT.md`, evaluates the file against it, and prints findings in your terminal. If the file is clean, VectorLint produces no output and exits with status `0`. If the file has violations, VectorLint prints each finding with its location and a suggested fix. +VectorLint reads your `VECTORLINT.md`, reviews the file against it, and prints findings in your terminal. If the file is clean, VectorLint produces no output and exits with status `0`. If the file has violations, VectorLint prints each finding with its location and a suggested fix. --- diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index bf29b1cd..20cf7430 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -1,19 +1,19 @@ --- title: Defining your style rules -description: Set global style instructions and create targeted LLM prompts to evaluate content against your style guide, terminology, and content standards. +description: Set global style instructions and create targeted LLM prompts to review content against your style guide, terminology, and content standards. --- -VectorLint gives you two ways to bring your style guide into evaluations. Understanding which to use and when to combine them is the starting point for any content quality workflow: +VectorLint gives you two ways to bring your style guide into reviews. Understanding which to use and when to combine them is the starting point for any content quality workflow: -- **`VECTORLINT.md` file**: the global style instructions in plain language. Create this file from your style guide and place it in your project root. VectorLint prepends its contents to the system prompt for every evaluation, making your tone, terminology, and baseline standards apply across all rules automatically. Use it for broad, always-applicable guidance. See [Project Configuration](/project-config) for details. +- **`VECTORLINT.md` file**: the global style instructions in plain language. Create this file from your style guide and place it in your project root. VectorLint prepends its contents to the system prompt for every review, making your tone, terminology, and baseline standards apply across all rules automatically. Use it for broad, always-applicable guidance. See [Project Configuration](/project-config) for details. -- **Rule pack files**: the targeted LLM prompts for specific checks. Create these as separate Markdown files as described below. Each file is a structured prompt that instructs the LLM to evaluate content against one specific standard: grammar, headline quality, AI pattern detection, technical accuracy, and so on. Rules are organized into packs and mapped to file patterns in `.vectorlint.ini`. Rule pack files live in subdirectories under `RulesPath` — see [File Structure Reference](#file-structure-reference) below. +- **Rule pack files**: the targeted LLM prompts for specific checks. Create these as separate Markdown files as described below. Each file is a structured prompt that instructs the LLM to review content against one specific standard: grammar, headline quality, AI pattern detection, unsupported claims, and so on. Rules are organized into packs and mapped to file patterns in `.vectorlint.ini`. Rule pack files live in subdirectories under `RulesPath` — see [File Structure Reference](#file-structure-reference) below. If you can write a prompt for it, you can lint for it with VectorLint. ## Creating your VECTORLINT.md -Place a `VECTORLINT.md` file in your project root to define global style instructions that apply to every evaluation. VectorLint prepends its contents to the system prompt for every rule it runs. +Place a `VECTORLINT.md` file in your project root to define global style instructions that apply to every review. VectorLint prepends its contents to the system prompt for every rule it runs. Keep this file concise. VectorLint emits a warning if the file exceeds approximately 4,000 tokens, as very large contexts can degrade performance and increase API costs. @@ -32,7 +32,7 @@ Rules for extraction: - Remove all tables, comparison columns, governing references, and checklists — extract only the actionable rules they contain. - Remove any rule that is structural (document architecture, heading - levels) rather than evaluable at the sentence or paragraph level. + levels) rather than reviewable at the sentence or paragraph level. - Use plain markdown bullets. No bold, no nested lists beyond one level. - Target output: under 600 words / ~800 tokens. - Do not include a preamble or closing statement. @@ -49,7 +49,7 @@ To help you get started, the VectorLint docs repository includes an [example VEC ## Creating rule pack files -Rule pack files are optional Markdown files, each containing a targeted LLM prompt for a specific check. Unlike `VECTORLINT.md`, which sets broad context for every evaluation, rule pack files enforce precise, measurable criteria — grammar, headline quality, AI pattern detection, technical accuracy, and so on. You can create as many as your content workflow requires and organize them into packs mapped to specific file patterns in `.vectorlint.ini`. +Rule pack files are optional Markdown files, each containing a targeted LLM prompt for a specific check. Unlike `VECTORLINT.md`, which sets broad context for every review, rule pack files enforce precise, measurable criteria — grammar, headline quality, AI pattern detection, unsupported claims, and so on. You can create as many as your content workflow requires and organize them into packs mapped to specific file patterns in `.vectorlint.ini`. Each rule file has two parts: YAML frontmatter that configures how VectorLint handles the result, and a Markdown body that is the actual prompt sent to the LLM. @@ -57,9 +57,8 @@ Each rule file has two parts: YAML frontmatter that configures how VectorLint ha ```markdown --- -id: MyEval -name: My Content Evaluator -evaluator: base +id: MyRule +name: My Content Reviewer severity: error --- @@ -77,9 +76,7 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| -| `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | -| `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | @@ -89,7 +86,6 @@ The LLM returns a list of specific violations. VectorLint scores the result usin ```markdown --- -evaluator: base id: GrammarChecker name: Grammar Checker severity: error @@ -111,7 +107,7 @@ Report any errors found with specific examples. ## Targeting Specific Content -The `target` field lets you evaluate a specific portion of a document — for example, only the H1 headline — rather than the full content. +The `target` field lets you review a specific portion of a document — for example, only the H1 headline — rather than the full content. ```yaml target: @@ -122,9 +118,9 @@ target: suggestion: Add an H1 headline for the article. ``` -**When `required: true`:** If the pattern doesn't match, VectorLint reports an immediate `error` with the `suggestion` text, and skips LLM evaluation. +**When `required: true`:** If the pattern doesn't match, VectorLint reports an immediate `error` with the `suggestion` text, and skips LLM review. -**When `required: false` (or omitted):** If the pattern doesn't match, VectorLint evaluates the full document instead. +**When `required: false` (or omitted):** If the pattern doesn't match, VectorLint reviews the full document instead. ## File Structure Reference @@ -134,9 +130,7 @@ project/ │ └── rules/ ← RulesPath in .vectorlint.ini │ ├── Acme/ ← Pack: "Acme" │ │ ├── grammar-checker.md -│ │ ├── headline-evaluator.md -│ │ └── Technical/ ← Nested subdirectory (supported) -│ │ └── technical-accuracy.md +│ │ └── headline-reviewer.md │ └── TechCorp/ ← Pack: "TechCorp" │ └── brand-voice.md └── .vectorlint.ini @@ -155,7 +149,6 @@ RunRules=Acme ```markdown --- -evaluator: base id: AIPatterns name: AI Pattern Detector severity: warning @@ -173,4 +166,4 @@ human-sounding rewrite. - [Customize style rules](/customize-style-rules) — learn how to write effective LLM prompts for your rule pack files - [Project Configuration](/project-config) — assign rule packs to file patterns -- [LLM Providers](/llm-providers) — configure the model used for evaluations +- [LLM Providers](/llm-providers) — configure the model used for reviews diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 677502d6..faa50747 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -1,6 +1,6 @@ --- title: Troubleshooting -description: Diagnose and fix common VectorLint issues with installation, configuration, API keys, and evaluation output. +description: Diagnose and fix common VectorLint issues with installation, configuration, API keys, and review output. --- @@ -99,24 +99,6 @@ See [Environment variables](/env-variables) for the full variable reference. --- -### Search provider not working for TechnicalAccuracy rules - -**Symptom:** Rules using `evaluator: technical-accuracy` produce no findings or return an error about search. - -**Cause:** No search provider is configured. - -**Fix:** Add a search provider to your config: - -```toml -[env] -SEARCH_PROVIDER = "perplexity" -PERPLEXITY_API_KEY = "pplx-..." -``` - -Without a search provider, `technical-accuracy` evaluators cannot verify facts against external sources and will return reduced-confidence or no results. - ---- - ## Configuration issues ### Rules not running on a file diff --git a/docs/use-cases.mdx b/docs/use-cases.mdx index 75091090..d0275ade 100644 --- a/docs/use-cases.mdx +++ b/docs/use-cases.mdx @@ -29,18 +29,6 @@ For content-specific tuning, see [Customizing style rules](/customize-style-rule --- -## Verify technical accuracy - -**The problem:** Outdated version numbers, deprecated API references, and incorrect command syntax are difficult category of errors to catch in manual review, because catching them requires domain expertise your reviewers may not have. - -**The path:** The `technical-accuracy` evaluator uses a search provider to check factual claims in your content against current external sources. Point it at your API reference pages or CLI documentation and it surfaces stale information before it ships. - -See [Defining your style rules](/style-guide) for how to configure a `technical-accuracy` rule. - -**Who does this:** Developer documentation teams, API reference authors, teams maintaining large technical content libraries. - ---- - ## Gate content quality in CI **The problem:** Teams can skip manual quality checks because the pipeline doesn't enforce it. @@ -86,6 +74,6 @@ See [Configuring a project](/project-config) for the full configuration referenc ## Next steps -- [Quickstart](/quickstart) — run your first evaluation in under ten minutes -- [How it works](/how-it-works) — understand the evaluation pipeline, PAT filtering, and scoring +- [Quickstart](/quickstart) — run your first review in under ten minutes +- [How it works](/how-it-works) — understand the review pipeline, PAT filtering, and scoring - [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules diff --git a/package-lock.json b/package-lock.json index 49e625dd..0c065650 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@ai-sdk/azure": "^3.0.34", "@ai-sdk/google": "^3.0.31", "@ai-sdk/openai": "^3.0.33", - "@ai-sdk/perplexity": "^1.0.0", "@langfuse/otel": "^5.1.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/sdk-node": "^0.214.0", @@ -334,51 +333,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@ai-sdk/perplexity": { - "version": "1.1.9", - "resolved": "https://registry.npmjs.org/@ai-sdk/perplexity/-/perplexity-1.1.9.tgz", - "integrity": "sha512-Ytolh/v2XupXbTvjE18EFBrHLoNMH0Ueji3lfSPhCoRUfkwrgZ2D9jlNxvCNCCRiGJG5kfinSHvzrH5vGDklYA==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "1.1.3", - "@ai-sdk/provider-utils": "2.2.8" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.0.0" - } - }, - "node_modules/@ai-sdk/provider": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", - "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@ai-sdk/provider-utils": { - "version": "2.2.8", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", - "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "1.1.3", - "nanoid": "^3.3.8", - "secure-json-parse": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.23.8" - } - }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -5417,6 +5371,7 @@ "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, "funding": [ { "type": "github", @@ -6113,12 +6068,6 @@ "node": ">=10" } }, - "node_modules/secure-json-parse": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", - "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", - "license": "BSD-3-Clause" - }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", diff --git a/package.json b/package.json index 3a73ad4d..14aaa18b 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,6 @@ "@ai-sdk/azure": "^3.0.34", "@ai-sdk/google": "^3.0.31", "@ai-sdk/openai": "^3.0.33", - "@ai-sdk/perplexity": "^1.0.0", "@langfuse/otel": "^5.1.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/sdk-node": "^0.214.0", diff --git a/presets/VectorLint/ai-pattern.md b/presets/VectorLint/ai-pattern.md index 9c0ec2ec..ca7e1f06 100644 --- a/presets/VectorLint/ai-pattern.md +++ b/presets/VectorLint/ai-pattern.md @@ -1,10 +1,8 @@ --- specVersion: 2.0.0 -evaluator: base id: AIPattern name: AI Pattern severity: warning -evaluateAs: document --- # AI Pattern diff --git a/presets/VectorLint/capitalization.md b/presets/VectorLint/capitalization.md index e454d51a..5c18dffd 100644 --- a/presets/VectorLint/capitalization.md +++ b/presets/VectorLint/capitalization.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Capitalization name: Capitalization severity: warning -evaluateAs: document --- # Capitalization diff --git a/presets/VectorLint/consistency.md b/presets/VectorLint/consistency.md index fb17a50d..e7ee8a5e 100644 --- a/presets/VectorLint/consistency.md +++ b/presets/VectorLint/consistency.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Consistency name: Consistency severity: warning -evaluateAs: document --- # Consistency diff --git a/presets/VectorLint/inclusivity.md b/presets/VectorLint/inclusivity.md index fab769ce..dc645519 100644 --- a/presets/VectorLint/inclusivity.md +++ b/presets/VectorLint/inclusivity.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Inclusivity name: Inclusivity severity: warning -evaluateAs: document --- # Inclusivity diff --git a/presets/VectorLint/passive-voice.md b/presets/VectorLint/passive-voice.md index e71c93d0..46e3f33b 100644 --- a/presets/VectorLint/passive-voice.md +++ b/presets/VectorLint/passive-voice.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: PassiveVoice name: Passive Voice severity: warning -evaluateAs: document --- # Passive Voice diff --git a/presets/VectorLint/repetition.md b/presets/VectorLint/repetition.md index 6849b9c1..ddb9c909 100644 --- a/presets/VectorLint/repetition.md +++ b/presets/VectorLint/repetition.md @@ -1,10 +1,8 @@ --- specVersion: 2.0.0 -evaluator: base id: Repetition name: Repetition severity: warning -evaluateAs: document --- # Repetition diff --git a/presets/VectorLint/unsupported-claims.md b/presets/VectorLint/unsupported-claims.md index c2146a38..af20a30f 100644 --- a/presets/VectorLint/unsupported-claims.md +++ b/presets/VectorLint/unsupported-claims.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: UnsupportedClaims name: Unsupported Claims severity: warning -evaluateAs: document --- # Unsupported Claims diff --git a/presets/VectorLint/wordiness.md b/presets/VectorLint/wordiness.md index 87d386f3..56203e4c 100644 --- a/presets/VectorLint/wordiness.md +++ b/presets/VectorLint/wordiness.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Wordiness name: Wordiness severity: warning -evaluateAs: document --- # Wordiness diff --git a/src/boundaries/rule-pack-loader.ts b/src/boundaries/rule-pack-loader.ts index 6769b1a6..a899c41b 100644 --- a/src/boundaries/rule-pack-loader.ts +++ b/src/boundaries/rule-pack-loader.ts @@ -64,9 +64,9 @@ export class RulePackLoader { } /** - * Recursively finds all evaluation files in a pack directory. - * @param packRoot The root directory of the eval pack - * @returns A list of absolute file paths to evaluation files + * Recursively finds all rule files in a pack directory. + * @param packRoot The root directory of the rule pack + * @returns A list of absolute file paths to rule files */ async findRuleFiles(packPath: string): Promise { const rules: string[] = []; diff --git a/src/chunking/merger.ts b/src/chunking/merger.ts index c76e17ff..c65f8aea 100644 --- a/src/chunking/merger.ts +++ b/src/chunking/merger.ts @@ -1,8 +1,8 @@ -import type { EvaluationItem } from "../prompts/schema"; +import type { ReviewItem } from "../prompts/schema"; export function mergeViolations( - chunkViolations: EvaluationItem[][] -): EvaluationItem[] { + chunkViolations: ReviewItem[][] +): ReviewItem[] { const all = chunkViolations.flat(); // Deduplicate using composite key (quoted_text + description + analysis) diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 05cce95c..e6258970 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -4,8 +4,6 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { dirname } from 'path'; import { createProvider } from '../providers/provider-factory'; -import { PerplexitySearchProvider } from '../providers/perplexity-provider'; -import type { SearchProvider } from '../providers/search-provider'; import { loadConfig } from '../boundaries/config-loader'; import { loadUserInstructions } from '../boundaries/user-instruction-loader'; import { loadRuleFile, type PromptFile } from '../prompts/prompt-loader'; @@ -17,7 +15,7 @@ import { loadDirective } from '../prompts/directive-loader'; import { resolveTargets } from '../scan/file-resolver'; import { parseCliOptions, parseEnvironment } from '../boundaries/index'; import { handleUnknownError } from '../errors/index'; -import { evaluateFiles } from './orchestrator'; +import { reviewFiles } from './orchestrator'; import { DEFAULT_REVIEW_MODEL_CALL, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; @@ -71,10 +69,7 @@ async function runOrExit( } } -/* - * Registers the main evaluation command with Commander. - * This is the default command that runs content evaluations against target files. - */ +/** Registers the main content-review command with Commander. */ export function registerMainCommand(program: Command): void { program .option('-v, --verbose', 'Enable verbose logging') @@ -182,7 +177,7 @@ export function registerMainCommand(program: Command): void { } if (targets.length === 0) { - console.error('Error: no target files found to evaluate.'); + console.error('Error: no target files found to review.'); process.exit(1); } @@ -210,18 +205,12 @@ export function registerMainCommand(program: Command): void { requestBuilder ); - // Create search provider if API key is available - const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY - ? new PerplexitySearchProvider({ logger: runtimeLogger }) - : undefined; - - // Run evaluations via orchestrator - const result = await evaluateFiles(targets, { + // Run reviews via the orchestrator. + const result = await reviewFiles(targets, { prompts, rulesPath, provider, requestBuilder, - ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, logger: runtimeLogger, diff --git a/src/cli/init-command.ts b/src/cli/init-command.ts index 66998623..14a428ac 100644 --- a/src/cli/init-command.ts +++ b/src/cli/init-command.ts @@ -20,7 +20,7 @@ RunRules=VectorLint const USER_INSTRUCTION_TEMPLATE = `# User Instructions @@ -117,7 +117,7 @@ export function registerInitCommand(program: Command): void { console.log(`Next steps:`); console.log(` 1. Open ${getGlobalConfigPath()} and configure your API keys (e.g., OPENAI_API_KEY)`); if (createUserInstructions) { - console.log(` 2. Edit ${USER_INSTRUCTION_FILENAME} to define your instructions for content evaluation`); + console.log(` 2. Edit ${USER_INSTRUCTION_FILENAME} to define your content-review instructions`); } console.log(` ${createUserInstructions ? '3' : '2'}. Run 'vectorlint ' to start linting your content`); if (createConfig) { diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index b93f0746..cc853ad0 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -7,7 +7,7 @@ import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; +import { printFileHeader, printIssueRow, printReviewSummaries, type ReviewSummary } from '../output/reporter'; import { handleUnknownError } from '../errors/index'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; import { OutputFormat } from './types'; @@ -16,14 +16,14 @@ import { chooseModelCall } from '../review/executor'; import { buildReviewRequest } from '../review/request-builder'; import type { ReviewResult, ReviewSeverity, ReviewTarget } from '../review/types'; import type { - EvaluationOptions, EvaluationResult, ReportIssueParams, EvaluateFileResult, + ReviewOptions, ReviewRunResult, ReportIssueParams, ReviewFileResult, } from './types'; import { calculateCost, TokenUsageStats } from '../providers/token-usage'; import { writeDebugRunArtifact } from '../debug/run-artifact'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } { const provider = process.env.LLM_PROVIDER; @@ -156,12 +156,12 @@ function writeReviewDebugArtifact(relFile: string, result: ReviewResult): void { * Determines the source-backed prompts that apply to a file, honoring the * configured scan paths. When no rules match but a VECTORLINT.md user * instruction guide exists, a synthetic rule is added so the reviewer model - * evaluates against the user instructions in the system prompt. + * reviews against the user instructions in the system prompt. */ function resolveApplicablePrompts( relFile: string, prompts: PromptFile[], - scanPaths: EvaluationOptions['scanPaths'], + scanPaths: ReviewOptions['scanPaths'], userInstructionContent: string | undefined, ): PromptFile[] { const toRun: PromptFile[] = []; @@ -214,16 +214,16 @@ function resolveApplicablePrompts( } /* - * Evaluates a single file: builds a ReviewRequest from the applicable + * Reviews a single file: builds a ReviewRequest from the applicable * source-backed prompts, resolves the model-call strategy, dispatches through * the selected ReviewExecutor, and routes the resulting ReviewResult to the * existing line/json/vale output sinks. */ -async function evaluateFile( +async function reviewFile( file: string, - options: EvaluationOptions, + options: ReviewOptions, jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter, -): Promise { +): Promise { const { prompts, scanPaths, @@ -232,7 +232,7 @@ async function evaluateFile( debugJson, } = options; - const allScores = new Map(); + const allScores = new Map(); const content = readFileSync(file, "utf-8"); const relFile = path.relative(process.cwd(), file) || file; @@ -251,7 +251,7 @@ async function evaluateFile( // No applicable rules: nothing to review for this file. if (toRun.length === 0) { if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); + printReviewSummaries(allScores); console.log(""); } return { @@ -344,7 +344,7 @@ async function evaluateFile( } if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); + printReviewSummaries(allScores); console.log(""); } @@ -362,10 +362,10 @@ async function evaluateFile( * Runs reviews across all target files, dispatching each through the executor * selected by the model-call strategy, and aggregates the results. */ -export async function evaluateFiles( +export async function reviewFiles( targets: string[], - options: EvaluationOptions -): Promise { + options: ReviewOptions +): Promise { const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; @@ -389,7 +389,7 @@ export async function evaluateFiles( for (const file of targets) { try { totalFiles += 1; - const fileResult = await evaluateFile(file, options, jsonFormatter); + const fileResult = await reviewFile(file, options, jsonFormatter); totalErrors += fileResult.errors; totalWarnings += fileResult.warnings; requestFailures += fileResult.requestFailures; diff --git a/src/cli/types.ts b/src/cli/types.ts index d1c1c154..15741a05 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,14 +1,13 @@ import type { PromptFile } from '../prompts/prompt-loader'; import type { StructuredModelClient } from '../providers/structured-model-client'; import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; -import type { SearchProvider } from '../providers/search-provider'; import type { RequestBuilder } from '../providers/request-builder'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import type { EvaluationSummary } from '../output/reporter'; +import type { ReviewSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; import type { ReviewModelCall } from '../review/types'; @@ -35,12 +34,10 @@ export const DEFAULT_OUTPUT_FORMAT = OUTPUT_FORMATS[0]; /** Default reviewer model-call strategy. */ export const DEFAULT_REVIEW_MODEL_CALL: ReviewModelCall = 'auto'; -export interface EvaluationOptions { +export interface ReviewOptions { prompts: PromptFile[]; rulesPath: string | undefined; provider: StructuredModelClient & ToolCallingModelClient; - /** Retained for the search-provider capability; the executor path reviews structured output. */ - searchProvider?: SearchProvider; requestBuilder: RequestBuilder; concurrency: number; verbose: boolean; @@ -53,7 +50,7 @@ export interface EvaluationOptions { logger?: Logger; } -export interface EvaluationResult { +export interface ReviewRunResult { totalFiles: number; totalErrors: number; totalWarnings: number; @@ -68,7 +65,7 @@ export interface ErrorTrackingResult { warnings: number; hadOperationalErrors: boolean; hadSeverityErrors: boolean; - scoreEntries?: EvaluationSummary[]; + scoreEntries?: ReviewSummary[]; } export interface ReportIssueParams { @@ -87,7 +84,7 @@ export interface ReportIssueParams { match?: string; } -export interface EvaluateFileResult extends ErrorTrackingResult { +export interface ReviewFileResult extends ErrorTrackingResult { requestFailures: number; tokenUsage?: TokenUsageStats; } diff --git a/src/cli/validate-command.ts b/src/cli/validate-command.ts index c05b6d64..56471959 100644 --- a/src/cli/validate-command.ts +++ b/src/cli/validate-command.ts @@ -19,7 +19,7 @@ const __dirname = dirname(__filename); /* * Registers the 'validate' command with Commander. - * This command validates prompt configuration files without running evaluations. + * This command validates prompt configuration files without running reviews. * It checks YAML frontmatter structure, schema compliance, and prompt completeness. * * Note: process.exit is intentional in CLI commands to set proper exit codes. diff --git a/src/config/global-config.ts b/src/config/global-config.ts index 6a694122..439b99cd 100644 --- a/src/config/global-config.ts +++ b/src/config/global-config.ts @@ -56,13 +56,6 @@ const DEFAULT_GLOBAL_CONFIG_TEMPLATE = `# VectorLint Environment Configuration # BEDROCK_MODEL = "global.anthropic.claude-sonnet-4-5-20250929-v1:0" # BEDROCK_TEMPERATURE = "0.2" -# ============================================ -# Search Provider Configuration (Optional) -# Enables technical accuracy verification -# ============================================ - -# SEARCH_PROVIDER = "perplexity" -# PERPLEXITY_API_KEY = "pplx-0000000000000000" `; /** diff --git a/src/debug/violation-filter.ts b/src/debug/violation-filter.ts index 2a93d35d..762bf9ae 100644 --- a/src/debug/violation-filter.ts +++ b/src/debug/violation-filter.ts @@ -1,4 +1,4 @@ export { computeFilterDecision, type FilterDecision, -} from "../evaluators/violation-filter"; +} from "../findings/filter-decision"; diff --git a/src/evaluators/accuracy-evaluator.ts b/src/evaluators/accuracy-evaluator.ts deleted file mode 100644 index 2634dffe..00000000 --- a/src/evaluators/accuracy-evaluator.ts +++ /dev/null @@ -1,254 +0,0 @@ -import { BaseEvaluator } from "./base-evaluator"; -import { registerEvaluator } from "./evaluator-registry"; -import type { LLMProvider } from "../providers/llm-provider"; -import type { SearchProvider } from "../providers/search-provider"; -import type { PromptFile } from "../schemas/prompt-schemas"; -import type { PromptEvaluationResult } from "../prompts/schema"; -import type { TokenUsage } from "../providers/token-usage"; -import { renderTemplate } from "../prompts/template-renderer"; -import { getPrompt } from "./prompt-loader"; -import { z } from "zod"; -import { Type, type Severity } from "./types"; -import { MissingDependencyError } from "../errors/index"; -import { countWords } from "../chunking"; - -// Schema for claim extraction response -const CLAIM_EXTRACTION_SCHEMA = z.object({ - claims: z.array(z.string()), -}); - -// Schema for search result -const SEARCH_RESULT_SCHEMA = z.object({ - snippet: z.string(), - url: z.string(), - title: z.string().optional(), -}); - -type SearchResult = z.infer; - -interface ClaimExtractionResult { - claims: string[]; - usage?: TokenUsage; -} - -/** - * Technical Accuracy Evaluator - Acts as an orchestrator only. - * - Evaluator (this class): Orchestrates data gathering (claims, search evidence) - * - Eval (prompt): Contains all evaluation logic via templates - * - * Architecture: - * 1. Extract claims from content (via LLM with claim-extraction prompt) - * 2. Search for evidence for each claim (via SearchProvider) - * 3. Pass content, claims, and evidence to the main eval prompt (via templates) - * 4. Return the structured evaluation result - * - * Evaluators should NOT contain evaluation logic - all evaluation is done by the prompt. - */ -export class TechnicalAccuracyEvaluator extends BaseEvaluator { - private static readonly CLAIM_EXTRACTION_PROMPT_KEY = "claim-extraction"; - - constructor( - llmProvider: LLMProvider, - prompt: PromptFile, - private searchProvider: SearchProvider, - defaultSeverity?: Severity - ) { - super(llmProvider, prompt, defaultSeverity); - } - - async evaluate(_file: string, content: string): Promise { - // Step 1: Extract factual claims from the content - const { claims, usage: claimUsage } = await this.extractClaims(content); - - // If no claims found, return success (empty items array, perfect score) - // Use the scoring module to calculate result - if (claims.length === 0) { - const wordCount = countWords(content) || 1; - const raw: PromptEvaluationResult = { - violations: [], - word_count: wordCount, - ...(claimUsage && { usage: claimUsage }), - }; - return raw; - } - - // Step 2: Search for evidence for each claim - const searchResults = await this.searchForEvidence(claims); - - // Step 3: Prepare template variables - const templateVars = { - content: content, - claims: this.formatClaimsForTemplate(claims), - searchResults: this.formatSearchResultsForTemplate(searchResults), - }; - - // Step 4: Render the prompt with template variables - const renderedPrompt = renderTemplate(this.getPromptBody(), templateVars); - - // Step 5: Create enriched prompt with rendered body - const enrichedPrompt: PromptFile = { - ...this.prompt, - body: renderedPrompt, - }; - - // Step 6: Use parent's evaluation logic with enriched prompt - const evaluator = new BaseEvaluator( - this.llmProvider, - enrichedPrompt, - this.defaultSeverity - ); - const result = await evaluator.evaluate(_file, content); - - // Aggregate token usage from claim extraction + evaluation - if (claimUsage) { - const totalUsage: TokenUsage = { - inputTokens: claimUsage.inputTokens + (result.usage?.inputTokens || 0), - outputTokens: - claimUsage.outputTokens + (result.usage?.outputTokens || 0), - }; - result.usage = totalUsage; - } - - return result; - } - - /** - * Extract factual claims from content using the claim extraction prompt. - */ - private async extractClaims(content: string): Promise { - try { - const claimSchema = { - name: "ClaimExtraction", - schema: { - type: "object", - properties: { - claims: { - type: "array", - items: { type: "string" }, - }, - }, - required: ["claims"], - }, - }; - - const claimExtractionPrompt = getPrompt( - TechnicalAccuracyEvaluator.CLAIM_EXTRACTION_PROMPT_KEY - ); - - const { data: claimData, usage } = - await this.llmProvider.runPromptStructured( - content, - claimExtractionPrompt, - claimSchema - ); - - // Validate the response with Zod schema - const claimResult = CLAIM_EXTRACTION_SCHEMA.parse(claimData); - - return { claims: claimResult.claims, ...(usage && { usage }) }; - } catch (e: unknown) { - const err = e instanceof Error ? e : new Error(String(e)); - console.warn(`[vectorlint] Claim extraction failed: ${err.message}`); - return { claims: [] }; - } - } - - /** - * Search for evidence for each claim. - * Returns a map of claim index to search results. - */ - private async searchForEvidence( - claims: string[] - ): Promise> { - const resultsMap = new Map(); - - for (let i = 0; i < claims.length; i++) { - const claim = claims[i]; - // Skip if claim is undefined (shouldn't happen, but TypeScript requires the check) - if (!claim) { - resultsMap.set(i, []); - continue; - } - - try { - const snippetsRaw: unknown = await this.searchProvider.search(claim); - - // Validate search results - const SEARCH_RESULTS_ARRAY_SCHEMA = z.array(SEARCH_RESULT_SCHEMA); - const snippets = SEARCH_RESULTS_ARRAY_SCHEMA.parse(snippetsRaw); - - resultsMap.set(i, snippets); - } catch (e: unknown) { - const err = e instanceof Error ? e : new Error(String(e)); - console.warn( - `[vectorlint] Search failed for claim "${claim}": ${err.message}` - ); - resultsMap.set(i, []); - } - } - - return resultsMap; - } - - /** - * Format claims as a numbered list for the template. - */ - private formatClaimsForTemplate(claims: string[]): string { - return claims.map((claim, index) => `${index + 1}. ${claim}`).join("\n"); - } - - /** - * Format search results grouped by claim for the template. - */ - private formatSearchResultsForTemplate( - resultsMap: Map - ): string { - const formatted: string[] = []; - - for (const [index, results] of resultsMap.entries()) { - const claimNum = index + 1; - formatted.push(`\n### Claim ${claimNum} Evidence:`); - - if (results.length === 0) { - formatted.push("No search results found."); - } else { - results.forEach((result, i) => { - formatted.push(`[${i + 1}] ${result.snippet} (${result.url})`); - }); - } - } - - return formatted.join("\n"); - } - - /** - * Get the prompt body, ensuring it's defined. - * Throws an error if the prompt body is missing. - */ - private getPromptBody(): string { - if (!this.prompt.body) { - throw new Error("Prompt body is empty or undefined"); - } - return this.prompt.body; - } -} - -// Self-register on module load using registerEvaluator directly -registerEvaluator( - Type.TECHNICAL_ACCURACY, - (llmProvider, prompt, searchProvider, defaultSeverity) => { - if (!searchProvider) { - throw new MissingDependencyError( - "technical-accuracy evaluator requires a search provider", - "search-provider", - "Configure TAVILY_API_KEY or PERPLEXITY_API_KEY in .env, or remove this eval" - ); - } - return new TechnicalAccuracyEvaluator( - llmProvider, - prompt, - searchProvider, - defaultSeverity - ); - } -); diff --git a/src/evaluators/base-evaluator.ts b/src/evaluators/base-evaluator.ts deleted file mode 100644 index 2b71ae5f..00000000 --- a/src/evaluators/base-evaluator.ts +++ /dev/null @@ -1,128 +0,0 @@ -import path from "path"; -import type { LLMProvider } from "../providers/llm-provider"; -import type { EvalContext } from "../providers/request-builder"; -import type { PromptFile } from "../schemas/prompt-schemas"; -import type { TokenUsage } from "../providers/token-usage"; -import { - buildEvaluationLLMSchema, - type EvaluationLLMResult, - type PromptEvaluationResult, -} from "../prompts/schema"; -import { registerEvaluator } from "./evaluator-registry"; -import type { Evaluator } from "./evaluator"; -import { Type, Severity } from "./types"; -import { - mergeViolations, - RecursiveChunker, - countWords, - type Chunk, -} from "../chunking"; -import { prependLineNumbers } from "../output/line-numbering"; - -const CHUNKING_THRESHOLD = 600; // Word count threshold for enabling chunking -const MAX_CHUNK_SIZE = 500; // Maximum words per chunk - -/** Evaluates rule violations, chunking large documents when needed. */ -export class BaseEvaluator implements Evaluator { - constructor( - protected llmProvider: LLMProvider, - protected prompt: PromptFile, - protected defaultSeverity?: Severity - ) { } - - async evaluate(file: string, content: string): Promise { - const ext = path.extname(file); - const context: EvalContext = ext ? { fileType: ext } : {}; - return this.runEvaluation(content, context); - } - - protected chunkContent(content: string): Chunk[] { - const wordCount = countWords(content) || 1; - - const chunkingEnabled = this.prompt.meta.evaluateAs !== "document"; - - if (!chunkingEnabled || wordCount <= CHUNKING_THRESHOLD) { - // Chunking disabled or content is small enough - return as single chunk - return [ - { - content, - index: 0, - }, - ]; - } - - const chunker = new RecursiveChunker(); - return chunker.chunk(content, { maxChunkSize: MAX_CHUNK_SIZE }); - } - - /** - * Aggregates token usage from multiple LLM calls. - */ - protected aggregateUsage( - usages: (TokenUsage | undefined)[] - ): TokenUsage | undefined { - const validUsages = usages.filter((u): u is TokenUsage => u !== undefined); - if (validUsages.length === 0) return undefined; - - return validUsages.reduce( - (acc, u) => ({ - inputTokens: acc.inputTokens + u.inputTokens, - outputTokens: acc.outputTokens + u.outputTokens, - }), - { inputTokens: 0, outputTokens: 0 } - ); - } - - protected async runEvaluation( - content: string, - context?: EvalContext - ): Promise { - const schema = buildEvaluationLLMSchema(); - - // Prepend line numbers for deterministic line reporting - const numberedContent = prependLineNumbers(content); - const chunks = this.chunkContent(numberedContent); - const totalWordCount = countWords(content) || 1; - - // Collect all violations from all chunks - const allChunkViolations: EvaluationLLMResult["violations"][] = []; - const rawChunkOutputs: EvaluationLLMResult[] = []; - const chunkReasonings: string[] = []; - const usages: (TokenUsage | undefined)[] = []; - - for (const chunk of chunks) { - const { data: llmResult, usage } = - await this.llmProvider.runPromptStructured( - chunk.content, - this.prompt.body, - schema, - context - ); - allChunkViolations.push(llmResult.violations); - rawChunkOutputs.push(llmResult); - if (llmResult.reasoning) chunkReasonings.push(llmResult.reasoning); - usages.push(usage); - } - - // Merge and deduplicate violations - const mergedViolations = mergeViolations(allChunkViolations); - - const aggregatedUsage = this.aggregateUsage(usages); - const reasoning = chunkReasonings.join(" ").trim() || undefined; - - return { - violations: mergedViolations, - word_count: totalWordCount, - ...(reasoning && { reasoning }), - raw_model_output: rawChunkOutputs.length === 1 ? rawChunkOutputs[0] : rawChunkOutputs, - ...(aggregatedUsage && { usage: aggregatedUsage }), - }; - } -} - -registerEvaluator( - Type.BASE, - (llmProvider, prompt, _searchProvider, defaultSeverity) => { - return new BaseEvaluator(llmProvider, prompt, defaultSeverity); - } -); diff --git a/src/evaluators/evaluator-registry.ts b/src/evaluators/evaluator-registry.ts deleted file mode 100644 index 9b135e9f..00000000 --- a/src/evaluators/evaluator-registry.ts +++ /dev/null @@ -1,81 +0,0 @@ -import type { Evaluator } from './evaluator'; -import type { LLMProvider } from '../providers/llm-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import type { PromptFile } from '../schemas/prompt-schemas'; - -/* - * Factory function signature for creating evaluators. - * Evaluators can optionally depend on search providers for fact verification. - */ -import type { Severity } from './types'; - -/* - * Factory function signature for creating evaluators. - * Evaluators can optionally depend on search providers for fact verification. - */ -export type EvaluatorFactory = ( - llmProvider: LLMProvider, - prompt: PromptFile, - searchProvider?: SearchProvider, - defaultSeverity?: Severity -) => Evaluator; - -/* - * EvaluatorRegistry manages evaluator type registration and instantiation. - * Evaluators self-register by calling registerEvaluator() in their module. - */ -class EvaluatorRegistry { - private registry = new Map(); - - register(type: string, factory: EvaluatorFactory): void { - if (this.registry.has(type)) { - throw new Error(`Evaluator type '${type}' is already registered`); - } - this.registry.set(type, factory); - } - - create( - type: string, - llmProvider: LLMProvider, - prompt: PromptFile, - searchProvider?: SearchProvider, - defaultSeverity?: Severity - ): Evaluator { - const factory = this.registry.get(type); - - if (!factory) { - const available = Array.from(this.registry.keys()).join(', '); - throw new Error( - `Unknown evaluator type: '${type}'. Available types: ${available || 'none'}` - ); - } - - return factory(llmProvider, prompt, searchProvider, defaultSeverity); - } - - getRegisteredTypes(): string[] { - return Array.from(this.registry.keys()); - } -} - -// Singleton instance -const REGISTRY = new EvaluatorRegistry(); - -// Public API -export function registerEvaluator(type: string, factory: EvaluatorFactory): void { - REGISTRY.register(type, factory); -} - -export function createEvaluator( - type: string, - llmProvider: LLMProvider, - prompt: PromptFile, - searchProvider?: SearchProvider, - defaultSeverity?: Severity -): Evaluator { - return REGISTRY.create(type, llmProvider, prompt, searchProvider, defaultSeverity); -} - -export function getRegisteredEvaluatorTypes(): string[] { - return REGISTRY.getRegisteredTypes(); -} diff --git a/src/evaluators/evaluator.ts b/src/evaluators/evaluator.ts deleted file mode 100644 index df0ebba1..00000000 --- a/src/evaluators/evaluator.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { PromptEvaluationResult } from '../prompts/schema'; - -/* - * Core evaluator interface for content evaluation. - * Implementations receive a file path and content, returning structured evaluation results. - */ -export interface Evaluator { - evaluate(file: string, content: string): Promise; -} diff --git a/src/evaluators/index.ts b/src/evaluators/index.ts deleted file mode 100644 index 35e0a1c8..00000000 --- a/src/evaluators/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Evaluators module - exports evaluator interface, base class, and registry. - * - * Import this module to: - * - Access the Evaluator interface for type definitions - * - Use BaseEvaluator as a base class for custom evaluators - * - Use registry functions to create and register evaluators - * - * Importing this module also triggers self-registration of all built-in evaluators. - */ - -// Core interface -export type { Evaluator } from './evaluator'; - -// Base evaluator class (also triggers 'base' registration on import) -export { BaseEvaluator } from './base-evaluator'; - -// Registry functions -export { - registerEvaluator, - createEvaluator, - getRegisteredEvaluatorTypes, - type EvaluatorFactory, -} from './evaluator-registry'; - -// Prompt loader for evaluator-specific prompts -export { getPrompt } from './prompt-loader'; - -// Import specialized evaluators to trigger their self-registration -// These must be imported after base-evaluator to ensure registry is ready -import './accuracy-evaluator'; diff --git a/src/evaluators/prompt-loader.ts b/src/evaluators/prompt-loader.ts deleted file mode 100644 index 16d437fc..00000000 --- a/src/evaluators/prompt-loader.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { z } from "zod"; -import promptsData from "./prompts.json"; - -/** - * Schema for evaluator prompts JSON file. - * Simple key-value mapping: prompt key -> prompt content string. - */ -const PROMPTS_SCHEMA = z.record(z.string(), z.string()); - -const PROMPTS = PROMPTS_SCHEMA.parse(promptsData); - -/** - * Get an evaluator prompt by key. - * Evaluators call this with their known prompt key to retrieve the prompt content. - * - * @param key - The prompt key (e.g., "claim-extraction") - * @returns The prompt content string - * @throws Error if the prompt key is not found - */ -export function getPrompt(key: string): string { - const prompt = PROMPTS[key]; - if (!prompt) { - const available = Object.keys(PROMPTS).join(", "); - throw new Error( - `Prompt '${key}' not found. Available prompts: ${available || "none"}` - ); - } - return prompt; -} diff --git a/src/evaluators/prompts.json b/src/evaluators/prompts.json deleted file mode 100644 index a436fe3f..00000000 --- a/src/evaluators/prompts.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "claim-extraction": "You are a **claim extraction agent** designed to identify verifiable factual statements from technical content.\n\n## Task\n\nAnalyze the provided content and extract all factual claims that can be verified against external sources.\n\n## What to Extract\n\nExtract statements that make claims about:\n\n- **Technical facts**: Specific features, capabilities, or behaviors of tools/technologies\n- **Quantitative data**: Statistics, performance metrics, version numbers, dates\n- **Attributions**: Statements about who created, maintains, or endorses something\n- **Historical facts**: When something was released, deprecated, or changed\n- **Comparisons**: Claims about relative performance, popularity, or capabilities\n\n## What to Skip\n\nDo NOT extract:\n\n- Opinions or preferences (\"I think...\", \"in my opinion...\")\n- Generic statements without specifics (\"many developers use...\")\n- Instructions or recommendations (\"you should...\", \"it's best to...\")\n- Questions\n- Examples or hypotheticals clearly marked as such\n\n## Guidelines\n\nEach extracted claim should be:\n\n- **Complete**: Include enough context to be independently verifiable\n- **Specific**: Avoid vague or general statements\n- **Factual**: Make a concrete assertion about reality\n\nExtract between 0 to 10 claims. If there are more than 10 verifiable claims, prioritize the most significant or impactful ones. Extract claims as they appear in the content, maintaining the original phrasing when possible.\n\nFocus on extracting claims that could be false or outdated, not obvious truths." -} diff --git a/src/evaluators/types.ts b/src/evaluators/types.ts deleted file mode 100644 index b17a1686..00000000 --- a/src/evaluators/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Evaluator type constants to avoid magic strings. - */ -export enum Type { - BASE = 'base', - TECHNICAL_ACCURACY = 'technical-accuracy', -} - -export enum Severity { - ERROR = 'error', - WARNING = 'warning', -} diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts index a186f404..a9cf46f6 100644 --- a/src/executors/agent-model-call-executor.ts +++ b/src/executors/agent-model-call-executor.ts @@ -1,8 +1,8 @@ import type { ToolCallDefinition, ToolCallingModelClient } from '../providers/tool-calling-model-client'; -import type { EvalContext, RequestBuilder } from '../providers/request-builder'; +import type { ReviewCallContext, RequestBuilder } from '../providers/request-builder'; import { - buildEvaluationLLMSchema, - type EvaluationLLMResult, + buildReviewLLMSchema, + type ReviewLLMResult, } from '../prompts/schema'; import { countWords } from '../chunking'; import { processFindings } from '../findings'; @@ -19,7 +19,8 @@ import type { ReviewExecutor } from '../review/executor'; import { TargetReadCapability, buildReadTargetSectionTool } from './target-read-capability-adapter'; import { budgetExceededDiagnostic, - buildEvalContext, + buildReviewCallContext, + buildReviewPrompt, buildReviewUsage, splitRuleId, toFindingSeverity, @@ -48,12 +49,12 @@ export class AgentModelCallExecutor implements ReviewExecutor { ) {} async run(request: ReviewRequest): Promise { - const schema = buildEvaluationLLMSchema(); + const schema = buildReviewLLMSchema(); const capability = new TargetReadCapability(request.target.content); // Exactly one executor-owned tool is exposed: target-section paging. const tools = buildReadTargetSectionTool(capability); const context = { - ...buildEvalContext(request.target.uri), + ...buildReviewCallContext(request.target.uri), recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, }; @@ -68,7 +69,7 @@ export class AgentModelCallExecutor implements ReviewExecutor { try { for (const rule of request.rules) { - const ruleResult = await this.reviewRule( + const contentReview = await this.reviewTargetWithRule( request, rule, schema, @@ -78,10 +79,10 @@ export class AgentModelCallExecutor implements ReviewExecutor { counters, elapsedMs, ); - findings.push(...ruleResult.findings); - scores.push(...ruleResult.scores); - diagnostics.push(...ruleResult.diagnostics); - if (ruleResult.hadOperationalErrors) { + findings.push(...contentReview.findings); + scores.push(...contentReview.scores); + diagnostics.push(...contentReview.diagnostics); + if (contentReview.hadOperationalErrors) { hadOperationalErrors = true; } } @@ -109,12 +110,12 @@ export class AgentModelCallExecutor implements ReviewExecutor { * model page through the target via `read_target_section`, then projects the * returned violations through {@link processFindings}. */ - private async reviewRule( + private async reviewTargetWithRule( request: ReviewRequest, rule: ReviewRule, - schema: ReturnType, + schema: ReturnType, tools: Record, - context: EvalContext, + context: ReviewCallContext, targetLineCount: number, counters: RunCounters, elapsedMs: () => number, @@ -127,11 +128,14 @@ export class AgentModelCallExecutor implements ReviewExecutor { elapsedMs: elapsedMs(), }); - const { data, usage } = await this.client.runWithTools({ + const { data, usage } = await this.client.runWithTools({ // The source-backed rule body, wrapped with the directive/user // instructions exactly as the single-call path does. No model-supplied // rule override is introduced. - systemPrompt: this.builder.buildPromptBodyForStructured(rule.body, context), + systemPrompt: this.builder.buildPromptBodyForStructured( + buildReviewPrompt(rule.body, request.context), + context, + ), prompt: this.buildTargetPrompt(request.target.uri, targetLineCount), tools, schema, diff --git a/src/executors/shared.ts b/src/executors/shared.ts index ad6ff76b..6eef5635 100644 --- a/src/executors/shared.ts +++ b/src/executors/shared.ts @@ -1,10 +1,11 @@ import path from 'path'; -import { Severity } from '../evaluators/types'; -import type { EvalContext } from '../providers/request-builder'; +import { Severity } from '../review/severity'; +import type { ReviewCallContext } from '../providers/request-builder'; import { BudgetExceededError } from '../review/errors'; import type { ReviewDiagnostic, + ReviewContext, ReviewRequest, ReviewSeverity, ReviewUsage, @@ -41,15 +42,45 @@ export function splitRuleId(id: string): { pack: string; ruleId: string } { } /** - * Builds the provider {@link EvalContext} (file-type hint) for a target URI. + * Builds the provider {@link ReviewCallContext} (file-type hint) for a target URI. * Shared by both executors so structured and tool-calling calls receive the * same file-type context for directive substitution. */ -export function buildEvalContext(uri: string): EvalContext { +export function buildReviewCallContext(uri: string): ReviewCallContext { const ext = path.extname(uri); return ext ? { fileType: ext } : {}; } +/** Adds caller-supplied reference context to a source-backed rule prompt. */ +export function buildReviewPrompt( + ruleBody: string, + context: readonly ReviewContext[] | undefined, +): string { + if (!context || context.length === 0) { + return ruleBody; + } + + const contextSections = context.map((item) => { + const metadata = [ + item.relation ? `Relation: ${item.relation}` : undefined, + item.uri ? `Source: ${item.uri}` : undefined, + ].filter((value): value is string => value !== undefined); + + return [ + `### ${item.label}`, + ...metadata, + item.content, + ].join('\n'); + }); + + return [ + ruleBody, + '', + '## Caller-supplied context', + ...contextSections, + ].join('\n'); +} + /** * Maps the review contract's plain {@link ReviewSeverity} union onto the * finding processor's {@link Severity} enum at the executor boundary (no diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index dba27a7d..baffc8d3 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -8,18 +8,19 @@ import type { ReviewRule, ReviewScore, } from '../review/types'; -import type { EvalContext } from '../providers/request-builder'; +import type { ReviewCallContext } from '../providers/request-builder'; import type { StructuredModelClient } from '../providers/structured-model-client'; import { - buildEvaluationLLMSchema, - type EvaluationLLMResult, + buildReviewLLMSchema, + type ReviewLLMResult, } from '../prompts/schema'; import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; import { prependLineNumbers } from '../output/line-numbering'; import { processFindings } from '../findings'; import { budgetExceededDiagnostic, - buildEvalContext, + buildReviewCallContext, + buildReviewPrompt, buildReviewUsage, splitRuleId, toFindingSeverity, @@ -28,9 +29,8 @@ import { /** * Word-count threshold above which the single-call executor chunks the target - * before reviewing it. Mirrors the check evaluator's chunking threshold so the - * single-call path preserves the existing chunk/merge behavior for large - * documents. + * before reviewing it. The threshold preserves the existing chunk/merge + * behavior for large documents. */ const CHUNKING_WORD_THRESHOLD = 600; const MAX_CHUNK_WORDS = 500; @@ -52,9 +52,9 @@ export class SingleModelCallExecutor implements ReviewExecutor { constructor(private readonly client: StructuredModelClient) {} async run(request: ReviewRequest): Promise { - const schema = buildEvaluationLLMSchema(); + const schema = buildReviewLLMSchema(); const context = { - ...buildEvalContext(request.target.uri), + ...buildReviewCallContext(request.target.uri), recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, }; @@ -69,7 +69,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { try { for (const rule of request.rules) { - const ruleResult = await this.reviewRule( + const contentReview = await this.reviewTargetWithRule( request, rule, schema, @@ -77,10 +77,10 @@ export class SingleModelCallExecutor implements ReviewExecutor { counters, elapsedMs, ); - findings.push(...ruleResult.findings); - scores.push(...ruleResult.scores); - diagnostics.push(...ruleResult.diagnostics); - if (ruleResult.hadOperationalErrors) { + findings.push(...contentReview.findings); + scores.push(...contentReview.scores); + diagnostics.push(...contentReview.diagnostics); + if (contentReview.hadOperationalErrors) { hadOperationalErrors = true; } } @@ -111,19 +111,20 @@ export class SingleModelCallExecutor implements ReviewExecutor { * structured model call per chunk, merges violations across chunks, and * projects the merged candidates through {@link processFindings}. */ - private async reviewRule( + private async reviewTargetWithRule( request: ReviewRequest, rule: ReviewRule, - schema: ReturnType, - context: EvalContext, + schema: ReturnType, + context: ReviewCallContext, counters: RunCounters, elapsedMs: () => number, ): Promise { const numberedContent = prependLineNumbers(request.target.content); const wordCount = countWords(request.target.content) || 1; const chunks = this.chunkTarget(numberedContent, wordCount, request.budget.maxChunksPerRule); + const reviewPrompt = buildReviewPrompt(rule.body, request.context); - const chunkViolations: EvaluationLLMResult['violations'][] = []; + const chunkViolations: ReviewLLMResult['violations'][] = []; for (const chunk of chunks) { // Enforce the model-call budget before committing to another call. The // prospective count (calls made so far plus this one) lets enforceBudget @@ -133,9 +134,9 @@ export class SingleModelCallExecutor implements ReviewExecutor { elapsedMs: elapsedMs(), }); - const { data, usage } = await this.client.runPromptStructured( + const { data, usage } = await this.client.runPromptStructured( chunk.content, - rule.body, + reviewPrompt, schema, context, ); diff --git a/src/evaluators/violation-filter.ts b/src/findings/filter-decision.ts similarity index 100% rename from src/evaluators/violation-filter.ts rename to src/findings/filter-decision.ts diff --git a/src/findings/processor.ts b/src/findings/processor.ts index af9a0641..9973e24d 100644 --- a/src/findings/processor.ts +++ b/src/findings/processor.ts @@ -4,7 +4,7 @@ import type { ReviewResult, ReviewScore, } from '../review/types'; -import { computeFilterDecision } from '../evaluators/violation-filter'; +import { computeFilterDecision } from '../findings/filter-decision'; import { verifyFindingEvidence, FINDING_EVIDENCE_NOT_LOCATABLE, diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts index 349a3b49..882cdf93 100644 --- a/src/findings/scorer.ts +++ b/src/findings/scorer.ts @@ -1,5 +1,5 @@ import { calculateScore, type ScoringOptions } from '../scoring'; -import type { ScoredEvaluation } from '../prompts/schema'; +import type { ScoredReview } from '../prompts/schema'; import { resolveSeverity } from './severity'; import type { RawViolation, RuleSeverity } from './types'; @@ -13,7 +13,7 @@ export interface ScoredFindings { scoreText: string; severity: RuleSeverity; findingCount: number; - scored: ScoredEvaluation; + scored: ScoredReview; } /** Scores verified findings using violation density. */ diff --git a/src/findings/severity.ts b/src/findings/severity.ts index 6bb448bc..74e38590 100644 --- a/src/findings/severity.ts +++ b/src/findings/severity.ts @@ -1,12 +1,12 @@ -import type { ScoredEvaluation } from '../prompts/schema'; +import type { ScoredReview } from '../prompts/schema'; import type { FindingsCriterion, RuleSeverity } from './types'; /** Input to {@link resolveSeverity}. */ export interface SeverityInput { - scored: ScoredEvaluation; + scored: ScoredReview; } -/** Resolves severity from a scored evaluation. */ +/** Resolves severity from a scored review. */ export function resolveSeverity(input: SeverityInput): RuleSeverity { return input.scored.severity; } diff --git a/src/findings/types.ts b/src/findings/types.ts index 9c8c0b85..2310e6a7 100644 --- a/src/findings/types.ts +++ b/src/findings/types.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; /** Severity assigned to a rule and its findings. */ export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; diff --git a/src/index.ts b/src/index.ts index 2f2be86d..c6e48d9c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,9 +10,6 @@ import { loadGlobalConfig } from "./config/global-config"; import { CLI_DESCRIPTION, CLI_VERSION } from "./config/constants"; -// Import evaluators module to trigger self-registration of all evaluators -import "./evaluators/index"; - /* * Loads environment variables from Global Config and .env files. * Hierarchy: CLI/Shell > Local.env > Global Config diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 7ed80e03..90cf82e8 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,8 +1,8 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'tool-calling'; + operation: 'structured-review' | 'tool-calling'; provider: string; model: string; - evaluator?: string; + reviewer?: string; rule?: string; recordPayloadTelemetry?: boolean; } diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index bae2d555..708ebb51 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -60,7 +60,7 @@ export class LangfuseObservability implements AIObservability { metadata: { provider: context.provider, model: context.model, - ...(context.evaluator ? { evaluator: context.evaluator } : {}), + ...(context.reviewer ? { reviewer: context.reviewer } : {}), ...(context.rule ? { rule: context.rule } : {}), }, recordInputs: context.recordPayloadTelemetry === true, diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index 0d589d14..e4909f66 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -1,4 +1,4 @@ -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import { CLI_VERSION } from '../config/constants'; export interface ScoreComponent { criterion?: string; @@ -8,7 +8,7 @@ export interface ScoreComponent { normalizedMaxScore: number; } -export interface EvaluationScore { +export interface ReviewScoreOutput { id: string; scores: ScoreComponent[]; } @@ -28,7 +28,7 @@ export interface Issue { export interface FileResult { issues: Issue[]; - evaluationScores: EvaluationScore[]; + reviewScores: ReviewScoreOutput[]; } export interface Result { @@ -51,7 +51,7 @@ export class JsonFormatter { addIssue(file: string, issue: Issue): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } this.files[file].issues.push(issue); @@ -62,11 +62,11 @@ export class JsonFormatter { } } - addEvaluationScore(file: string, score: EvaluationScore): void { + addReviewScore(file: string, score: ReviewScoreOutput): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } - this.files[file].evaluationScores.push(score); + this.files[file].reviewScores.push(score); } toJson(): string { diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts index d1daed7e..946008ed 100644 --- a/src/output/rdjson-formatter.ts +++ b/src/output/rdjson-formatter.ts @@ -1,5 +1,5 @@ -import type { Issue, EvaluationScore } from './json-formatter'; -import { Severity } from '../evaluators/types'; +import type { Issue, ReviewScoreOutput } from './json-formatter'; +import { Severity } from '../review/severity'; export interface RdJsonResult { source: { @@ -48,7 +48,7 @@ export interface RdJsonSuggestion { interface FileResult { issues: Issue[]; - evaluationScores: EvaluationScore[]; + reviewScores: ReviewScoreOutput[]; } export class RdJsonFormatter { @@ -56,16 +56,16 @@ export class RdJsonFormatter { addIssue(file: string, issue: Issue): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } this.files[file].issues.push(issue); } - addEvaluationScore(file: string, score: EvaluationScore): void { + addReviewScore(file: string, score: ReviewScoreOutput): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } - this.files[file].evaluationScores.push(score); + this.files[file].reviewScores.push(score); } toRdJsonFormat(): RdJsonResult { diff --git a/src/output/reporter.ts b/src/output/reporter.ts index 9e69124a..ccd32992 100644 --- a/src/output/reporter.ts +++ b/src/output/reporter.ts @@ -1,10 +1,10 @@ import chalk from 'chalk'; import stripAnsi from 'strip-ansi'; import path from 'path'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import { TokenUsageStats } from '../providers/token-usage'; -export interface EvaluationSummary { +export interface ReviewSummary { id: string; scoreText: string; score?: number; @@ -135,16 +135,16 @@ export function printGlobalSummary(files: number, errors: number, warnings: numb } } -export function printEvaluationSummaries( - summaries: Map +export function printReviewSummaries( + summaries: Map ) { if (summaries.size === 0) return; console.log(''); console.log(chalk.bold('\nQuality Scores:')); - for (const [evalName, items] of summaries) { - console.log(` ${chalk.cyan(evalName)}:`); + for (const [reviewName, items] of summaries) { + console.log(` ${chalk.cyan(reviewName)}:`); // Find max ID length for alignment const maxIdLen = Math.max(...items.map(i => i.id.length)); diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index 13e28ca4..a70a4fab 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -2,7 +2,7 @@ import { existsSync, readFileSync } from "fs"; import path from "path"; /** - * Load a directive to append to evaluation prompts. + * Load a directive to append to review prompts. * Precedence: * 1) Project override: .vectorlint/directive.md in current working directory * 2) Built-in: prompts/directive.md shipped with the CLI @@ -10,7 +10,7 @@ import path from "path"; */ const DEFAULT_DIRECTIVE = ` ## Role -You are VectorLint. You evaluate technical content and flag issues based on a user's style guide and rules, putting into context that the content is a technical documentation. +You are VectorLint. You review technical content and flag issues based on a user's style guide and rules, accounting for the conventions of technical documentation. Your goal is to surface issues that would @@ -32,7 +32,7 @@ flowing text, not across structural boundaries. In plain markdown or text files, unless separated by headings or horizontal rules. - + The user-defined Rule is your only criteria for flagging violations. Do not flag issues if they aren't mentioned in the user's rule or style guide. The goal and context sections exist solely to help you determine whether a pattern match is worth surfacing. @@ -47,7 +47,7 @@ In practice this means: lower the confidence — do not omit the finding entirely Treat the goal and context as your system guidance. Do not leak your internal knowledge to the user. - + A finding is worth surfacing when: @@ -60,7 +60,7 @@ Assign lower confidence (≤ 0.5) when the content's structure or domain makes t ## Task -Evaluate the provided Input against the Rule, identifying +Review the provided Input against the Rule, identifying every instance where the content violates the specified standards. diff --git a/src/prompts/prompt-loader.ts b/src/prompts/prompt-loader.ts index f0fc5e3b..987d0a66 100644 --- a/src/prompts/prompt-loader.ts +++ b/src/prompts/prompt-loader.ts @@ -2,7 +2,7 @@ import { readdirSync, readFileSync, statSync } from 'fs'; import path from 'path'; import YAML from 'yaml'; import { PROMPT_META_SCHEMA, type PromptFile, type PromptMeta } from '../schemas/prompt-schemas'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; // Re-export types for backward compatibility export type { PromptFile, PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; diff --git a/src/prompts/prompt-validator.ts b/src/prompts/prompt-validator.ts index 6275f731..617ee8c8 100644 --- a/src/prompts/prompt-validator.ts +++ b/src/prompts/prompt-validator.ts @@ -1,5 +1,5 @@ import { PromptFile, PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import { DEFAULT_TARGET_FLAGS } from './target'; export type ValidationLevel = Severity; diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index a370f52c..37027609 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -1,4 +1,4 @@ -import { Severity } from "../evaluators/types"; +import { Severity } from "../review/severity"; import type { TokenUsage } from "../providers/token-usage"; export type GateChecks = { @@ -19,9 +19,9 @@ export type GateCheckNotes = { fix_preserves_meaning: string; }; -export function buildEvaluationLLMSchema() { +export function buildReviewLLMSchema() { return { - name: "vectorlint_evaluation_result", + name: "vectorlint_review_result", strict: true, schema: { type: "object", @@ -132,7 +132,7 @@ export function buildEvaluationLLMSchema() { } as const; } -export type EvaluationLLMResult = { +export type ReviewLLMResult = { reasoning: string; violations: Array<{ line: number; @@ -151,7 +151,7 @@ export type EvaluationLLMResult = { }>; }; -export type EvaluationItem = { +export type ReviewItem = { line?: number; description?: string; analysis: string; @@ -167,23 +167,15 @@ export type EvaluationItem = { confidence?: number; }; -export type ScoredEvaluation = { +export type ScoredReview = { final_score: number; // 1-10 percentage: number; violation_count: number; - items: Array; + items: Array; severity: typeof Severity.WARNING | typeof Severity.ERROR; message: string; reasoning?: string; - violations: Array; - usage?: TokenUsage; - raw_model_output?: unknown; -}; - -export type PromptEvaluationResult = { - violations: ScoredEvaluation["violations"]; - word_count: number; - reasoning?: string; + violations: Array; usage?: TokenUsage; raw_model_output?: unknown; }; diff --git a/src/providers/index.ts b/src/providers/index.ts index 2c02f6e0..ae8bb341 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -6,8 +6,6 @@ export { type ToolCallRunOptions, } from './tool-calling-model-client'; export { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; -export { SearchProvider } from './search-provider'; -export { PerplexitySearchProvider, type PerplexitySearchConfig } from './perplexity-provider'; export { createProvider, type ProviderOptions, ProviderType } from './provider-factory'; export { RequestBuilder, DefaultRequestBuilder } from './request-builder'; export { TokenUsage, TokenUsageStats, PricingConfig, calculateCost } from './token-usage'; diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts deleted file mode 100644 index 432f8c09..00000000 --- a/src/providers/perplexity-provider.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { generateText } from 'ai'; -import type { LanguageModel } from 'ai'; -import { z } from 'zod'; -import { createPerplexity } from '@ai-sdk/perplexity'; -import type { SearchProvider } from './search-provider'; -import type { PerplexityResult } from '../schemas/perplexity-responses'; -import { createNoopLogger, type Logger } from '../logging/logger'; -import { handleUnknownError } from '../errors'; - -// Boundary validation schema for Perplexity source data. -// The AI SDK's typed Source may not include provider-specific fields (text, publishedDate), -// so we validate the raw data at the boundary to safely extract them. -const PERPLEXITY_SOURCE_SCHEMA = z.object({ - title: z.string().optional(), - text: z.string().optional(), - url: z.string().optional(), - publishedDate: z.string().optional(), -}).passthrough(); -const PERPLEXITY_SOURCES_SCHEMA = z.array(PERPLEXITY_SOURCE_SCHEMA); - -export interface PerplexitySearchConfig { - apiKey?: string; - maxResults?: number; - logger?: Logger; -} - -export class PerplexitySearchProvider implements SearchProvider { - private client: ReturnType; - private maxResults: number; - private logger: Logger; - - constructor(config: PerplexitySearchConfig = {}) { - // Use provided API key or fall back to environment variable - const apiKey = config.apiKey || process.env.PERPLEXITY_API_KEY; - if (!apiKey) { - throw new Error('Perplexity API key is required. Set PERPLEXITY_API_KEY environment variable or pass apiKey in config.'); - } - this.client = createPerplexity({ apiKey }); - this.maxResults = config.maxResults ?? 5; - this.logger = config.logger ?? createNoopLogger(); - } - - async search(query: string): Promise { - if (!query?.trim()) throw new Error('Search query cannot be empty.'); - - this.logger.debug('Perplexity search started', { query }); - - try { - const result = await generateText({ - // @ai-sdk/perplexity@1 exposes LanguageModelV1 models, while ai@6's - // generateText types require a LanguageModel (V2/V3). This is a - // third-party SDK version skew, not a repo type hole; resolve by - // upgrading @ai-sdk/perplexity to a V2 release in a follow-up. - model: this.client('sonar-pro') as unknown as LanguageModel, - prompt: query, - }); - - // Validate sources at the boundary — the SDK may include provider-specific - // fields not present in the typed Source interface - const rawSources: unknown[] = Array.isArray(result.sources) ? result.sources.slice(0, this.maxResults) : []; - const parseResult = PERPLEXITY_SOURCES_SCHEMA.safeParse(rawSources); - if (!parseResult.success) { - this.logger.warn('Perplexity source validation failed', { - error: parseResult.error.message, - }); - } - const sources = parseResult.success ? parseResult.data : []; - - const results: PerplexityResult[] = sources.map(source => ({ - title: source.title || 'Untitled', - snippet: source.text || '', - url: source.url || '', - date: source.publishedDate || '', - })); - - this.logger.debug('Perplexity search completed', { resultCount: results.length }); - this.logger.debug('Perplexity result preview', { - results: results.slice(0, 2), - }); - - return results; - } catch (e: unknown) { - const err = handleUnknownError(e, 'Perplexity API call'); - throw new Error(`Perplexity API call failed: ${err.message}`); - } - } -} diff --git a/src/providers/request-builder.ts b/src/providers/request-builder.ts index 9e0b239c..f79c6de6 100644 --- a/src/providers/request-builder.ts +++ b/src/providers/request-builder.ts @@ -1,12 +1,12 @@ // Centralized request construction for provider-agnostic use -export interface EvalContext { +export interface ReviewCallContext { fileType?: string; recordPayloadTelemetry?: boolean; } export interface RequestBuilder { - buildPromptBodyForStructured(originalBody: string, context?: EvalContext): string; + buildPromptBodyForStructured(originalBody: string, context?: ReviewCallContext): string; } export class DefaultRequestBuilder implements RequestBuilder { @@ -18,7 +18,7 @@ export class DefaultRequestBuilder implements RequestBuilder { this.userInstructions = (userInstructions || '').trim(); } - buildPromptBodyForStructured(originalBody: string, context?: EvalContext): string { + buildPromptBodyForStructured(originalBody: string, context?: ReviewCallContext): string { let directive = this.directive; if (directive) { directive = directive.replaceAll('{{file_type}}', context?.fileType ?? ''); diff --git a/src/providers/search-provider.ts b/src/providers/search-provider.ts deleted file mode 100644 index dfac8818..00000000 --- a/src/providers/search-provider.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Search provider interface for fact verification and research. - * Implementations query external search APIs and return results. - */ -export interface SearchProvider { - search(query: string): Promise; -} diff --git a/src/providers/structured-model-client.ts b/src/providers/structured-model-client.ts index 16335277..beb123f6 100644 --- a/src/providers/structured-model-client.ts +++ b/src/providers/structured-model-client.ts @@ -1,5 +1,5 @@ import type { TokenUsage } from './token-usage'; -import type { EvalContext } from './request-builder'; +import type { ReviewCallContext } from './request-builder'; /** * Result of a structured model call: validated output plus optional usage. @@ -15,6 +15,6 @@ export interface StructuredModelClient { content: string, promptText: string, schema: { name: string; schema: Record }, - context?: EvalContext, + context?: ReviewCallContext, ): Promise>; } diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 1bb3a48a..022985b1 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -51,7 +51,7 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { content: string, promptText: string, schema: { name: string; schema: Record }, - context?: import('./request-builder').EvalContext + context?: import('./request-builder').ReviewCallContext ): Promise> { const systemPrompt = this.builder.buildPromptBodyForStructured(promptText, context); @@ -82,13 +82,13 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { } try { - const evaluator = this.extractContextValue(context, 'evaluatorName', 'evaluator'); + const reviewer = this.extractContextValue(context, 'reviewerName', 'reviewer'); const rule = this.extractContextValue(context, 'ruleName', 'rule'); const observabilityOptions = this.getObservabilityOptions({ - operation: 'structured-eval', + operation: 'structured-review', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', - ...(evaluator ? { evaluator } : {}), + ...(reviewer ? { reviewer } : {}), ...(rule ? { rule } : {}), ...(context?.recordPayloadTelemetry !== undefined ? { recordPayloadTelemetry: context.recordPayloadTelemetry } @@ -270,7 +270,7 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { } private extractContextValue( - context: import('./request-builder').EvalContext | undefined, + context: import('./request-builder').ReviewCallContext | undefined, ...keys: string[] ): string | undefined { if (!context) { diff --git a/src/review/severity.ts b/src/review/severity.ts new file mode 100644 index 00000000..0dad5967 --- /dev/null +++ b/src/review/severity.ts @@ -0,0 +1,5 @@ +/** Severity assigned to rules and surfaced findings. */ +export enum Severity { + ERROR = 'error', + WARNING = 'warning', +} diff --git a/src/schemas/index.ts b/src/schemas/index.ts index d17675be..052c1e80 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -3,4 +3,3 @@ export * from './prompt-schemas'; export * from './cli-schemas'; export * from './config-schemas'; export * from './env-schemas'; -export * from './perplexity-responses'; diff --git a/src/schemas/perplexity-responses.ts b/src/schemas/perplexity-responses.ts deleted file mode 100644 index c7b5d0b7..00000000 --- a/src/schemas/perplexity-responses.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { z } from 'zod'; -export const PERPLEXITY_RESULT_SCHEMA = z.object({ - title: z.string().default('Untitled'), - snippet: z.string().default(''), - url: z.string().default(''), - date: z.string().default(''), -}); - -export const PERPLEXITY_RESPONSE_SCHEMA = z.object({ - results: z.array(PERPLEXITY_RESULT_SCHEMA).default([]), -}); - -export type PerplexityResult = z.infer; -export type PerplexityResponse = z.infer; diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index cb246335..b0133947 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { Severity } from "../evaluators/types"; +import { Severity } from "../review/severity"; // Target specification schema for regex matching export const TARGET_SPEC_SCHEMA = z @@ -21,7 +21,6 @@ export const PROMPT_CRITERION_SCHEMA = z.object({ // Prompt metadata schema for YAML frontmatter. export const PROMPT_META_SCHEMA = z.object({ specVersion: z.union([z.string(), z.number()]).optional(), - evaluator: z.enum(["base", "technical-accuracy"]).optional(), id: z.string(), name: z.string(), severity: z.nativeEnum(Severity).optional(), @@ -30,7 +29,6 @@ export const PROMPT_META_SCHEMA = z.object({ .optional(), target: TARGET_SPEC_SCHEMA.optional(), criteria: z.array(PROMPT_CRITERION_SCHEMA).optional(), - evaluateAs: z.enum(["document", "chunk"]).optional(), }); diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index b0560b52..1b5383cb 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -1,5 +1,5 @@ -import type { EvaluationItem, ScoredEvaluation } from "../prompts/schema"; -import { Severity } from "../evaluators/types"; +import type { ReviewItem, ScoredReview } from "../prompts/schema"; +import { Severity } from "../review/severity"; export interface ScoringOptions { strictness?: number | "lenient" | "strict" | "standard" | undefined; @@ -30,10 +30,10 @@ function resolveStrictness( * Formula: Score = (100 - (violations/wordCount * 100 * strictness)) / 10 */ export function calculateScore( - violations: EvaluationItem[], + violations: ReviewItem[], wordCount: number, options: ScoringOptions = {} -): ScoredEvaluation { +): ScoredReview { const strictness = resolveStrictness(options.strictness); const mappedViolations = violations.map((item) => ({ ...item, diff --git a/src/types/external.d.ts b/src/types/external.d.ts deleted file mode 100644 index b7935cc5..00000000 --- a/src/types/external.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -declare module '@perplexity-ai/perplexity_ai' { - export default class Perplexity { - constructor(); - search: { - create(params: { - query: string; - max_results?: number; - max_tokens_per_page?: number; - }): Promise; - }; - } -} diff --git a/tests/config-loader-integration.test.ts b/tests/config-loader-integration.test.ts index f5efedae..0028101c 100644 --- a/tests/config-loader-integration.test.ts +++ b/tests/config-loader-integration.test.ts @@ -24,7 +24,7 @@ RulesPath = ./prompts [docs/**/*.md] RunRules = VectorLint -technical-accuracy.strictness = 9 +clarity.strictness = 9 [blog/**/*.md] RunRules = BlogPack, SEOPack @@ -41,7 +41,7 @@ readability.severity = error expect(config.scanPaths[0]!.pattern).toBe('docs/**/*.md'); expect(config.scanPaths[0]!.runRules).toEqual(['VectorLint']); expect(config.scanPaths[0]!.overrides).toEqual({ - 'technical-accuracy.strictness': '9' + 'clarity.strictness': '9' }); // Second section @@ -63,7 +63,7 @@ strictness = 7 [content/api/**/*.md] RunRules = APIPack strictness = 9 -technical-accuracy.depth = high +clarity.depth = high [content/archived/**/*.md] RunRules = diff --git a/tests/evaluations/README.md b/tests/evaluations/README.md deleted file mode 100644 index bc6fb185..00000000 --- a/tests/evaluations/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Manual Evaluations - -This directory contains test fixtures for running VectorLint manually against real LLM providers. Use it to evaluate accuracy, compare models, and inspect gate check behavior. - -## Contents - -``` -tests/evaluations/ -├── .vectorlint.ini # Config pointing at test-rules/ -├── TEST_FILE.md # Sample document to evaluate -└── test-rules/ - └── Test/ # Rule pack with general-purpose test rules - ├── clarity.md - ├── consistency.md - ├── passive-voice.md - ├── readability.md - └── wordiness.md -``` - -## Running an evaluation - -From the repo root: - -```bash -# Basic run -npm run dev -- tests/evaluations/TEST_FILE.md \ - --config tests/evaluations/.vectorlint.ini - -# With debug artifacts (writes raw model output + gate check decisions) -npm run dev -- tests/evaluations/TEST_FILE.md \ - --config tests/evaluations/.vectorlint.ini \ - --debug-json - -# With verbose output -npm run dev -- tests/evaluations/TEST_FILE.md \ - --config tests/evaluations/.vectorlint.ini \ - --debug-json --verbose -``` - -Debug artifacts are written to `.vectorlint/runs//.json` and are gitignored. - -## Switching models - -Set your provider and model via environment variables before running: - -```bash -# OpenAI -LLM_PROVIDER=openai OPENAI_MODEL=gpt-4o npm run dev -- ... - -# Anthropic -LLM_PROVIDER=anthropic ANTHROPIC_MODEL=claude-sonnet-4-6 npm run dev -- ... - -# Gemini -LLM_PROVIDER=gemini GEMINI_MODEL=gemini-1.5-pro npm run dev -- ... -``` - -Or configure them in `~/.vectorlint/config.toml`. - -## Inspecting debug artifacts - -Each artifact under `.vectorlint/runs/` contains: - -- `raw_model_output` — exact JSON returned by the model, including all gate check fields -- `filter_decisions` — deterministic surface/hide decision per violation candidate with reasons -- `surfaced_violations` — candidates that passed all gates - -Use these to compare how different models respond to the same rules and content, and to tune gate check thresholds. diff --git a/tests/evaluator.test.ts b/tests/evaluator.test.ts deleted file mode 100644 index 79f0bb91..00000000 --- a/tests/evaluator.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, mkdirSync, writeFileSync } from 'fs'; -import path from 'path'; -import { tmpdir } from 'os'; -import { loadRules } from '../src/prompts/prompt-loader.js'; - -// Fake provider implementing LLMProvider -class FakeProvider { - calls: Array<{ content: string; prompt: string }> = []; - runPromptStructured(content: string, promptText: string): Promise { - this.calls.push({ content, prompt: promptText }); - const injected = content && content.length > 0 ? 'OK' : 'MISSING'; - return Promise.resolve({ result: `RESULT:${injected}` } as T); - } -} - -function setupEnv() { - const root = mkdtempSync(path.join(tmpdir(), 'vlint-')); - const promptsDir = path.join(root, 'prompts'); - mkdirSync(promptsDir, { recursive: true }); - // Two prompts with minimal frontmatter criteria - writeFileSync( - path.join(promptsDir, 'p1.md'), - `---\nid: P1\nname: Prompt 1\ncriteria:\n - name: A\n id: A\n---\nBody 1\n` - ); - writeFileSync( - path.join(promptsDir, 'p2.md'), - `---\nid: P2\nname: Prompt 2\ncriteria:\n - name: B\n id: B\n---\nBody 2\n` - ); - const files = [ - path.join(root, 'a.md'), - path.join(root, 'b.txt'), - ]; - writeFileSync(files[0], '# title'); - writeFileSync(files[1], 'plain'); - return { root, promptsDir, files }; -} - -describe('Evaluation aggregation', () => { - it('runs all prompts for all files and injects content if placeholder exists', async () => { - const { promptsDir, files } = setupEnv(); - const provider = new FakeProvider(); - const { prompts } = loadRules(promptsDir); - // Simulate aggregation: 2 prompts x 2 files = 4 calls - for (let i = 0; i < files.length; i++) { - const content = '# test'; - for (const p of prompts) { - await provider.runPromptStructured(content, p.body); - } - } - expect(provider.calls.length).toBe(4); - }); -}); diff --git a/tests/executors/agent-model-call-executor.test.ts b/tests/executors/agent-model-call-executor.test.ts index 1e253347..170b0970 100644 --- a/tests/executors/agent-model-call-executor.test.ts +++ b/tests/executors/agent-model-call-executor.test.ts @@ -7,7 +7,7 @@ import { DefaultRequestBuilder } from '../../src/providers/request-builder'; import type { ToolCallDefinition, ToolCallingModelClient, ToolCallRunOptions } from '../../src/providers/tool-calling-model-client'; import type { LLMResult } from '../../src/providers/structured-model-client'; import type { TokenUsage } from '../../src/providers/token-usage'; -import type { EvaluationLLMResult } from '../../src/prompts/schema'; +import type { ReviewLLMResult } from '../../src/prompts/schema'; import type { TargetSectionErrorResult, TargetSectionResult } from '../../src/executors/target-read-capability-adapter'; const SUPPORTED_CHECKS = { @@ -28,7 +28,7 @@ const SUPPORTED_NOTES = { fix_preserves_meaning: 'yes', }; -type ModelViolation = EvaluationLLMResult['violations'][number]; +type ModelViolation = ReviewLLMResult['violations'][number]; function modelViolation(overrides: Partial = {}): ModelViolation { return { @@ -62,7 +62,7 @@ interface FakeToolClient extends ToolCallingModelClient { } function makeFakeClient( - respond: () => { data: EvaluationLLMResult; usage?: TokenUsage }, + respond: () => { data: ReviewLLMResult; usage?: TokenUsage }, ): FakeToolClient { const fake = { calls: 0, @@ -183,6 +183,25 @@ describe('AgentModelCallExecutor', () => { expect(client.captured[0]!.prompt).toContain('file:///repo/docs/guide.md'); }); + it('includes caller-supplied context in the system prompt', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule()], { + context: [{ + label: 'Current API contract', + relation: 'reference', + content: 'The endpoint returns HTTP 202.', + }], + })); + + expect(client.captured[0]!.systemPrompt).toContain('## Caller-supplied context'); + expect(client.captured[0]!.systemPrompt).toContain('### Current API contract'); + expect(client.captured[0]!.systemPrompt).toContain('The endpoint returns HTTP 202.'); + }); + it('stops and records an operational error when the model-call budget is exhausted', async () => { const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] }, diff --git a/tests/executors/single-model-call-executor.test.ts b/tests/executors/single-model-call-executor.test.ts index 2ac746b5..0b5ee7f2 100644 --- a/tests/executors/single-model-call-executor.test.ts +++ b/tests/executors/single-model-call-executor.test.ts @@ -9,7 +9,7 @@ import type { } from '../../src/review'; import type { LLMResult, StructuredModelClient } from '../../src/providers/structured-model-client'; import type { TokenUsage } from '../../src/providers/token-usage'; -import type { EvaluationLLMResult } from '../../src/prompts/schema'; +import type { ReviewLLMResult } from '../../src/prompts/schema'; const SUPPORTED_CHECKS = { rule_supports_claim: true, @@ -29,7 +29,7 @@ const SUPPORTED_NOTES = { fix_preserves_meaning: 'yes', }; -type ModelViolation = EvaluationLLMResult['violations'][number]; +type ModelViolation = ReviewLLMResult['violations'][number]; function modelViolation(overrides: Partial = {}): ModelViolation { return { @@ -58,7 +58,7 @@ type FakeStructuredClient = StructuredModelClient & { }; function makeFakeClient( - respond: (content: string, promptText: string) => { data: EvaluationLLMResult; usage?: TokenUsage }, + respond: (content: string, promptText: string) => { data: ReviewLLMResult; usage?: TokenUsage }, ): FakeStructuredClient { const client = { structuredCalls: 0, @@ -143,6 +143,26 @@ describe('SingleModelCallExecutor', () => { expect(result.usage?.modelCalls).toBe(1); }); + it('includes caller-supplied context in the review prompt', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + await executor.run(makeRequest([makeRule()], { + context: [{ + label: 'Current API contract', + relation: 'reference', + uri: 'file:///repo/openapi.md', + content: 'The endpoint returns HTTP 202.', + }], + })); + + expect(client.lastPromptText).toContain('## Caller-supplied context'); + expect(client.lastPromptText).toContain('### Current API contract'); + expect(client.lastPromptText).toContain('The endpoint returns HTTP 202.'); + }); + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { const client = makeFakeClient(() => ({ data: { diff --git a/tests/file-sections.test.ts b/tests/file-sections.test.ts index e29998a9..61b0ec2d 100644 --- a/tests/file-sections.test.ts +++ b/tests/file-sections.test.ts @@ -51,7 +51,7 @@ describe('File-centric configuration (File Sections)', () => { const config = { 'critical/**/*.md': { RunRules: 'VectorLint', - 'technical-accuracy.strictness': '9', + 'clarity.strictness': '9', 'readability.severity': 'error' } }; @@ -59,7 +59,7 @@ describe('File-centric configuration (File Sections)', () => { const sections = parser.parseSections(config); expect(sections[0].overrides).toEqual({ - 'technical-accuracy.strictness': '9', + 'clarity.strictness': '9', 'readability.severity': 'error' }); }); @@ -142,15 +142,15 @@ describe('File-centric configuration (File Sections)', () => { describe('Integration: Pattern priority and override merging', () => { it('applies prompts with overrides based on file path patterns', () => { const sections: FilePatternConfig[] = [ - createFilePatternConfig('**/*.md', ['VectorLint'], { 'technical-accuracy.strictness': 7 }), - createFilePatternConfig('docs/api/**/*.md', ['APIPack'], { 'technical-accuracy.strictness': 9 }) + createFilePatternConfig('**/*.md', ['VectorLint'], { 'clarity.strictness': 7 }), + createFilePatternConfig('docs/api/**/*.md', ['APIPack'], { 'clarity.strictness': 9 }) ]; const result = resolver.resolveConfiguration('docs/api/users.md', sections); // Cascading: APIPack applies on top of VectorLint expect(result.packs).toEqual(['VectorLint', 'APIPack']); - expect(result.overrides['technical-accuracy.strictness']).toBe(9); // Later wins + expect(result.overrides['clarity.strictness']).toBe(9); // Later wins }); }); }); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts index 8e660bf2..7edffc11 100644 --- a/tests/findings/scorer.test.ts +++ b/tests/findings/scorer.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { calculateScore } from '../../src/scoring'; -import { Severity } from '../../src/evaluators/types'; +import { Severity } from '../../src/review/severity'; import { scoreFindings } from '../../src/findings/scorer'; import type { RawViolation } from '../../src/findings/types'; diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts index 031115f1..dde92f43 100644 --- a/tests/findings/severity.test.ts +++ b/tests/findings/severity.test.ts @@ -4,10 +4,10 @@ import { resolveCriterionId, resolveSeverity, } from '../../src/findings/severity'; -import { Severity } from '../../src/evaluators/types'; -import type { ScoredEvaluation } from '../../src/prompts/schema'; +import { Severity } from '../../src/review/severity'; +import type { ScoredReview } from '../../src/prompts/schema'; -function makeScoredEvaluation(severity: Severity): ScoredEvaluation { +function makeScoredReview(severity: Severity): ScoredReview { return { final_score: 5, percentage: 50, @@ -20,11 +20,11 @@ function makeScoredEvaluation(severity: Severity): ScoredEvaluation { } describe('resolveSeverity', () => { - it('returns the density-derived severity from the scored evaluation', () => { - expect(resolveSeverity({ scored: makeScoredEvaluation(Severity.WARNING) })).toBe( + it('returns the density-derived severity from the scored review', () => { + expect(resolveSeverity({ scored: makeScoredReview(Severity.WARNING) })).toBe( Severity.WARNING, ); - expect(resolveSeverity({ scored: makeScoredEvaluation(Severity.ERROR) })).toBe( + expect(resolveSeverity({ scored: makeScoredReview(Severity.ERROR) })).toBe( Severity.ERROR, ); }); diff --git a/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md b/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md index bdb8cae5..b590be1f 100644 --- a/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md +++ b/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Capitalization name: Capitalization severity: warning -evaluateAs: document --- # Capitalization diff --git a/tests/fixtures/consistency/rules/consistency-rules/consistency.md b/tests/fixtures/consistency/rules/consistency-rules/consistency.md index 219d1022..04c9e717 100644 --- a/tests/fixtures/consistency/rules/consistency-rules/consistency.md +++ b/tests/fixtures/consistency/rules/consistency-rules/consistency.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Consistency name: Consistency severity: warning -evaluateAs: document --- # Consistency diff --git a/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md b/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md index ab5bda8f..a0bf644f 100644 --- a/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md +++ b/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Inclusivity name: Inclusivity severity: warning -evaluateAs: document --- # Inclusivity diff --git a/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md b/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md index bfa21666..9bcda8dd 100644 --- a/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md +++ b/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md @@ -1,9 +1,7 @@ --- -evaluator: base id: PassiveVoice name: Passive Voice severity: warning -evaluateAs: document --- # Passive Voice diff --git a/tests/fixtures/repetition/rules/repetition-rules/repetition.md b/tests/fixtures/repetition/rules/repetition-rules/repetition.md index d102e343..66e87219 100644 --- a/tests/fixtures/repetition/rules/repetition-rules/repetition.md +++ b/tests/fixtures/repetition/rules/repetition-rules/repetition.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Repetition name: Repetition severity: warning -evaluateAs: document --- # Repetition diff --git a/tests/fixtures/technical-accuracy/test.md b/tests/fixtures/technical-accuracy/test.md deleted file mode 100644 index 391efe75..00000000 --- a/tests/fixtures/technical-accuracy/test.md +++ /dev/null @@ -1,31 +0,0 @@ -# Evaluating Modern AI-Assisted Developer Tools - -AI-driven developer productivity has surged in 2025, with new tools claiming to automate everything from documentation to deployment. - -**1. GitHub Copilot** has become one of the most widely adopted AI pair-programming tools, integrated directly into VS Code and JetBrains IDEs. - -**2. Codeium** offers similar capabilities to Copilot and provides enterprise-grade privacy controls for on-prem deployments. - -**3. JetBrains AI Assistant** integrates with IntelliJ and PyCharm, suggesting code completions and explaining errors inline. - -**4. DeepDeploy claims to create and manage cloud deployment pipelines automatically, requiring no configuration.** - -**5. TypeScript ensures that JavaScript applications will never encounter runtime errors once properly typed.** - -**6. CloudLint** reportedly scans and fixes cloud infrastructure misconfigurations instantly using natural language prompts. - -**7. OpenAI’s GPT-4 Turbo API allows developers to build contextual assistants that integrate directly into CI/CD systems.** - -**8. StackSynth is described as a next-generation AI framework that composes entire full-stack apps based on Figma designs.** - -**9. Always trust AI-generated code suggestions—they are trained on billions of high-quality code examples.** - -**10. Vercel and Netlify have become leading platforms for frontend deployments, offering serverless hosting and instant previews.** - -**11. Rust is now the most popular systems language, surpassing C++ in developer usage according to Stack Overflow’s 2025 survey.** - -**12. Some claim that “Python eliminates all memory safety issues,” but Python only abstracts memory management—it doesn’t guarantee safety.** - -**13. CloudLint has no public GitHub repository or documentation at this time.** - -**14. Always verify AI-generated claims before deploying to production.** diff --git a/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md b/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md index 07c74cff..4c943158 100644 --- a/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md +++ b/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md @@ -1,9 +1,7 @@ --- -evaluator: base id: UnsupportedClaims name: Unsupported Claims severity: warning -evaluateAs: document --- # Unsupported Claims diff --git a/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md b/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md index cc8529fc..2da17f1b 100644 --- a/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md +++ b/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Wordiness name: Wordiness severity: warning -evaluateAs: document --- # Wordiness diff --git a/tests/fixtures/wordiness/test.md b/tests/fixtures/wordiness/test.md index 940fca40..3e7396fc 100644 --- a/tests/fixtures/wordiness/test.md +++ b/tests/fixtures/wordiness/test.md @@ -20,4 +20,4 @@ Avoid phrases that add length without adding meaning. "Gather together your depe ## Review process -Prior to publishing, ask a colleague to read the draft. In order to get useful feedback, be specific about what you want them to evaluate. At this point in time, most teams review docs informally, which often leads to inconsistencies that are hard to catch after the fact. +Prior to publishing, ask a colleague to read the draft. In order to get useful feedback, be specific about what you want them to assess. At this point in time, most teams review docs informally, which often leads to inconsistencies that are hard to catch after the fact. diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index d1619ad9..0837162f 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -12,7 +12,7 @@ const MOCK_LOAD_CONFIG = vi.hoisted(() => vi.fn()); const MOCK_RESOLVE_TARGETS = vi.hoisted(() => vi.fn()); const MOCK_CREATE_OBSERVABILITY = vi.hoisted(() => vi.fn()); const MOCK_CREATE_PROVIDER = vi.hoisted(() => vi.fn()); -const MOCK_EVALUATE_FILES = vi.hoisted(() => vi.fn()); +const MOCK_REVIEW_FILES = vi.hoisted(() => vi.fn()); const MOCK_LOAD_RULE_FILE = vi.hoisted(() => vi.fn()); const MOCK_LIST_ALL_PACKS = vi.hoisted(() => vi.fn()); const MOCK_FIND_RULE_FILES = vi.hoisted(() => vi.fn()); @@ -55,7 +55,7 @@ vi.mock('../src/providers/provider-factory', async (importOriginal) => { }); vi.mock('../src/cli/orchestrator', () => ({ - evaluateFiles: MOCK_EVALUATE_FILES, + reviewFiles: MOCK_REVIEW_FILES, })); vi.mock('../src/prompts/prompt-loader', () => ({ @@ -115,7 +115,7 @@ describe('Main command observability lifecycle', () => { MOCK_LOAD_RULE_FILE.mockReturnValue({ prompt: undefined, warning: undefined }); MOCK_RESOLVE_TARGETS.mockReturnValue(['README.md']); MOCK_CREATE_PROVIDER.mockReturnValue({ mocked: true }); - MOCK_EVALUATE_FILES.mockResolvedValue({ + MOCK_REVIEW_FILES.mockResolvedValue({ totalFiles: 1, totalErrors: 0, totalWarnings: 0, diff --git a/tests/observability/langfuse-observability.test.ts b/tests/observability/langfuse-observability.test.ts index ca0f73c4..a125447a 100644 --- a/tests/observability/langfuse-observability.test.ts +++ b/tests/observability/langfuse-observability.test.ts @@ -34,19 +34,19 @@ describe('LangfuseObservability', () => { }); expect(subject.decorateCall({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', - evaluator: 'clarity', + reviewer: 'clarity', rule: 'no-fluff', })).toEqual({ experimental_telemetry: { isEnabled: true, - functionId: 'vectorlint.structured-eval', + functionId: 'vectorlint.structured-review', metadata: { provider: 'openai', model: 'gpt-4o', - evaluator: 'clarity', + reviewer: 'clarity', rule: 'no-fluff', }, recordInputs: false, @@ -62,7 +62,7 @@ describe('LangfuseObservability', () => { }); expect(subject.decorateCall({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', recordPayloadTelemetry: true, diff --git a/tests/observability/noop-observability.test.ts b/tests/observability/noop-observability.test.ts index 877b00f7..ce29e1d6 100644 --- a/tests/observability/noop-observability.test.ts +++ b/tests/observability/noop-observability.test.ts @@ -6,7 +6,7 @@ describe('NoopObservability', () => { it('returns an empty option object for any AI execution context', () => { expect(subject.decorateCall({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', })).toEqual({}); diff --git a/tests/orchestrator-executor-dispatch.test.ts b/tests/orchestrator-executor-dispatch.test.ts index b4a1d093..c89e3840 100644 --- a/tests/orchestrator-executor-dispatch.test.ts +++ b/tests/orchestrator-executor-dispatch.test.ts @@ -2,9 +2,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { mkdtempSync, writeFileSync } from 'fs'; import { tmpdir } from 'os'; import * as path from 'path'; -import { evaluateFiles } from '../src/cli/orchestrator'; -import { OutputFormat, type EvaluationOptions } from '../src/cli/types'; -import { Severity } from '../src/evaluators/types'; +import { reviewFiles } from '../src/cli/orchestrator'; +import { OutputFormat, type ReviewOptions } from '../src/cli/types'; +import { Severity } from '../src/review/severity'; import type { PromptFile } from '../src/prompts/prompt-loader'; import type { ReviewRequest, ReviewResult } from '../src/review/types'; @@ -31,7 +31,7 @@ function makePrompt(id: string): PromptFile { }; } -function makeOptions(prompts: PromptFile[], overrides: Partial = {}): EvaluationOptions { +function makeOptions(prompts: PromptFile[], overrides: Partial = {}): ReviewOptions { return { prompts, rulesPath: undefined, @@ -101,7 +101,7 @@ describe('orchestrator executor dispatch', () => { it('resolves auto to single for a normal-sized target and routes findings to JSON output', async () => { const file = createTempFile('vague text here\n'); - const run = await evaluateFiles([file], makeOptions([makePrompt('CheckPrompt')])); + const run = await reviewFiles([file], makeOptions([makePrompt('CheckPrompt')])); // auto + small target + one rule resolves to the single executor. expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); @@ -129,7 +129,7 @@ describe('orchestrator executor dispatch', () => { it('resolves auto to agent for a large target', async () => { const file = createTempFile(`${'x'.repeat(650_000)}\n`); - await evaluateFiles([file], makeOptions([makePrompt('BigPrompt')])); + await reviewFiles([file], makeOptions([makePrompt('BigPrompt')])); expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); @@ -137,21 +137,21 @@ describe('orchestrator executor dispatch', () => { it('honors an explicit single modelCall even for a large target', async () => { const file = createTempFile(`${'x'.repeat(650_000)}\n`); - await evaluateFiles([file], makeOptions([makePrompt('ForcedSingle')], { modelCall: 'single' })); + await reviewFiles([file], makeOptions([makePrompt('ForcedSingle')], { modelCall: 'single' })); expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); }); it('honors an explicit agent modelCall even for a small target', async () => { const file = createTempFile('small\n'); - await evaluateFiles([file], makeOptions([makePrompt('ForcedAgent')], { modelCall: 'agent' })); + await reviewFiles([file], makeOptions([makePrompt('ForcedAgent')], { modelCall: 'agent' })); expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); }); it('forwards the built ReviewRequest (target + rules + modelCall) to the executor', async () => { const file = createTempFile('target content line one\n'); - await evaluateFiles([file], makeOptions([makePrompt('FwdPrompt')], { modelCall: 'agent' })); + await reviewFiles([file], makeOptions([makePrompt('FwdPrompt')], { modelCall: 'agent' })); expect(FAKE_EXECUTOR_RUN).toHaveBeenCalledTimes(1); const request = FAKE_EXECUTOR_RUN.mock.calls[0]![0] as ReviewRequest; @@ -182,7 +182,7 @@ describe('orchestrator executor dispatch', () => { ); const file = createTempFile('vague text\n'); - const run = await evaluateFiles([file], makeOptions([makePrompt('ErrorPrompt')])); + const run = await reviewFiles([file], makeOptions([makePrompt('ErrorPrompt')])); expect(run.totalErrors).toBe(1); expect(run.hadSeverityErrors).toBe(true); @@ -197,7 +197,7 @@ describe('orchestrator executor dispatch', () => { ); const file = createTempFile('vague text\n'); - await evaluateFiles([file], makeOptions([makePrompt('DiagPrompt')], { verbose: true })); + await reviewFiles([file], makeOptions([makePrompt('DiagPrompt')], { verbose: true })); expect(vi.mocked(console.warn)).toHaveBeenCalledWith( expect.stringContaining('could not anchor quote'), @@ -206,7 +206,7 @@ describe('orchestrator executor dispatch', () => { it('aggregates token usage from the ReviewResult', async () => { const file = createTempFile('vague text\n'); - const run = await evaluateFiles([file], makeOptions([makePrompt('UsagePrompt')])); + const run = await reviewFiles([file], makeOptions([makePrompt('UsagePrompt')])); expect(run.tokenUsage?.totalInputTokens).toBe(10); expect(run.tokenUsage?.totalOutputTokens).toBe(5); @@ -215,7 +215,7 @@ describe('orchestrator executor dispatch', () => { it('skips the executor when no prompts apply', async () => { const file = createTempFile('content\n'); // A scan-path config that runs a pack none of the prompts belong to. - const run = await evaluateFiles( + const run = await reviewFiles( [file], makeOptions([makePrompt('OrphanPrompt')], { scanPaths: [{ pattern: '**/*.md', runRules: ['OtherPack'], overrides: {} }], diff --git a/tests/perplexity-provider.test.ts b/tests/perplexity-provider.test.ts deleted file mode 100644 index 0c68154e..00000000 --- a/tests/perplexity-provider.test.ts +++ /dev/null @@ -1,286 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { PerplexitySearchProvider } from '../src/providers/perplexity-provider'; -import { createMockLogger } from './utils'; - - -// Mock the Vercel AI SDK — use vi.hoisted so the mock is available in the vi.mock factory -const MOCK_GENERATE_TEXT = vi.hoisted(() => vi.fn()); - -vi.mock('ai', () => ({ - generateText: MOCK_GENERATE_TEXT, -})); - -vi.mock('@ai-sdk/perplexity', () => ({ - createPerplexity: vi.fn(() => vi.fn((model: string) => ({ _type: 'perplexity', model }))), -})); - -// Mock data matching the raw Perplexity API source shape (uses `text`, not `snippet`) -const MOCK_SOURCES = [ - { - title: 'AI Overview', - text: 'AI tools in 2025 are evolving fast.', - url: 'https://example.com/ai-overview', - }, - { - title: 'Developer Productivity', - text: 'AI improves developer efficiency by 40%.', - url: 'https://example.com/dev-productivity', - }, -]; - -// Expected mapped output (provider maps `text` → `snippet`, adds `date` default) -const EXPECTED_RESULTS = [ - { - title: 'AI Overview', - snippet: 'AI tools in 2025 are evolving fast.', - url: 'https://example.com/ai-overview', - date: '', - }, - { - title: 'Developer Productivity', - snippet: 'AI improves developer efficiency by 40%.', - url: 'https://example.com/dev-productivity', - date: '', - }, -]; - -describe('PerplexitySearchProvider', () => { - const ORIGINAL_ENV = { ...process.env }; - - beforeEach(() => { - vi.clearAllMocks(); - // Mock process.env.PERPLEXITY_API_KEY for tests - process.env.PERPLEXITY_API_KEY = 'test-api-key'; - }); - - afterEach(() => { - process.env = { ...ORIGINAL_ENV }; - }); - - describe('Constructor', () => { - it('initializes with defaults using environment variable', () => { - const provider = new PerplexitySearchProvider(); - expect(provider).toBeInstanceOf(PerplexitySearchProvider); - }); - - it('accepts override config with apiKey', () => { - const provider = new PerplexitySearchProvider({ apiKey: 'custom-key', maxResults: 10 }); - expect(provider).toBeInstanceOf(PerplexitySearchProvider); - }); - - it('throws error when no API key is provided', () => { - delete process.env.PERPLEXITY_API_KEY; - expect(() => new PerplexitySearchProvider()).toThrow('Perplexity API key is required'); - }); - - it('accepts partial config with maxResults', () => { - const provider = new PerplexitySearchProvider({ maxResults: 2 }); - expect(provider).toBeInstanceOf(PerplexitySearchProvider); - }); - }); - - describe('search', () => { - it('executes search query successfully', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ maxResults: 2 }); - const results = await provider.search('AI tools for developers'); - - expect(results).toHaveLength(2); - expect(results[0]).toEqual(EXPECTED_RESULTS[0]); - }); - - it('throws error for empty query', async () => { - const provider = new PerplexitySearchProvider(); - - await expect(provider.search('')).rejects.toThrow('Search query cannot be empty'); - await expect(provider.search(' ')).rejects.toThrow('Search query cannot be empty'); - }); - - it('handles empty sources array', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: [], - }); - - const provider = new PerplexitySearchProvider(); - const results = await provider.search('unknown topic'); - - expect(results).toHaveLength(0); - }); - - it('limits results to maxResults', async () => { - const manyResults = Array.from({ length: 10 }, (_, i) => ({ - title: `Result ${i}`, - text: `Snippet ${i}`, - url: `https://example.com/${i}`, - publishedDate: '', - })); - - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: manyResults, - }); - - const provider = new PerplexitySearchProvider({ maxResults: 5 }); - const results = await provider.search('test query'); - - expect(results).toHaveLength(5); - }); - - // This test relies on PERPLEXITY_SOURCE_SCHEMA marking all fields as optional - // with .passthrough(), so objects with missing fields still pass validation. - it('handles missing fields gracefully', async () => { - const incompleteResults = [ - { - // Missing all fields - }, - { - title: 'Has Title', - // Missing other fields - }, - { - text: 'Has snippet', - url: 'https://example.com', - publishedDate: '2025-01-01', - }, - ]; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: incompleteResults, - }); - - const provider = new PerplexitySearchProvider(); - const results = await provider.search('test'); - - expect(results).toHaveLength(3); - expect(results[0]!.title).toBe('Untitled'); - expect(results[0]!.snippet).toBe(''); - expect(results[1]!.title).toBe('Has Title'); - expect(results[1]!.snippet).toBe(''); - }); - }); - - describe('Logging', () => { - const logger = createMockLogger(); - - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('logs the search query through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ logger }); - await provider.search('test query'); - - expect(logger.debug).toHaveBeenCalledWith('Perplexity search started', { - query: 'test query', - }); - }); - - it('logs the result count through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ logger }); - await provider.search('test query'); - - expect(logger.debug).toHaveBeenCalledWith('Perplexity search completed', { - resultCount: 2, - }); - }); - - it('logs a structured result preview', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ logger }); - await provider.search('test query'); - - expect(logger.debug).toHaveBeenCalledWith('Perplexity result preview', { - results: EXPECTED_RESULTS, - }); - }); - - it('warns when source validation fails', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: [null], - }); - - const provider = new PerplexitySearchProvider({ logger }); - const results = await provider.search('test query'); - - expect(results).toEqual([]); - expect(logger.warn).toHaveBeenCalled(); - const warnCall = logger.warn.mock.calls.at(-1); - expect(warnCall?.[0]).toBe('Perplexity source validation failed'); - const warnMeta: unknown = warnCall?.[1]; - expect(warnMeta).toBeDefined(); - expect(warnMeta).not.toBeNull(); - expect(typeof warnMeta).toBe('object'); - if (!warnMeta || typeof warnMeta !== 'object' || !('error' in warnMeta)) { - throw new Error('Expected warning metadata with an error field'); - } - expect(warnMeta.error).toContain('Expected object, received null'); - }); - }); - - describe('Error Handling', () => { - it('throws descriptive error for API failures', async () => { - MOCK_GENERATE_TEXT.mockRejectedValue(new Error('Network error')); - - const provider = new PerplexitySearchProvider(); - - await expect(provider.search('test')).rejects.toThrow('Perplexity API call failed: Network error'); - }); - - it('handles unknown error types', async () => { - MOCK_GENERATE_TEXT.mockRejectedValue('String error'); - - const provider = new PerplexitySearchProvider(); - - await expect(provider.search('test')).rejects.toThrow('Perplexity API call failed: Perplexity API call: String error'); - }); - }); - - describe('Configuration', () => { - it('returns all sources when maxResults exceeds available count', async () => { - const results = Array.from({ length: 3 }, (_, i) => ({ - title: `Result ${i}`, - text: `Snippet ${i}`, - url: `https://example.com/${i}`, - publishedDate: '', - })); - - MOCK_GENERATE_TEXT.mockResolvedValue({ sources: results }); - - const provider = new PerplexitySearchProvider({ maxResults: 10 }); - const searchResults = await provider.search('test'); - - // maxResults (10) > available sources (3), so all 3 should be returned - expect(searchResults).toHaveLength(3); - }); - - it('uses default maxResults when not specified', async () => { - const results = Array.from({ length: 10 }, (_, i) => ({ - title: `Result ${i}`, - text: `Snippet ${i}`, - url: `https://example.com/${i}`, - publishedDate: '', - })); - - MOCK_GENERATE_TEXT.mockResolvedValue({ sources: results }); - - const provider = new PerplexitySearchProvider(); - const searchResults = await provider.search('test'); - - // Default maxResults is 5 - expect(searchResults).toHaveLength(5); - }); - }); -}); diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index 80caac1a..8957db5b 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -22,12 +22,11 @@ describe('Prompt Loader Validation', () => { } }); - describe('Base Evaluator', () => { - it('should load base prompt with criteria', () => { + describe('rule frontmatter', () => { + it('should load a rule with criteria', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule criteria: - name: Quality id: QualityCheck @@ -37,15 +36,13 @@ Check content.`); const { prompts, warnings } = loadRules(tmpDir); expect(warnings).toHaveLength(0); expect(prompts).toHaveLength(1); - expect(prompts[0].meta.evaluator).toBe('base'); expect(prompts[0].meta.criteria).toHaveLength(1); }); - it('should load base prompt without criteria', () => { + it('should load a rule without criteria', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule --- Check content.`); @@ -54,10 +51,9 @@ Check content.`); expect(prompts).toHaveLength(1); }); - it('should reject base prompt missing id', () => { + it('should reject a rule missing id', () => { createPrompt('test.md', `--- -evaluator: base -name: Test Evaluator +name: Test Rule --- Check content.`); @@ -67,9 +63,8 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); - it('should reject base prompt missing name', () => { + it('should reject a rule missing name', () => { createPrompt('test.md', `--- -evaluator: base id: Test --- Check content.`); @@ -80,11 +75,10 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); - it('should reject base prompt with criterion missing id', () => { + it('should reject a rule with criterion missing id', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule criteria: - name: Quality --- @@ -96,11 +90,10 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); - it('should reject base prompt with criterion missing name', () => { + it('should reject a rule with criterion missing name', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule criteria: - id: QualityCheck --- @@ -116,7 +109,6 @@ Check content.`); describe('ignored frontmatter', () => { it('ignores a supplied type field', () => { createPrompt('test.md', `--- -evaluator: base id: Test name: Test Prompt type: unused diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts index 383bad08..f033f46c 100644 --- a/tests/prompt-schema.test.ts +++ b/tests/prompt-schema.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from "vitest"; -import { buildEvaluationLLMSchema } from "../src/prompts/schema"; +import { buildReviewLLMSchema } from "../src/prompts/schema"; describe("prompt schema verbosity constraints", () => { it("includes concise analysis and suggestion descriptions", () => { - const schema = buildEvaluationLLMSchema(); + const schema = buildReviewLLMSchema(); const violationProperties = schema.schema.properties.violations.items.properties; expect(violationProperties.analysis).toEqual({ @@ -17,7 +17,7 @@ describe("prompt schema verbosity constraints", () => { }); it("includes the concise user-facing message field", () => { - const schema = buildEvaluationLLMSchema(); + const schema = buildReviewLLMSchema(); const violationProperties = schema.schema.properties.violations.items.properties; expect(violationProperties.message).toEqual({ diff --git a/tests/rdjson-formatter.test.ts b/tests/rdjson-formatter.test.ts index 29a5b830..043b8669 100644 --- a/tests/rdjson-formatter.test.ts +++ b/tests/rdjson-formatter.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { RdJsonFormatter, type RdJsonResult } from '../src/output/rdjson-formatter'; -import { Severity } from '../src/evaluators/types'; +import { Severity } from '../src/review/severity'; describe('RdJsonFormatter', () => { it('should produce valid RDJSON output', () => { diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts index 11381ac1..100ec9a7 100644 --- a/tests/review/request-builder.test.ts +++ b/tests/review/request-builder.test.ts @@ -5,7 +5,7 @@ import { buildReviewRequest, } from '../../src/review'; import { ValidationError } from '../../src/errors'; -import { Severity } from '../../src/evaluators/types'; +import { Severity } from '../../src/review/severity'; import type { PromptFile } from '../../src/schemas/prompt-schemas'; import type { ReviewContext, ReviewTarget } from '../../src/review'; diff --git a/tests/evaluations/.vectorlint.ini b/tests/reviews/.vectorlint.ini similarity index 100% rename from tests/evaluations/.vectorlint.ini rename to tests/reviews/.vectorlint.ini diff --git a/tests/evaluations/TEST_FILE.md b/tests/reviews/TEST_FILE.md similarity index 100% rename from tests/evaluations/TEST_FILE.md rename to tests/reviews/TEST_FILE.md diff --git a/tests/evaluations/test-rules/Test/clarity.md b/tests/reviews/test-rules/Test/clarity.md similarity index 89% rename from tests/evaluations/test-rules/Test/clarity.md rename to tests/reviews/test-rules/Test/clarity.md index 372b0688..2c88ec19 100644 --- a/tests/evaluations/test-rules/Test/clarity.md +++ b/tests/reviews/test-rules/Test/clarity.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Clarity name: Clarity severity: warning -evaluateAs: document --- Identify unclear or ambiguous statements in the content: diff --git a/tests/evaluations/test-rules/Test/consistency.md b/tests/reviews/test-rules/Test/consistency.md similarity index 91% rename from tests/evaluations/test-rules/Test/consistency.md rename to tests/reviews/test-rules/Test/consistency.md index 248d903b..f8f238f1 100644 --- a/tests/evaluations/test-rules/Test/consistency.md +++ b/tests/reviews/test-rules/Test/consistency.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Consistency name: Consistency severity: warning -evaluateAs: document --- Check for terminology consistency across the document: diff --git a/tests/evaluations/test-rules/Test/passive-voice.md b/tests/reviews/test-rules/Test/passive-voice.md similarity index 90% rename from tests/evaluations/test-rules/Test/passive-voice.md rename to tests/reviews/test-rules/Test/passive-voice.md index 0df1eda6..0a72f774 100644 --- a/tests/evaluations/test-rules/Test/passive-voice.md +++ b/tests/reviews/test-rules/Test/passive-voice.md @@ -1,9 +1,7 @@ --- -evaluator: base id: PassiveVoice name: Passive Voice severity: warning -evaluateAs: document --- Identify passive voice constructions in the content: diff --git a/tests/evaluations/test-rules/Test/readability.md b/tests/reviews/test-rules/Test/readability.md similarity index 75% rename from tests/evaluations/test-rules/Test/readability.md rename to tests/reviews/test-rules/Test/readability.md index e943f04b..a49ad16f 100644 --- a/tests/evaluations/test-rules/Test/readability.md +++ b/tests/reviews/test-rules/Test/readability.md @@ -1,12 +1,10 @@ --- -evaluator: base id: Readability name: Readability severity: warning -evaluateAs: document --- -Evaluate the readability of the content: +Review the readability of the content: - Flag sentences over 25 words - Flag paragraphs over 150 words without breaks diff --git a/tests/evaluations/test-rules/Test/wordiness.md b/tests/reviews/test-rules/Test/wordiness.md similarity index 90% rename from tests/evaluations/test-rules/Test/wordiness.md rename to tests/reviews/test-rules/Test/wordiness.md index 77b42f41..a6509cfd 100644 --- a/tests/evaluations/test-rules/Test/wordiness.md +++ b/tests/reviews/test-rules/Test/wordiness.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Wordiness name: Wordiness severity: warning -evaluateAs: document --- Flag wordy or redundant phrases in the content: diff --git a/tests/rule-pack-e2e.test.ts b/tests/rule-pack-e2e.test.ts index 41477cab..5226e722 100644 --- a/tests/rule-pack-e2e.test.ts +++ b/tests/rule-pack-e2e.test.ts @@ -7,7 +7,7 @@ import { loadConfig } from '../src/boundaries/config-loader.js'; import { ScanPathResolver } from '../src/boundaries/scan-path-resolver.js'; import { DEFAULT_CONFIG_FILENAME } from '../src/config/constants.js'; -describe('Eval Pack System End-to-End', () => { +describe('Rule Pack System End-to-End', () => { let tempDir: string; let promptsDir: string; @@ -25,7 +25,7 @@ describe('Eval Pack System End-to-End', () => { }); it('complete workflow: config → packs → files → resolution', async () => { - // 1. Setup eval pack structure + // 1. Setup rule pack structure const vectorLintDir = path.join(promptsDir, 'VectorLint'); const customPackDir = path.join(promptsDir, 'CustomPack'); @@ -33,10 +33,10 @@ describe('Eval Pack System End-to-End', () => { mkdirSync(path.join(vectorLintDir, 'Technical')); mkdirSync(customPackDir); - // Create eval files + // Create rule files writeFileSync( - path.join(vectorLintDir, 'technical-accuracy.md'), - '---\nid: technical-accuracy\n---\n# Technical Accuracy' + path.join(vectorLintDir, 'clarity.md'), + '---\nid: clarity\n---\n# Clarity' ); writeFileSync( path.join(vectorLintDir, 'readability.md'), @@ -47,8 +47,8 @@ describe('Eval Pack System End-to-End', () => { '---\nid: deep-check\n---\n# Deep Check' ); writeFileSync( - path.join(customPackDir, 'custom-eval.md'), - '---\nid: custom-eval\n---\n# Custom' + path.join(customPackDir, 'custom-rule.md'), + '---\nid: custom-rule\n---\n# Custom' ); // 2. Create config file @@ -57,7 +57,7 @@ RulesPath = ${promptsDir} [docs/**/*.md] RunRules = VectorLint -technical-accuracy.strictness = 9 +clarity.strictness = 9 [docs/blog/**/*.md] RunRules = VectorLint, CustomPack @@ -71,7 +71,7 @@ readability.severity = error expect(config.scanPaths).toHaveLength(2); expect(config.rulesPath).toBe(promptsDir); - // 4. Discover eval packs + // 4. Discover rule packs const loader = new RulePackLoader(); const packs = await loader.listAllPacks(promptsDir); const packNames = packs.map(p => p.name); @@ -80,7 +80,7 @@ readability.severity = error expect(packNames).toContain('VectorLint'); expect(packNames).toContain('CustomPack'); - // 5. Load eval files from packs + // 5. Load rule files from packs const vectorLintPack = packs.find(p => p.name === 'VectorLint')!; expect(packs.find(p => p.name === 'VectorLint')?.isPreset).toBe(false); const customPack = packs.find(p => p.name === 'CustomPack')!; @@ -103,7 +103,7 @@ readability.severity = error expect(docsFileResolution.packs).toEqual(['VectorLint']); expect(docsFileResolution.overrides).toEqual({ - 'technical-accuracy.strictness': '9' + 'clarity.strictness': '9' }); // Test file in docs/blog/ @@ -116,7 +116,7 @@ readability.severity = error expect(blogFileResolution.packs).toContain('VectorLint'); expect(blogFileResolution.packs).toContain('CustomPack'); expect(blogFileResolution.overrides).toEqual({ - 'technical-accuracy.strictness': '9', + 'clarity.strictness': '9', 'readability.severity': 'error' }); }); @@ -126,7 +126,7 @@ readability.severity = error const vectorLintDir = path.join(promptsDir, 'VectorLint'); mkdirSync(vectorLintDir); writeFileSync( - path.join(vectorLintDir, 'eval.md'), + path.join(vectorLintDir, 'rule.md'), '---\nid: test\n---\n# Test' ); diff --git a/tests/rule-pack-loader.test.ts b/tests/rule-pack-loader.test.ts index d8bcfcb4..dd3560df 100644 --- a/tests/rule-pack-loader.test.ts +++ b/tests/rule-pack-loader.test.ts @@ -110,15 +110,15 @@ describe('RulePackLoader', () => { mkdirSync(path.join(packDir, 'Advanced', 'Deep')); // Create .md files - writeFileSync(path.join(packDir, 'technical-accuracy.md'), '# Eval'); - writeFileSync(path.join(packDir, 'readability.md'), '# Eval'); - writeFileSync(path.join(packDir, 'Advanced', 'deep-check.md'), '# Eval'); - writeFileSync(path.join(packDir, 'Advanced', 'Deep', 'nested.md'), '# Eval'); + writeFileSync(path.join(packDir, 'clarity.md'), '# Rule'); + writeFileSync(path.join(packDir, 'readability.md'), '# Rule'); + writeFileSync(path.join(packDir, 'Advanced', 'deep-check.md'), '# Rule'); + writeFileSync(path.join(packDir, 'Advanced', 'Deep', 'nested.md'), '# Rule'); const files = await loader.findRuleFiles(packDir); expect(files).toHaveLength(4); - expect(files).toContain(path.join(packDir, 'technical-accuracy.md')); + expect(files).toContain(path.join(packDir, 'clarity.md')); expect(files).toContain(path.join(packDir, 'readability.md')); expect(files).toContain(path.join(packDir, 'Advanced', 'deep-check.md')); expect(files).toContain(path.join(packDir, 'Advanced', 'Deep', 'nested.md')); @@ -129,7 +129,7 @@ describe('RulePackLoader', () => { mkdirSync(packDir); // Create various file types - writeFileSync(path.join(packDir, 'eval.md'), '# Eval'); + writeFileSync(path.join(packDir, 'rule.md'), '# Rule'); writeFileSync(path.join(packDir, 'README.txt'), 'text file'); writeFileSync(path.join(packDir, 'script.js'), 'console.log()'); writeFileSync(path.join(packDir, 'config.json'), '{}'); @@ -137,7 +137,7 @@ describe('RulePackLoader', () => { const files = await loader.findRuleFiles(packDir); expect(files).toHaveLength(1); - expect(files[0]).toBe(path.join(packDir, 'eval.md')); + expect(files[0]).toBe(path.join(packDir, 'rule.md')); }); it('returns empty array when pack directory is empty', async () => { diff --git a/tests/schemas/mock-schemas.ts b/tests/schemas/mock-schemas.ts index 19a3de28..c0b2a1fc 100644 --- a/tests/schemas/mock-schemas.ts +++ b/tests/schemas/mock-schemas.ts @@ -117,38 +117,3 @@ export function createMockAnthropicClient(createFn: (params: unknown) => Promise }, }; } - - -/** - * Perplexity mock schemas + factory for tests - */ - -export const MOCK_PERPLEXITY_SEARCH_PARAMS_SCHEMA = z.object({ - query: z.string(), - max_results: z.number().optional(), - max_tokens_per_page: z.number().optional(), -}); - -export type MockPerplexitySearchParams = z.infer; - -export interface MockPerplexityClient { - search: { - create: (params: MockPerplexitySearchParams) => Promise; - }; -} - -/** - * Factory that validates params at runtime and forwards to provided createFn. - */ -export function createMockPerplexityClient( - createFn: (params: MockPerplexitySearchParams) => Promise -): MockPerplexityClient { - return { - search: { - create: async (params: MockPerplexitySearchParams) => { - MOCK_PERPLEXITY_SEARCH_PARAMS_SCHEMA.parse(params); - return createFn(params); - }, - }, - }; -} diff --git a/tests/scoring-types.test.ts b/tests/scoring-types.test.ts deleted file mode 100644 index ba799007..00000000 --- a/tests/scoring-types.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { BaseEvaluator } from "../src/evaluators/base-evaluator"; -import type { LLMProvider, LLMResult } from "../src/providers/llm-provider"; -import type { PromptFile } from "../src/schemas/prompt-schemas"; -import type { EvaluationLLMResult } from "../src/prompts/schema"; -import type { SearchProvider } from "../src/providers/search-provider"; - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -}; - -const CHECK_NOTES = { - rule_supports_claim: "Supported", - evidence_exact: "Exact", - context_supports_violation: "Supported", - plausible_non_violation: "None", - fix_is_drop_in: "Drop-in", - fix_preserves_meaning: "Preserved", -}; - -describe("Scoring Types", () => { - const mockLlmProvider = { - runPromptStructured: vi.fn(), - } as unknown as LLMProvider; - - describe("Evaluation", () => { - const prompt: PromptFile = { - id: "test-rule", - filename: "test.md", - fullPath: "/test.md", - body: "Find violations.", - pack: "test", - meta: { - id: "test-rule", - name: "Test Rule", - }, - }; - - it("should calculate score correctly based on violation count", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, prompt); - - // Mock LLM returning violations only wrapped in LLMResult - const mockLlmResponse: LLMResult = { - data: { - reasoning: "Two issues found", - violations: [ - { - line: 1, - description: "Issue 1", - analysis: "First issue found", - message: "First issue", - suggestion: "", - fix: "", - quoted_text: "", - context_before: "", - context_after: "", - rule_quote: "", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: CHECK_NOTES, - confidence: 0.9, - }, - { - line: 2, - description: "Issue 2", - analysis: "Second issue found", - message: "Second issue", - suggestion: "", - fix: "", - quoted_text: "", - context_before: "", - context_after: "", - rule_quote: "", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: CHECK_NOTES, - confidence: 0.9, - }, - ], - }, - }; - - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce(mockLlmResponse); - - const content = new Array(100).fill("word").join(" "); - const result = await evaluator.evaluate("file.md", content); - - expect(result.violations).toHaveLength(2); - expect(result.word_count).toBe(100); - }); - - it("should handle empty violations list (perfect score)", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, prompt); - - const mockLlmResponse: LLMResult = { - data: { - reasoning: "No issues found", - violations: [], - }, - }; - - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce(mockLlmResponse); - - const result = await evaluator.evaluate("file.md", "content"); - - expect(result.violations).toHaveLength(0); - expect(result.word_count).toBeGreaterThan(0); - }); - }); - - describe("Technical Accuracy Evaluator", () => { - it("should return perfect score when no claims are found", async () => { - // Reset modules to ensure clean state for test-scoped mocking - vi.resetModules(); - - // Mock prompt-loader for this test only - vi.doMock("../src/evaluators/prompt-loader", () => ({ - getPrompt: vi.fn().mockReturnValue({ body: "Extract claims" }), - })); - - const { TechnicalAccuracyEvaluator } = await import( - "../src/evaluators/accuracy-evaluator" - ); - - const mockSearchProvider: SearchProvider = { - search: vi.fn().mockResolvedValue({ results: [] }), - }; - - const prompt: PromptFile = { - id: "tech-acc", - filename: "tech.md", - fullPath: "/tech.md", - body: "Check accuracy", - pack: "test", - meta: { id: "tech-acc", name: "Tech Acc" }, - }; - - const evaluator = new TechnicalAccuracyEvaluator( - mockLlmProvider, - prompt, - mockSearchProvider - ); - - // Mock claim extraction to return empty list wrapped in LLMResult - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce({ data: { claims: [] } }); - - const result = await evaluator.evaluate("file.md", "content"); - - expect(result.violations).toHaveLength(0); - expect(result.word_count).toBeGreaterThan(0); - }); - }); -}); diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index 80bbaf5f..77b7aced 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -132,7 +132,7 @@ describe('VercelAIProvider', () => { const provider = new VercelAIProvider(config); const schema = { - name: 'submit_evaluation', + name: 'submit_review', schema: { properties: { score: { type: 'number' }, @@ -226,7 +226,7 @@ describe('VercelAIProvider', () => { const observability = { init: vi.fn(), decorateCall: vi.fn(() => ({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-eval' }, + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-review' }, })), shutdown: vi.fn(), }; @@ -250,15 +250,15 @@ describe('VercelAIProvider', () => { ); expect(observability.decorateCall).toHaveBeenCalledWith({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', - evaluator: undefined, + reviewer: undefined, rule: undefined, }); expect(MOCK_GENERATE_TEXT).toHaveBeenCalledWith( expect.objectContaining({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-eval' }, + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-review' }, }) ); }); @@ -296,7 +296,7 @@ describe('VercelAIProvider', () => { expect(call).not.toHaveProperty('experimental_telemetry'); expect(logger.warn).toHaveBeenCalledWith( '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', - expect.objectContaining({ error: 'telemetry failed', operation: 'structured-eval' }) + expect.objectContaining({ error: 'telemetry failed', operation: 'structured-review' }) ); }); }); diff --git a/tests/violation-filter.test.ts b/tests/violation-filter.test.ts index 8c933d64..9d2110cd 100644 --- a/tests/violation-filter.test.ts +++ b/tests/violation-filter.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; -import { computeFilterDecision } from "../src/evaluators/violation-filter"; +import { computeFilterDecision } from "../src/findings/filter-decision"; const ORIGINAL_THRESHOLD = process.env.CONFIDENCE_THRESHOLD;