From 3c04c62bd7449bed25d42f62035afab878e87f66 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:39:07 +0100 Subject: [PATCH 01/34] Add typecheck script and clear tsc --noEmit baseline - Add npm run typecheck (tsc --noEmit) as the source-of-truth type gate - Narrow exactOptionalPropertyTypes gaps via conditional spreads and optional-with-undefined fields across agent executor, orchestrator, observability, providers, and scorer - Make CheckItem.line/description optional to match runtime reality (reported items carry optional fields; output formatters use a separate Issue type, so display is unaffected) - Confine @ai-sdk/perplexity LanguageModelV1 -> LanguageModel skew to a single cast; ai@6 dropped V1 types and a provider upgrade is a separate, out-of-scope dependency change - No strict compiler options relaxed; src/agent/* left compiling only --- package.json | 1 + src/agent/executor.ts | 36 ++++++++++----------- src/cli/orchestrator.ts | 2 +- src/evaluators/violation-filter.ts | 4 +-- src/observability/factory.ts | 2 +- src/observability/langfuse-observability.ts | 4 +-- src/prompts/schema.ts | 4 +-- src/providers/perplexity-provider.ts | 7 +++- src/providers/vercel-ai-provider.ts | 22 +++++++------ src/scoring/scorer.ts | 5 ++- 10 files changed, 49 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 74f474d5..d9e16b86 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "tsx src/index.ts", "build": "tsup", + "typecheck": "tsc --noEmit", "start": "node dist/index.js", "lint": "eslint .", "lint:fix": "eslint . --fix", diff --git a/src/agent/executor.ts b/src/agent/executor.ts index 432c6d64..749a5685 100644 --- a/src/agent/executor.ts +++ b/src/agent/executor.ts @@ -1,7 +1,7 @@ import { readdir, readFile } from 'fs/promises'; import * as os from 'os'; import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type PromptEvaluationResult } from '../prompts/schema'; +import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; import type { PromptFile } from '../prompts/prompt-loader'; import { Type, Severity } from '../evaluators/types'; import { computeFilterDecision } from '../evaluators/violation-filter'; @@ -14,6 +14,7 @@ import { createReviewSessionStore } from './review-session-store'; import { buildAgentSystemPrompt } from './prompt-builder'; import { AgentToolError } from '../errors'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; +import type { FilePatternConfig } from '../boundaries/file-section-parser'; import { createAgentTools, listAvailableTools, @@ -53,7 +54,7 @@ export interface RunAgentExecutorParams { prompts: PromptFile[]; provider: LLMProvider; workspaceRoot: string; - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }>; + scanPaths: FilePatternConfig[]; outputFormat: OutputFormat; printMode: boolean; sessionHomeDir?: string; @@ -178,7 +179,7 @@ function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { function buildFileRuleMatches( relativeTargets: string[], prompts: PromptFile[], - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }> + scanPaths: FilePatternConfig[] ): Array<{ file: string; ruleSource: string }> { const resolver = new ScanPathResolver(); const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); @@ -301,12 +302,13 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'read_file', path, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `read_file:${path}`, }; } @@ -317,10 +319,11 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = currentProgressFile ?? defaultProgressFile; return { toolName: 'list_directory', path, - progressFile: currentProgressFile ?? defaultProgressFile, + ...(progressFile ? { progressFile } : {}), signature: `list_directory:${path}`, }; } @@ -333,14 +336,15 @@ function resolveVisibleToolContext(params: { const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); const path = resolvedPath ?? parsed.data.file; const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'lint', path, ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), ruleText, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, }; } @@ -390,11 +394,7 @@ type FindingLikeViolation = { suggestion?: string; fix?: string; confidence?: number; - checks?: { - plausible_non_violation?: boolean; - context_supports_violation?: boolean; - rule_supports_claim?: boolean; - }; + checks?: GateChecks; }; async function appendInlineFinding(params: { @@ -516,8 +516,8 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, promptBySource, targetFiles, - currentProgressFile, - defaultProgressFile: relativeTargets[0], + ...(currentProgressFile ? { currentProgressFile } : {}), + ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), }); if (visibleToolContext?.progressFile) { @@ -798,7 +798,7 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, fileRuleMatches, availableTools, - userInstructions, + ...(userInstructions ? { userInstructions } : {}), }), prompt: [ `Workspace root: ${workspaceRoot}`, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f4325095..6ae4acd4 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1194,7 +1194,7 @@ async function evaluateFilesInAgentMode( progressReporter, maxParallelToolCalls: 3, maxRetries: options.agentMaxRetries ?? 10, - userInstructions: options.userInstructionContent, + ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), }); let totalErrors = 0; diff --git a/src/evaluators/violation-filter.ts b/src/evaluators/violation-filter.ts index 2d3fc295..c4daa9b5 100644 --- a/src/evaluators/violation-filter.ts +++ b/src/evaluators/violation-filter.ts @@ -56,8 +56,8 @@ export function computeFilterDecision(v: GateViolationLike): FilterDecision { const hasFix = (v.fix ?? "").trim() !== ""; - const hasConfidence = typeof v.confidence === "number"; - const passesConfidence = hasConfidence && v.confidence >= confidenceThreshold; + const confidence = typeof v.confidence === "number" ? v.confidence : undefined; + const passesConfidence = confidence !== undefined && confidence >= confidenceThreshold; if (!passesConfidence) reasons.push(`confidence<${confidenceThreshold}`); const surface = diff --git a/src/observability/factory.ts b/src/observability/factory.ts index 84ec671a..c646fc08 100644 --- a/src/observability/factory.ts +++ b/src/observability/factory.ts @@ -19,6 +19,6 @@ export function createObservability(env: EnvConfig, logger?: Logger): AIObservab publicKey: env.LANGFUSE_PUBLIC_KEY, secretKey: env.LANGFUSE_SECRET_KEY, ...(env.LANGFUSE_BASE_URL ? { baseUrl: env.LANGFUSE_BASE_URL } : {}), - logger, + ...(logger ? { logger } : {}), }); } diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index b8898127..25f4d32c 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -12,8 +12,8 @@ export interface LangfuseObservabilityConfig { } export class LangfuseObservability implements AIObservability { - private sdk?: NodeSDK; - private initPromise?: Promise; + private sdk?: NodeSDK | undefined; + private initPromise?: Promise | undefined; private readonly logger: Logger; constructor(private readonly config: LangfuseObservabilityConfig) { diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 26c137e1..bd9fd9ee 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -337,8 +337,8 @@ export type JudgeResult = { }; export type CheckItem = { - line: number; - description: string; + line?: number; + description?: string; analysis: string; message?: string; suggestion?: string; diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts index c6a05499..432f8c09 100644 --- a/src/providers/perplexity-provider.ts +++ b/src/providers/perplexity-provider.ts @@ -1,4 +1,5 @@ import { generateText } from 'ai'; +import type { LanguageModel } from 'ai'; import { z } from 'zod'; import { createPerplexity } from '@ai-sdk/perplexity'; import type { SearchProvider } from './search-provider'; @@ -46,7 +47,11 @@ export class PerplexitySearchProvider implements SearchProvider { try { const result = await generateText({ - model: this.client('sonar-pro'), + // @ai-sdk/perplexity@1 exposes LanguageModelV1 models, while ai@6's + // generateText types require a LanguageModel (V2/V3). This is a + // third-party SDK version skew, not a repo type hole; resolve by + // upgrading @ai-sdk/perplexity to a V2 release in a follow-up. + model: this.client('sonar-pro') as unknown as LanguageModel, prompt: query, }); diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 689a19c0..5cd833fd 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -25,7 +25,7 @@ export class VercelAIProvider implements LLMProvider { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; - private observability?: AIObservability; + private observability?: AIObservability | undefined; constructor(config: VercelAIConfig, builder?: RequestBuilder) { this.config = { @@ -73,12 +73,14 @@ export class VercelAIProvider implements LLMProvider { } try { + const evaluator = this.extractContextValue(context, 'evaluatorName', 'evaluator'); + const rule = this.extractContextValue(context, 'ruleName', 'rule'); const observabilityOptions = this.getObservabilityOptions({ operation: 'structured-eval', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', - evaluator: this.extractContextValue(context, 'evaluatorName', 'evaluator'), - rule: this.extractContextValue(context, 'ruleName', 'rule'), + ...(evaluator ? { evaluator } : {}), + ...(rule ? { rule } : {}), }); const result = await generateText({ @@ -187,14 +189,14 @@ export class VercelAIProvider implements LLMProvider { } } - return { - usage: result.usage - ? { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, + return result.usage + ? { + usage: { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + }, } - : undefined, - }; + : {}; } private getObservabilityOptions(context: AIExecutionContext): Record { diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index ecea0db6..24f0ced5 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -48,7 +48,10 @@ export function calculateCheckScore( ): CheckResult { const strictness = resolveStrictness(options.strictness); - const mappedViolations = violations.map((item) => ({ ...item, criterionName: item.description })); + const mappedViolations = violations.map((item) => ({ + ...item, + ...(item.description !== undefined ? { criterionName: item.description } : {}), + })); // Density Calculation: Violations per 100 words const density = (mappedViolations.length / wordCount) * 100; From e319bba09fb1e07af044a55161da89ebb6510313 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:40:19 +0100 Subject: [PATCH 02/34] Exclude docs/research from eslint - Add docs/research/** to eslint ignores; the directory holds local, untracked throwaway research scripts that are not part of the shipped package and should not be type-checked by lint --- eslint.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 8549cd13..4bec57ff 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -10,7 +10,7 @@ import unicorn from "eslint-plugin-unicorn"; export default defineConfig([ // Ignored + housekeeping - { ignores: ["node_modules", "coverage", "dist", "build"] }, + { ignores: ["node_modules", "coverage", "dist", "build", "docs/research/**"] }, { linterOptions: { reportUnusedDisableDirectives: true } }, // Make resolver settings global From 20e0acc8204f1b67858ab7b31bf3f5b023c2799a Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:25 +0100 Subject: [PATCH 03/34] Add vitest.config.ts for stable test resolution - Project had no vitest config; vitest ran on pure defaults - Pin test discovery to tests/**/*.test.ts and inline ora, @langfuse/otel, and @opentelemetry/sdk-node so suites that transitively import agent/observability modules resolve reliably - No change to the 45 suites / 323 tests baseline --- vitest.config.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 vitest.config.ts diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 00000000..b5f7a7b4 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // Tests live under tests/ and follow the *.test.ts convention. + include: ['tests/**/*.test.ts'], + // Inline heavy/ESM-only deps so suites that transitively import the + // agent (ora) or observability (@langfuse/otel, @opentelemetry/sdk-node) + // modules resolve reliably without relying on scattered per-file mocks. + server: { + deps: { + inline: ['ora', '@langfuse/otel', '@opentelemetry/sdk-node'], + }, + }, + }, +}); From 3580e4971c2e6cfa8ce2308d350803c58d43ddba Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:59 +0100 Subject: [PATCH 04/34] Add npm run verify aggregate gate - Runs typecheck, lint, and test:run in sequence - Single command for CI and downstream refactor phases to prove the verification baseline --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d9e16b86..3a73ad4d 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint:fix": "eslint . --fix", "test": "vitest", "test:run": "vitest run", - "test:ci": "vitest run --coverage" + "test:ci": "vitest run --coverage", + "verify": "npm run typecheck && npm run lint && npm run test:run" }, "bin": { "vectorlint": "./dist/index.js", From 0fe8700b2e4d2ded0f51ef10e30e1f7fb4f39af5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:47:58 +0100 Subject: [PATCH 05/34] Add typecheck CI job and align test workflow Node version - New typecheck.yml runs tsc --noEmit on push/PR for main and release-docs - Bump test.yml from Node 18 to 20 to satisfy package.json engines>=20.6 - Catches type regressions in CI before merge, not just via ESLint --- .github/workflows/test.yml | 2 +- .github/workflows/typecheck.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/typecheck.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcf03d6f..06411499 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'npm' - name: Install dependencies diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 00000000..73dad374 --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,28 @@ +name: Typecheck + +on: + push: + branches: [main, release-docs] + pull_request: + branches: [main, release-docs] + +jobs: + typecheck: + name: tsc --noEmit + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run typecheck + run: npm run typecheck From 6c271f121b80e05e5e3d7ae2d7b6c02352755abd Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:58:45 +0100 Subject: [PATCH 06/34] Deprecate --mode agent and fall back to standard - README Agent Mode section replaced with under-review notice linking the audit - --mode agent now warns through the injected logger and runs standard evaluation; the agent executor code is retained (unreachable from the CLI) pending Phase 4 removal - Add logger to EvaluationOptions so the deprecation warning routes through the logging abstraction instead of console - Rewrite orchestrator-agent-output tests to assert deprecation + fallback BREAKING: --mode agent no longer runs the autonomous workspace-agent loop --- README.md | 29 +- src/cli/commands.ts | 3 +- src/cli/orchestrator.ts | 9 +- src/cli/types.ts | 2 + tests/orchestrator-agent-output.test.ts | 1153 ++--------------------- 5 files changed, 80 insertions(+), 1116 deletions(-) diff --git a/README.md b/README.md index 859ea51a..f6511e6a 100644 --- a/README.md +++ b/README.md @@ -145,29 +145,16 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode +## Agent Mode (under review) -Agent mode is for reviews that need context from several files such as -documentation drift and cross-file accuracy. - -Run VectorLint in autonomous agent mode: - -```bash -vectorlint doc.md --mode agent -``` - -For machine-parseable output: - -```bash -vectorlint doc.md --mode agent --output json -``` - -To suppress interactive progress in line output: - -```bash -vectorlint doc.md --mode agent --print -``` +The `--mode agent` flag is under active rework. It currently enables an +autonomous cross-file review mode that is being **removed** in favor of a +bounded harness model. See +[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) +for the decision and the refactor plan. +Until the refactor lands, `--mode agent` prints a deprecation warning and +falls back to standard mode. Do not build integrations against it. ## Contributing diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 369eda1f..50dbffc8 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -82,7 +82,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default) or agent', DEFAULT_REVIEW_MODE) + .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) .option('-p, --print', 'Suppress interactive progress output in agent mode') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') @@ -217,6 +217,7 @@ export function registerMainCommand(program: Command): void { ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, + logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, mode: cliOptions.mode, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 6ae4acd4..f8df367a 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1168,6 +1168,9 @@ async function buildAgentRuleScores( return results; } +// Retained in quarantine: unreachable from the CLI after --mode agent was +// deprecated (now falls back to standard evaluation). Removed in Phase 4. +// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine async function evaluateFilesInAgentMode( targets: string[], options: EvaluationOptions, @@ -1322,7 +1325,11 @@ export async function evaluateFiles( } if (mode === AGENT_REVIEW_MODE) { - return evaluateFilesInAgentMode(targets, options, outputFormat, jsonFormatter); + options.logger?.warn( + '--mode agent is deprecated and now falls back to standard mode. ' + + 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', + ); + // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } for (const file of targets) { diff --git a/src/cli/types.ts b/src/cli/types.ts index a73f55e6..7f95192a 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -10,6 +10,7 @@ import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; +import type { Logger } from '../logging/logger'; export enum OutputFormat { Line = "line", @@ -48,6 +49,7 @@ export interface EvaluationOptions { agentMaxRetries?: number; pricing?: PricingConfig; userInstructionContent?: string; + logger?: Logger; } export interface EvaluationResult { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index e08bbdaa..0b61ea63 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -2,28 +2,21 @@ import { mkdtempSync, rmSync, writeFileSync } from 'fs'; import * as path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; +import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; import type { PromptFile } from '../src/prompts/prompt-loader'; import type { LLMProvider } from '../src/providers/llm-provider'; +import type { Logger } from '../src/logging/logger'; import { Severity } from '../src/evaluators/types'; -function makePrompt(params?: { - id?: string; - name?: string; - source?: string; - body?: string; -}): PromptFile { - const id = params?.id ?? 'consistency'; - const name = params?.name ?? 'Consistency'; - const source = params?.source ?? 'packs/default/consistency.md'; - const body = params?.body ?? 'Find inconsistent wording'; - +function makePrompt(): PromptFile { + const id = 'consistency'; + const name = 'Consistency'; return { id, filename: `${id}.md`, - fullPath: source, + fullPath: 'packs/default/consistency.md', pack: 'Default', - body, + body: 'Find inconsistent wording', meta: { id: name, name, @@ -33,91 +26,39 @@ function makePrompt(params?: { }; } -function makeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; -} +type LoggerSpy = Logger & { warn: ReturnType }; -function makeTopLevelOnlyProvider(): LLMProvider { +function makeLogger(): LoggerSpy { return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Top-level without references', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as unknown as LoggerSpy; } -function makeCrossFileTopLevelProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'other.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; +interface StandardProviderSpies { + provider: LLMProvider; + runPromptStructured: ReturnType; + runAgentToolLoop: ReturnType; } -function makeNoFinalizeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Finding recorded before session ended', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - // intentionally omit finalize_review to simulate missing-finalize scenario - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; +function makeStandardProvider(): StandardProviderSpies { + const runPromptStructured = vi.fn().mockResolvedValue({ + data: { reasoning: 'ok', violations: [] }, + }); + const runAgentToolLoop = vi.fn().mockResolvedValue({ + usage: { inputTokens: 0, outputTokens: 0 }, + }); + const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; + return { provider, runPromptStructured, runAgentToolLoop }; } -describe('agent orchestrator output', () => { - const originalIsTTY = Object.getOwnPropertyDescriptor(process.stderr, 'isTTY'); +// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no +// longer reaches the autonomous agent executor: it warns through the injected +// logger and falls back to standard evaluation. The retained agent executor +// code is covered directly by tests/agent/* and removed in Phase 4. +describe('agent mode deprecation', () => { const tempRepos: string[] = []; function createTempRepo(): string { @@ -137,55 +78,15 @@ describe('agent orchestrator output', () => { for (const repo of tempRepos.splice(0, tempRepos.length)) { rmSync(repo, { recursive: true, force: true }); } - if (originalIsTTY) { - Object.defineProperty(process.stderr, 'isTTY', originalIsTTY); - } }); - it('produces agent-mode findings and summary when mode is set to agent', async () => { + it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalFiles).toBe(1); - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.hadOperationalErrors).toBe(false); - }); - - it('includes nested lint usage in the final agent-mode token totals', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { reasoning: 'ok', violations: [] }, - usage: { inputTokens: 7, outputTokens: 3 }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -194,338 +95,31 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.tokenUsage).toEqual({ - totalInputTokens: 18, - totalOutputTokens: 8, + logger, }); - }); - - it('keeps json output shape consistent with formatter-based structure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const payload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record; - summary?: { files?: number; errors?: number; warnings?: number }; - metadata?: { version?: string; timestamp?: string }; - }; - - expect(payload.files).toBeDefined(); - expect(payload.summary).toBeDefined(); - expect(typeof payload.summary?.files).toBe('number'); - expect(typeof payload.summary?.errors).toBe('number'); - expect(typeof payload.summary?.warnings).toBe('number'); - expect(payload.metadata?.timestamp).toBeTruthy(); - }); - - it('keeps top-level json keys consistent between standard and agent modes', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const standardPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const agentPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - expect(Object.keys(agentPayload).sort()).toEqual( - Object.keys(standardPayload).sort() + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('deprecated'), ); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), + ); + // Standard evaluation ran; the autonomous executor did not. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); - it('emits configured agent progress messages in line output mode', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain(' └ Found no issues in doc.md'); - expect(stderrOutput).not.toContain('[vectorlint]'); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders visible tool invocations and results while hiding internal agent tools', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.search_files.execute({ pattern: '**/*.md' }); - await tools.read_file.execute({ path: 'doc.md' }); - await tools.list_directory.execute({ path: '.' }); - await tools.search_content.execute({ pattern: 'bad phrase', path: '.', glob: '**/*.md' }); - await tools.finalize_review.execute({}); - - return { usage: { inputTokens: 6, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Lint("Find inconsistent wording...")'); - expect(stderrOutput).toContain('Found no issues in doc.md'); - expect(stderrOutput).toContain('Read(doc.md)'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('List(.)'); - expect(stderrOutput).toContain('Listed 1 entry in .'); - expect(stderrOutput).not.toContain('SearchFiles('); - expect(stderrOutput).not.toContain('SearchContent('); - expect(stderrOutput).not.toContain('Finalize('); - }); - - it('renders interactive tool lines without a trailing newline', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const firstToolLine = stderrSpy.mock.calls - .map((call) => String(call[0])) - .find((chunk) => chunk.includes(' └ ')); - - expect(firstToolLine).toBeDefined(); - expect(firstToolLine?.endsWith('\n')).toBe(false); - }); - - it('uses in-place progress updates for repeated tool calls instead of appending only plain lines', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/wordiness.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - makePrompt({ id: 'wordiness', name: 'Wordiness', source: 'packs/default/wordiness.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('\x1b[1A'); - }); - - it('updates progress rule labels as the active lint rule changes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('for AI Pattern'); - expect(stderrOutput).toContain('for Consistency'); - }); - - it('shows visible tool retry status after a visible tool failure', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('does not invoke the agent executor in line output mode', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); - const retriable = path.join(repo, 'retriable.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: 'retriable.md' })).rejects.toThrow(); - writeFileSync(retriable, 'hello\n', 'utf8'); - await tools.read_file.execute({ path: 'retriable.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); await evaluateFiles([file], { prompts: [makePrompt()], @@ -534,115 +128,23 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: false, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading retriable.md'); - expect(stderrOutput).toContain('Retrying Read(retriable.md)...'); - expect(stderrOutput).toContain('Read 1 line from retriable.md'); - }); - - it('shows visible-tool path errors even when path validation fails before file access', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, + logger, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading ../outside.md'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); }); - it('shows quality scores in agent line output and keeps operational failures explicit when finalize_review is missing', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('stays silent in standard mode without a logger', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 6, outputTokens: 3 } }; - }, - } as unknown as LLMProvider; + const { provider, runAgentToolLoop } = makeStandardProvider(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -650,549 +152,14 @@ describe('agent orchestrator output', () => { provider, concurrency: 1, verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(0); - expect(stdout).toContain('Use consistent wording'); - expect(stdout).toContain('Quality Scores:'); - expect(stderrSpy).toHaveBeenCalled(); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Review failed after'); - expect(stderrOutput).not.toContain('Completed review in'); - }); - - it('suppresses progress output when print mode is enabled', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('returns machine-parseable json output without progress text contamination', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stdout).not.toContain('Completed review.'); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('appends a new two-line block when agent work moves to the next file', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const fileOne = path.join(repo, 'doc.md'); - const fileTwo = path.join(repo, 'doc2.md'); - writeFileSync(fileOne, 'bad phrase\n', 'utf8'); - writeFileSync(fileTwo, 'another bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.read_file.execute({ path: 'doc2.md' }); - await tools.lint.execute({ file: 'doc2.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 4, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([fileOne, fileTwo], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('Reviewing doc2.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc2.md'); - const firstFileIndex = stderrOutput.indexOf('Reviewing doc.md for Consistency'); - const secondFileIndex = stderrOutput.indexOf('Reviewing doc2.md for Consistency'); - expect(firstFileIndex).toBeGreaterThan(-1); - expect(secondFileIndex).toBeGreaterThan(firstFileIndex); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders top-level findings without explicit references at 1:1 in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeTopLevelOnlyProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('Top-level without references'); - expect(stdout).toContain('1:1'); - }); - - it('prints lazy file headers for non-target referenced findings in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - const otherFile = path.join(repo, 'other.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - writeFileSync(otherFile, 'other content\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeCrossFileTopLevelProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('other.md'); - expect(stdout).toContain('Cross-file issue found'); - }); - - it('scores against every matched file, including clean files without findings', async () => { - const repo = createTempRepo(); - const firstFile = path.join(repo, 'doc.md'); - const secondFile = path.join(repo, 'other.md'); - writeFileSync(firstFile, 'one two three four five six seven eight nine ten\n', 'utf8'); - writeFileSync(secondFile, 'alpha beta gamma delta epsilon zeta eta theta iota kappa\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'one', - context_before: '', - context_after: '', - description: 'Issue found', - analysis: 'Needs cleanup.', - message: 'Fix the wording', - suggestion: 'Use clearer wording', - fix: 'clear wording', - rule_quote: 'Be precise', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([firstFile, secondFile], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('5.0/10'); - }); - - it('keeps canonical rule identity and warning severity aligned across json and rdjson outputs', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: DEFAULT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const jsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record }>; - }; - - const jsonIssue = - jsonPayload.files?.['doc.md']?.issues?.find( - (issue) => issue.rule === 'Default.Consistency' - ) ?? jsonPayload.files?.['doc.md']?.issues?.[0]; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const rdjsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - diagnostics?: Array<{ code?: { value?: string }; severity?: string }>; - }; - - const rdjsonDiagnostic = - rdjsonPayload.diagnostics?.find( - (diagnostic) => diagnostic.code?.value === 'Default.Consistency' - ) ?? rdjsonPayload.diagnostics?.[0]; - - expect(jsonIssue?.rule).toBe('Default.Consistency'); - expect(jsonIssue?.severity).toBe(Severity.WARNING); - expect(rdjsonDiagnostic?.code?.value).toBe('Default.Consistency'); - expect(rdjsonDiagnostic?.severity).toBe(Severity.WARNING); - }); - - it('keeps rdjson output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('keeps vale-json output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.ValeJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('surfaces findings recorded before missing finalize while still reporting an operational failure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeNoFinalizeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.requestFailures).toBe(0); - expect(result.hadOperationalErrors).toBe(true); - }); - - it('passes userInstructionContent into the agent system prompt', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let capturedSystemPrompt = ''; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const systemPrompt = params.systemPrompt; - capturedSystemPrompt = typeof systemPrompt === 'string' ? systemPrompt : ''; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - userInstructionContent: 'Always enforce concise phrasing.', - } as never); - - expect(capturedSystemPrompt).toContain('User Instructions (from VECTORLINT.md):'); - expect(capturedSystemPrompt).toContain('Always enforce concise phrasing.'); - }); - - it('counts provider tool-loop failures as request failures in agent mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop() { - return Promise.reject(new Error('provider request failed')); - }, - } as unknown as LLMProvider; - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(1); - }); - - it('passes a default agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(10); - }); - - it('passes configured agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - agentMaxRetries: 4, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(4); + // No deprecation warning and no executor invocation in standard mode. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); }); From a2e9f25c8935163285235736ac70ff5926d47b95 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:10:35 +0100 Subject: [PATCH 07/34] Append Phase 1 completion note to harness audit - Record the shifted 2026-07-13 baseline: only tsc --noEmit was failing; lint and test:run were already green, so the audit's lint and four-suite module-resolution failures are stale - Note the durable Phase 1 gates (typecheck, verify, vitest.config, docs/research lint exclusion, Node 20 typecheck/test workflows) - Point to the --mode agent deprecation and standard fallback as the precondition for Phases 2-5; record that no spec or architecture doc is superseded in this appendix --- ...0-vectorlint-harness-architecture-audit.md | 380 ++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md new file mode 100644 index 00000000..51161ce0 --- /dev/null +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -0,0 +1,380 @@ +# VectorLint Harness Architecture Audit + +Date: 2026-07-10 + +Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. + +## Product Decision + +VectorLint should not keep the current autonomous workspace-agent mode. + +The retained agent-like capability should be a constrained reader executor: a model or headless agent can read the target content in sections when that is useful for context management, but it should not search the workspace, inspect arbitrary files, perform top-level workspace checks, or rewrite the rule being evaluated. + +The intended execution model is: + +- `direct`: send target content and rule in one structured review call. +- `reader`: give the executor a read-section capability over the target content only. +- `auto`: choose `direct` for normal-sized inputs and `reader` for large inputs or explicit context-management needs. + +This is an execution strategy decision, not a return to VectorLint-as-agent. + +## Product Direction Under Review + +VectorLint should become a programmable, bounded, on-page content review harness that another caller can invoke. + +The caller can be Codex, Claude Code, another coding agent, CI, a local CLI wrapper, or a future service. That caller owns exploration, cross-page reasoning, research, and context gathering. VectorLint owns the constrained review of target content against source-backed rules and returns structured findings, scores, diagnostics, and usage metadata. + +In practical terms: + +- VectorLint reviews the target page or explicitly supplied content. +- External context is caller-supplied, not discovered by VectorLint. +- Rules define review behavior, granularity, severity, and scoring constraints. +- Executors can be API models, headless local agents, or constrained reader executors, but they receive the same review request contract. +- VectorLint should not expose a model-controlled workspace tool loop as its core product surface. + +## Objective System Audit Summary + +Separate from product direction, the current codebase has several system-level liabilities: + +- Build health is weak: `npx tsc --noEmit`, `npm run lint`, and parts of `npm run test:run` fail in the current workspace. +- Runtime contracts are duplicated across hand-written TypeScript, JSON-schema objects, Zod schemas, and formatter-specific shapes. +- Standard mode and agent mode project the same underlying findings differently. +- CLI orchestration owns too many responsibilities: config, file matching, evaluation, scoring, output formatting, debug artifacts, and agent routing. +- Evidence verification and counting are inconsistent enough to affect exit behavior. +- Observability and debug paths can record full prompts, content, and outputs. +- Documentation and implementation disagree on the current agent-mode topology and tool contracts. + +## Architecture Verdict + +The current codebase contains two execution models at once: + +1. The older evaluator path: one structured model call per rule or chunk. +2. The newer autonomous workspace-agent path: VectorLint gives a model tools to read/search the workspace and call lint as a nested tool. + +The second path should not continue as-is. The useful piece is not "agent mode" broadly; it is target-aware reading for context management. The current implementation also duplicates result projection, scoring, evidence validation, output formatting, and configuration behavior from the first path. That is why the architecture feels harder to maintain: the code is not just messy; it is serving two different ownership models. + +The highest-leverage change is to define a neutral review-domain contract and make every executor implement that contract. + +## Proposed Core Contract + +The contract should be centered on review, not providers or agents. + +```ts +interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; +} + +interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; +} +``` + +API models, reasoning models, and headless CLI agents become adapter implementations behind this contract. The CLI becomes orchestration around request creation, result formatting, and exit behavior. + +## Major Findings + +### 1. The Current Agent Surface Is Broader Than The Decided Execution Model + +Evidence: + +- `README.md:148` presents `--mode agent` as autonomous cross-file review mode. +- `src/agent/prompt-builder.ts:43` instructs the model to perform workspace-level checks. +- `src/agent/executor.ts:670` exposes `read_file`. +- `src/agent/executor.ts:715` exposes `search_content`. + +Impact: + +VectorLint still behaves like it owns exploration. That puts it in competition with external coding agents instead of becoming a harness those agents can use. It also overshoots the decided reader-executor model, which only needs target-scoped reading for context management. + +Recommendation: + +Remove the current autonomous workspace-agent surface. Preserve only a constrained reader executor if needed: + +- It may read sections of the target content. +- It may not search the workspace. +- It may not read arbitrary files. +- It may not perform top-level workspace findings. +- It may not override source-backed rules. + +### 2. The Provider Interface Mixes Structured Review With Autonomous Tool Loops + +Evidence: + +- `src/providers/llm-provider.ts:28` requires both `runPromptStructured` and `runAgentToolLoop`. +- `src/providers/vercel-ai-provider.ts:136` implements the agent loop on the same provider. +- `src/agent/executor.ts:796` depends on the provider-level agent loop. + +Impact: + +Every future executor has to pretend to be both a structured model provider and an autonomous agent runner. That is the wrong abstraction for headless CLI support and for a constrained reader executor. + +Recommendation: + +Split the contracts: + +- `StructuredModelClient` for API structured output. +- `HeadlessReviewExecutor` for local CLI agents. +- `ReaderReviewExecutor` for target-scoped read-section execution. +- `ReviewExecutor` as the stable domain-level interface. +- `TelemetrySink` or observability decoration as an optional cross-cutting concern. + +### 3. Runtime Type Contracts Are Not Strong Enough For A Harness + +Evidence: + +- `src/providers/vercel-ai-provider.ts:119` returns `output as T`. +- `src/providers/llm-provider.ts:9` defines tool input and output as `unknown`. +- `src/agent/tools-registry.ts:21` exposes tools without typed output contracts. +- `npx tsc --noEmit` currently fails in core files. + +Impact: + +The code presents types as guarantees, but several are only assertions. A programmable harness needs reliable runtime validation because external callers and headless adapters will depend on these contracts. + +Recommendation: + +Tie structured execution to concrete Zod schemas or canonical parsers. Introduce typed tool contracts only if tools remain. Add a `typecheck` script and wire it into CI and build. + +### 4. Standard Mode And Agent Mode Project Results Differently + +Evidence: + +- `src/cli/orchestrator.ts:626` derives standard check severity from scoring. +- `src/agent/executor.ts:431` stamps agent findings with prompt severity. +- `src/agent/executor.ts:598` flattens judge criteria before recording findings. +- `src/cli/orchestrator.ts:1216` builds agent scores only for line output. +- `src/cli/orchestrator.ts:1237` emits structured output without agent scores or diagnostics. + +Impact: + +The same underlying review can produce different severity, rule identity, score data, JSON shape, and exit behavior depending on mode. That makes VectorLint hard to trust as a machine-facing gate. + +Recommendation: + +Extract one shared result projection pipeline: + +```text +PromptEvaluationResult + -> verified findings + -> rule and criterion identity + -> severity and score + -> diagnostics + -> output formatter +``` + +Every executor should return into this pipeline. + +### 5. The On-Page Boundary Is Not Enforced + +Evidence: + +- `src/agent/executor.ts:585` lets `lint` resolve any file inside `workspaceRoot`. +- `src/agent/executor.ts:670` lets `read_file` read any workspace file. +- `src/agent/executor.ts:715` lets `search_content` scan workspace content. +- `src/agent/executor.ts:134` allows model-supplied `reviewInstruction`. +- `src/agent/executor.ts:583` replaces the source-backed prompt body with the model-supplied override. + +Impact: + +Prompt-injected page content can indirectly steer the model to read non-target files or rewrite the rule being evaluated. That breaks deterministic review semantics and creates privacy and security risk. + +Recommendation: + +For the new harness, enforce a target/context allowlist: + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller explicitly includes those files as context. +- Reader execution can page through target content only. +- Rule bodies are source-backed and caller-authored, not model-authored. + +### 6. Evidence Handling Can Misreport Findings + +Evidence: + +- Standard mode skips unverifiable quotes but can still count original surfaced violation totals in some paths. +- `src/agent/executor.ts:415` attempts to locate evidence. +- `src/agent/executor.ts:426` falls back to the model-provided line when location fails. + +Impact: + +Standard mode can fail a run without printing the issue that caused the failure. Agent mode can surface hallucinated evidence as if it were verified. + +Recommendation: + +Use one evidence verifier. Count only emitted, verified findings. If evidence cannot be located, return a diagnostic and either skip the finding or mark the run operationally failed. + +### 7. Cost And Work Are Not Bounded Enough + +Evidence: + +- `src/providers/vercel-ai-provider.ts:161` defaults agent loops to 1000 steps. +- `src/agent/executor.ts:577` allows repeated nested `lint` calls. +- `src/evaluators/base-evaluator.ts:213` runs one model call per chunk. +- `src/agent/executor.ts:680` and `src/agent/executor.ts:715` glob before applying result caps. + +Impact: + +A single review can multiply cost through agent steps, nested lint calls, rules, chunks, and workspace search. Even a constrained reader executor will be slower than a direct call, so `auto` must have clear thresholds and budgets. + +Recommendation: + +Add explicit budgets: + +- Max target bytes. +- Max caller context bytes. +- Max chunks per rule. +- Max model calls per review. +- Max findings per rule. +- Max wall-clock duration. +- Max headless executor retries. + +### 8. Documentation Still Points At The Old Product + +Evidence: + +- `README.md:148` documents autonomous workspace-agent mode. +- `docs/specs/2026-03-17-agentic-capabilities-design.md:24` frames the roadmap around cross-document agent mode. +- The same spec describes implementation details that are not true now: one agent per rule, final `AgentFindingSchema`, AbortSignal propagation, ripgrep JSON, and Vale JSON omission of agent findings. + +Impact: + +Future contributors will optimize toward the wrong architecture and trust contracts that the code does not implement. + +Recommendation: + +Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and migration plan. + +### 9. Secrets And Sensitive Content Need Better Handling + +Evidence: + +- `src/config/global-config.ts:13` says the global config stores API keys. +- `src/config/global-config.ts:76` creates the config directory with default permissions. +- `src/config/global-config.ts:81` writes the config file with default permissions. +- `src/observability/langfuse-observability.ts:66` records inputs. +- `src/observability/langfuse-observability.ts:67` records outputs. +- `src/providers/vercel-ai-provider.ts:59` can log full prompts and content. + +Impact: + +Provider keys, unreleased docs, caller context, and model outputs can leak through filesystem permissions, telemetry, debug logs, or persisted artifacts. + +Recommendation: + +Use private file modes for config and review artifacts. Make payload telemetry a separate opt-in from metadata telemetry. Add redaction or safe debug modes. + +## Verification Results + +Commands run from `/Users/klinsmann/Projects/TinyRocketLabs/vectorlint`: + +```bash +npm run lint +npm run test:run +npx tsc --noEmit +``` + +Results: + +- `npm run lint` failed before linting because typed ESLint rules were applied to a `.cjs` research script without parser type information. +- `npm run test:run` passed most tests but failed 4 suites during module resolution for `ora` and `@langfuse/otel`. +- `npx tsc --noEmit` failed with contract errors in agent, orchestrator, observability, logging, and provider modules. + +## Recommended Refactor Sequence + +### Phase 1: Stop The Bleeding + +1. Remove or hide the current autonomous workspace-agent mode. +2. Fix `tsc --noEmit`. +3. Fix test module resolution. +4. Fix lint configuration for non-TypeScript research scripts. +5. Add `typecheck` to CI and build validation. + +### Phase 2: Define The Harness Contract + +1. Create a neutral review-domain module. +2. Define `ReviewRequest`, `ReviewRule`, `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, and `ReviewDiagnostic`. +3. Define target/context boundary rules. +4. Define structured output shape once. +5. Define execution strategy: `direct`, `reader`, and `auto`. + +### Phase 3: Share Result Projection + +1. Extract evidence location and verification. +2. Extract filtering. +3. Extract severity and scoring. +4. Extract output formatter inputs. +5. Make standard mode and any executor mode use the same projection path. + +### Phase 4: Replace The Agent Loop With Executors + +1. Keep API model execution as direct execution. +2. Add target-scoped reader execution as another executor. +3. Add headless CLI execution behind the same review contract if still useful. +4. Pass the same review request to each executor. +5. Remove model-controlled rule overrides. +6. Remove workspace read/search tools from core review. + +### Phase 5: Update Documentation + +1. Mark the agentic capabilities spec superseded. +2. Write a new harness architecture spec. +3. Update README and CLI docs. +4. Document migration from autonomous workspace-agent mode to direct/reader/auto execution. + +## Suggested Target Architecture + +```text +CLI / API / External Agent + -> ReviewRequestBuilder + -> ReviewExecutor + -> ApiModelExecutor + -> ReaderExecutor + -> HeadlessCliExecutor + -> ResultProjection + -> EvidenceVerifier + -> ViolationFilter + -> ScoreCalculator + -> Diagnostics + -> OutputFormatter + -> line + -> json + -> rdjson + -> vale-json +``` + +The important inversion is that VectorLint no longer asks, "What tools should this agent use?" It asks, "Given this target, these rules, this context, this execution strategy, and this budget, what findings can be verified?" + +## Final Recommendation + +The decided direction is cleaner than the current architecture. The current pain is not a sign that VectorLint is too complex; it is a sign that the codebase is carrying two ownership models and several duplicated runtime contracts. + +Make VectorLint the harness. Let external agents be agents. Keep reader execution only as a bounded way to manage target-content context. + +--- + +## Appendix: Phase 1 — Completed (2026-07-13) + +Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on branch `codex/ci/harness-stop-the-bleeding`. It shifts the verification baseline recorded in "Verification Results" above and addresses Finding 1 at the CLI boundary. + +**Shifted baseline.** The 2026-07-13 pre-work capture differed from the original verification results: `npm run lint` and `npm run test:run` already passed, so the earlier lint failure and the four `ora` / `@langfuse/otel` module-resolution failures were stale. Only `npx tsc --noEmit` was still failing. Phase 1 cleared those remaining type errors (narrowed at boundaries; no strict compiler options relaxed) and added durable gates so the baseline cannot silently regress: + +- `npm run typecheck` (`tsc --noEmit`) and `npm run verify` (typecheck + lint + test:run) scripts. +- `vitest.config.ts` for stable test module resolution. +- `docs/research/**` excluded from linting. +- `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. + +**Current state.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the autonomous workspace-agent surface is deprecated at the CLI boundary: `--mode agent` emits a deprecation warning and falls back to standard evaluation. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. The deprecation notice and pointer to this audit live in the README `## Agent Mode` section and the `--mode` help text. + +This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. From 888f0271e30b0e0e1fa5d2773ca89f94cc415d58 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:22:49 +0100 Subject: [PATCH 08/34] Add root .vectorlint.ini for reproducible smoke validation - Commit a repository-native .vectorlint.ini using the bundled VectorLint preset (verbatim `vectorlint init` template). - The required validation commands `npm start -- README.md --output line` and `npm start -- README.md --mode agent` previously failed from the committed branch with "Missing configuration file"; they had relied on an untracked, deleted VECTORLINT.md. - Both smoke commands now run from a clean checkout with no untracked setup files; --mode agent still warns and falls back to standard mode. Refs: .agent-runs/2026-07-13-223741-harness-refactor/reports/01-stop-the-bleeding.md --- .vectorlint.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vectorlint.ini diff --git a/.vectorlint.ini b/.vectorlint.ini new file mode 100644 index 00000000..d5b0d845 --- /dev/null +++ b/.vectorlint.ini @@ -0,0 +1,9 @@ +# VectorLint Configuration +# Global settings +RulesPath= +Concurrency=4 +DefaultSeverity=warning + +# Default rules for all markdown files +[**/*.md] +RunRules=VectorLint From 99f73c22ae728cb3dfb904dd0208b881322ac09b Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:56:45 +0100 Subject: [PATCH 09/34] Add neutral review-domain contract module - Introduce src/review/ with typed + Zod-validated ReviewRequest/Result contracts, boundary helpers, budget defaults/enforcement, model-call selection, ReviewExecutor interface, and a PromptFile request builder - Every external shape has a paired strict Zod schema; legacy scoring-mode, rubric, and model-authored rule-override fields are rejected - modelCall is single | agent | auto; chooseModelCall() resolves auto - On-page boundary (target + caller context only) via buildScope/isInScope with pure lexical URI normalization; no filesystem reads - BudgetExceededError extends the repository VectorlintError base - Purely additive: no changes outside src/review/ and tests/review/ - tests/review/ covers surface, schemas, budget, boundary, results, model-call selection, and the request builder (46 tests) --- src/review/README.md | 42 ++++++ src/review/boundary.ts | 67 ++++++++++ src/review/budget.ts | 83 ++++++++++++ src/review/executor.ts | 53 ++++++++ src/review/index.ts | 15 +++ src/review/request-builder.ts | 88 ++++++++++++ src/review/schemas.ts | 135 +++++++++++++++++++ src/review/types.ts | 192 +++++++++++++++++++++++++++ tests/review/boundary.test.ts | 39 ++++++ tests/review/budget.test.ts | 67 ++++++++++ tests/review/executor.test.ts | 46 +++++++ tests/review/module-surface.test.ts | 34 +++++ tests/review/request-builder.test.ts | 97 ++++++++++++++ tests/review/result.test.ts | 95 +++++++++++++ tests/review/types.test.ts | 74 +++++++++++ 15 files changed, 1127 insertions(+) create mode 100644 src/review/README.md create mode 100644 src/review/boundary.ts create mode 100644 src/review/budget.ts create mode 100644 src/review/executor.ts create mode 100644 src/review/index.ts create mode 100644 src/review/request-builder.ts create mode 100644 src/review/schemas.ts create mode 100644 src/review/types.ts create mode 100644 tests/review/boundary.test.ts create mode 100644 tests/review/budget.test.ts create mode 100644 tests/review/executor.test.ts create mode 100644 tests/review/module-surface.test.ts create mode 100644 tests/review/request-builder.test.ts create mode 100644 tests/review/result.test.ts create mode 100644 tests/review/types.test.ts diff --git a/src/review/README.md b/src/review/README.md new file mode 100644 index 00000000..bc2a120b --- /dev/null +++ b/src/review/README.md @@ -0,0 +1,42 @@ +# `src/review/` — Review-Domain Contract + +The neutral, implementation-neutral contract every executor programs against. +A caller builds a `ReviewRequest`; an executor returns a `ReviewResult`. + +## What lives here + +- `types.ts` — all review-domain interfaces (`ReviewTarget`, `ReviewRule`, + `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, + `ReviewDiagnostic`, `ReviewResult`, `ReviewRequest`, `ReviewModelCall`). +- `schemas.ts` — Zod schemas mirroring every external shape (boundary + validation). All schemas are strict; legacy scoring-mode, rubric, and + model-authored rule-override fields are rejected. +- `budget.ts` — `DEFAULT_REVIEW_BUDGET`, `REVIEW_BUDGET_SCHEMA`, + `enforceBudget()`, and `BudgetExceededError` (extends `VectorlintError`). +- `boundary.ts` — `buildScope()` / `isInScope()` enforcing the on-page boundary. +- `executor.ts` — `ReviewExecutor` interface, `REVIEW_MODEL_CALLS` + (`single | agent | auto`), `chooseModelCall()`. +- `request-builder.ts` — `buildReviewRequest()` bridging `PromptFile` to + `ReviewRequest`. + +## On-page boundary (audit Finding #5) + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller includes them as context. +- Agent model calls page through target content only. +- Rule bodies are source-backed and caller-authored, never model-authored. + +## Execution strategy + +`modelCall: single | agent | auto` selects how the reviewer model is invoked, +not how rules are scored. `single` sends target + rule in one structured call; +`agent` gives the executor a target-scoped read-section capability; `auto` +picks `single` for normal-sized inputs and `agent` for large ones. + +## Wiring status + +Phase 2 (this module) is purely additive and is **not** wired into the CLI. +Phase 3 emits `ReviewResult` via shared finding processing; Phase 4 implements +executors behind `ReviewExecutor` and removes the old agent loop. See +`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`. diff --git a/src/review/boundary.ts b/src/review/boundary.ts new file mode 100644 index 00000000..8dc5bb0c --- /dev/null +++ b/src/review/boundary.ts @@ -0,0 +1,67 @@ +import type { ReviewScope } from './types'; + +const FILE_URI_RE = /^file:\/\/(?[^/]*)(?\/.*)?$/; + +/** + * Lexically resolves '.' and '..' segments in a path string. Pure: performs no + * filesystem reads. The normalization philosophy is adapted from + * src/agent/path-utils.ts (resolveWithinRoot), which is removed in Phase 4; + * copied here so src/review/ has no agent import and no filesystem access. + * + * Symlink canonicalization is intentionally NOT performed inside the contract: + * callers are responsible for canonicalizing real files into target/context + * URIs before building a ReviewRequest. + */ +function resolveDotSegments(input: string): string { + const segments: string[] = []; + for (const segment of input.split('/')) { + if (segment === '' || segment === '.') continue; + if (segment === '..') { + segments.pop(); + continue; + } + segments.push(segment); + } + return (input.startsWith('/') ? '/' : '') + segments.join('/'); +} + +/** + * Normalizes a review URI by resolving '.'/'..' segments in its path. file:// + * URIs are normalized structurally; other (virtual, in-memory) URIs are + * returned as-is after dot resolution of their path-like suffix. + */ +export function normalizeReviewUri(uri: string): string { + const match = FILE_URI_RE.exec(uri); + if (match?.groups) { + const authority = match.groups['authority'] ?? ''; + const rawPath = match.groups['path'] ?? '/'; + return `file://${authority}${resolveDotSegments(rawPath)}`; + } + return uri; +} + +/** + * Builds the on-page boundary scope (audit Finding #5) from the target URI and + * any caller-supplied context URIs. Only these URIs are in scope; arbitrary + * workspace files are out of scope unless the caller explicitly includes them. + */ +export function buildScope(params: { + targetUri: string; + contextUris?: readonly string[]; +}): ReviewScope { + const allowedUris = new Set(); + allowedUris.add(normalizeReviewUri(params.targetUri)); + for (const uri of params.contextUris ?? []) { + allowedUris.add(normalizeReviewUri(uri)); + } + return { allowedUris }; +} + +/** + * Returns true iff `uri` (after normalization) is the target or caller-supplied + * context. Path-traversal segments are resolved before comparison, so a + * traversal-style path cannot masquerade as an in-scope URI. + */ +export function isInScope(scope: ReviewScope, uri: string): boolean { + return scope.allowedUris.has(normalizeReviewUri(uri)); +} diff --git a/src/review/budget.ts b/src/review/budget.ts new file mode 100644 index 00000000..e7d4cd8b --- /dev/null +++ b/src/review/budget.ts @@ -0,0 +1,83 @@ +import { z } from 'zod'; +import type { ReviewBudget } from './types'; +import { VectorlintError } from '../errors'; + +/** + * Sensible default bounds for a single review (audit Finding #7). Frozen so a + * shared default object cannot be accidentally mutated by a caller. + * + * Note: there is intentionally no finding cap (`maxFindingsPerRule`) — + * VectorLint reports every verified issue it finds — and no headless retry + * budget (`maxHeadlessRetries`) because the decided architecture has no + * headless executor. + */ +export const DEFAULT_REVIEW_BUDGET: Readonly = Object.freeze({ + maxTargetBytes: 1_000_000, + maxCallerContextBytes: 500_000, + maxChunksPerRule: 20, + maxModelCallsPerReview: 50, + maxWallClockMs: 5 * 60_000, +}); + +/** Zod schema mirroring {@link ReviewBudget}, applying defaults for omitted fields. */ +export const REVIEW_BUDGET_SCHEMA = z + .object({ + maxTargetBytes: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxTargetBytes), + maxCallerContextBytes: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxCallerContextBytes), + maxChunksPerRule: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxChunksPerRule), + maxModelCallsPerReview: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview), + maxWallClockMs: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxWallClockMs), + }) + .strict(); + +/** Current resource usage, checked against a {@link ReviewBudget}. */ +export interface BudgetUsage { + modelCalls: number; + elapsedMs: number; +} + +/** + * Thrown by {@link enforceBudget} when current usage violates a hard limit. + * Extends the repository's canonical error base (VectorlintError). + */ +export class BudgetExceededError extends VectorlintError { + constructor( + message: string, + public readonly limit: keyof ReviewBudget, + public readonly actual: number, + ) { + super(message, 'BUDGET_EXCEEDED'); + this.name = 'BudgetExceededError'; + } +} + +/** + * Throws {@link BudgetExceededError} if current usage violates a hard limit. + * Executors call this before/after each model call and on timeout checks. + * Only runtime counters (model calls, wall clock) are enforced here; size + * limits (target/context bytes, chunks) are enforced at request-build time. + */ +export function enforceBudget(budget: ReviewBudget, usage: BudgetUsage): void { + if (usage.modelCalls > budget.maxModelCallsPerReview) { + throw new BudgetExceededError( + `model calls (${usage.modelCalls}) exceed maxModelCallsPerReview (${budget.maxModelCallsPerReview})`, + 'maxModelCallsPerReview', + usage.modelCalls, + ); + } + if (usage.elapsedMs > budget.maxWallClockMs) { + throw new BudgetExceededError( + `elapsed time (${usage.elapsedMs}ms) exceeds maxWallClockMs (${budget.maxWallClockMs}ms)`, + 'maxWallClockMs', + usage.elapsedMs, + ); + } +} diff --git a/src/review/executor.ts b/src/review/executor.ts new file mode 100644 index 00000000..2eef9011 --- /dev/null +++ b/src/review/executor.ts @@ -0,0 +1,53 @@ +import type { ReviewModelCall, ReviewRequest, ReviewResult } from './types'; + +/** + * Above this target byte size (roughly the existing chunking threshold in + * bytes), `auto` model-call selection prefers the agent executor so the + * reviewer can page through target content for context management. + */ +export const AGENT_MODEL_CALL_BYTE_THRESHOLD = 600_000; + +/** + * The single source of truth for reviewer model-call strategies. Kept in sync + * with the {@link ReviewModelCall} union via `satisfies`. + */ +export const REVIEW_MODEL_CALLS = ['single', 'agent', 'auto'] as const satisfies readonly ReviewModelCall[]; + +/** + * The stable domain-level interface every executor implements. Single and + * agent executors are implementations behind this contract (audit Finding #2). + * A headless/autonomous workspace executor is explicitly NOT part of this + * contract. + */ +export interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +/** + * Agent-call capability: a bounded way to page through target content for + * context management. NOT a workspace tool (audit Product Decision). An agent + * executor receives this capability scoped to the target only. + */ +export interface ReviewTargetReadCapability { + /** Read a 1-based [startLine, endLine] window of the target content. */ + readTargetSection( + startLine: number, + endLine: number, + ): Promise<{ startLine: number; endLine: number; content: string }>; +} + +/** + * Resolves 'auto' to 'single' or 'agent'. Single for normal-sized inputs; + * agent for large inputs or multi-rule runs that benefit from paging. + */ +export function chooseModelCall( + modelCall: ReviewModelCall, + signal: { targetBytes: number; rules: number }, +): 'single' | 'agent' { + if (modelCall === 'single') return 'single'; + if (modelCall === 'agent') return 'agent'; + if (signal.targetBytes > AGENT_MODEL_CALL_BYTE_THRESHOLD || signal.rules > 5) { + return 'agent'; + } + return 'single'; +} diff --git a/src/review/index.ts b/src/review/index.ts new file mode 100644 index 00000000..c44dbe66 --- /dev/null +++ b/src/review/index.ts @@ -0,0 +1,15 @@ +/** + * Neutral review-domain contract for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. The CLI, headless adapters, + * and external callers all build a ReviewRequest and receive a ReviewResult. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + */ +export * from './types'; +export * from './schemas'; +export * from './budget'; +export * from './boundary'; +export * from './executor'; +export * from './request-builder'; diff --git a/src/review/request-builder.ts b/src/review/request-builder.ts new file mode 100644 index 00000000..72500596 --- /dev/null +++ b/src/review/request-builder.ts @@ -0,0 +1,88 @@ +import type { PromptFile } from '../schemas/prompt-schemas'; +import type { + ReviewBudget, + ReviewContext, + ReviewModelCall, + ReviewOutputPolicy, + ReviewRequest, + ReviewRule, + ReviewTarget, +} from './types'; +import { DEFAULT_REVIEW_BUDGET } from './budget'; +import { ValidationError } from '../errors'; + +/** Default output policy: include usage metadata, never record payloads. */ +export const DEFAULT_REVIEW_OUTPUT_POLICY: Readonly = Object.freeze({ + includeUsage: true, + recordPayloadTelemetry: false, +}); + +/** Optional overrides applied by {@link buildReviewRequest}. */ +export interface ReviewRequestBuilderConfig { + budget?: ReviewBudget; + outputPolicy?: ReviewOutputPolicy; + modelCall?: ReviewModelCall; +} + +export interface BuildReviewRequestParams { + target: ReviewTarget; + /** Existing prompt files to convert into source-backed ReviewRules. */ + prompts: readonly PromptFile[]; + /** Caller-supplied, in-scope context, passed through unchanged. */ + context?: ReviewContext[]; + config?: ReviewRequestBuilderConfig; +} + +/** Builds the stable ReviewRule id formatted Pack.Rule (mirrors src/agent/rule-id.ts). */ +function toReviewRuleId(prompt: PromptFile): string { + const pack = prompt.pack || 'Default'; + return `${pack}.${prompt.meta.id}`; +} + +/** + * Converts a single PromptFile into a source-backed ReviewRule, mapping only + * fields that belong in the neutral contract. Legacy `meta.type`, `evaluator`, + * `criteria`, and judge/rubric fields are intentionally NOT copied. + */ +function toReviewRule(prompt: PromptFile): ReviewRule { + const rule: ReviewRule = { + id: toReviewRuleId(prompt), + source: prompt.fullPath, + body: prompt.body, + }; + if (prompt.meta.name !== undefined) { + rule.name = prompt.meta.name; + } + // Severity is a string enum whose values ('error' | 'warning') match + // ReviewSeverity exactly, so it maps cleanly without transformation. + if (prompt.meta.severity !== undefined) { + rule.severity = prompt.meta.severity; + } + return rule; +} + +/** + * Builds a {@link ReviewRequest} from existing {@link PromptFile}s plus a + * target. This is the conservative bridge Phase 4 uses to convert the current + * prompt pipeline into the review contract without changing how prompts load. + * Throws a {@link ValidationError} when no prompts are supplied. + */ +export function buildReviewRequest(params: BuildReviewRequestParams): ReviewRequest { + const { target, prompts, context } = params; + if (prompts.length === 0) { + throw new ValidationError('buildReviewRequest requires at least one prompt.'); + } + + const config = params.config; + const request: ReviewRequest = { + target, + rules: prompts.map(toReviewRule), + budget: config?.budget ?? DEFAULT_REVIEW_BUDGET, + outputPolicy: config?.outputPolicy ?? DEFAULT_REVIEW_OUTPUT_POLICY, + modelCall: config?.modelCall ?? 'auto', + }; + if (context !== undefined) { + request.context = context; + } + return request; +} diff --git a/src/review/schemas.ts b/src/review/schemas.ts new file mode 100644 index 00000000..a1132cd3 --- /dev/null +++ b/src/review/schemas.ts @@ -0,0 +1,135 @@ +import { z } from 'zod'; +import { REVIEW_BUDGET_SCHEMA } from './budget'; + +/** + * Zod schemas mirroring src/review/types.ts (boundary validation). + * + * Every external review-domain shape has a paired schema here so callers and + * external adapters can validate untrusted input at the system boundary + * (audit Finding #3). Schemas are intentionally strict: unknown keys are + * rejected so legacy scoring-mode, rubric, and model-authored rule-override + * fields cannot leak into the new contract. + */ + +export const REVIEW_SEVERITY_SCHEMA = z.enum(['error', 'warning']); + +export const REVIEW_VIOLATION_CONDITION_SCHEMA = z + .object({ + id: z.string().min(1), + description: z.string().min(1), + }) + .strict(); + +export const REVIEW_TARGET_SCHEMA = z + .object({ + uri: z.string().min(1), + content: z.string(), + contentType: z.string().min(1), + byteLength: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RULE_SCHEMA = z + .object({ + id: z.string().min(1), + source: z.string().min(1), + body: z.string(), + name: z.string().optional(), + severity: REVIEW_SEVERITY_SCHEMA.default('warning'), + violationConditions: z.array(REVIEW_VIOLATION_CONDITION_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_CONTEXT_SCHEMA = z + .object({ + label: z.string().min(1), + content: z.string(), + relation: z.string().optional(), + uri: z.string().optional(), + }) + .strict(); + +export const REVIEW_OUTPUT_POLICY_SCHEMA = z + .object({ + includeUsage: z.boolean(), + recordPayloadTelemetry: z.boolean(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_LEVEL_SCHEMA = z.enum(['info', 'warn', 'error']); + +export const REVIEW_SCORE_COMPONENT_SCHEMA = z + .object({ + id: z.string().min(1), + scoreText: z.string(), + score: z.number(), + weight: z.number().optional(), + }) + .strict(); + +export const REVIEW_SCORE_SCHEMA = z + .object({ + ruleId: z.string().min(1), + score: z.number(), + scoreText: z.string(), + severity: REVIEW_SEVERITY_SCHEMA, + findingCount: z.number().int().nonnegative().optional(), + components: z.array(REVIEW_SCORE_COMPONENT_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_FINDING_SCHEMA = z + .object({ + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + severity: REVIEW_SEVERITY_SCHEMA, + message: z.string(), + line: z.number().int().positive(), + column: z.number().int().positive(), + match: z.string(), + analysis: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_SCHEMA = z + .object({ + level: REVIEW_DIAGNOSTIC_LEVEL_SCHEMA, + code: z.string().min(1), + message: z.string(), + ruleId: z.string().optional(), + context: z.record(z.string(), z.unknown()).optional(), + }) + .strict(); + +export const REVIEW_USAGE_SCHEMA = z + .object({ + inputTokens: z.number().int().nonnegative().optional(), + outputTokens: z.number().int().nonnegative().optional(), + modelCalls: z.number().int().nonnegative(), + costUsd: z.number().nonnegative().optional(), + wallClockMs: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RESULT_SCHEMA = z + .object({ + findings: z.array(REVIEW_FINDING_SCHEMA), + scores: z.array(REVIEW_SCORE_SCHEMA), + diagnostics: z.array(REVIEW_DIAGNOSTIC_SCHEMA), + usage: REVIEW_USAGE_SCHEMA.optional(), + hadOperationalErrors: z.boolean().optional(), + }) + .strict(); + +export const REVIEW_REQUEST_SCHEMA = z + .object({ + target: REVIEW_TARGET_SCHEMA, + rules: z.array(REVIEW_RULE_SCHEMA).min(1), + context: z.array(REVIEW_CONTEXT_SCHEMA).optional(), + budget: REVIEW_BUDGET_SCHEMA, + outputPolicy: REVIEW_OUTPUT_POLICY_SCHEMA, + modelCall: z.enum(['single', 'agent', 'auto']), + }) + .strict(); diff --git a/src/review/types.ts b/src/review/types.ts new file mode 100644 index 00000000..97e22039 --- /dev/null +++ b/src/review/types.ts @@ -0,0 +1,192 @@ +/** + * Neutral review-domain contract types for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. Callers build a + * {@link ReviewRequest} and executors return a {@link ReviewResult}. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + * + * This module is implementation-neutral: it deliberately exposes no legacy + * scoring-mode, rubric, or model-authored rule-override surface. `modelCall` + * selects how the reviewer model is invoked, not how rules are scored. + */ + +/** Finding/Rule severity. */ +export type ReviewSeverity = 'error' | 'warning'; + +/** Objective Via Negativa condition that counts as a violation when present. */ +export interface ReviewViolationCondition { + id: string; + description: string; +} + +/** + * Target content under review. The on-page boundary (audit Finding #5) is + * enforced against this: executors may only read sections of `content`. + */ +export interface ReviewTarget { + /** Stable absolute URI (file:// or virtual scheme for in-memory content). */ + uri: string; + /** Full target content. May be paged by an agent executor. */ + content: string; + /** MIME-ish content type, e.g. 'text/markdown'. */ + contentType: string; + /** Optional byte length hint; computed from content if absent. */ + byteLength?: number; +} + +/** + * A source-backed, caller-authored rule. Model-authored rule overrides are + * explicitly disallowed (audit Finding #1, #5). + */ +export interface ReviewRule { + /** Stable id, formatted Pack.Rule[.Criterion] in output. */ + id: string; + /** Canonical source path (where the rule body came from). */ + source: string; + /** The rule prompt body. Source-backed; never model-authored. */ + body: string; + /** Human-readable name. */ + name?: string; + /** 'error' | 'warning'. Defaults to 'warning'. */ + severity?: ReviewSeverity; + /** Optional structured Via Negativa violation conditions. */ + violationConditions?: ReviewViolationCondition[]; +} + +/** + * Caller-supplied context. Explicitly in scope (audit boundary rule): + * VectorLint does NOT discover workspace files as context on its own; the + * caller owns exploration and context gathering. + */ +export interface ReviewContext { + /** Short label used in diagnostics/tracing. */ + label: string; + /** The context content itself. */ + content: string; + /** How this context relates to the target, e.g. 'reference' | 'glossary'. */ + relation?: string; + /** Optional source URI for provenance. */ + uri?: string; +} + +/** + * The on-page boundary scope: the normalized URIs an executor may read. + * Built by {@link buildScope} (see boundary.ts) and checked by + * {@link isInScope}. + */ +export interface ReviewScope { + /** Normalized absolute URIs that are in scope (target + caller context). */ + readonly allowedUris: ReadonlySet; +} + +/** + * Hard bounds on a single review (audit Finding #7). These limit work, not + * output. Executors MUST check these and surface a ReviewDiagnostic (or fail + * the run) when exceeded. + */ +export interface ReviewBudget { + maxTargetBytes: number; + maxCallerContextBytes: number; + maxChunksPerRule: number; + maxModelCallsPerReview: number; + /** Maximum elapsed review time in milliseconds. This is the run timeout. */ + maxWallClockMs: number; +} + +/** + * Controls optional output/telemetry behavior. Diagnostics are always part of + * ReviewResult; this policy does not hide operational warnings. Audit + * Finding #9: payload telemetry must be a separate opt-in from metadata + * telemetry. + */ +export interface ReviewOutputPolicy { + /** Include usage/cost in the result. */ + includeUsage: boolean; + /** Opt-in: record prompt/content payloads to telemetry. Default false. */ + recordPayloadTelemetry: boolean; +} + +/** A verified finding anchored in the target content. */ +export interface ReviewFinding { + ruleId: string; + ruleSource: string; + severity: ReviewSeverity; + message: string; + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. Unverified finding evidence is a diagnostic. */ + match: string; + analysis?: string; + suggestion?: string; + fix?: string; +} + +/** A per-rule score produced through shared finding processing. */ +export interface ReviewScore { + ruleId: string; + score: number; + /** Human-readable score, e.g. "8.0/10". */ + scoreText: string; + severity: ReviewSeverity; + /** Count of verified findings that contributed to this score, if applicable. */ + findingCount?: number; + components?: ReviewScoreComponent[]; +} + +export interface ReviewScoreComponent { + id: string; + scoreText: string; + score: number; + weight?: number; +} + +export type ReviewDiagnosticLevel = 'info' | 'warn' | 'error'; + +/** An operational or finding-processing note. Always part of ReviewResult. */ +export interface ReviewDiagnostic { + level: ReviewDiagnosticLevel; + /** Stable machine code, e.g. 'finding-evidence-not-locatable'. */ + code: string; + message: string; + ruleId?: string; + /** Extra machine-readable detail (model-call count, evidence preview, ...). */ + context?: Record; +} + +/** Aggregated resource usage for a review run. */ +export interface ReviewUsage { + inputTokens?: number; + outputTokens?: number; + modelCalls: number; + costUsd?: number; + wallClockMs?: number; +} + +/** The output from any executor, produced through shared finding processing. */ +export interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; + /** True if the run hit an operational error but still returned partial results. */ + hadOperationalErrors?: boolean; +} + +/** How to call the reviewer model. 'auto' resolves via chooseModelCall(). */ +export type ReviewModelCall = 'single' | 'agent' | 'auto'; + +/** The complete input to any executor. */ +export interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + /** Caller-supplied, in-scope context (never workspace-discovered). */ + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; + /** Reviewer model call shape. 'auto' resolves via chooseModelCall(). */ + modelCall: ReviewModelCall; +} diff --git a/tests/review/boundary.test.ts b/tests/review/boundary.test.ts new file mode 100644 index 00000000..5ff92dad --- /dev/null +++ b/tests/review/boundary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { buildScope, isInScope, normalizeReviewUri } from '../../src/review'; + +describe('review boundary', () => { + const scope = buildScope({ + targetUri: 'file:///repo/docs/guide.md', + contextUris: ['file:///repo/docs/glossary.md'], + }); + + it('target is always in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/guide.md')).toBe(true); + }); + + it('caller-supplied context uri is in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/glossary.md')).toBe(true); + }); + + it('arbitrary workspace file is NOT in scope', () => { + expect(isInScope(scope, 'file:///repo/src/index.ts')).toBe(false); + }); + + it('rejects path traversal that escapes the target', () => { + expect(isInScope(scope, 'file:///repo/docs/../src/index.ts')).toBe(false); + }); + + it('normalizes traversal that resolves back onto an in-scope uri', () => { + expect(isInScope(scope, 'file:///repo/docs/sub/../guide.md')).toBe(true); + }); +}); + +describe('normalizeReviewUri', () => { + it('resolves dot segments in file uris', () => { + expect(normalizeReviewUri('file:///repo/a/../b/./c.md')).toBe('file:///repo/b/c.md'); + }); + + it('preserves authority', () => { + expect(normalizeReviewUri('file://host/x/../y.md')).toBe('file://host/y.md'); + }); +}); diff --git a/tests/review/budget.test.ts b/tests/review/budget.test.ts new file mode 100644 index 00000000..a63bcc24 --- /dev/null +++ b/tests/review/budget.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { + BudgetExceededError, + DEFAULT_REVIEW_BUDGET, + REVIEW_BUDGET_SCHEMA, + enforceBudget, +} from '../../src/review'; +import { VectorlintError } from '../../src/errors'; + +describe('review budget defaults', () => { + it('provides sensible positive defaults', () => { + expect(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxWallClockMs).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxTargetBytes).toBeGreaterThan(0); + }); + + it('does not cap findings or headless retries', () => { + expect('maxFindingsPerRule' in DEFAULT_REVIEW_BUDGET).toBe(false); + expect('maxHeadlessRetries' in DEFAULT_REVIEW_BUDGET).toBe(false); + }); +}); + +describe('REVIEW_BUDGET_SCHEMA', () => { + it('applies defaults for omitted fields', () => { + const parsed = REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 5 }); + expect(parsed.maxWallClockMs).toBe(DEFAULT_REVIEW_BUDGET.maxWallClockMs); + expect(parsed.maxModelCallsPerReview).toBe(5); + }); + + it('rejects non-positive limits', () => { + expect(() => REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 0 })).toThrow(); + }); +}); + +describe('enforceBudget', () => { + it('is silent within limits', () => { + expect(() => + enforceBudget(DEFAULT_REVIEW_BUDGET, { modelCalls: 1, elapsedMs: 1000 }), + ).not.toThrow(); + }); + + it('throws BudgetExceededError when model calls exceed the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 3 }, + { modelCalls: 4, elapsedMs: 0 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('throws BudgetExceededError when wall clock exceeds the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxWallClockMs: 100 }, + { modelCalls: 0, elapsedMs: 101 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('extends the repository error base', () => { + const err = new BudgetExceededError('boom', 'maxModelCallsPerReview', 9); + expect(err).toBeInstanceOf(VectorlintError); + expect(err.limit).toBe('maxModelCallsPerReview'); + expect(err.actual).toBe(9); + expect(err.code).toBe('BUDGET_EXCEEDED'); + }); +}); diff --git a/tests/review/executor.test.ts b/tests/review/executor.test.ts new file mode 100644 index 00000000..c15e670b --- /dev/null +++ b/tests/review/executor.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_MODEL_CALLS, + chooseModelCall, + type ReviewExecutor, + type ReviewRequest, +} from '../../src/review'; + +describe('REVIEW_MODEL_CALLS', () => { + it('exposes single | agent | auto', () => { + expect(REVIEW_MODEL_CALLS).toEqual(['single', 'agent', 'auto']); + }); +}); + +describe('chooseModelCall', () => { + it('respects an explicit single call', () => { + expect(chooseModelCall('single', { targetBytes: 2_000_000, rules: 10 })).toBe('single'); + }); + + it('respects an explicit agent call', () => { + expect(chooseModelCall('agent', { targetBytes: 10, rules: 1 })).toBe('agent'); + }); + + it('returns single for small inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 10_000, rules: 1 })).toBe('single'); + }); + + it('returns agent for large inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 2_000_000, rules: 1 })).toBe('agent'); + }); + + it('returns agent for many rules under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 1_000, rules: 6 })).toBe('agent'); + }); +}); + +describe('ReviewExecutor contract', () => { + it('can be implemented and run() returns a ReviewResult', async () => { + const fake: ReviewExecutor = { + run: () => Promise.resolve({ findings: [], scores: [], diagnostics: [] }), + }; + const result = await fake.run({} as unknown as ReviewRequest); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + }); +}); diff --git a/tests/review/module-surface.test.ts b/tests/review/module-surface.test.ts new file mode 100644 index 00000000..6c03ad30 --- /dev/null +++ b/tests/review/module-surface.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import * as review from '../../src/review'; + +describe('src/review public surface', () => { + it('exports the core contract values and functions', () => { + // Runtime presence checks for values; types are checked by tsc. + expect(review.REVIEW_MODEL_CALLS).toBeDefined(); + expect(review.DEFAULT_REVIEW_BUDGET).toBeDefined(); + expect(typeof review.chooseModelCall).toBe('function'); + expect(typeof review.isInScope).toBe('function'); + expect(typeof review.buildReviewRequest).toBe('function'); + }); + + it('exposes the boundary, budget, and error helpers', () => { + expect(typeof review.buildScope).toBe('function'); + expect(typeof review.enforceBudget).toBe('function'); + expect(typeof review.normalizeReviewUri).toBe('function'); + expect(review.BudgetExceededError).toBeDefined(); + }); + + it('exposes the Zod schemas for every external shape', () => { + expect(review.REVIEW_TARGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_RULE_SCHEMA).toBeDefined(); + expect(review.REVIEW_CONTEXT_SCHEMA).toBeDefined(); + expect(review.REVIEW_BUDGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_OUTPUT_POLICY_SCHEMA).toBeDefined(); + expect(review.REVIEW_FINDING_SCHEMA).toBeDefined(); + expect(review.REVIEW_SCORE_SCHEMA).toBeDefined(); + expect(review.REVIEW_DIAGNOSTIC_SCHEMA).toBeDefined(); + expect(review.REVIEW_USAGE_SCHEMA).toBeDefined(); + expect(review.REVIEW_RESULT_SCHEMA).toBeDefined(); + expect(review.REVIEW_REQUEST_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts new file mode 100644 index 00000000..618efa03 --- /dev/null +++ b/tests/review/request-builder.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_REVIEW_BUDGET, + REVIEW_RULE_SCHEMA, + buildReviewRequest, +} from '../../src/review'; +import { ValidationError } from '../../src/errors'; +import { Severity } from '../../src/evaluators/types'; +import type { PromptFile } from '../../src/schemas/prompt-schemas'; +import type { ReviewContext, ReviewTarget } from '../../src/review'; + +function makePrompt(overrides: Partial = {}): PromptFile { + return { + id: 'pseudo-advice', + filename: 'pseudo-advice.md', + fullPath: '/repo/presets/VectorLint/pseudo-advice.md', + meta: { + id: 'PseudoAdvice', + name: 'Pseudo Advice', + severity: Severity.WARNING, + type: 'check', + criteria: [{ id: 'Vague', name: 'Vague advice' }], + }, + body: 'Check for pseudo advice.', + pack: 'VectorLint', + ...overrides, + }; +} + +describe('buildReviewRequest', () => { + const target: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: '# Guide', + contentType: 'text/markdown', + }; + + it('maps one PromptFile to one ReviewRule with default budget and auto modelCall', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + expect(request.modelCall).toBe('auto'); + expect(request.budget).toEqual(DEFAULT_REVIEW_BUDGET); + expect(request.rules).toHaveLength(1); + + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule?.id).toBe('VectorLint.PseudoAdvice'); + expect(rule?.source).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(rule?.body).toBe('Check for pseudo advice.'); + expect(rule?.severity).toBe('warning'); + expect(rule?.name).toBe('Pseudo Advice'); + // Strict parse proves no legacy evaluator/criteria/type fields leaked. + expect(() => REVIEW_RULE_SCHEMA.parse(rule)).not.toThrow(); + }); + + it('does not copy legacy meta.type, evaluator, or criteria fields', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'type' in rule).toBe(false); + expect(rule && 'evaluator' in rule).toBe(false); + expect(rule && 'criteria' in rule).toBe(false); + }); + + it('throws a ValidationError when no prompts are supplied', () => { + expect(() => buildReviewRequest({ target, prompts: [] })).toThrow(ValidationError); + }); + + it('passes caller-supplied context through unchanged', () => { + const context: ReviewContext[] = [ + { label: 'glossary', content: 'term a means ...', relation: 'reference' }, + ]; + const request = buildReviewRequest({ target, prompts: [makePrompt()], context }); + expect(request.context).toBe(context); + }); + + it('honors an explicit modelCall override', () => { + const request = buildReviewRequest({ + target, + prompts: [makePrompt()], + config: { modelCall: 'agent' }, + }); + expect(request.modelCall).toBe('agent'); + }); + + it('omits severity on the rule when the prompt has none', () => { + const request = buildReviewRequest({ + target, + prompts: [ + makePrompt({ + meta: { id: 'PseudoAdvice', name: 'Pseudo Advice' }, + }), + ], + }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'severity' in rule).toBe(false); + }); +}); diff --git a/tests/review/result.test.ts b/tests/review/result.test.ts new file mode 100644 index 00000000..3be3767a --- /dev/null +++ b/tests/review/result.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_DIAGNOSTIC_SCHEMA, + REVIEW_FINDING_SCHEMA, + REVIEW_RESULT_SCHEMA, +} from '../../src/review'; + +describe('ReviewFinding schema', () => { + it('parses a complete finding', () => { + const finding = REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'VectorLint.Consistency', + ruleSource: '/repo/presets/VectorLint/consistency.md', + severity: 'warning', + message: 'Vague advice.', + line: 5, + column: 1, + match: 'consider leveraging', + analysis: 'too vague', + suggestion: 'be specific', + fix: 'consider using concrete terms', + }); + expect(finding.line).toBe(5); + expect(finding.match).toBe('consider leveraging'); + expect(finding.severity).toBe('warning'); + }); + + it('rejects a finding with an unknown severity', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'critical', + message: 'x', + line: 1, + column: 1, + match: 'x', + }), + ).toThrow(); + }); + + it('rejects a finding missing required anchored match evidence', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'warning', + message: 'x', + line: 1, + column: 1, + }), + ).toThrow(); + }); +}); + +describe('ReviewDiagnostic schema', () => { + it('parses a diagnostic with level/code/message', () => { + const diag = REVIEW_DIAGNOSTIC_SCHEMA.parse({ + level: 'warn', + code: 'finding-evidence-not-locatable', + message: 'could not anchor the finding', + }); + expect(diag.level).toBe('warn'); + }); + + it('rejects an unknown level', () => { + expect(() => + REVIEW_DIAGNOSTIC_SCHEMA.parse({ level: 'fatal', code: 'x', message: 'y' }), + ).toThrow(); + }); +}); + +describe('ReviewResult schema', () => { + it('accepts findings/scores/diagnostics and defaults usage to undefined', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [], diagnostics: [] }); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + expect(result.usage).toBeUndefined(); + }); + + it('keeps diagnostics mandatory', () => { + expect(() => + REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [] }), + ).toThrow(); + }); + + it('accepts usage when provided', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ + findings: [], + scores: [], + diagnostics: [], + usage: { modelCalls: 2, inputTokens: 10, outputTokens: 5 }, + }); + expect(result.usage?.modelCalls).toBe(2); + }); +}); diff --git a/tests/review/types.test.ts b/tests/review/types.test.ts new file mode 100644 index 00000000..d3bee632 --- /dev/null +++ b/tests/review/types.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { REVIEW_CONTEXT_SCHEMA, REVIEW_RULE_SCHEMA, REVIEW_TARGET_SCHEMA } from '../../src/review'; + +describe('ReviewTarget schema', () => { + it('parses a valid target', () => { + const target = REVIEW_TARGET_SCHEMA.parse({ + uri: 'file:///repo/docs/guide.md', + content: '# Guide\n\nBody text.', + contentType: 'text/markdown', + }); + expect(target.uri).toBe('file:///repo/docs/guide.md'); + }); + + it('allows empty content (streaming target)', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ uri: 'file:///x', content: '', contentType: 'text/plain' }), + ).not.toThrow(); + }); + + it('rejects a target missing a uri', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ content: 'x', contentType: 'text/plain' }), + ).toThrow(); + }); +}); + +describe('ReviewRule schema', () => { + it('parses a rule with required fields and defaults severity to warning', () => { + const rule = REVIEW_RULE_SCHEMA.parse({ + id: 'Consistency', + source: 'VectorLint/consistency.md', + body: 'Check internal consistency.', + violationConditions: [ + { id: 'Contradiction', description: 'The target makes mutually inconsistent claims.' }, + ], + }); + expect(rule.severity).toBe('warning'); + expect(rule.violationConditions?.[0]?.id).toBe('Contradiction'); + }); + + it('rejects legacy evaluator and judge criteria fields', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'Judge whether the tone is good.', + evaluator: 'judge', + criteria: [{ id: 'Tone', name: 'Tone quality' }], + }), + ).toThrow(); + }); + + it('rejects an unknown severity', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'x', + severity: 'critical', + }), + ).toThrow(); + }); +}); + +describe('ReviewContext schema', () => { + it('parses caller-supplied scoped content', () => { + const ctx = REVIEW_CONTEXT_SCHEMA.parse({ + label: 'related-glossary', + content: 'Term A means ...', + relation: 'reference', + }); + expect(ctx.label).toBe('related-glossary'); + }); +}); From e4259e7865a1604a4091d6432bcc04c516b9d2ee Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:11:06 +0100 Subject: [PATCH 10/34] Extract single finding evidence verifier - Add src/findings/finding-evidence-verifier.ts wrapping locateQuotedText from src/output/location.ts as the single source of truth (audit #6). - Anchored quotes return verified line/column/match; unanchored quotes return a finding-evidence-not-locatable warn diagnostic and no finding. - Never falls back to a model-provided line number the way the old agent path did. - Cover anchored, unanchored, no-line-hint, and truncation cases. --- src/findings/finding-evidence-verifier.ts | 84 +++++++++++++++++++ .../finding-evidence-verifier.test.ts | 69 +++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/findings/finding-evidence-verifier.ts create mode 100644 tests/findings/finding-evidence-verifier.test.ts diff --git a/src/findings/finding-evidence-verifier.ts b/src/findings/finding-evidence-verifier.ts new file mode 100644 index 00000000..7a6ffb6a --- /dev/null +++ b/src/findings/finding-evidence-verifier.ts @@ -0,0 +1,84 @@ +import { locateQuotedText, type LocationWithMatch } from '../output/location'; + +/** Stable machine code emitted when finding evidence cannot be anchored. */ +export const FINDING_EVIDENCE_NOT_LOCATABLE = 'finding-evidence-not-locatable' as const; + +/** Evidence markers a model returns for a candidate finding. */ +export interface FindingEvidenceInput { + quoted_text: string; + context_before?: string; + context_after?: string; + /** Optional 1-based line hint from the model; never used as a fallback. */ + line?: number; +} + +/** Located coordinates for a verified finding, anchored in target content. */ +export interface VerifiedFindingCoords { + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. */ + match: string; +} + +/** Operational note produced when evidence cannot be verified. */ +export interface FindingEvidenceDiagnostic { + code: string; + level: 'warn' | 'error'; + message: string; +} + +/** The outcome of verifying a single candidate finding's evidence. */ +export interface FindingEvidenceVerification { + verified: boolean; + finding?: VerifiedFindingCoords; + diagnostic?: FindingEvidenceDiagnostic; +} + +/** + * The single source of truth for finding evidence verification + * (audit Finding #6). + * + * Wraps the existing `locateQuotedText` fuzzy matcher; it never reimplements + * matching. On a successful anchor it returns a verified finding with the + * located line/column/match. When the quoted text cannot be anchored it + * returns a `finding-evidence-not-locatable` warn diagnostic and NO finding — + * it never falls back to a model-provided line number the way the old agent + * path did. + */ +export function verifyFindingEvidence( + content: string, + findingEvidence: FindingEvidenceInput, +): FindingEvidenceVerification { + const location: LocationWithMatch | null = locateQuotedText( + content, + { + quoted_text: findingEvidence.quoted_text || '', + context_before: findingEvidence.context_before || '', + context_after: findingEvidence.context_after || '', + }, + 80, + findingEvidence.line, + ); + + if (!location) { + return { + verified: false, + diagnostic: { + code: FINDING_EVIDENCE_NOT_LOCATABLE, + level: 'warn', + message: `Could not locate quoted text "${(findingEvidence.quoted_text || '').slice(0, 60)}" in target content.`, + }, + }; + } + + return { + verified: true, + finding: { + line: location.line, + column: location.column, + match: location.match, + }, + }; +} diff --git a/tests/findings/finding-evidence-verifier.test.ts b/tests/findings/finding-evidence-verifier.test.ts new file mode 100644 index 00000000..64d38209 --- /dev/null +++ b/tests/findings/finding-evidence-verifier.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, +} from '../../src/findings/finding-evidence-verifier'; + +const CONTENT = 'line one\nline two has the quote\nline three'; + +describe('verifyFindingEvidence', () => { + it('locates an exact quote and returns a verified finding', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'the quote', + context_before: '', + context_after: '', + line: 2, + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(2); + expect(result.finding?.match).toBe('the quote'); + expect(result.diagnostic).toBeUndefined(); + }); + + it('returns a warn diagnostic (not a fallback finding) when evidence cannot be located', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'this does not exist anywhere', + context_before: '', + context_after: '', + line: 99, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + expect(result.diagnostic?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostic?.level).toBe('warn'); + }); + + it('never returns a finding whose line came from the model when the quote did not anchor', () => { + // The old agent path fell back to the model-provided line (here 1) and + // surfaced the raw quoted_text as a verified match. Assert we do not. + // (Use a lexically distant quote that locateQuotedText cannot anchor.) + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'definitely not present xyzzy', + context_before: '', + context_after: '', + line: 1, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + }); + + it('still anchors a quote when no line hint is provided', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'line three', + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(3); + expect(result.finding?.column).toBe(1); + }); + + it('produces a diagnostic that mentions the (truncated) quoted text', () => { + const longQuote = 'x'.repeat(120); + const result = verifyFindingEvidence(CONTENT, { + quoted_text: longQuote, + }); + expect(result.verified).toBe(false); + expect(result.diagnostic?.message).toContain('x'.repeat(60)); + // Truncated, not the full 120 chars. + expect(result.diagnostic?.message).not.toContain('x'.repeat(61)); + }); +}); From 25ec49aa43ec112c56785ef41a20207ebb4c9c88 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:13:50 +0100 Subject: [PATCH 11/34] Add shared finding severity and scoring helpers - Add src/findings/types.ts with the FindingProcessingInput contract and strict Zod schemas; no modelCall/mode/agent/evaluator/judge/rubric fields, so legacy judge/evaluator-shaped input is rejected. - Add src/findings/severity.ts: resolveSeverity derives severity from the calculateCheckScore result (one path, no agent stamping); buildRuleId matches Pack.Rule[.Criterion] naming without importing src/agent/. - Add src/findings/scorer.ts as a thin wrapper over calculateCheckScore; the verified finding count drives the score. - Cover schema parse/reject, severity, rule-id, criterion resolution, and score numerics against calculateCheckScore. --- src/findings/scorer.ts | 60 ++++++++++++ src/findings/severity.ts | 52 +++++++++++ src/findings/types.ts | 160 ++++++++++++++++++++++++++++++++ tests/findings/scorer.test.ts | 94 +++++++++++++++++++ tests/findings/severity.test.ts | 73 +++++++++++++++ tests/findings/types.test.ts | 145 +++++++++++++++++++++++++++++ 6 files changed, 584 insertions(+) create mode 100644 src/findings/scorer.ts create mode 100644 src/findings/severity.ts create mode 100644 src/findings/types.ts create mode 100644 tests/findings/scorer.test.ts create mode 100644 tests/findings/severity.test.ts create mode 100644 tests/findings/types.test.ts diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts new file mode 100644 index 00000000..b5fb93fa --- /dev/null +++ b/src/findings/scorer.ts @@ -0,0 +1,60 @@ +import { calculateCheckScore, type CheckScoringOptions } from '../scoring'; +import type { CheckResult } from '../prompts/schema'; +import { resolveSeverity } from './severity'; +import type { RawViolation, RuleSeverity } from './types'; + +/** Optional scoring knobs mirroring {@link CheckScoringOptions}. */ +export interface ScoreCheckOptions { + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +} + +/** + * The count/density score result for a rule's verified findings. Mirrors the + * fields the processor needs to assemble a `ReviewScore`. The numeric values + * come straight from {@link calculateCheckScore}; this layer never + * reimplements the scoring math (audit Finding #4). + */ +export interface ScoredCheck { + score: number; + scoreText: string; + severity: RuleSeverity; + /** Count of verified findings that drove the score. */ + findingCount: number; + /** The raw check result, used to resolve severity. */ + scored: CheckResult; +} + +/** + * Scores a rule's verified findings using the existing check density formula. + * + * The score is driven by the **verified** finding count, not the raw candidate + * count (audit Finding #6): callers must pass only findings whose evidence has + * been anchored. This is a thin adapter over {@link calculateCheckScore}; it + * only normalizes the result into the shared score shape. + */ +export function scoreCheck(params: { + verifiedViolations: RawViolation[]; + wordCount: number; + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +}): ScoredCheck { + const scored = calculateCheckScore( + params.verifiedViolations, + params.wordCount, + { + ...(params.strictness !== undefined ? { strictness: params.strictness } : {}), + ...(params.promptSeverity !== undefined + ? { promptSeverity: params.promptSeverity } + : {}), + }, + ); + + return { + score: scored.final_score, + scoreText: `${scored.final_score.toFixed(1)}/10`, + severity: resolveSeverity({ scored }), + findingCount: scored.violation_count, + scored, + }; +} diff --git a/src/findings/severity.ts b/src/findings/severity.ts new file mode 100644 index 00000000..cca534f0 --- /dev/null +++ b/src/findings/severity.ts @@ -0,0 +1,52 @@ +import type { CheckResult } from '../prompts/schema'; +import type { FindingsCriterion, RuleSeverity } from './types'; + +/** Input to {@link resolveSeverity}. */ +export interface SeverityInput { + scored: CheckResult; +} + +/** + * The one severity resolution path for objective violation checks + * (audit Finding #4). Severity is derived from the existing count/density + * score result (`CheckResult.severity`); there is no agent-specific or + * mode-specific severity stamping here. + */ +export function resolveSeverity(input: SeverityInput): RuleSeverity { + return input.scored.severity; +} + +/** + * Builds the hierarchical output rule id matching the current standard + * orchestrator naming (`buildRuleName`): `Pack.Rule` or, when a violation is + * attributed to a criterion, `Pack.Rule.Criterion`. This is a faithful port of + * that behavior; it does not import from `src/agent/`. + */ +export function buildRuleId( + pack: string, + ruleId: string, + criterionId?: string, +): string { + const parts = [pack, ruleId]; + if (criterionId) { + parts.push(criterionId); + } + return parts.join('.'); +} + +/** + * Resolves a criterion id from the rule's declared criteria by matching a + * `criterionName` a violation carries to a declared criterion's `name`. + * Returns `undefined` when the violation is not attributed to a declared + * criterion (the common case for objective check violations). + */ +export function resolveCriterionId( + criteria: readonly FindingsCriterion[] | undefined, + criterionName: string | undefined, +): string | undefined { + if (!criterionName || !criteria) { + return undefined; + } + const match = criteria.find((c) => c.name === criterionName); + return match?.id; +} diff --git a/src/findings/types.ts b/src/findings/types.ts new file mode 100644 index 00000000..9a6360c6 --- /dev/null +++ b/src/findings/types.ts @@ -0,0 +1,160 @@ +import { z } from 'zod'; +import { Severity } from '../evaluators/types'; + +/** + * Input contract for shared finding processing (Phase 3, audit Finding #6). + * + * This module is the boundary between "what a model returned" (raw candidate + * violations) and "what the formatters receive" (`ReviewResult`). It is + * intentionally independent of model call (`single` | `agent`): there is no + * `modelCall`, `mode`, `agent`, `evaluator`, `judge`, or rubric field here. + * + * Following the `src/review/` pattern, the domain types below are explicit + * interfaces (so their optional properties stay assignable under + * `exactOptionalPropertyTypes` to `CheckItem`/`GateViolationLike`), and the + * Zod schemas mirror them for boundary validation. The schemas are strict so + * legacy `evaluator`/`judge`/`modelCall`/`mode`/`agent`/rubric payloads are + * rejected. + */ + +/** + * The two severity levels objective violation checks can resolve to. Mirrors + * `ReviewSeverity`; values are identical (`'error'` | `'warning'`). + */ +export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; + +/** Optional strictness knob for check density scoring. */ +export type Strictness = number | 'lenient' | 'standard' | 'strict'; + +/** + * The six boolean evidence gates a model returns per violation. Structurally + * identical to `GateChecks` in `src/prompts/schema.ts`. + */ +export interface GateChecks { + rule_supports_claim: boolean; + evidence_exact: boolean; + context_supports_violation: boolean; + plausible_non_violation: boolean; + fix_is_drop_in: boolean; + fix_preserves_meaning: boolean; +} + +/** + * A raw candidate finding returned by a reviewer model, before evidence + * verification or filtering. `analysis` is the only required descriptive + * field; everything else is optional to tolerate partial model output while + * still failing closed at the filter and verifier stages. + */ +export interface RawViolation { + line?: number; + quoted_text?: string; + context_before?: string; + context_after?: string; + description?: string; + /** Optional criterion attribution used to build `Pack.Rule.Criterion` ids. */ + criterionName?: string; + analysis: string; + message?: string; + suggestion?: string; + fix?: string; + rule_quote?: string; + confidence?: number; + checks?: GateChecks; +} + +/** + * Criterion metadata used ONLY to name output rule ids + * (`Pack.Rule.Criterion`). Weight and target are judge/rubric concerns and are + * deliberately absent so the finding-processing contract cannot leak them. + */ +export interface FindingsCriterion { + id: string; + name: string; +} + +/** + * The narrow slice of prompt metadata that finding processing needs: severity + * for scoring, strictness for density, and criteria for rule-id naming. It + * excludes `evaluator`, `type`, `evaluateAs`, `target`, and other legacy + * fields that do not affect objective violation-check processing. + */ +export interface PromptMetaForFindings { + severity?: RuleSeverity; + strictness?: Strictness; + criteria?: FindingsCriterion[]; +} + +/** + * The full finding-processing input. There is deliberately no `outputFormat`: + * the processor emits a format-agnostic `ReviewResult`; the caller routes it + * to a formatter. + */ +export interface FindingProcessingInput { + pack: string; + ruleId: string; + ruleSource: string; + candidateFindings: RawViolation[]; + wordCount: number; + promptMeta: PromptMetaForFindings; + targetContent: string; +} + +// --- Boundary validation schemas (mirror the interfaces above) --------------- + +export const GATE_CHECKS_SCHEMA = z + .object({ + rule_supports_claim: z.boolean(), + evidence_exact: z.boolean(), + context_supports_violation: z.boolean(), + plausible_non_violation: z.boolean(), + fix_is_drop_in: z.boolean(), + fix_preserves_meaning: z.boolean(), + }) + .strict(); + +export const RAW_VIOLATION_SCHEMA = z + .object({ + line: z.number().optional(), + quoted_text: z.string().optional(), + context_before: z.string().optional(), + context_after: z.string().optional(), + description: z.string().optional(), + criterionName: z.string().optional(), + analysis: z.string(), + message: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + rule_quote: z.string().optional(), + confidence: z.number().optional(), + checks: GATE_CHECKS_SCHEMA.optional(), + }) + .strict(); + +export const FINDINGS_CRITERION_SCHEMA = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + }) + .strict(); + +export const PROMPT_META_FOR_FINDINGS_SCHEMA = z + .object({ + severity: z.nativeEnum(Severity).optional(), + strictness: z + .union([z.number().positive(), z.enum(['lenient', 'standard', 'strict'])]) + .optional(), + criteria: z.array(FINDINGS_CRITERION_SCHEMA).optional(), + }) + .strict(); + +export const FINDING_PROCESSING_INPUT_SCHEMA = z + .object({ + pack: z.string().min(1), + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + candidateFindings: z.array(RAW_VIOLATION_SCHEMA), + wordCount: z.number().nonnegative(), + promptMeta: PROMPT_META_FOR_FINDINGS_SCHEMA, + targetContent: z.string(), + }) + .strict(); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts new file mode 100644 index 00000000..7e8147b0 --- /dev/null +++ b/tests/findings/scorer.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { calculateCheckScore } from '../../src/scoring'; +import { Severity } from '../../src/evaluators/types'; +import { scoreCheck } from '../../src/findings/scorer'; +import type { RawViolation } from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + analysis: 'Issue', + message: 'Issue message', + fix: 'Fix', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +describe('scoreCheck', () => { + it('returns the same numeric result as calculateCheckScore for the same density', () => { + const verified = [violation(), violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + const direct = calculateCheckScore(verified, 100, { + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + expect(result.score).toBe(direct.final_score); + expect(result.scoreText).toBe(`${direct.final_score.toFixed(1)}/10`); + expect(result.severity).toBe(direct.severity); + expect(result.findingCount).toBe(verified.length); + expect(result.findingCount).toBe(direct.violation_count); + }); + + it('is driven by the verified finding count, not a raw candidate count', () => { + // 1 verified violation over 100 words at standard strictness: + // density 1 -> 100 - 1*10 = 90 -> 9.0/10 + const one = scoreCheck({ + verifiedViolations: [violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(one.score).toBe(9.0); + expect(one.scoreText).toBe('9.0/10'); + + // 2 verified violations -> density 2 -> 100 - 2*10 = 80 -> 8.0/10 + const two = scoreCheck({ + verifiedViolations: [violation(), violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(two.score).toBe(8.0); + expect(two.findingCount).toBe(2); + }); + + it('resolves error severity from the density score when score is low and prompt severity is error', () => { + // 20 violations over 100 words at standard strictness: + // density 20 -> 100 - 20*10 = -100 -> clamp 0 -> 0.0/10 -> prompt severity + const result = scoreCheck({ + verifiedViolations: Array.from({ length: 20 }, () => violation()), + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.ERROR, + }); + expect(result.score).toBe(0); + expect(result.severity).toBe(Severity.ERROR); + }); + + it('forwards strictness and prompt severity options to calculateCheckScore', () => { + const verified = [violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'strict', + }); + const direct = calculateCheckScore(verified, 100, { strictness: 'strict' }); + expect(result.score).toBe(direct.final_score); + }); +}); diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts new file mode 100644 index 00000000..46e0c93a --- /dev/null +++ b/tests/findings/severity.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from 'vitest'; +import { + buildRuleId, + resolveCriterionId, + resolveSeverity, +} from '../../src/findings/severity'; +import { EvaluationType, Severity } from '../../src/evaluators/types'; +import type { CheckResult } from '../../src/prompts/schema'; + +function makeCheckResult(severity: Severity): CheckResult { + return { + type: EvaluationType.CHECK, + final_score: 5, + percentage: 50, + violation_count: 1, + items: [], + severity, + message: 'Found 1 issue', + violations: [], + }; +} + +describe('resolveSeverity', () => { + it('returns the density-derived severity from the check score result', () => { + expect(resolveSeverity({ scored: makeCheckResult(Severity.WARNING) })).toBe( + Severity.WARNING, + ); + expect(resolveSeverity({ scored: makeCheckResult(Severity.ERROR) })).toBe( + Severity.ERROR, + ); + }); +}); + +describe('buildRuleId', () => { + it('builds Pack.Rule when no criterion is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern')).toBe('VectorLint.AiPattern'); + }); + + it('builds Pack.Rule.Criterion when a criterion id is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern', 'Hedging')).toBe( + 'VectorLint.AiPattern.Hedging', + ); + }); + + it('ignores an empty criterion id', () => { + expect(buildRuleId('VectorLint', 'AiPattern', '')).toBe( + 'VectorLint.AiPattern', + ); + }); +}); + +describe('resolveCriterionId', () => { + const criteria = [ + { id: 'Hedging', name: 'Hedge words' }, + { id: 'Filler', name: 'Filler phrases' }, + ]; + + it('maps a criterion name to its declared id', () => { + expect(resolveCriterionId(criteria, 'Hedge words')).toBe('Hedging'); + }); + + it('returns undefined when the name does not match a criterion', () => { + expect(resolveCriterionId(criteria, 'Unknown')).toBeUndefined(); + }); + + it('returns undefined when no criterion name is provided', () => { + expect(resolveCriterionId(criteria, undefined)).toBeUndefined(); + }); + + it('returns undefined when no criteria are declared', () => { + expect(resolveCriterionId(undefined, 'Hedge words')).toBeUndefined(); + }); +}); diff --git a/tests/findings/types.test.ts b/tests/findings/types.test.ts new file mode 100644 index 00000000..4309ead6 --- /dev/null +++ b/tests/findings/types.test.ts @@ -0,0 +1,145 @@ +import { describe, it, expect } from 'vitest'; +import { + FINDING_PROCESSING_INPUT_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function validInput() { + return { + pack: 'VectorLint', + ruleId: 'AiPattern', + ruleSource: '/repo/presets/VectorLint/ai-pattern.md', + candidateFindings: [ + { + line: 1, + quoted_text: 'leverage synergies', + context_before: '', + context_after: '', + description: 'cliché phrase', + analysis: 'vague corporate cliché', + message: 'Avoid cliché phrase', + suggestion: 'Use concrete verbs', + fix: 'work together', + rule_quote: 'Avoid clichés', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + }, + ], + wordCount: 100, + promptMeta: { + severity: 'warning' as const, + strictness: 'standard' as const, + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + targetContent: 'leverage synergies everywhere', + }; +} + +describe('RAW_VIOLATION_SCHEMA', () => { + it('parses a representative candidate finding', () => { + const parsed = RAW_VIOLATION_SCHEMA.parse(validInput().candidateFindings[0]); + expect(parsed.analysis).toBe('vague corporate cliché'); + expect(parsed.checks?.evidence_exact).toBe(true); + }); + + it('requires analysis', () => { + expect(() => + RAW_VIOLATION_SCHEMA.parse({ message: 'no analysis field' }), + ).toThrow(); + }); +}); + +describe('PROMPT_META_FOR_FINDINGS_SCHEMA', () => { + it('rejects legacy evaluator/type/evaluateAs/target fields', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + severity: 'warning', + evaluator: 'base', + }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ type: 'judge' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ evaluateAs: 'document' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ target: { regex: 'x' } }), + ).toThrow(); + }); + + it('rejects rubric weight on criteria', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + criteria: [{ id: 'Hedging', name: 'Hedge words', weight: 2 }], + }), + ).toThrow(); + }); +}); + +describe('FINDING_PROCESSING_INPUT_SCHEMA', () => { + it('parses a representative supported finding-processing input', () => { + const parsed = FINDING_PROCESSING_INPUT_SCHEMA.parse(validInput()); + expect(parsed.pack).toBe('VectorLint'); + expect(parsed.candidateFindings).toHaveLength(1); + expect(parsed.promptMeta.criteria?.[0]?.id).toBe('Hedging'); + }); + + it('has no modelCall/mode/agent field and rejects them', () => { + for (const legacyKey of ['modelCall', 'mode', 'agent'] as const) { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), [legacyKey]: 'agent' }), + ).toThrow(); + } + }); + + it('rejects legacy judge/evaluator/rubric-shaped top-level input', () => { + // Legacy judge result shape: criteria array with scores at the top level. + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + pack: 'VectorLint', + ruleId: 'Tone', + ruleSource: '/x.md', + wordCount: 100, + targetContent: 'x', + promptMeta: {}, + evaluator: 'judge', + criteria: [{ name: 'Tone', score: 2, violations: [] }], + }), + ).toThrow(); + }); + + it('rejects a legacy evaluator field nested under promptMeta', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + promptMeta: { ...validInput().promptMeta, evaluator: 'base' }, + }), + ).toThrow(); + }); + + it('rejects malformed candidate findings', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + candidateFindings: [{ message: 'no analysis field' }], + }), + ).toThrow(); + }); + + it('rejects an unknown top-level key', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), judge: true }), + ).toThrow(); + }); +}); From 20d42ff7d9441f3269d6fdea53f527ead80342b8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:15:37 +0100 Subject: [PATCH 12/34] Implement shared finding processor contract - Add src/findings/processor.ts: processFindings runs the single pipeline (filter -> verify -> dedupe -> score -> severity -> diagnostics) and returns the Phase 2 ReviewResult from src/review/types.ts. - Verified finding count, not raw candidate count, drives the score; unanchored evidence becomes a warn diagnostic and emits no finding; hadOperationalErrors is true only for error-level diagnostics. - Add src/findings/index.ts barrel export. - Golden processor test asserts findings/score match the standard check path; cover diagnostics, filtering, verified-count fix, dedupe, rule-id attribution, and ReviewResult contract validation. --- src/findings/index.ts | 48 +++++ src/findings/processor.ts | 167 ++++++++++++++++ tests/findings/module-surface.test.ts | 27 +++ tests/findings/processor.test.ts | 278 ++++++++++++++++++++++++++ 4 files changed, 520 insertions(+) create mode 100644 src/findings/index.ts create mode 100644 src/findings/processor.ts create mode 100644 tests/findings/module-surface.test.ts create mode 100644 tests/findings/processor.test.ts diff --git a/src/findings/index.ts b/src/findings/index.ts new file mode 100644 index 00000000..095b878c --- /dev/null +++ b/src/findings/index.ts @@ -0,0 +1,48 @@ +/** + * Shared finding-processing layer (Phase 3). + * + * Transforms raw candidate violation findings into a formatter-ready + * `ReviewResult` through one pipeline: evidence verification, filtering, + * count/density scoring, severity resolution, and diagnostics. + * + * This module is independent of model call (`single` | `agent`) and contains + * no legacy `judge`, `evaluator`, rubric, or autonomous-loop output + * compatibility code. See + * docs/plans/2026-07-10-phase-3-share-result-projection.md. + */ +export { + FINDING_EVIDENCE_NOT_LOCATABLE, + verifyFindingEvidence, + type FindingEvidenceDiagnostic, + type FindingEvidenceInput, + type FindingEvidenceVerification, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; + +export { + buildRuleId, + resolveCriterionId, + resolveSeverity, + type SeverityInput, +} from './severity'; + +export { scoreCheck, type ScoreCheckOptions, type ScoredCheck } from './scorer'; + +export { + processFindings, +} from './processor'; + +export { + FINDING_PROCESSING_INPUT_SCHEMA, + FINDINGS_CRITERION_SCHEMA, + GATE_CHECKS_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, + type FindingsCriterion, + type FindingProcessingInput, + type GateChecks, + type PromptMetaForFindings, + type RawViolation, + type RuleSeverity, + type Strictness, +} from './types'; diff --git a/src/findings/processor.ts b/src/findings/processor.ts new file mode 100644 index 00000000..45a1c7ae --- /dev/null +++ b/src/findings/processor.ts @@ -0,0 +1,167 @@ +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewResult, + ReviewScore, +} from '../review/types'; +import { computeFilterDecision } from '../evaluators/violation-filter'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; +import { scoreCheck } from './scorer'; +import { buildRuleId, resolveCriterionId } from './severity'; +import type { FindingProcessingInput, RawViolation } from './types'; + +interface VerifiedEntry { + raw: RawViolation; + coords: VerifiedFindingCoords; +} + +/** + * The single finding-processing pipeline for objective Via Negativa violation + * checks (Phase 3). Transforms raw candidate findings into a formatter-ready + * {@link ReviewResult}. + * + * Order (audit lines 167-174): + * 1. filter candidates by the evidence gate (`computeFilterDecision`); + * 2. verify each surfaced finding's evidence (anchored quote or diagnostic); + * 3. deduplicate verified findings by `(quoted_text, line)`; + * 4. score by verified finding count/density; + * 5. resolve severity from the density score; + * 6. assemble findings, score, and diagnostics. + * + * The verified finding count — not the raw candidate count — drives the score + * (audit Finding #6). Unanchored evidence becomes a + * `finding-evidence-not-locatable` warn diagnostic and emits no finding. + * `hadOperationalErrors` is `true` only when a diagnostic is `error` level. + */ +export function processFindings(input: FindingProcessingInput): ReviewResult { + const surfaced = filterCandidates(input.candidateFindings); + const { verified, evidenceDiagnostics } = verifyAndDedupe( + surfaced, + input.targetContent, + ); + + const scored = scoreCheck({ + verifiedViolations: verified.map((entry) => entry.raw), + wordCount: input.wordCount, + ...(input.promptMeta.strictness !== undefined + ? { strictness: input.promptMeta.strictness } + : {}), + ...(input.promptMeta.severity !== undefined + ? { promptSeverity: input.promptMeta.severity } + : {}), + }); + + const severity = scored.severity; + const scoreRuleId = buildRuleId(input.pack, input.ruleId); + + const findings: ReviewFinding[] = verified.map(({ raw, coords }) => { + const criterionId = resolveCriterionId( + input.promptMeta.criteria, + raw.criterionName, + ); + const ruleId = buildRuleId(input.pack, input.ruleId, criterionId); + const message = (raw.message || raw.analysis || '').trim(); + return { + ruleId, + ruleSource: input.ruleSource, + severity, + message, + line: coords.line, + column: coords.column, + match: coords.match, + ...(raw.analysis ? { analysis: raw.analysis } : {}), + ...(raw.suggestion ? { suggestion: raw.suggestion } : {}), + ...(raw.fix ? { fix: raw.fix } : {}), + }; + }); + + const scores: ReviewScore[] = [ + { + ruleId: scoreRuleId, + score: scored.score, + scoreText: scored.scoreText, + severity, + findingCount: verified.length, + }, + ]; + + const diagnostics: ReviewDiagnostic[] = [...evidenceDiagnostics]; + const hadOperationalErrors = diagnostics.some( + (diagnostic) => diagnostic.level === 'error', + ); + + return { findings, scores, diagnostics, hadOperationalErrors }; +} + +/** + * Drops candidate findings that fail the evidence gate. Filtered candidates + * contribute neither findings nor diagnostics, matching the current standard + * orchestrator behavior. + */ +function filterCandidates(candidates: readonly RawViolation[]): RawViolation[] { + const surfaced: RawViolation[] = []; + for (const candidate of candidates) { + const decision = computeFilterDecision(candidate); + if (decision.surface) { + surfaced.push(candidate); + } + } + return surfaced; +} + +/** + * Verifies each surfaced finding's evidence and deduplicates verified findings + * by `(quoted_text, line)`. Unanchored findings become warn diagnostics and are + * not emitted. + */ +function verifyAndDedupe( + surfaced: readonly RawViolation[], + targetContent: string, +): { verified: VerifiedEntry[]; evidenceDiagnostics: ReviewDiagnostic[] } { + const verified: VerifiedEntry[] = []; + const evidenceDiagnostics: ReviewDiagnostic[] = []; + const seen = new Set(); + + for (const candidate of surfaced) { + const verification = verifyFindingEvidence(targetContent, { + quoted_text: candidate.quoted_text || '', + ...(candidate.context_before !== undefined + ? { context_before: candidate.context_before } + : {}), + ...(candidate.context_after !== undefined + ? { context_after: candidate.context_after } + : {}), + ...(candidate.line !== undefined ? { line: candidate.line } : {}), + }); + + if (!verification.verified || !verification.finding) { + evidenceDiagnostics.push({ + level: 'warn', + code: FINDING_EVIDENCE_NOT_LOCATABLE, + message: + verification.diagnostic?.message ?? + 'Could not locate finding evidence in target content.', + }); + continue; + } + + const coords = verification.finding; + const dedupeKey = candidate.quoted_text + ? `${candidate.quoted_text}:${coords.line}` + : null; + if (dedupeKey) { + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + } + + verified.push({ raw: candidate, coords }); + } + + return { verified, evidenceDiagnostics }; +} diff --git a/tests/findings/module-surface.test.ts b/tests/findings/module-surface.test.ts new file mode 100644 index 00000000..a0e5c481 --- /dev/null +++ b/tests/findings/module-surface.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import * as findings from '../../src/findings'; + +describe('src/findings public surface', () => { + it('exports the processing entry point and helpers', () => { + expect(typeof findings.processFindings).toBe('function'); + expect(typeof findings.verifyFindingEvidence).toBe('function'); + expect(typeof findings.resolveSeverity).toBe('function'); + expect(typeof findings.buildRuleId).toBe('function'); + expect(typeof findings.resolveCriterionId).toBe('function'); + expect(typeof findings.scoreCheck).toBe('function'); + }); + + it('exports the stable diagnostic code constant', () => { + expect(findings.FINDING_EVIDENCE_NOT_LOCATABLE).toBe( + 'finding-evidence-not-locatable', + ); + }); + + it('exports the Zod boundary schemas', () => { + expect(findings.FINDING_PROCESSING_INPUT_SCHEMA).toBeDefined(); + expect(findings.RAW_VIOLATION_SCHEMA).toBeDefined(); + expect(findings.GATE_CHECKS_SCHEMA).toBeDefined(); + expect(findings.FINDINGS_CRITERION_SCHEMA).toBeDefined(); + expect(findings.PROMPT_META_FOR_FINDINGS_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/findings/processor.test.ts b/tests/findings/processor.test.ts new file mode 100644 index 00000000..90f9641d --- /dev/null +++ b/tests/findings/processor.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { processFindings } from '../../src/findings/processor'; +import { FINDING_EVIDENCE_NOT_LOCATABLE } from '../../src/findings/finding-evidence-verifier'; +import { REVIEW_RESULT_SCHEMA } from '../../src/review'; +import type { + FindingProcessingInput, + RawViolation, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + line: 1, + quoted_text: 'Alpha text', + context_before: '', + context_after: '', + analysis: 'Issue 1', + message: 'Issue 1', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +function baseInput( + candidateFindings: RawViolation[], + overrides: Partial = {}, +): FindingProcessingInput { + return { + pack: 'TestPack', + ruleId: 'CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + candidateFindings, + wordCount: 100, + promptMeta: { severity: 'warning', strictness: 'standard' }, + targetContent: 'Alpha text\nBeta text\n', + ...overrides, + }; +} + +describe('processFindings', () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + // Default threshold (0.75) unless a test opts in. + delete process.env.CONFIDENCE_THRESHOLD; + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + }); + + describe('golden objective-check output', () => { + beforeEach(() => { + // Surface every gate-passing candidate regardless of confidence. + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('produces byte-for-byte findings/score matching the standard orchestrator path', () => { + const input = baseInput([ + violation(), + violation({ + line: 2, + quoted_text: 'Beta text', + analysis: 'Issue 2', + message: 'Issue 2', + suggestion: 'Suggestion 2', + fix: 'Fix 2', + }), + ]); + + const result = processFindings(input); + + // Score: density 2/100 at standard strictness (10) -> 100 - 20 -> 8.0/10. + expect(result.scores).toEqual([ + { + ruleId: 'TestPack.CheckPrompt', + score: 8.0, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 2, + }, + ]); + + // Findings: Pack.Rule (check violations carry no criterion name), + // anchored lines/columns/match. + expect(result.findings).toHaveLength(2); + expect(result.findings[0]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + severity: 'warning', + message: 'Issue 1', + line: 1, + column: 1, + match: 'Alpha text', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + }); + expect(result.findings[1]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + severity: 'warning', + message: 'Issue 2', + line: 2, + column: 1, + match: 'Beta text', + }); + + expect(result.diagnostics).toEqual([]); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('returns a ReviewResult that validates against the review contract', () => { + const result = processFindings(baseInput([violation()])); + expect(() => REVIEW_RESULT_SCHEMA.parse(result)).not.toThrow(); + }); + }); + + describe('evidence verification (audit Finding #6)', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('turns unanchored evidence into a warn diagnostic and emits no finding', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'this quote is not in the content' }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostics[0]?.level).toBe('warn'); + // Score reflects 0 verified findings -> perfect 10.0/10. + expect(result.scores[0]?.score).toBe(10.0); + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('counts only verified findings toward the score, not raw candidate count', () => { + // One anchored + one unanchored candidate. Verified count = 1. + // Density 1/100 -> 100 - 10 -> 9.0/10 (the intentional fix). + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text' }), + violation({ quoted_text: 'nowhere to be found' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.score).toBe(9.0); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.diagnostics).toHaveLength(1); + }); + + it('does not set hadOperationalErrors for warn-level evidence diagnostics', () => { + const result = processFindings( + baseInput([violation({ quoted_text: 'missing' })]), + ); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('deduplicates verified findings by quoted_text and line', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', line: 1 }), + violation({ quoted_text: 'Alpha text', line: 1, message: 'dup' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + }); + }); + + describe('candidate filtering', () => { + it('drops candidates that fail computeFilterDecision without a diagnostic', () => { + // Default threshold 0.75: confidence 0.2 is filtered out. + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.2 }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + // No verified findings -> 10.0/10. + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('filtered candidates do not contribute to the score count', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.9 }), + violation({ quoted_text: 'Beta text', line: 2, confidence: 0.2 }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.scores[0]?.score).toBe(9.0); + }); + }); + + describe('rule id building', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('attributes findings to Pack.Rule when violations carry no criterion name', () => { + const result = processFindings(baseInput([violation({ quoted_text: 'Alpha text' })])); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt'); + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + + it('attributes findings to Pack.Rule.Criterion when a violation names a declared criterion', () => { + const result = processFindings( + baseInput( + [ + violation({ + quoted_text: 'Alpha text', + criterionName: 'Hedge words', + }), + ], + { + promptMeta: { + severity: 'warning', + strictness: 'standard', + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + }, + ), + ); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt.Hedging'); + // The score stays at Pack.Rule (no criterion), matching the orchestrator. + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + }); + + describe('severity resolution', () => { + it('resolves error severity from the density score and stamps it on findings', () => { + // 20 verified violations over 100 words -> 0.0/10 -> prompt severity error. + const result = processFindings( + baseInput( + Array.from({ length: 20 }, (_, i) => + violation({ + quoted_text: `Alpha text${i}`, + analysis: `Issue ${i}`, + message: `Issue ${i}`, + }), + ), + { + targetContent: Array.from({ length: 20 }, (_, i) => `Alpha text${i}`).join('\n') + '\n', + promptMeta: { severity: 'error', strictness: 'standard' }, + }, + ), + ); + + expect(result.scores[0]?.severity).toBe('error'); + expect(result.findings.every((f) => f.severity === 'error')).toBe(true); + }); + }); +}); From 671da9cd5e1cfae9863970274e338c541fc63893 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 03:24:02 +0100 Subject: [PATCH 13/34] Route standard check evaluation through shared processor - Build FindingProcessingInput and call processFindings in the standard objective-check branch of routePromptResult - Feed returned findings/scores/diagnostics through the existing line/json/rdjson/vale sinks and quality-score output - Remove inline filter/score/severity/quote-location logic from the check path; computeFilterDecision kept only for the debug-run artifact - Verified finding count now drives counts and score (audit Finding #6); unanchored quotes become warn diagnostics and no longer flag operational errors - Add orchestrator regression tests covering finding/score output, JSON/Vale shape, and exit behavior for supported objective-check reviews --- src/cli/orchestrator.ts | 120 ++++--- tests/orchestrator-check-processor.test.ts | 356 +++++++++++++++++++++ 2 files changed, 413 insertions(+), 63 deletions(-) create mode 100644 tests/orchestrator-check-processor.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f8df367a..a8e24436 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -12,6 +12,7 @@ import { printFileHeader, printIssueRow, printEvaluationSummaries, type Evaluati import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; +import { processFindings } from '../findings'; import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; @@ -616,79 +617,72 @@ function routePromptResult( let promptErrors = 0; let promptWarnings = 0; - // Handle Check Result + // Handle Check Result — routed through the shared finding processor + // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, + // filters, deduplicates, scores, and resolves severity; the orchestrator + // only feeds the returned ReviewResult to the existing formatter sinks. if (!isJudgeResult(result)) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - - // Score calculated from surfaced violations only — matches what user sees - const scored = calculateCheckScore( - surfacedViolations, - result.word_count, - { - strictness: promptFile.meta.strictness, - promptSeverity: promptFile.meta.severity, - } - ); - const severity = scored.severity; - // Group violations by criterionName - const violationsByCriterion = new Map< - string | undefined, - typeof surfacedViolations - >(); - for (const v of surfacedViolations) { - const criterionName = v.criterionName; - if (!violationsByCriterion.has(criterionName)) { - violationsByCriterion.set(criterionName, []); - } - violationsByCriterion.get(criterionName)!.push(v); - } - - // Report violations grouped by criterion - let totalErrors = 0; - let totalWarnings = 0; + const reviewResult = processFindings({ + pack: promptFile.pack, + ruleId: promptId, + ruleSource: promptFile.fullPath, + candidateFindings: result.violations, + wordCount: result.word_count, + promptMeta: { + ...(meta.severity !== undefined ? { severity: meta.severity } : {}), + ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), + ...(meta.criteria ? { criteria: meta.criteria } : {}), + }, + targetContent: content, + }); - for (const [criterionName, violations] of violationsByCriterion) { - // Find criterion ID from meta - let criterionId: string | undefined; - if (criterionName && meta.criteria) { - const criterion = meta.criteria.find(c => c.name === criterionName); - criterionId = criterion?.id; - } + // processFindings always returns exactly one score entry (processor contract). + const ruleScore = reviewResult.scores[0]!; + const severity = + ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - const ruleName = buildRuleName(promptFile.pack, promptId, criterionId); - - if (violations.length > 0) { - const violationResult = locateAndReportViolations({ - violations, - content, - relFile, - severity, - ruleName, - scoreText: '', - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; + // Report only verified findings through the existing line/json/rdjson/vale sink. + for (const finding of reviewResult.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity, + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); + } - if (severity === Severity.ERROR) { - totalErrors += violations.length; - } else { - totalWarnings += violations.length; - } + // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, + // consistent with prior operational reporting. They are warn-level and do not + // flag the run as operationally failed (audit Finding #6). + if (verbose) { + for (const diagnostic of reviewResult.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } } - // Create scoreEntry for Quality Scores display + // Verified finding count drives the counts (audit Finding #6). + const findingCount = reviewResult.findings.length; + const totalErrors = severity === Severity.ERROR ? findingCount : 0; + const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; + const scoreEntry: EvaluationSummary = { - id: buildRuleName(promptFile.pack, promptId, undefined), - scoreText: `${scored.final_score.toFixed(1)}/10`, - score: scored.final_score, + id: ruleScore.ruleId, + scoreText: ruleScore.scoreText, + score: ruleScore.score, }; if (debugJson) { + const { decisions, surfacedViolations } = getViolationFilterResults( + result.violations + ); const runId = randomUUID(); const model = getModelInfoFromEnv(); @@ -721,7 +715,7 @@ function routePromptResult( return { errors: totalErrors, warnings: totalWarnings, - hadOperationalErrors, + hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, scoreEntries: [scoreEntry], }; diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts new file mode 100644 index 00000000..eaaa1529 --- /dev/null +++ b/tests/orchestrator-check-processor.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { Result } from "../src/output/json-formatter"; +import type { ValeOutput } from "../src/schemas/vale-responses"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; + +const { EVALUATE_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), +})); + +type CheckViolation = RawCheckResult["violations"][number]; + +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +} as const; + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +/** + * Builds a check violation that passes the confidence gate by default. Real + * model output always carries `message`; the helper sets it explicitly so the + * test reflects production behavior rather than the analysis fallback. + */ +function makeCheckViolation( + overrides: Partial = {} +): CheckViolation { + return { + line: 1, + analysis: "Issue 1", + message: "Issue 1", + suggestion: "Suggestion 1", + fix: "Fix 1", + quoted_text: "Alpha text", + context_before: "", + context_after: "", + rule_quote: "Rule quote", + checks: FULLY_SUPPORTED_CHECKS, + confidence: 0.9, + ...overrides, + }; +} + +function makeCheckResult(params: { + violations: CheckViolation[]; + wordCount?: number; +}): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: params.violations, + word_count: params.wordCount ?? 100, + }; +} + +describe("standard check evaluation via the shared finding processor", () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + delete process.env.CONFIDENCE_THRESHOLD; + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + vi.restoreAllMocks(); + }); + + it("reports fully locatable findings, score, and counts unchanged", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckPrompt", + name: "Check Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First issue", analysis: "First issue" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second issue", + analysis: "Second issue", + suggestion: "Suggestion 2", + fix: "Fix 2", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. + expect(run.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("8.0/10"); + // The line summary prints the last segment of the Pack.Rule score id. + expect(scoreLine).toContain("CheckPrompt"); + }); + + it("counts only verified findings and excludes unanchored quotes from the score", async () => { + // One anchored + one unanchored quote. Both pass the confidence gate, so + // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CountingFixPrompt", + name: "Counting Fix Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "this quote is not anywhere in the content", + message: "Ghost issue", + analysis: "Ghost issue", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a + // finding; it contributes neither to the count nor the score. + expect(run.totalWarnings).toBe(1); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("9.0/10"); + }); + + it("does not flag severity errors when no finding can be verified", async () => { + // severity=error, but the only quote cannot anchor -> 0 verified findings -> + // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorNoVerifiedPrompt", + name: "Error No Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ + line: 9, + quoted_text: "nowhere to be found", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it("flags severity errors when verified error-severity findings exist", async () => { + // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> + // prompt severity error applies. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorVerifiedPrompt", + name: "Error Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [makeCheckViolation({ quoted_text: "Alpha text" })], + wordCount: 10, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it("emits verified findings with anchored location through the JSON sink", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckJsonPrompt", + name: "Check JSON Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First", analysis: "First" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second", + analysis: "Second", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.Json, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as Result; + const issues = Object.values(parsed.files).flatMap((file) => file.issues); + + expect(issues).toHaveLength(2); + expect(issues[0]).toMatchObject({ + line: 1, + column: 1, + severity: Severity.WARNING, + message: "First", + rule: "TestPack.CheckJsonPrompt", + match: "Alpha text", + }); + expect(issues[1]).toMatchObject({ + line: 2, + message: "Second", + match: "Beta text", + }); + }); + + it("omits unanchored quotes from the Vale JSON output", async () => { + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CheckValePrompt", + name: "Check Vale Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "missing quote", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.ValeJson, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as ValeOutput; + const issues = Object.values(parsed).flat(); + + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + Check: "TestPack.CheckValePrompt", + Line: 1, + Match: "Alpha text", + Severity: "warning", + }); + }); +}); From 9fe6440c7cc768ec0da72cb2a7fd2d73330b51d5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:36 +0100 Subject: [PATCH 14/34] Reject judge prompt type at the loader boundary - PROMPT_META_SCHEMA now rejects `type: judge` and its deprecated alias `subjective` via a superRefine, so judge-typed prompts fail to load with a descriptive warning instead of reaching evaluation. - superRefine (not refine) keeps the inferred union including "judge" so the legacy BaseEvaluator type-checks unchanged until Phase 4 deletes that path. - Add loader tests covering judge/subjective rejection and that check still loads; subjective and semi-objective are documented as deprecated aliases. --- src/schemas/prompt-schemas.ts | 26 ++++++++++--- tests/prompt-loader-validation.test.ts | 51 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index f8689645..2dadd85e 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -27,12 +27,15 @@ export const PROMPT_CRITERION_SCHEMA = z.object({ * - 'technical-accuracy': specialized evaluator with claim extraction + search * * Evaluation type: - * - 'judge': 1-4 scores per criterion, normalized to 1-10 * - 'check': density-based scoring (errors per 100 words) + * - 'semi-objective': deprecated alias for 'check' * - * Deprecated aliases (still supported): - * - 'subjective' → 'judge' - * - 'semi-objective' → 'check' + * 'judge' and its deprecated alias 'subjective' are no longer supported + * (Phase 3): subjective rubric scoring is not a future-facing review type. + * The enum still admits them so the Zod error is descriptive and the inferred + * type stays compatible with the legacy BaseEvaluator until Phase 4 deletes + * that path; a superRefine rejects them at this boundary so judge-typed prompts + * fail to load. * * Strictness factor for check scoring: * - Determines penalty weight per 1% error density. @@ -44,11 +47,24 @@ export const PROMPT_META_SCHEMA = z.object({ type: z .enum(["judge", "check", "subjective", "semi-objective"]) .transform((val) => { - // Map deprecated values to new canonical values + // Map deprecated values to their canonical forms. if (val === "subjective") return "judge" as const; if (val === "semi-objective") return "check" as const; return val; }) + .superRefine((val, ctx) => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). + // superRefine (not refine) so the inferred union still includes 'judge' + // and stays compatible with the legacy BaseEvaluator until Phase 4 deletes + // that path; the value is rejected at this boundary regardless. + if (val === "judge") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "judge evaluation is no longer supported; use check (Via Negativa) rules", + }); + } + }) .optional(), id: z.string(), name: z.string(), diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index 094ad7cb..fc34cc1d 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -130,4 +130,55 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); }); + + describe('Judge boundary rejection (Phase 3)', () => { + it('rejects judge-typed prompts at the loader boundary', () => { + createPrompt('judge.md', `--- +evaluator: base +id: Judge +name: Judge Prompt +type: judge +criteria: + - name: Clarity + id: Clarity +--- +Judge content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('judge.md'); + expect(warnings[0]).toMatch(/judge/i); + }); + + it('rejects subjective-typed prompts (deprecated judge alias)', () => { + createPrompt('subjective.md', `--- +evaluator: base +id: Subjective +name: Subjective Prompt +type: subjective +--- +Subjective content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('subjective.md'); + }); + + it('still loads check-typed prompts', () => { + createPrompt('check.md', `--- +evaluator: base +id: Check +name: Check Prompt +type: check +--- +Check content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(warnings).toHaveLength(0); + expect(prompts).toHaveLength(1); + expect(prompts[0].meta.type).toBe('check'); + }); + }); }); From d36d06dd139439667ac765b4adfe1e1162a76e42 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:54 +0100 Subject: [PATCH 15/34] Remove legacy judge review path - Drop the judge branch of routePromptResult and the now-dead criterion extraction/reporting helpers (extractAndReportCriterion, validateCriteriaCompleteness, validateScores, locateAndReportViolations, buildRuleName); subjective rubric scoring is not a future-facing review type. - A JudgeResult that still reaches the orchestrator is refused explicitly (operational error, no findings) rather than misprojected as a check result; no adapter is added for old judge metadata. - Standard objective-check orchestration still routes through processFindings unchanged. - Replace the judge-filtering orchestrator test with a judge-rejection test. --- src/cli/orchestrator.ts | 545 +-------------------------- tests/orchestrator-filtering.test.ts | 41 +- 2 files changed, 28 insertions(+), 558 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index a8e24436..4d09e917 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -6,10 +6,9 @@ import * as os from 'os'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; -import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { processFindings } from '../findings'; @@ -21,8 +20,7 @@ import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '. import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; import type { EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessViolationsParams, - ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, + ReportIssueParams, ProcessPromptResultParams, RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, RunPromptEvaluationResultSuccess } from './types'; @@ -33,7 +31,6 @@ import { import { calculateCheckScore } from '../scoring'; import { countWords } from '../chunking/utils'; import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { locateQuotedText } from "../output/location"; import { computeFilterDecision, type FilterDecision, @@ -64,23 +61,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } -/* - * Constructs a hierarchical rule name following the pattern: - * - With criterion: PackName.RuleId.CriterionId - * - Without criterion: PackName.RuleId - */ -function buildRuleName( - packName: string, - ruleId: string, - criterionId: string | undefined -): string { - const parts = [packName, ruleId]; - if (criterionId) { - parts.push(criterionId); - } - return parts.join('.'); -} - /* * Generic concurrency runner that executes workers in parallel up to a specified limit. * Preserves result order matching input order. @@ -174,119 +154,6 @@ function reportIssue(params: ReportIssueParams): void { } } -/* - * Locates and reports each violation using pre/post evidence markers. - * If location matching fails (missing markers, content mismatch), logs warning - * and continues processing. Returns hadOperationalErrors=true if any violations - * couldn't be located, signaling text matching issues vs. content quality issues. - */ -function locateAndReportViolations(params: ProcessViolationsParams): { - hadOperationalErrors: boolean; -} { - const { - violations, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose, - } = params; - - let hadOperationalErrors = false; - - // Locate all violations and filter out those that can't be verified - // Then de-duplicate by (quoted_text, line) - const seen = new Set(); - const verifiedViolations: Array<{ - v: (typeof violations)[0]; - line: number; - column: number; - matchedText: string; - rowSummary: string; - }> = []; - - for (const v of violations) { - if (!v) continue; - - const rowSummary = (v.message || "").trim(); - - try { - const locWithMatch = locateQuotedText( - content, - { - quoted_text: v.quoted_text || "", - context_before: v.context_before || "", - context_after: v.context_after || "", - }, - 80, - v.line - ); - - if (!locWithMatch) { - // Can't verify this quote exists - skip it entirely - if (verbose) { - console.warn( - `[vectorlint] Skipping unverifiable quote: "${v.quoted_text}"` - ); - } - hadOperationalErrors = true; - continue; - } - - const line = locWithMatch.line; - const column = locWithMatch.column; - const matchedText = locWithMatch.match || ""; - - // De-duplicate by (quoted_text, line) - skip if quoted_text is empty - const dedupeKey = v.quoted_text ? `${v.quoted_text}:${line}` : null; - if (dedupeKey && seen.has(dedupeKey)) { - continue; // Skip duplicate - } - if (dedupeKey) { - seen.add(dedupeKey); - } - - verifiedViolations.push({ v, line, column, matchedText, rowSummary }); - } catch (e: unknown) { - const err = handleUnknownError(e, "Locating evidence"); - if (verbose) { - console.warn(`[vectorlint] Error locating evidence: ${err.message}`); - } - hadOperationalErrors = true; - } - } - - // Report only verified, unique violations - for (const { - v, - line, - column, - matchedText, - rowSummary, - } of verifiedViolations) { - reportIssue({ - file: relFile, - line, - column, - severity, - summary: rowSummary, - ruleName, - outputFormat, - jsonFormatter, - ...(v.analysis !== undefined && { analysis: v.analysis }), - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), - ...(v.fix !== undefined && { fix: v.fix }), - scoreText, - match: matchedText, - }); - } - - return { hadOperationalErrors }; -} - function getViolationFilterResults< TViolation extends Parameters[0] >( @@ -304,297 +171,10 @@ function getViolationFilterResults< } /* - * Extracts pre-calculated scores from a subjective evaluation criterion and reports surfaced violations. - * Violations that do not pass computeFilterDecision are not reported. - * Returns error/warning counts, score entry for Quality Scores, and score components for JSON. - */ -function extractAndReportCriterion( - params: ProcessCriterionParams -): ProcessCriterionResult { - const { - exp, - result, - content, - relFile, - packName, - promptId, - meta, - outputFormat, - jsonFormatter, - verbose, - } = params; - let hadOperationalErrors = false; - let hadSeverityErrors = false; - - const nameKey = String(exp.name || exp.id || ""); - const criterionId = exp.id - ? String(exp.id) - : exp.name - ? String(exp.name) - .replace(/[^A-Za-z0-9]+/g, " ") - .split(" ") - .filter(Boolean) - .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) - .join("") - : ""; - const ruleName = buildRuleName(packName, promptId, criterionId); - - const weightNum = exp.weight || 1; - const maxScore = weightNum; - - // Target gating (deterministic precheck) - const metaTargetSpec = meta.target; - const expTargetSpec = exp.target; - const targetCheck = checkTarget(content, metaTargetSpec, expTargetSpec); - const missingTarget = targetCheck.missing; - - if (missingTarget) { - hadSeverityErrors = true; - const summary = "target not found"; - const suggestion = - targetCheck.suggestion || - expTargetSpec?.suggestion || - metaTargetSpec?.suggestion || - "Add the required target section."; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity: Severity.ERROR, - summary, - ruleName, - outputFormat, - jsonFormatter, - suggestion, - scoreText: "nil", - match: "", - }); - return { - errors: 1, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "0.0/10", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const got = result.criteria.find( - (c) => c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) { - return { - errors: 0, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "-", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const score = Number(got.score); - - // Use pre-calculated values from evaluator - const rawWeighted = got.weighted_points; - const normalizedScore = got.normalized_score; - const userScore = rawWeighted; - const violations = got.violations; - const { surfacedViolations } = getViolationFilterResults(violations); - - // Display normalized score (1-10) in CLI output - const scoreText = `${normalizedScore.toFixed(1)}/10`; - - // Determine severity based on violations - // If there are violations, use evaluator's scoring to determine severity - // Score <= 1 = error, score = 2 = warning, score > 2 = no severity needed (but we still create scoreEntry) - let errors = 0; - let warnings = 0; - let severity: Severity | undefined; - - if (surfacedViolations.length > 0) { - // Determine severity from score for violations - if (score <= 1) { - severity = Severity.ERROR; - hadSeverityErrors = true; - errors = surfacedViolations.length; - } else if (score === 2) { - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } else { - // Score > 2 but has violations - this is informational - // Use WARNING as default for informational violations - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } - - // Report surfaced violations only - const violationResult = locateAndReportViolations({ - violations: surfacedViolations as Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - analysis?: string; - suggestion?: string; - }>, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = - hadOperationalErrors || violationResult.hadOperationalErrors; - } else if (score <= 2) { - // No violations but low score - report with summary - severity = score <= 1 ? Severity.ERROR : Severity.WARNING; - if (severity === Severity.ERROR) { - hadSeverityErrors = true; - errors = 1; - } else { - warnings = 1; - } - - const sum = got.summary.trim(); - const words = sum.split(/\s+/).filter(Boolean); - const limited = words.slice(0, 15).join(" "); - const summaryText = limited || "No findings"; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: summaryText, - ruleName, - outputFormat, - jsonFormatter, - scoreText, - match: "", - }); - } - - return { - errors, - warnings, - userScore, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText, score: normalizedScore }, - scoreComponent: { - criterion: nameKey, - rawScore: score, - maxScore: 4, - weightedScore: rawWeighted, - weightedMaxScore: weightNum, - normalizedScore: normalizedScore, - normalizedMaxScore: 10, - }, - }; -} - -/* - * Validates that all expected criteria are present in the result. - */ -function validateCriteriaCompleteness(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - const expectedNames = new Set( - (meta.criteria || []).map((c) => String(c.name || c.id || "")) - ); - const returnedNames = new Set( - result.criteria.map((c: { name: string }) => c.name) - ); - - // Create normalized maps for case-insensitive lookup - const expectedNormalized = new Set(); - const expectedOriginalMap = new Map(); - for (const name of expectedNames) { - const norm = name.toLowerCase(); - expectedNormalized.add(norm); - expectedOriginalMap.set(norm, name); - } - - const returnedNormalized = new Set(); - for (const name of returnedNames) { - returnedNormalized.add(name.toLowerCase()); - } - - for (const norm of expectedNormalized) { - if (!returnedNormalized.has(norm)) { - console.error( - `Missing criterion in model output: ${expectedOriginalMap.get(norm)}` - ); - hadErrors = true; - } - } - - for (const name of returnedNames) { - if (!expectedNormalized.has(name.toLowerCase())) { - console.warn( - `[vectorlint] Extra criterion returned by model (ignored): ${name}` - ); - } - } - - return hadErrors; -} - -/* - * Validates that all criterion scores are within valid range. - */ -function validateScores(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - for (const exp of meta.criteria || []) { - const nameKey = String(exp.name || exp.id || ""); - const got = result.criteria.find( - (c) => - c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) continue; - - const score = Number(got.score); - if (!Number.isFinite(score) || score < 0 || score > 4) { - console.error(`Invalid score for ${nameKey}: ${score}`); - hadErrors = true; - } - } - - return hadErrors; -} - -/* - * Routes evaluation results through check or judge processing paths. - * Check: Reports violations, creates scoreEntry using final_score. - * Judge: Iterates through criteria, validates scores, creates scoreEntry per criterion. - * Both paths generate scoreEntries for Quality Scores display. + * Routes an evaluation result through the shared finding processor. + * Check results are verified, filtered, scored, and reported; judge/rubric + * results are rejected (Phase 3) because subjective scoring is not a + * future-facing review type. */ function routePromptResult( params: ProcessPromptResultParams @@ -612,11 +192,6 @@ function routePromptResult( const meta = promptFile.meta; const promptId = (meta.id || "").toString(); - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let promptErrors = 0; - let promptWarnings = 0; - // Handle Check Result — routed through the shared finding processor // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, // filters, deduplicates, scores, and resolves severity; the orchestrator @@ -721,105 +296,21 @@ function routePromptResult( }; } - // Handle Judge Result - // Validate criterion completeness and scores - hadOperationalErrors = - validateCriteriaCompleteness({ meta, result }) || hadOperationalErrors; - hadOperationalErrors = - validateScores({ meta, result }) || hadOperationalErrors; - - // Reset promptErrors and promptWarnings for subjective results - promptErrors = 0; - promptWarnings = 0; - const criterionScores: EvaluationSummary[] = []; - const scoreComponents: ScoreComponent[] = []; - - // Iterate through each criterion - for (const exp of meta.criteria || []) { - const criterionResult = extractAndReportCriterion({ - exp, - result, - content, - relFile, - packName: promptFile.pack, - promptId, - promptFilename: promptFile.filename, - meta, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - - promptErrors += criterionResult.errors; - promptWarnings += criterionResult.warnings; - hadOperationalErrors = - hadOperationalErrors || criterionResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || criterionResult.hadSeverityErrors; - criterionScores.push(criterionResult.scoreEntry); - - if (criterionResult.scoreComponent) { - scoreComponents.push(criterionResult.scoreComponent); - } - } - - if (outputFormat === OutputFormat.Json && scoreComponents.length > 0) { - (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore( - relFile, - { - id: promptId || promptFile.filename.replace(/\.md$/, ""), - scores: scoreComponents, - } + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a + // caller bypassed that boundary. Refuse it explicitly rather than + // misprojecting it as a check result. No adapter is added for judge metadata. + if (verbose) { + console.warn( + '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' ); } - - if (debugJson) { - const runId = randomUUID(); - const flat = result.criteria.flatMap((c) => - (c.violations || []).map((v, i) => ({ - criterion: c.name, - index: i, - violation: v, - decision: computeFilterDecision(v), - })) - ); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "judge", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: flat.map((x) => ({ - criterion: x.criterion, - index: x.index, - surface: x.decision.surface, - reasons: x.decision.reasons, - })), - surfaced_violations: flat.filter((x) => x.decision.surface).map((x) => ({ - criterion: x.criterion, - violation: x.violation, - })), - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - return { - errors: promptErrors, - warnings: promptWarnings, - hadOperationalErrors, - hadSeverityErrors, - scoreEntries: criterionScores, + errors: 0, + warnings: 0, + hadOperationalErrors: true, + hadSeverityErrors: false, + scoreEntries: [], }; } diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index 73e56119..b4d2cf89 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -307,7 +307,11 @@ describe("CLI violation filtering", () => { expect(scoreLine).not.toContain("8.0/10"); }); - it("filters low-confidence judge violations from CLI counts by default", async () => { + it("rejects judge/rubric results instead of projecting them as check findings", async () => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // orchestrator refuses a JudgeResult rather than running the old criterion + // extraction/reporting path: no findings are projected and the run is + // flagged with an operational error. const targetFile = createTempFile("Alpha text\nBeta text\n"); const prompt = createPrompt({ id: "JudgePrompt", @@ -323,42 +327,17 @@ describe("CLI violation filtering", () => { makeJudgeViolation({ line: 2, quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", confidence: 0.2, }), ]) ); - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - confidence: 0.2, - }), - ]) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.totalWarnings).toBe(0); + expect(run.hadOperationalErrors).toBe(true); + expect(run.hadSeverityErrors).toBe(false); }); it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { From d4ea9a7764f682e1288a09303f7fa94d0c20c599 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:20 +0100 Subject: [PATCH 16/34] Remove orphaned orchestrator param types from CLI contract - Drop ProcessViolationsParams, ProcessCriterionParams, ProcessCriterionResult, and ValidationParams from src/cli/types.ts. They were orphaned when the orchestrator's inline violation/criterion processing moved to src/findings (Tasks 2-3) and the judge path was removed, and are referenced nowhere else. - Remove the now-unused imports PromptMeta, PromptCriterionSpec, ScoreComponent, and JudgeResult. Severity and PromptEvaluationResult are retained (still used by ReportIssueParams / ProcessPromptResultParams). --- src/cli/types.ts | 42 ++---------------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/src/cli/types.ts b/src/cli/types.ts index 7f95192a..a083f2e2 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,13 +1,12 @@ import type { PromptFile } from '../prompts/prompt-loader'; import type { LLMProvider } from '../providers/llm-provider'; import type { SearchProvider } from '../providers/search-provider'; -import type { PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; -import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; +import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; @@ -95,43 +94,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessViolationsParams extends EvaluationContext { - violations: Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - message?: string; - analysis?: string; - suggestion?: string; - fix?: string; - }>; - severity: Severity; - ruleName: string; - scoreText: string; -} - -export interface ProcessCriterionParams extends EvaluationContext { - exp: PromptCriterionSpec; - result: JudgeResult; - packName: string; - promptId: string; - promptFilename: string; - meta: PromptMeta; -} - -export interface ProcessCriterionResult extends ErrorTrackingResult { - userScore: number; - maxScore: number; - scoreEntry: { id: string; scoreText: string; score?: number }; - scoreComponent?: ScoreComponent; -} - -export interface ValidationParams { - meta: PromptMeta; - result: JudgeResult; -} - export interface ProcessPromptResultParams extends EvaluationContext { promptFile: PromptFile; result: PromptEvaluationResult; From 15ec7a3e146df0820d39b9680e06b0369d5dd33e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:33 +0100 Subject: [PATCH 17/34] Document finding-processing behavior and clean stale judge docs - Add tests/findings/README.md mapping the behavior moved into src/findings (filtering, evidence verification, density scoring, rule-id/severity, formatter routing) and the intentionally removed judge/rubric and agent behavior, with agent tests marked out of scope for Phase 3 (Phase 3 Task 8). - Stop describing type: judge/subjective as a valid future-facing prompt type in CREATING_RULES.md, docs/frontmatter-fields.mdx, and docs/style-guide.mdx; note that judge/subjective are now rejected at load. - Drop the stale rubric-based scoring feature blurb from README.md. --- CREATING_RULES.md | 197 +++--------------------------------- README.md | 3 +- docs/frontmatter-fields.mdx | 80 ++------------- docs/style-guide.mdx | 113 ++------------------- tests/findings/README.md | 83 +++++++++++++++ 5 files changed, 117 insertions(+), 359 deletions(-) create mode 100644 tests/findings/README.md diff --git a/CREATING_RULES.md b/CREATING_RULES.md index ec90954f..7b800c9d 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -8,7 +8,6 @@ A comprehensive guide to creating powerful, reusable content evaluations using V - [Rule Anatomy](#rule-anatomy) - [Evaluation Modes](#evaluation-modes) - [Check Rules](#check-rules) -- [Judge Rules](#judge-rules) - [Target Specification](#target-specification) - [Configuration Reference](#configuration-reference) - [Best Practices](#best-practices) @@ -25,7 +24,7 @@ VectorLint rules are Markdown files with YAML frontmatter that define how your c - **Rule = Prompt file** (`.md` file organized in rule packs) - **Pack** = Subdirectory containing related rules (typically named after a company/style guide) - **Criteria** = Individual quality checks within a rule -- **Score** = LLM-assigned rating (1-4 scale for judge, density-based for check) +- **Score** = Density-based quality score derived from violation count - **Severity** = How failures are reported (`error` or `warning`) @@ -74,27 +73,20 @@ project/ ## Evaluation Modes -VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in two distinct modes, determined by the `type` field: +VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in one mode, determined by the `type` field: | Mode | `type` | Use Case | Scoring | Output | | ---------- | ------- | ------------------------------------- | --------------------------- | ----------------------- | | **Check** | `check` | Pass/fail checks, counting violations | 10 points - 1 per violation | List of specific issues | -| **Judge** | `judge` | Multi-dimensional quality scoring | 0-4 scale per criterion | Weighted average score | -### When to Use Each +The deprecated `type: judge` (alias `subjective`) is no longer supported; a rule using it is rejected at load. -**Use Check when:** +### When to Use Check - You need to find specific errors (e.g., "Find all grammar mistakes") - The check is binary (Pass/Fail) for each item - You want a list of specific violations to fix -**Use Judge when:** - -- You're measuring quality on a spectrum (e.g., "How engaging is this?") -- You have multiple dimensions (Clarity, Tone, Depth) -- You need weighted importance (some criteria matter more) - --- ## Check Rules @@ -141,71 +133,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes --- -## Judge Rules - -Judge rules use weighted criteria and a 1-4 rubric for sophisticated quality measurement. - -### Structure - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator - -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator... [Your detailed instructions] - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit - -### Good - -Clear benefit but less specific impact - -... -``` - -### The 1-4 Scoring Scale - -VectorLint uses a **1-4 scale** for all judge criteria, which is then normalized to a 1-10 scale: - -| LLM Rating | Meaning | Normalized Score | -| :--------- | :-------- | :--------------- | -| **4** | Excellent | **10.0** | -| **3** | Good | **7.0** | -| **2** | Fair | **4.0** | -| **1** | Poor | **1.0** | - -### Score Calculation - -1. **Normalization**: We map the 1-4 rating to a 1-10 score using the formula: `1 + ((Rating - 1) / 3) * 9`. -2. **Weighted Average**: The final score is the weighted average of all normalized criterion scores. - -**Example:** - -- Criterion: "Value Communication" (weight=12) -- Rating: 3 (Good) -> Normalized: 7.0 -- Weighted Points: 7.0 \* 12 = 84 points - ---- - ## Target Specification The `target` field allows you to: @@ -243,27 +170,17 @@ target: ### Frontmatter Fields -| Field | Type | Required | Description | -| ------------- | ------------- | -------- | ------------------------------------------------------------------ | -| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | -| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | -| `type` | string | No | Mode: `judge` or `check` (default: `check`) | -| `id` | string | **Yes** | Unique identifier (used in error reporting) | -| `name` | string | **Yes** | Human-readable name | +| Field | Type | Required | Description | +| ------------- | ------------- | -------- | -------------------------------------------------------------- | +| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | +| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | +| `type` | string | No | Mode: `check` (default: `check`). `judge`/`subjective` rejected at load | +| `id` | string | **Yes** | Unique identifier (used in error reporting) | +| `name` | string | **Yes** | Human-readable name | | `severity` | string | No | `error` or `warning` (default: `warning`) | | `evaluateAs` | string | No | `document` or `chunk` - whether to evaluate content as a whole or in chunks (default: `chunk`) | | `target` | object | No | Content matching specification | -| `criteria` | array | **Yes\*** | List of evaluation criteria (\*required for judge) | - -### Criterion Fields - -| Field | Type | Required | Description | -| -------- | ------ | -------- | ------------------------------------------ | -| `name` | string | **Yes** | Human-readable criterion name | -| `id` | string | **Yes** | Unique identifier (PascalCase recommended) | -| `weight` | number | No | Importance weight (default: 1) | -| `target` | object | No | Criterion-specific content matching | --- @@ -288,25 +205,10 @@ You are a headline evaluator for developer blog posts. Assess whether the headli 2. Uses natural, conversational language (avoid buzzwords) 3. Creates curiosity without being clickbait -For each criterion, provide a score (0-4) and specific examples from the text. -``` - -### 2. **Use Meaningful Weights (Subjective)** - -Scale weights to reflect real-world importance: - -```yaml -criteria: - # Technical accuracy is critical - - name: Technical Accuracy - weight: 40 - - # Readability is important - - name: Readability - weight: 30 +For each violation, quote the exact text and suggest a concrete fix. ``` -### 3. **Provide Context in Prompts** +### 2. **Provide Context in Prompts** Help the LLM understand your domain: @@ -338,79 +240,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes Report any errors found with specific examples. ``` -### Example 2: Headline Evaluator (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: Headline -name: Headline Evaluator - -severity: error -target: - regex: '^#\s+(.+)$' - flags: "mu" - group: 1 - required: true - suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 10 - - name: Language Authenticity - id: LanguageAuthenticity - weight: 5 ---- - -You are a headline evaluator. Assess the H1 headline for: - -1. **Value Communication** (10 points): Does it clearly state what the reader gains? -2. **Language Authenticity** (5 points): Does it use natural, conversational language? - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit clearly stated - -... -``` - -### Example 3: AI Pattern Detector (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: AIPatterns -name: AI Pattern Detector - -severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 ---- - -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Buzzwords**: leverage, synergy, elevate -2. **Formulaic transitions**: Moreover, Furthermore - ... -``` - ## Resources - [VectorLint README](./README.md) - Installation and basic usage diff --git a/README.md b/README.md index f6511e6a..662ac7f0 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,9 @@ If you can write a prompt for it, you can lint it with VectorLint. ## Quality Scores -VectorLint scores your content using error density and a rubric-based system, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. +VectorLint scores your content using error density, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. - **Density-Based Scoring:** For errors that can be counted, scores are calculated based on **error density (errors per 100 words)**, making quality assessment fair across documents of any length. -- **Rubric-Based Scoring:** For more nuanced quality standards, like flow and completeness, scores are graded on a 1-4 rubric system and then normalized to a **1-10 scale**. ## How VectorLint Reduces False Positives diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index 82e7d93c..d840d7b9 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -27,14 +27,12 @@ Your LLM prompt goes here. | Field | Default | Description | |---|---|---| -| `specVersion` | _(none)_ | Rule spec version. Set to `1.0.0` for rules using the full judge schema. | +| `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | | `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | -| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` scores content against weighted criteria on a 1–4 rubric. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations and scores them by error density. `judge` (and its deprecated `subjective` alias) is no longer supported — a rule with `type: judge` or `type: subjective` is rejected at load. | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | | `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | | `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | -| `criteria` | _(none)_ | Required when `type` is `judge`. Defines the scoring dimensions and their weights. See [Criteria fields](#criteria-fields) below. | - ## Target fields The `target` field narrows evaluation to a specific part of the document. It is an object with the following sub-fields: @@ -58,40 +56,16 @@ target: suggestion: Add an H1 headline for the article. ``` -## Criteria fields - -The `criteria` field is an array of criterion objects used with `type: judge`. Each criterion defines one scoring dimension. - -| Field | Type | Required | Description | -|---|---|---|---| -| `name` | string | Yes | Human-readable criterion name shown in CLI output. | -| `id` | string | Yes | Unique identifier for this criterion. PascalCase recommended (e.g., `TechnicalAccuracy`). Referenced in the Markdown body rubric as `# CriterionName `. | -| `weight` | number | No | Relative importance of this criterion in the weighted average. Higher values increase the criterion's influence on the final score. Defaults to `1`. | -| `target` | object | No | Criterion-specific content matching. Uses the same sub-fields as the top-level `target`. Overrides the rule-level target for this criterion only. | - -**Example — judge rule with two weighted criteria:** - -```yaml -criteria: - - name: Technical Accuracy - id: TechnicalAccuracy - weight: 40 - - name: Readability - id: Readability - weight: 30 -``` - ## Complete example -The following rule uses all commonly-used fields. It targets the H1 headline, judges it against two criteria, and requires the headline to exist before evaluation runs. +The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before evaluation runs. ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator +type: check +id: HeadlineQuality +name: Headline Quality severity: error evaluateAs: document target: @@ -100,44 +74,10 @@ target: group: 1 required: true suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 --- -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. +You are a headline reviewer for developer blog posts. Evaluate the H1 headline +and report specific, fixable violations — for example: a vague benefit, a +missing subject, or passive voice. For each violation, quote the exact text and +suggest a concrete rewrite. ``` diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index 1fcaccb1..3dfe0c20 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -79,21 +79,19 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| | `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | -| `type` | `check` | Evaluation mode: `check` or `judge`. See below. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` (and its `subjective` alias) is no longer supported and is rejected at load. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | | `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | -| `criteria` | _(none)_ | Required for `judge` rules. Defines the scoring dimensions. | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | ## Evaluation Modes -VectorLint supports two evaluation modes, chosen with the `type` field. +VectorLint uses a single evaluation mode, set with the `type` field. | Mode | `type` value | Best for | |------|-------------|----------| | Check | `check` | Finding specific, countable violations (grammar errors, banned terms) | -| Judge | `judge` | Measuring quality on a spectrum (clarity, tone, completeness) | ### Check rules @@ -122,69 +120,6 @@ Report any errors found with specific examples. | `standard` (4–7) | ~10 points | | `strict` (8–10) | ~20 points | -### Judge rules - -The LLM scores content against multiple weighted criteria using a 1–4 rubric. VectorLint normalizes each score to a 1–10 scale and computes a weighted average. - -| LLM rating | Meaning | Normalized | -|------------|---------|------------| -| 4 | Excellent | 10.0 | -| 3 | Good | 7.0 | -| 2 | Fair | 4.0 | -| 1 | Poor | 1.0 | - -Define criteria in the frontmatter and expand each into a rubric section in the Markdown body: - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. -``` - ## Targeting Specific Content The `target` field lets you evaluate a specific portion of a document — for example, only the H1 headline — rather than the full content. @@ -227,51 +162,23 @@ RunRules=Acme ## Examples -### AI Pattern Detector (Judge) +### AI Pattern Detector (Check) ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge +type: check id: AIPatterns name: AI Pattern Detector severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 --- -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Overused buzzwords**: leverage, synergy, elevate, unlock, empower -2. **Formulaic transitions**: Moreover, Furthermore, In conclusion -3. **Hollow openings**: "In today's fast-paced world..." -4. **Excessive hedging**: "It's worth noting that...", "It's important to mention..." - -## RUBRIC - -# Language Authenticity - -### Excellent -Natural, human voice throughout. No AI patterns detected. - -### Good -Mostly natural. One or two minor patterns present. - -### Fair -Several AI patterns reduce authenticity. - -### Poor -Pervasive AI patterns throughout. +Detect AI-generated writing patterns in this content. Report each occurrence as a +violation: overused buzzwords (leverage, synergy, elevate, unlock, empower), +formulaic transitions (Moreover, Furthermore, In conclusion), hollow openings +("In today's fast-paced world..."), and excessive hedging ("It's worth noting +that..."). For each violation, quote the exact text and suggest a concrete, +human-sounding rewrite. ``` ## Next Steps diff --git a/tests/findings/README.md b/tests/findings/README.md new file mode 100644 index 00000000..f48cc664 --- /dev/null +++ b/tests/findings/README.md @@ -0,0 +1,83 @@ +# Finding Processing — Test Coverage & Behavior Mapping + +This directory owns the regression coverage for the **shared objective +violation-check finding-processing pipeline** (`src/findings/`). It documents +which behaviors moved out of `src/cli/orchestrator.ts` (and adjacent output +tests) into `src/findings/`, which behaviors were **intentionally removed**, and +which behaviors are **out of scope** for this phase. + +The pipeline (`processFindings`) composes a single path: + +```text +candidate findings ──▶ verify evidence ──▶ filter ──▶ dedupe ──▶ score ──▶ severity ──▶ ReviewResult +``` + +It is independent of model call (`single` / `agent` / `auto`) and carries no +`judge`, `evaluator`, rubric, or autonomous-agent compatibility path. + +--- + +## Behavior-preservation mapping + +| Behavior (origin) | Preserved by | Notes | +|---|---|---| +| Violation filtering / surfacing (`orchestrator.ts` `computeFilterDecision`) | `processor.test.ts` — *drops candidates that fail computeFilterDecision without a diagnostic*; *filtered candidates do not contribute to the score count* | Filter logic itself stays in `src/debug/violation-filter.ts`; the processor is a caller. `orchestrator-filtering.test.ts` still guards the end-to-end CLI surfacing. | +| Fuzzy finding-evidence location (`locateQuotedText`) | `finding-evidence-verifier.test.ts` — exact-quote location, warn diagnostic on miss, **no model-line fallback**, anchors without a line hint, diagnostic names the quoted text | One verifier (audit Finding #6). The agent path's model-line fallback is **not** preserved (see Removed). | +| Verified-finding counting & density scoring (`calculateCheckScore`) | `scorer.test.ts` — same numeric result as `calculateCheckScore`; score driven by verified count, not raw candidate count; resolves error severity; forwards strictness/severity | `scorer.ts` is a thin wrapper; no reimplemented math. | +| Severity + rule-id attribution | `severity.test.ts` — `resolveSeverity`, `buildRuleId` (`Pack.Rule` / `Pack.Rule.Criterion`), `resolveCriterionId`; `processor.test.ts` — stamps severity, attributes to pack rule / criterion | One severity path; one rule-id builder (audit Finding #4). | +| Golden check output (byte-for-byte) | `processor.test.ts` — *produces byte-for-byte findings/score matching the standard orchestrator path* | Proves no regression before the orchestrator was rewired (Task 2). | +| Unanchored-evidence diagnostics & exit behavior | `processor.test.ts` — *turns unanchored evidence into a warn diagnostic and emits no finding*; *counts only verified findings toward the score*; *does not set hadOperationalErrors for warn-level evidence diagnostics* | Intentional Phase 3 fix: unanchored quotes no longer flag the run as operationally failed. | +| Deduplication | `processor.test.ts` — *deduplicates verified findings by quoted_text and line* | | +| Contract / input boundary | `types.test.ts` — `FINDING_PROCESSING_INPUT_SCHEMA`, `RAW_VIOLATION_SCHEMA`, `PROMPT_META_FOR_FINDINGS_SCHEMA` parse supported input and reject legacy `evaluator`/`judge`/`rubric weight`/`agent`/`modelCall`/`mode`; `module-surface.test.ts` — public surface + diagnostic code constant | Strict Zod at the boundary; no `any`. | +| Formatter routing (line / JSON / RDJSON / Vale) | `orchestrator-check-processor.test.ts` — locatable findings + score + counts unchanged; verified findings through the JSON sink; unanchored quotes omitted from Vale JSON. `orchestrator-filtering.test.ts` — no dummy JSON/Vale issues when nothing is surfaced | End-to-end coverage of the rewired check path feeding existing formatters from `ReviewResult`. | + +--- + +## Intentionally removed behavior (no replacement fixture) + +These were legacy paths that the Phase 3 contract no longer supports. They are +covered by **rejection tests**, not behavior-parity tests. + +| Removed behavior | Rejection coverage | +|---|---| +| Subjective `judge` / rubric criterion evaluation | `prompt-loader-validation.test.ts` — *Judge boundary rejection (Phase 3)* rejects `type: judge` and the `subjective` alias at the loader boundary; `orchestrator-filtering.test.ts` — *rejects judge/rubric results instead of projecting them as check findings* | +| Legacy `evaluator` / `judge` / rubric criteria in the review contract | `review/types.test.ts` — *rejects legacy evaluator and judge criteria fields*; `findings/types.test.ts` — input contract rejects judge/evaluator/rubric shapes | +| Agent finding-evidence model-line fallback | `finding-evidence-verifier.test.ts` — *never returns a finding whose line came from the model when the quote did not anchor* (the removed behavior is asserted as absent) | + +Judge criterion output behavior (1–4 rubric, weighted average) has **no** +replacement fixture in this phase. Objective rule coverage belongs to the +objective-check test inventory, not a ported rubric. + +--- + +## Out of scope for Phase 3 (Phase 4 territory) + +These are **not** behavior-preservation targets for this phase and are +intentionally left untouched: + +- `tests/agent/*`, the orchestrator test covering legacy agent output, and + `tests/providers/*agent-loop*.test.ts` — the old autonomous agent surface. + Phase 4 deletes or replaces these with agent-model-call executor tests. +- `BaseEvaluator`'s internal judge mode (`runJudgeEvaluation`), + `buildJudgeLLMSchema` (`src/prompts/schema.ts`), the judge scoring helpers in + `src/scoring/`, `tests/scoring-types.test.ts`, and the judge cases in + `tests/prompt-schema.test.ts` — evaluator internals that are now **unreachable + from the CLI** (judge prompts fail to load) but are scheduled for deletion in + Phase 4 (executors / provider interface / agent deletion). + +Phase 3 added **no** compatibility code for the legacy agent surface — no +agent tool-loop wiring, no review-instruction handler, and no agent result +projection — to `src/findings/`. + +--- + +## Files + +| File | Covers | +|---|---| +| `finding-evidence-verifier.test.ts` | Single evidence verifier; no model-line fallback. | +| `severity.test.ts` | `resolveSeverity`, `buildRuleId`, `resolveCriterionId`. | +| `scorer.test.ts` | `scoreCheck` numerics vs `calculateCheckScore`; verified-count scoring. | +| `types.test.ts` | Zod input contract: parses supported input, rejects legacy shapes. | +| `processor.test.ts` | `processFindings` golden output, diagnostics, counting fix, dedupe, rule-id attribution, `ReviewResult` validation. | +| `module-surface.test.ts` | Public barrel surface + stable diagnostic code constant. | From 4fa07011105f3f2b84df3ae22d41fd774e27749e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:50:55 +0100 Subject: [PATCH 18/34] Sanitize prompt criteria at the findings boundary - Map meta.criteria to { id, name } before processFindings so legacy PromptCriterionSpec rubric fields (weight, target) cannot cross the standard check orchestration -> findings contract - Add tests/orchestrator-finding-criteria.test.ts mocking processFindings to assert weight/target are stripped on the production path - prompt-loader criteria support is unchanged; criterion id/name still drive output rule-id attribution --- src/cli/orchestrator.ts | 6 +- tests/orchestrator-finding-criteria.test.ts | 126 ++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 tests/orchestrator-finding-criteria.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 4d09e917..1e61a47d 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -206,7 +206,11 @@ function routePromptResult( promptMeta: { ...(meta.severity !== undefined ? { severity: meta.severity } : {}), ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - ...(meta.criteria ? { criteria: meta.criteria } : {}), + // Sanitize to the findings contract ({ id, name }): meta.criteria is + // PromptCriterionSpec[], which can carry legacy rubric weight/target. + ...(meta.criteria + ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } + : {}), }, targetContent: content, }); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts new file mode 100644 index 00000000..dc0b1ad8 --- /dev/null +++ b/tests/orchestrator-finding-criteria.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; +import type { ReviewResult } from "../src/review/types"; + +const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), + PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), +})); + +// createEvaluator returns a controllable evaluator so evaluateFiles reaches +// routePromptResult -> processFindings without a real model call. +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +// Replace only processFindings (the findings boundary); keep every other +// findings export real so unrelated imports stay intact. +vi.mock("../src/findings", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; +}); + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +function makeCheckResult(): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: [], + word_count: 100, + }; +} + +describe("standard check orchestration sanitizes criteria at the findings boundary", () => { + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + PROCESS_FINDINGS_MOCK.mockReset(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("drops rubric weight/target from criteria before reaching processFindings", async () => { + const targetFile = createTempFile("Alpha text\n"); + // PromptCriterionSpec admits legacy rubric fields (weight, target). The + // orchestrator must strip them so only { id, name } crosses the boundary. + const prompt = createPrompt({ + id: "SanitizePrompt", + name: "Sanitize Prompt", + type: "check", + severity: Severity.WARNING, + criteria: [ + { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, + ], + }); + + EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); + PROCESS_FINDINGS_MOCK.mockReturnValue({ + findings: [], + scores: [ + { + ruleId: "TestPack.SanitizePrompt", + score: 10, + scoreText: "10.0/10", + severity: "warning", + }, + ], + diagnostics: [], + }); + + await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); + const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { + promptMeta: { criteria?: Array> }; + }; + + // toEqual fails if any extra (weight/target) key survives the mapping. + expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); + for (const criterion of input.promptMeta.criteria ?? []) { + expect(criterion).not.toHaveProperty("weight"); + expect(criterion).not.toHaveProperty("target"); + } + }); +}); From 699f3557e31ec86c0ac62d64ce8270e15f31221f Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 09:23:27 +0100 Subject: [PATCH 19/34] Define structured and tool-calling model clients - Add StructuredModelClient (runPromptStructured + LLMResult/TokenUsage) as the permanent single structured-output capability - Add ToolCallingModelClient: bounded, executor-owned tool transport (runWithTools) that names no workspace/agent-finding concepts - Make LLMProvider extend StructuredModelClient; keep runAgentToolLoop only as a temporary compile bridge for src/agent/ (deleted in a later task) - VercelAIProvider implements both capabilities via a bounded runWithTools transport (caller-owned tools + caller-set step budget), no product loop - Extend AIExecutionContext.operation with a transport-shaped 'tool-calling' label - Add provider contract tests asserting the new surfaces and the absence of legacy workspace-agent concepts Refs: docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md (Finding #2; Product Decision) --- src/observability/ai-observability.ts | 2 +- src/providers/index.ts | 8 +- src/providers/llm-provider.ts | 23 ++-- src/providers/structured-model-client.ts | 28 +++++ src/providers/tool-calling-model-client.ts | 53 ++++++++ src/providers/vercel-ai-provider.ts | 119 +++++++++++++++++- .../providers/structured-model-client.test.ts | 67 ++++++++++ .../tool-calling-model-client.test.ts | 95 ++++++++++++++ 8 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 src/providers/structured-model-client.ts create mode 100644 src/providers/tool-calling-model-client.ts create mode 100644 tests/providers/structured-model-client.test.ts create mode 100644 tests/providers/tool-calling-model-client.test.ts diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 5369c639..e16a3c9d 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,5 +1,5 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'agent-tool-loop'; + operation: 'structured-eval' | 'tool-calling' | 'agent-tool-loop'; provider: string; model: string; evaluator?: string; diff --git a/src/providers/index.ts b/src/providers/index.ts index 4bdb538c..2c02f6e0 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,4 +1,10 @@ -export { LLMProvider, type LLMResult } from './llm-provider'; +export { LLMProvider } from './llm-provider'; +export { type StructuredModelClient, type LLMResult } from './structured-model-client'; +export { + type ToolCallingModelClient, + type ToolCallDefinition, + type ToolCallRunOptions, +} from './tool-calling-model-client'; export { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; export { SearchProvider } from './search-provider'; export { PerplexitySearchProvider, type PerplexitySearchConfig } from './perplexity-provider'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index f87304dd..635d5ecd 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,10 +1,12 @@ import type { TokenUsage } from './token-usage'; -import type { EvalContext } from './request-builder'; +import type { StructuredModelClient } from './structured-model-client'; -export interface LLMResult { - data: T; - usage?: TokenUsage; -} +/** + * `LLMResult` now lives on the permanent structured-output capability. It is + * re-exported here so existing deep imports (`from './llm-provider'`) keep + * compiling until the legacy provider surface is removed in a later task. + */ +export type { LLMResult } from './structured-model-client'; export interface AgentToolDefinition { description: string; @@ -25,7 +27,14 @@ export interface AgentToolLoopResult { usage?: TokenUsage; } -export interface LLMProvider { - runPromptStructured(content: string, promptText: string, schema: { name: string; schema: Record }, context?: EvalContext): Promise>; +/** + * Temporary compile bridge, preserved only until later tasks remove the + * autonomous agent surface. It extends the permanent + * {@link StructuredModelClient} (single structured output) with the legacy + * `runAgentToolLoop` method still consumed by `src/agent/`. Do not expand this + * surface; `runAgentToolLoop` and the `Agent*` types above are deleted once the + * agent tree is removed (audit Finding #2; Product Decision). + */ +export interface LLMProvider extends StructuredModelClient { runAgentToolLoop(params: AgentToolLoopParams): Promise; } diff --git a/src/providers/structured-model-client.ts b/src/providers/structured-model-client.ts new file mode 100644 index 00000000..99b1a1e0 --- /dev/null +++ b/src/providers/structured-model-client.ts @@ -0,0 +1,28 @@ +import type { TokenUsage } from './token-usage'; +import type { EvalContext } from './request-builder'; + +/** + * Result of a structured model call: validated output plus optional usage. + */ +export interface LLMResult { + data: T; + usage?: TokenUsage; +} + +/** + * Permanent structured-output model capability (audit Finding #2). + * + * A {@link StructuredModelClient} makes a single structured model call and + * returns validated output. It owns no tool surface and no autonomous agent + * loop; that product-level autonomy is removed from the provider surface + * (audit Product Decision). The single structured-call executor builds on this + * capability. + */ +export interface StructuredModelClient { + runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record }, + context?: EvalContext, + ): Promise>; +} diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts new file mode 100644 index 00000000..2139c8f2 --- /dev/null +++ b/src/providers/tool-calling-model-client.ts @@ -0,0 +1,53 @@ +import type { ZodType } from 'zod'; +import type { LLMResult } from './structured-model-client'; + +/** + * An executor-owned tool definition supplied to the bounded tool-calling + * transport. The provider is transport only: it does not define or name + * product tools. Descriptions and parameter schemas come from the caller (the + * agent executor), keyed by the caller-supplied tool name, never from the + * provider (audit Finding #2). The object key in the tools map is the tool + * name, matching the existing provider tool-mapping convention. + */ +export interface ToolCallDefinition { + description: string; + parameters: ZodType; + execute: (input: unknown) => Promise; +} + +/** + * Bounded execution metadata for a single tool-calling run. The caller + * (executor) sets the bounds from the review budget (audit Finding #7); the + * transport honors them. It does not invent its own product-level loop. + */ +export interface ToolCallRunOptions { + /** Maximum number of model steps (tool-call rounds) before the run stops. */ + maxSteps?: number; + /** Provider retry budget for transient failures. */ + maxRetries?: number; + /** Maximum concurrent tool executions per model step. Defaults to 1. */ + maxParallelToolCalls?: number; + /** Optional abort signal for cooperative cancellation. */ + signal?: AbortSignal; +} + +/** + * Permanent bounded tool-calling model capability (audit Finding #2). + * + * A {@link ToolCallingModelClient} performs a single bounded generation that + * may execute caller-supplied tools and then emits structured output. The tool + * map is executor-owned and transport-shaped: this interface deliberately + * names no product, workspace, or agent-finding concept (audit Product + * Decision; Finding #1). The autonomous product loop is removed; bounded + * orchestration lives in the executor, which supplies the only tool(s) it + * permits (for example a target-scoped reader). + */ +export interface ToolCallingModelClient { + runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise>; +} diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 5cd833fd..ed314001 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -3,11 +3,20 @@ import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from '../observability/ai-observability'; import { handleUnknownError } from '../errors'; +/** + * Conservative default step cap for a single bounded tool-calling run when the + * caller does not supply one. Executors should pass an explicit `maxSteps` + * derived from the review budget (audit Finding #7) rather than relying on + * this default. + */ +const DEFAULT_TOOL_CALLING_MAX_STEPS = 10; + export interface VercelAIConfig { model: LanguageModel; providerName?: string; @@ -21,7 +30,7 @@ export interface VercelAIConfig { observability?: AIObservability; } -export class VercelAIProvider implements LLMProvider { +export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; @@ -135,6 +144,114 @@ export class VercelAIProvider implements LLMProvider { } } + /** + * Bounded tool-calling transport (audit Finding #2). Performs a single + * bounded generation that may execute caller-supplied tools and then emits + * structured output. The tool map, step budget, and schema are all + * executor-owned; the provider defines no product tools and runs no + * autonomous product loop (audit Product Decision). + */ + async runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise> { + const options = params.options ?? {}; + const maxParallel = options.maxParallelToolCalls ?? 1; + const limit = pLimit(maxParallel); + + const zodSchema = this.jsonSchemaToZod(params.schema); + + const mappedTools = Object.fromEntries( + Object.entries(params.tools).map(([name, definition]) => [ + name, + tool({ + description: definition.description, + inputSchema: definition.parameters, + execute: (input: unknown) => limit(() => definition.execute(input)), + }), + ]), + ); + + if (this.config.debug) { + this.logger.debug('[vectorlint] Sending tool-calling request via Vercel AI SDK', { + model: this.config.model, + toolNames: Object.keys(params.tools), + maxSteps: options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS, + }); + } + + try { + const result = await generateText({ + model: this.config.model, + system: params.systemPrompt, + prompt: params.prompt, + ...(this.config.temperature !== undefined && { temperature: this.config.temperature }), + ...(this.config.maxTokens !== undefined && { maxTokens: this.config.maxTokens }), + ...(options.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + ...(options.signal !== undefined ? { abortSignal: options.signal } : {}), + ...this.getObservabilityOptions({ + operation: 'tool-calling', + provider: this.config.providerName ?? 'unknown', + model: this.config.modelName ?? 'unknown', + }), + stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), + providerOptions: { + openai: { + parallelToolCalls: maxParallel > 1, + }, + }, + tools: mappedTools, + output: Output.object({ + schema: zodSchema, + }), + }); + + if (this.config.debug && result.usage) { + this.logger.debug('[vectorlint] tool-calling LLM response meta', { + usage: { + input_tokens: result.usage.inputTokens, + output_tokens: result.usage.outputTokens, + total_tokens: result.usage.totalTokens, + }, + finish_reason: result.finishReason, + steps: result.steps.length, + }); + } + + const usage = result.usage + ? { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + } + : undefined; + + const output: unknown = result.output; + if (output === undefined || output === null) { + throw new Error( + `LLM returned no structured output. Raw text: ${result.text?.slice(0, 500) ?? '(empty)'}` + ); + } + + const llmResult: LLMResult = { data: output as T }; + if (usage) { + llmResult.usage = usage; + } + return llmResult; + } catch (e: unknown) { + if (NoObjectGeneratedError.isInstance(e)) { + const rawText = e instanceof Error && 'text' in e ? String(e.text) : 'unknown'; + throw new Error( + `LLM failed to generate valid structured output. Raw text: ${rawText}` + ); + } + const err = e instanceof Error ? e : new Error(String(e)); + throw new Error(`Vercel AI SDK tool-calling call failed: ${err.message}`); + } + } + runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { const maxParallel = params.maxParallelToolCalls ?? 1; const limit = pLimit(maxParallel); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts new file mode 100644 index 00000000..7fa6b61d --- /dev/null +++ b/tests/providers/structured-model-client.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import type { + LLMResult, + StructuredModelClient, +} from '../../src/providers/structured-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Schema', schema: { type: 'object' } }; + +describe('StructuredModelClient contract', () => { + it('exposes runPromptStructured and returns an LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: { ok: true } }), + }; + + expect(typeof client.runPromptStructured).toBe('function'); + + const result = await client.runPromptStructured('content', 'prompt', SCHEMA); + expect(result.data).toEqual({ ok: true }); + expect(result.usage).toBeUndefined(); + }); + + it('carries optional token usage on LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => + Promise.resolve({ data: 1, usage: { inputTokens: 3, outputTokens: 4 } }), + }; + + const result: LLMResult = await client.runPromptStructured( + 'content', + 'prompt', + SCHEMA, + ); + + expect(result.data).toBe(1); + expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 4 }); + }); + + it('does not declare an autonomous agent-loop method', () => { + // Compile-time guard: the capability surface is exactly runPromptStructured. + // If a second member were added, this conditional type would resolve to + // `false` and the assignment below would fail tsc. + type StructuredKeys = keyof StructuredModelClient; + const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; + expect(onlyRunPromptStructured).toBe(true); + + // Runtime guard: a minimal implementation exposes only runPromptStructured. + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: null }), + }; + expect( + (client as unknown as Record).runAgentToolLoop, + ).toBeUndefined(); + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy StructuredModelClient). + const client: StructuredModelClient = provider; + expect(typeof client.runPromptStructured).toBe('function'); + }); +}); diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts new file mode 100644 index 00000000..ccf84f68 --- /dev/null +++ b/tests/providers/tool-calling-model-client.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import { z } from 'zod'; +import type { + ToolCallDefinition, + ToolCallRunOptions, + ToolCallingModelClient, +} from '../../src/providers/tool-calling-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Findings', schema: { type: 'object' } }; + +describe('ToolCallingModelClient contract', () => { + it('exposes a single bounded runWithTools method', () => { + // Compile-time guard: the capability surface is exactly runWithTools. + type ToolCallingKeys = keyof ToolCallingModelClient; + const onlyRunWithTools: ToolCallingKeys extends 'runWithTools' ? true : false = true; + expect(onlyRunWithTools).toBe(true); + + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: { findings: [] } }), + }; + expect(typeof client.runWithTools).toBe('function'); + }); + + it('returns structured output plus optional usage', async () => { + const client: ToolCallingModelClient = { + runWithTools: () => + Promise.resolve({ + data: { findings: [{ line: 1 }] }, + usage: { inputTokens: 5, outputTokens: 6 }, + }), + }; + + const result = await client.runWithTools({ + systemPrompt: 'system', + prompt: 'prompt', + tools: { read: makeTool() }, + schema: SCHEMA, + options: { maxSteps: 3, maxParallelToolCalls: 1 } satisfies ToolCallRunOptions, + }); + + expect(result.data).toEqual({ findings: [{ line: 1 }] }); + expect(result.usage).toEqual({ inputTokens: 5, outputTokens: 6 }); + }); + + it('accepts executor-owned tool definitions with typed parameters', () => { + const parameters = z.object({ startLine: z.number().int().positive() }); + const definition: ToolCallDefinition = { + description: 'paged read', + parameters, + execute: (input) => Promise.resolve(input), + }; + + expect(definition.parameters).toBe(parameters); + expect(typeof definition.execute).toBe('function'); + }); + + it('does not name workspace-agent or product-finding concepts on the capability surface', () => { + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: {} }), + }; + const keys = Object.keys(client as Record); + + const forbidden = [ + 'read_file', + 'search_content', + 'report_finding', + 'reviewInstruction', + 'runAgentToolLoop', + ]; + for (const concept of forbidden) { + expect(keys).not.toContain(concept); + } + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy ToolCallingModelClient). + const client: ToolCallingModelClient = provider; + expect(typeof client.runWithTools).toBe('function'); + }); +}); + +function makeTool(): ToolCallDefinition { + return { + description: 'a bounded executor-owned tool', + parameters: z.object({ value: z.number().int().positive() }), + execute: () => Promise.resolve({ ok: true }), + }; +} From 74957bcb6528d35988c11f66705d4875ade1f503 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 09:54:05 +0100 Subject: [PATCH 20/34] Implement SingleModelCallExecutor for single modelCall - Add SingleModelCallExecutor behind the Phase 2 ReviewExecutor contract, reviewing source-backed rules with one structured model call per (rule, chunk) through an injected StructuredModelClient. - Build rule prompts verbatim from request.rules[i].body; no tool surface and no reviewInstruction. - Reuse buildCheckLLMSchema, RecursiveChunker, mergeViolations, prependLineNumbers, and processFindings instead of parallel projection. - Enforce ReviewBudget.maxModelCallsPerReview via enforceBudget before each call; on exhaustion record an error diagnostic + hadOperationalErrors and return partial results rather than throwing past the contract. - Record usage (modelCalls, wallClockMs always; input/output tokens gated on outputPolicy.includeUsage) consistent with ReviewResult.usage. - Add focused tests: one call per rule, projected findings/scores/diagnostics, unanchored-evidence diagnostic, budget exhaustion, no tool calls, and large-target chunking/merging. --- src/executors/single-model-call-executor.ts | 227 ++++++++++++++++ .../single-model-call-executor.test.ts | 248 ++++++++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 src/executors/single-model-call-executor.ts create mode 100644 tests/executors/single-model-call-executor.test.ts diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts new file mode 100644 index 00000000..de3957cb --- /dev/null +++ b/src/executors/single-model-call-executor.ts @@ -0,0 +1,227 @@ +import path from 'path'; + +import type { ReviewExecutor } from '../review/executor'; +import { BudgetExceededError, enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, + ReviewUsage, +} from '../review/types'; +import type { EvalContext } from '../providers/request-builder'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import { Severity } from '../evaluators/types'; +import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; +import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; +import { prependLineNumbers } from '../output/line-numbering'; +import { processFindings } from '../findings'; + +/** + * Word-count threshold above which the single-call executor chunks the target + * before reviewing it. Mirrors the check evaluator's chunking threshold so the + * single-call path preserves the existing chunk/merge behavior for large + * documents. + */ +const CHUNKING_WORD_THRESHOLD = 600; +const MAX_CHUNK_WORDS = 500; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules and chunks. + */ +interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * The single modelCall {@link ReviewExecutor} (audit Finding #2). + * + * Reviews target content against source-backed rules with one structured model + * call per (rule, chunk) through an injected {@link StructuredModelClient}. It + * owns no tool surface and no autonomous loop: rule prompts come verbatim from + * {@link ReviewRule.body}, candidate findings flow through the shared + * {@link processFindings} pipeline, and the model-call budget is enforced via + * the review budget module before every call. + * + * This is the bounded, transport-only review strategy selected by + * `--model-call single` (and by `auto` for normal-sized inputs). + */ +export class SingleModelCallExecutor implements ReviewExecutor { + constructor(private readonly client: StructuredModelClient) {} + + async run(request: ReviewRequest): Promise { + const schema = buildCheckLLMSchema(); + const context = this.buildContext(request.target.uri); + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const ruleResult = await this.reviewRule( + request, + rule, + schema, + context, + counters, + elapsedMs, + ); + findings.push(...ruleResult.findings); + scores.push(...ruleResult.scores); + diagnostics.push(...ruleResult.diagnostics); + if (ruleResult.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + if (error instanceof BudgetExceededError) { + // Surface the existing budget error pattern as an operational failure: + // the run returns partial results plus an error diagnostic rather than + // throwing past the ReviewExecutor contract. + hadOperationalErrors = true; + diagnostics.push({ + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: this.buildUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: chunks the line-numbered target, makes one + * structured model call per chunk, merges violations across chunks, and + * projects the merged candidates through {@link processFindings}. + */ + private async reviewRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + context: EvalContext, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + const numberedContent = prependLineNumbers(request.target.content); + const wordCount = countWords(request.target.content) || 1; + const chunks = this.chunkTarget(numberedContent, wordCount, request.budget.maxChunksPerRule); + + const chunkViolations: CheckLLMResult['violations'][] = []; + for (const chunk of chunks) { + // Enforce the model-call budget before committing to another call. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the call that would push the run over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runPromptStructured( + chunk.content, + rule.body, + schema, + context, + ); + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + chunkViolations.push(data.violations); + } + + const { pack, ruleId } = splitRuleId(rule.id); + const mergedViolations = mergeViolations(chunkViolations); + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: mergedViolations, + wordCount, + // Map the review contract's plain severity union onto the finding + // processor's Severity enum at this boundary. + promptMeta: { + ...(rule.severity !== undefined + ? { severity: rule.severity === 'error' ? Severity.ERROR : Severity.WARNING } + : {}), + }, + targetContent: request.target.content, + }); + } + + /** + * Chunks line-numbered target content for context management. Small targets + * review as a single chunk; larger targets are split recursively and capped + * at the review budget's per-rule chunk limit. + */ + private chunkTarget( + numberedContent: string, + wordCount: number, + maxChunks: number, + ): Chunk[] { + if (wordCount <= CHUNKING_WORD_THRESHOLD) { + return [{ content: numberedContent, index: 0 }]; + } + const chunker = new RecursiveChunker(); + return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); + } + + private buildContext(uri: string): EvalContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; + } + + private buildUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, + ): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; + } +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} diff --git a/tests/executors/single-model-call-executor.test.ts b/tests/executors/single-model-call-executor.test.ts new file mode 100644 index 00000000..5172f9b0 --- /dev/null +++ b/tests/executors/single-model-call-executor.test.ts @@ -0,0 +1,248 @@ +import { describe, expect, it } from 'vitest'; + +import { SingleModelCallExecutor } from '../../src/executors/single-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { + ReviewRequest, + ReviewRule, + ReviewTarget, +} from '../../src/review'; +import type { LLMResult, StructuredModelClient } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { CheckLLMResult } from '../../src/prompts/schema'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = CheckLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +type FakeStructuredClient = StructuredModelClient & { + runWithTools: () => Promise; + structuredCalls: number; + toolCalls: number; + lastPromptText?: string; +}; + +function makeFakeClient( + respond: (content: string, promptText: string) => { data: CheckLLMResult; usage?: TokenUsage }, +): FakeStructuredClient { + const client = { + structuredCalls: 0, + toolCalls: 0, + lastPromptText: undefined as string | undefined, + runPromptStructured: ( + content: string, + promptText: string, + ): Promise> => { + client.structuredCalls += 1; + client.lastPromptText = promptText; + const { data, usage } = respond(content, promptText); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + runWithTools: (): Promise => { + client.toolCalls += 1; + return Promise.reject( + new Error('SingleModelCallExecutor must not call runWithTools'), + ); + }, + }; + return client; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'single', + ...overrides, + }; +} + +describe('SingleModelCallExecutor', () => { + it('makes one structured call per rule and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + // One structured call, with the rule body as the source-backed prompt. + expect(client.structuredCalls).toBe(1); + expect(client.lastPromptText).toBe('Flag vague advice.'); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.line).toBeGreaterThanOrEqual(1); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('reviews each rule independently and aggregates usage across calls', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.structuredCalls).toBe(2); + expect(result.scores).toHaveLength(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's single call fits within maxModelCallsPerReview = 1. + expect(client.structuredCalls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('never invokes the tool-calling surface', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + await executor.run(makeRequest([makeRule()])); + + expect(client.toolCalls).toBe(0); + expect(client.structuredCalls).toBe(1); + }); + + it('chunks large targets into multiple model calls and merges per-chunk violations', async () => { + // > 600 words triggers recursive chunking. + const targetContent = Array.from({ length: 200 }, (_, i) => `vague text line ${i}`).join('\n'); + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text line 0' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([makeRule()], { + target: { uri: 'file:///repo/docs/big.md', content: targetContent, contentType: 'text/markdown' }, + }), + ); + + expect(client.structuredCalls).toBeGreaterThan(1); + expect(result.usage?.modelCalls).toBe(client.structuredCalls); + expect(result.scores).toHaveLength(1); + // Identical violations across chunks dedupe to a single anchored finding. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + }); +}); From 8bf8e86c4012b41680bc24be3d7b98e0887ad166 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 11:10:34 +0100 Subject: [PATCH 21/34] Implement AgentModelCallExecutor with target-only read tool - Add AgentModelCallExecutor implementing ReviewExecutor via one bounded ToolCallingModelClient.runWithTools call per rule - Expose exactly one executor-owned tool, read_target_section, through TargetReadCapability bound to in-memory request.target.content (no filesystem or non-target URI access) - Map out-of-range/invalid ranges to model-visible error results so the bounded run continues instead of aborting - Enforce ReviewBudget.maxModelCallsPerReview via enforceBudget before each run, surfacing exhaustion as an error diagnostic with partial results - Pass request.rules[i].body verbatim as the source-backed prompt; no reviewInstruction is accepted or introduced - Project results through src/findings/processor.ts so anchored findings, scores, and diagnostics match the shared pipeline - Extract shared executor helpers (splitRuleId, buildEvalContext, buildReviewUsage, toFindingSeverity, budgetExceededDiagnostic, RunCounters) into src/executors/shared.ts and reuse from the single executor to remove duplication - Add focused tests for the agent executor and target-read adapter --- src/executors/agent-model-call-executor.ts | 172 +++++++++++ src/executors/shared.ts | 96 ++++++ src/executors/single-model-call-executor.ts | 87 ++---- .../target-read-capability-adapter.ts | 162 ++++++++++ .../agent-model-call-executor.test.ts | 277 ++++++++++++++++++ .../target-read-capability-adapter.test.ts | 108 +++++++ 6 files changed, 833 insertions(+), 69 deletions(-) create mode 100644 src/executors/agent-model-call-executor.ts create mode 100644 src/executors/shared.ts create mode 100644 src/executors/target-read-capability-adapter.ts create mode 100644 tests/executors/agent-model-call-executor.test.ts create mode 100644 tests/executors/target-read-capability-adapter.test.ts diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts new file mode 100644 index 00000000..1b09f388 --- /dev/null +++ b/src/executors/agent-model-call-executor.ts @@ -0,0 +1,172 @@ +import type { ToolCallDefinition, ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { EvalContext, RequestBuilder } from '../providers/request-builder'; +import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; +import { countWords } from '../chunking'; +import { processFindings } from '../findings'; +import { enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, +} from '../review/types'; +import type { ReviewExecutor } from '../review/executor'; +import { TargetReadCapability, buildReadTargetSectionTool } from './target-read-capability-adapter'; +import { + budgetExceededDiagnostic, + buildEvalContext, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; + +/** + * The agent `modelCall` {@link ReviewExecutor} (audit Product Decision; Finding #2). + * + * Reviews target content against source-backed rules through a single bounded + * tool-calling run per rule via an injected {@link ToolCallingModelClient}. The + * only executor-owned tool exposed to the model is `read_target_section`, + * which pages through the in-memory `request.target.content` (the on-page + * boundary, audit Finding #5). Rule prompts come verbatim from + * {@link ReviewRule.body} — no model-supplied rule override is introduced — + * candidate findings flow through the shared {@link processFindings} pipeline, + * and the model-call budget is enforced via the review budget module before + * every call. + * + * This is the bounded, target-only review strategy selected by + * `--model-call agent` (and by `auto` for large inputs). + */ +export class AgentModelCallExecutor implements ReviewExecutor { + constructor( + private readonly client: ToolCallingModelClient, + private readonly builder: RequestBuilder, + ) {} + + async run(request: ReviewRequest): Promise { + const schema = buildCheckLLMSchema(); + const capability = new TargetReadCapability(request.target.content); + // Exactly one executor-owned tool is exposed: target-section paging. + const tools = buildReadTargetSectionTool(capability); + const context = buildEvalContext(request.target.uri); + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const ruleResult = await this.reviewRule( + request, + rule, + schema, + tools, + context, + capability.lineCount, + counters, + elapsedMs, + ); + findings.push(...ruleResult.findings); + scores.push(...ruleResult.scores); + diagnostics.push(...ruleResult.diagnostics); + if (ruleResult.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { + hadOperationalErrors = true; + diagnostics.push(diagnostic); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: buildReviewUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: makes one bounded tool-calling run that lets the + * model page through the target via `read_target_section`, then projects the + * returned violations through {@link processFindings}. + */ + private async reviewRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + tools: Record, + context: EvalContext, + targetLineCount: number, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + // Enforce the model-call budget before committing to the run. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the run that would push the review over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runWithTools({ + // The source-backed rule body, wrapped with the directive/user + // instructions exactly as the single-call path does. No model-supplied + // rule override is introduced. + systemPrompt: this.builder.buildPromptBodyForStructured(rule.body, context), + prompt: this.buildTargetPrompt(request.target.uri, targetLineCount), + tools, + schema, + options: { + // Each run is bounded by the budget's per-rule section limit: the + // model may page through at most maxChunksPerRule sections before + // emitting its structured finding set. + maxSteps: request.budget.maxChunksPerRule, + maxParallelToolCalls: 1, + }, + }); + + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + + const { pack, ruleId } = splitRuleId(rule.id); + const wordCount = countWords(request.target.content) || 1; + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: data.violations, + wordCount, + promptMeta: { + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), + }, + targetContent: request.target.content, + }); + } + + private buildTargetPrompt(uri: string, lineCount: number): string { + return [ + `Target URI: ${uri}`, + `Target lines: ${lineCount}`, + '', + 'Page through the target with the read_target_section tool, then report verified violations in the structured output.', + ].join('\n'); + } +} diff --git a/src/executors/shared.ts b/src/executors/shared.ts new file mode 100644 index 00000000..e9e24ef9 --- /dev/null +++ b/src/executors/shared.ts @@ -0,0 +1,96 @@ +import path from 'path'; + +import { Severity } from '../evaluators/types'; +import type { EvalContext } from '../providers/request-builder'; +import { BudgetExceededError } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewRequest, + ReviewSeverity, + ReviewUsage, +} from '../review/types'; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +export const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules (and chunks/sections) by both + * the single and agent model-call executors. + */ +export interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +export function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} + +/** + * Builds the provider {@link EvalContext} (file-type hint) for a target URI. + * Shared by both executors so structured and tool-calling calls receive the + * same file-type context for directive substitution. + */ +export function buildEvalContext(uri: string): EvalContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; +} + +/** + * Maps the review contract's plain {@link ReviewSeverity} union onto the + * finding processor's {@link Severity} enum at the executor boundary (no + * unsafe cast). + */ +export function toFindingSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; +} + +/** + * Aggregates run-wide counters into a {@link ReviewUsage}, including token + * counts only when the output policy opts in to usage reporting. + */ +export function buildReviewUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, +): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; +} + +/** + * Returns a `review-budget-exceeded` error diagnostic when `error` is a + * {@link BudgetExceededError}; otherwise returns `undefined` so the caller can + * rethrow non-budget errors unchanged. Shared by both executors' run loops so + * budget exhaustion surfaces as a partial-result operational failure instead + * of throwing past the {@link ReviewExecutor} contract. + */ +export function budgetExceededDiagnostic(error: unknown): ReviewDiagnostic | undefined { + if (!(error instanceof BudgetExceededError)) { + return undefined; + } + return { + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }; +} diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index de3957cb..3809cbb3 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -1,7 +1,5 @@ -import path from 'path'; - import type { ReviewExecutor } from '../review/executor'; -import { BudgetExceededError, enforceBudget } from '../review/budget'; +import { enforceBudget } from '../review/budget'; import type { ReviewDiagnostic, ReviewFinding, @@ -9,15 +7,21 @@ import type { ReviewResult, ReviewRule, ReviewScore, - ReviewUsage, } from '../review/types'; import type { EvalContext } from '../providers/request-builder'; import type { StructuredModelClient } from '../providers/structured-model-client'; -import { Severity } from '../evaluators/types'; import { buildCheckLLMSchema, type CheckLLMResult } from '../prompts/schema'; import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; import { prependLineNumbers } from '../output/line-numbering'; import { processFindings } from '../findings'; +import { + budgetExceededDiagnostic, + buildEvalContext, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; /** * Word-count threshold above which the single-call executor chunks the target @@ -28,21 +32,6 @@ import { processFindings } from '../findings'; const CHUNKING_WORD_THRESHOLD = 600; const MAX_CHUNK_WORDS = 500; -/** - * Stable diagnostic code recorded when a run stops because the model-call - * budget was exhausted before every rule could be reviewed. - */ -const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; - -/** - * Mutable run-wide counters shared across rules and chunks. - */ -interface RunCounters { - modelCalls: number; - inputTokens: number; - outputTokens: number; -} - /** * The single modelCall {@link ReviewExecutor} (audit Finding #2). * @@ -61,7 +50,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { async run(request: ReviewRequest): Promise { const schema = buildCheckLLMSchema(); - const context = this.buildContext(request.target.uri); + const context = buildEvalContext(request.target.uri); const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; @@ -90,17 +79,13 @@ export class SingleModelCallExecutor implements ReviewExecutor { } } } catch (error: unknown) { - if (error instanceof BudgetExceededError) { - // Surface the existing budget error pattern as an operational failure: - // the run returns partial results plus an error diagnostic rather than - // throwing past the ReviewExecutor contract. + // Surface budget exhaustion as an operational failure: the run returns + // partial results plus an error diagnostic rather than throwing past the + // ReviewExecutor contract. Non-budget errors propagate unchanged. + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { hadOperationalErrors = true; - diagnostics.push({ - level: 'error', - code: REVIEW_BUDGET_EXCEEDED_CODE, - message: error.message, - context: { limit: error.limit, actual: error.actual }, - }); + diagnostics.push(diagnostic); } else { throw error; } @@ -111,7 +96,7 @@ export class SingleModelCallExecutor implements ReviewExecutor { scores, diagnostics, hadOperationalErrors, - usage: this.buildUsage(request, counters, elapsedMs()), + usage: buildReviewUsage(request, counters, elapsedMs()), }; } @@ -165,12 +150,8 @@ export class SingleModelCallExecutor implements ReviewExecutor { ruleSource: rule.source, candidateFindings: mergedViolations, wordCount, - // Map the review contract's plain severity union onto the finding - // processor's Severity enum at this boundary. promptMeta: { - ...(rule.severity !== undefined - ? { severity: rule.severity === 'error' ? Severity.ERROR : Severity.WARNING } - : {}), + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), }, targetContent: request.target.content, }); @@ -192,36 +173,4 @@ export class SingleModelCallExecutor implements ReviewExecutor { const chunker = new RecursiveChunker(); return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); } - - private buildContext(uri: string): EvalContext { - const ext = path.extname(uri); - return ext ? { fileType: ext } : {}; - } - - private buildUsage( - request: ReviewRequest, - counters: RunCounters, - wallClockMs: number, - ): ReviewUsage { - const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; - if (request.outputPolicy.includeUsage) { - usage.inputTokens = counters.inputTokens; - usage.outputTokens = counters.outputTokens; - } - return usage; - } -} - -/** - * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. - * The review contract carries the composite id, while {@link processFindings} - * rebuilds the same id from the parts via `buildRuleId`. Splits on the first - * dot; pack names are single path segments and rule ids are PascalCase. - */ -function splitRuleId(id: string): { pack: string; ruleId: string } { - const dot = id.indexOf('.'); - if (dot === -1) { - return { pack: id, ruleId: id }; - } - return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; } diff --git a/src/executors/target-read-capability-adapter.ts b/src/executors/target-read-capability-adapter.ts new file mode 100644 index 00000000..041c286a --- /dev/null +++ b/src/executors/target-read-capability-adapter.ts @@ -0,0 +1,162 @@ +import { z } from 'zod'; + +import { VectorlintError } from '../errors'; +import type { ToolCallDefinition } from '../providers/tool-calling-model-client'; +import type { ReviewTargetReadCapability } from '../review/executor'; + +/** + * The single executor-owned tool name exposed to the bounded tool-calling + * transport (audit Product Decision): page through the target under review by + * 1-based line range. This is the only agent-like capability that survives the + * Phase 4 agent removal. + */ +export const READ_TARGET_SECTION_TOOL_NAME = 'read_target_section'; + +/** + * Zod schema for {@link TargetReadCapability.readTargetSection} arguments: + * 1-based positive integers. The tool adapter parses model input against this + * before slicing, so malformed arguments become model-visible errors. + */ +export const READ_TARGET_SECTION_PARAMETERS = z + .object({ + startLine: z.number().int().positive(), + endLine: z.number().int().positive(), + }) + .strict(); + +const READ_TARGET_SECTION_DESCRIPTION = [ + 'Read a 1-based inclusive [startLine, endLine] window of the target content under review.', + 'Returns the window with original line numbers prepended so findings can cite exact lines.', + 'A window outside [1, targetLineCount], or with startLine > endLine, returns an error result', + 'describing the valid range instead of aborting the review.', +].join(' '); + +/** + * Thrown by {@link TargetReadCapability.readTargetSection} when the requested + * window is outside the target's valid line range. The tool adapter translates + * this into a model-visible error result so the bounded run continues. + */ +export class TargetSectionRangeError extends VectorlintError { + constructor( + message: string, + public readonly lineCount: number, + ) { + super(message, 'TARGET_SECTION_RANGE'); + this.name = 'TargetSectionRangeError'; + } +} + +/** Success result of a {@link TargetReadCapability.readTargetSection} call. */ +export interface TargetSectionResult { + startLine: number; + endLine: number; + /** The requested window, each line prefixed with its 1-based line number. */ + content: string; +} + +/** Model-visible error result returned in place of a window for invalid calls. */ +export interface TargetSectionErrorResult { + error: string; + /** Total lines in the target, so the model can retry with a valid range. */ + lineCount: number; +} + +/** + * The on-page boundary adapter (audit Finding #5): a target-only + * {@link ReviewTargetReadCapability} bound to the in-memory + * `ReviewTarget.content`. It performs NO filesystem access and reads no URI + * other than the target it was constructed from. The agent executor exposes + * this single capability to the model via {@link buildReadTargetSectionTool}. + */ +export class TargetReadCapability implements ReviewTargetReadCapability { + private readonly lines: readonly string[]; + + constructor(content: string) { + // Mirrors prependLineNumbers: a trailing newline yields a final empty line. + this.lines = content.split('\n'); + } + + /** Number of addressable lines in the target. */ + get lineCount(): number { + return this.lines.length; + } + + readTargetSection(startLine: number, endLine: number): Promise { + this.assertValidRange(startLine, endLine); + const slice = this.lines.slice(startLine - 1, endLine); + const content = slice.map((line, index) => `${startLine + index}\t${line}`).join('\n'); + return Promise.resolve({ startLine, endLine, content }); + } + + private assertValidRange(startLine: number, endLine: number): void { + const lineCount = this.lineCount; + if (!Number.isInteger(startLine) || !Number.isInteger(endLine)) { + throw new TargetSectionRangeError( + `read_target_section requires integer line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine < 1 || endLine < 1) { + throw new TargetSectionRangeError( + `read_target_section requires 1-based positive line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > endLine) { + throw new TargetSectionRangeError( + `read_target_section requires startLine <= endLine; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section startLine=${startLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (endLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section endLine=${endLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + } +} + +/** + * Builds the single executor-owned tool map exposed to the + * {@link ToolCallingModelClient}: exactly one entry, `read_target_section`, + * bound to the target-only {@link TargetReadCapability}. Argument-parse and + * range failures are returned as model-visible error results (never thrown + * past the tool boundary) so the bounded review run continues. + */ +export function buildReadTargetSectionTool( + capability: TargetReadCapability, +): Record { + return { + [READ_TARGET_SECTION_TOOL_NAME]: { + description: READ_TARGET_SECTION_DESCRIPTION, + parameters: READ_TARGET_SECTION_PARAMETERS, + execute: async ( + input: unknown, + ): Promise => { + const parsed = READ_TARGET_SECTION_PARAMETERS.safeParse(input); + if (!parsed.success) { + return { + error: `Invalid read_target_section arguments: ${parsed.error.message}.`, + lineCount: capability.lineCount, + }; + } + const { startLine, endLine } = parsed.data; + try { + return await capability.readTargetSection(startLine, endLine); + } catch (error: unknown) { + if (error instanceof TargetSectionRangeError) { + return { error: error.message, lineCount: error.lineCount }; + } + throw error; + } + }, + }, + }; +} diff --git a/tests/executors/agent-model-call-executor.test.ts b/tests/executors/agent-model-call-executor.test.ts new file mode 100644 index 00000000..9ade0f1a --- /dev/null +++ b/tests/executors/agent-model-call-executor.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from 'vitest'; + +import { AgentModelCallExecutor } from '../../src/executors/agent-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { ReviewRequest, ReviewRule, ReviewTarget } from '../../src/review'; +import { DefaultRequestBuilder } from '../../src/providers/request-builder'; +import type { ToolCallDefinition, ToolCallingModelClient, ToolCallRunOptions } from '../../src/providers/tool-calling-model-client'; +import type { LLMResult } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { CheckLLMResult } from '../../src/prompts/schema'; +import type { TargetSectionErrorResult, TargetSectionResult } from '../../src/executors/target-read-capability-adapter'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = CheckLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +interface CapturedCall { + systemPrompt: string; + prompt: string; + tools: Record; + options?: ToolCallRunOptions; +} + +interface FakeToolClient extends ToolCallingModelClient { + calls: number; + captured: CapturedCall[]; +} + +function makeFakeClient( + respond: () => { data: CheckLLMResult; usage?: TokenUsage }, +): FakeToolClient { + const fake = { + calls: 0, + captured: [] as CapturedCall[], + runWithTools: ( + params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }, + ): Promise> => { + fake.calls += 1; + fake.captured.push({ + systemPrompt: params.systemPrompt, + prompt: params.prompt, + tools: params.tools, + ...(params.options !== undefined ? { options: params.options } : {}), + }); + const { data, usage } = respond(); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + }; + return fake; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\nthird line\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'agent', + ...overrides, + }; +} + +describe('AgentModelCallExecutor', () => { + it('exposes exactly one tool (read_target_section) and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + // Exactly one bounded model run, exposing exactly one tool. + expect(client.calls).toBe(1); + expect(Object.keys(client.captured[0]!.tools)).toEqual(['read_target_section']); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('passes the source-backed rule body verbatim as the system prompt (no model-supplied rule override)', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule({ body: 'Flag vague advice.' })])); + + // With an empty directive/user-instructions builder, the system prompt is + // the rule body verbatim — no model-supplied rule override is introduced. + expect(client.captured[0]!.systemPrompt).toBe('Flag vague advice.'); + // The agent instruction names the only tool and the target. + expect(client.captured[0]!.prompt).toContain('read_target_section'); + expect(client.captured[0]!.prompt).toContain('file:///repo/docs/guide.md'); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's run fits within maxModelCallsPerReview = 1. + expect(client.calls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('reviews each rule independently and aggregates usage across runs', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.calls).toBe(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('exposes a read_target_section that slices only request.target.content', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule()])); + + const tool = client.captured[0]!.tools['read_target_section']!; + const result = (await tool.execute({ startLine: 2, endLine: 3 })) as TargetSectionResult; + + // The window matches the in-memory target lines with their real line numbers. + expect(result).toEqual({ + startLine: 2, + endLine: 3, + content: '2\tsecond line is fine\n3\tthird line', + }); + }); + + it('returns a model-visible error for out-of-range windows without aborting the run', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + // The run completes and returns a normal ReviewResult despite the model + // (simulated below) requesting an out-of-range section. + const result = await executor.run(makeRequest([makeRule()])); + expect(result.hadOperationalErrors).toBe(false); + + const tool = client.captured[0]!.tools['read_target_section']!; + const errorResult = (await tool.execute({ startLine: 1, endLine: 999 })) as TargetSectionErrorResult; + + expect(errorResult.error).toContain('999'); + expect(errorResult.lineCount).toBe(4); + }); + + it('bounds each run by the budget maxChunksPerRule step limit', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run( + makeRequest([makeRule()], { budget: { ...DEFAULT_REVIEW_BUDGET, maxChunksPerRule: 7 } }), + ); + + expect(client.captured[0]!.options?.maxSteps).toBe(7); + expect(client.captured[0]!.options?.maxParallelToolCalls).toBe(1); + }); +}); diff --git a/tests/executors/target-read-capability-adapter.test.ts b/tests/executors/target-read-capability-adapter.test.ts new file mode 100644 index 00000000..c46016af --- /dev/null +++ b/tests/executors/target-read-capability-adapter.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildReadTargetSectionTool, + READ_TARGET_SECTION_TOOL_NAME, + TargetReadCapability, + type TargetSectionErrorResult, + type TargetSectionResult, +} from '../../src/executors/target-read-capability-adapter'; + +const CONTENT = 'alpha line\nbeta line\ngamma line\n'; + +describe('TargetReadCapability', () => { + it('counts lines including the trailing empty line (mirrors prependLineNumbers)', () => { + expect(new TargetReadCapability(CONTENT).lineCount).toBe(4); + expect(new TargetReadCapability('only one').lineCount).toBe(1); + }); + + it('slices only the in-memory target content with original line numbers', async () => { + const capability = new TargetReadCapability(CONTENT); + + const first = await capability.readTargetSection(1, 2); + expect(first).toEqual({ + startLine: 1, + endLine: 2, + content: '1\talpha line\n2\tbeta line', + }); + + const tail = await capability.readTargetSection(3, 4); + expect(tail).toEqual({ + startLine: 3, + endLine: 4, + content: '3\tgamma line\n4\t', + }); + }); +}); + +describe('buildReadTargetSectionTool', () => { + function toolFor(content: string) { + return buildReadTargetSectionTool(new TargetReadCapability(content)); + } + + it('exposes exactly one tool named read_target_section', () => { + const tools = toolFor(CONTENT); + expect(Object.keys(tools)).toEqual([READ_TARGET_SECTION_TOOL_NAME]); + }); + + it('returns numbered target windows for valid ranges (target-only slicing)', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 3, + })) as TargetSectionResult; + + expect(result).toEqual({ + startLine: 1, + endLine: 3, + content: '1\talpha line\n2\tbeta line\n3\tgamma line', + }); + }); + + it('returns a model-visible error (never throws) when endLine exceeds the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 99, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('99'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error when startLine is past the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 10, + endLine: 12, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('10'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error for an inverted range', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 3, + endLine: 1, + })) as TargetSectionErrorResult; + + expect(result.error).toMatch(/startLine <= endLine/); + }); + + it('returns a model-visible error for malformed arguments', async () => { + const tools = toolFor(CONTENT); + const execute = tools[READ_TARGET_SECTION_TOOL_NAME]!.execute; + + const missing = (await execute({ startLine: 1 })) as TargetSectionErrorResult; + expect(missing.error).toMatch(/Invalid read_target_section arguments/); + expect(missing.lineCount).toBe(4); + + const negative = (await execute({ startLine: 0, endLine: 2 })) as TargetSectionErrorResult; + expect(negative.error).toMatch(/Invalid read_target_section arguments/); + + const floats = (await execute({ startLine: 1.5, endLine: 2 })) as TargetSectionErrorResult; + expect(floats.error).toMatch(/Invalid read_target_section arguments/); + }); +}); From 8bfb47de8264fd7e75440347d64e3c87e2b2f4fb Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:18:00 +0100 Subject: [PATCH 22/34] Wire --model-call executors into the CLI and remove the agent surface (core) --- src/agent/executor.ts | 843 ------------------ src/agent/path-utils.ts | 73 -- src/agent/progress.ts | 282 ------ src/agent/prompt-builder.ts | 57 -- src/agent/review-session-store.ts | 94 -- src/agent/rule-id.ts | 11 - src/agent/tools-registry.ts | 80 -- src/agent/types.ts | 134 --- src/cli/commands.ts | 22 +- src/cli/orchestrator.ts | 772 ++++------------ src/cli/types.ts | 65 +- src/executors/index.ts | 38 + src/providers/llm-provider.ts | 42 +- src/providers/provider-factory.ts | 5 +- src/providers/vercel-ai-provider.ts | 67 +- src/schemas/cli-schemas.ts | 5 +- tests/agent/agent-executor.test.ts | 841 ----------------- tests/agent/progress.test.ts | 90 -- tests/agent/prompt-builder.test.ts | 51 -- tests/agent/review-session-store.test.ts | 142 --- tests/agent/types-contract.test.ts | 83 -- tests/main-command-observability.test.ts | 6 +- tests/orchestrator-agent-output.test.ts | 165 ---- tests/orchestrator-check-processor.test.ts | 356 -------- tests/orchestrator-filtering.test.ts | 412 --------- tests/orchestrator-finding-criteria.test.ts | 126 --- tests/providers/llm-provider-contract.test.ts | 14 - .../providers/structured-model-client.test.ts | 6 +- .../tool-calling-model-client.test.ts | 2 - .../vercel-ai-provider-agent-loop.test.ts | 314 ------- 30 files changed, 263 insertions(+), 4935 deletions(-) delete mode 100644 src/agent/executor.ts delete mode 100644 src/agent/path-utils.ts delete mode 100644 src/agent/progress.ts delete mode 100644 src/agent/prompt-builder.ts delete mode 100644 src/agent/review-session-store.ts delete mode 100644 src/agent/rule-id.ts delete mode 100644 src/agent/tools-registry.ts delete mode 100644 src/agent/types.ts create mode 100644 src/executors/index.ts delete mode 100644 tests/agent/agent-executor.test.ts delete mode 100644 tests/agent/progress.test.ts delete mode 100644 tests/agent/prompt-builder.test.ts delete mode 100644 tests/agent/review-session-store.test.ts delete mode 100644 tests/agent/types-contract.test.ts delete mode 100644 tests/orchestrator-agent-output.test.ts delete mode 100644 tests/orchestrator-check-processor.test.ts delete mode 100644 tests/orchestrator-filtering.test.ts delete mode 100644 tests/orchestrator-finding-criteria.test.ts delete mode 100644 tests/providers/llm-provider-contract.test.ts delete mode 100644 tests/providers/vercel-ai-provider-agent-loop.test.ts diff --git a/src/agent/executor.ts b/src/agent/executor.ts deleted file mode 100644 index 749a5685..00000000 --- a/src/agent/executor.ts +++ /dev/null @@ -1,843 +0,0 @@ -import { readdir, readFile } from 'fs/promises'; -import * as os from 'os'; -import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; -import type { PromptFile } from '../prompts/prompt-loader'; -import { Type, Severity } from '../evaluators/types'; -import { computeFilterDecision } from '../evaluators/violation-filter'; -import { locateQuotedText } from '../output/location'; -import type { AgentToolLoopResult, LLMProvider } from '../providers/llm-provider'; -import type { TokenUsage } from '../providers/token-usage'; -import type { OutputFormat } from '../cli/types'; -import { createEvaluator } from '../evaluators'; -import { createReviewSessionStore } from './review-session-store'; -import { buildAgentSystemPrompt } from './prompt-builder'; -import { AgentToolError } from '../errors'; -import { ScanPathResolver } from '../boundaries/scan-path-resolver'; -import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import { - createAgentTools, - listAvailableTools, - type AgentToolHandler, - type AgentToolName, -} from './tools-registry'; -import { - LINT_TOOL_INPUT_SCHEMA, - FINALIZE_REVIEW_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, - SESSION_EVENT_TYPE, - type SessionEvent, -} from './types'; -import { resolveGlobPatternWithinRoot, resolveWithinRoot, toRelativePathFromRoot } from './path-utils'; -import type { AgentProgressReporter, VisibleToolName, VisibleToolProgress } from './progress'; -import { buildRuleId, normalizeRuleSource } from './rule-id'; -export interface AgentFinding { - file: string; - line: number; - column: number; - severity: Severity; - message: string; - ruleId: string; - ruleSource: string; - analysis?: string; - suggestion?: string; - fix?: string; - match?: string; -} - -export interface RunAgentExecutorParams { - targets: string[]; - prompts: PromptFile[]; - provider: LLMProvider; - workspaceRoot: string; - scanPaths: FilePatternConfig[]; - outputFormat: OutputFormat; - printMode: boolean; - sessionHomeDir?: string; - progressReporter?: AgentProgressReporter; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; - userInstructions?: string; -} - -export interface AgentExecutorResult { - findings: AgentFinding[]; - events: SessionEvent[]; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - requestFailures: number; - hadOperationalErrors: boolean; - errorMessage?: string; - usage?: AgentToolLoopResult['usage']; -} - -const MAX_SEARCH_FILE_RESULTS = 500; -const MAX_CONTENT_MATCH_RESULTS = 200; -const VISIBLE_TOOL_NAMES = new Set(['read_file', 'list_directory', 'lint']); - -function mergeTokenUsage(left?: TokenUsage, right?: TokenUsage): TokenUsage | undefined { - if (!left && !right) { - return undefined; - } - - return { - inputTokens: (left?.inputTokens ?? 0) + (right?.inputTokens ?? 0), - outputTokens: (left?.outputTokens ?? 0) + (right?.outputTokens ?? 0), - }; -} - -function buildUnknownRuleSourceError(ruleSource: string, validSources: string[]): Error { - const validHint = validSources.length > 0 ? validSources.join(', ') : '(none)'; - return new AgentToolError( - `Unknown ruleSource "${ruleSource}". Valid sources: ${validHint}`, - 'UNKNOWN_RULE_SOURCE' - ); -} - -function fallbackMessage(result: PromptEvaluationResult): string { - if (!isJudgeResult(result) && result.reasoning) { - return result.reasoning; - } - return 'Potential issue detected'; -} - -function severityFromPrompt(prompt: PromptFile): Severity { - return prompt.meta.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING; -} - -function resolvePromptBySource( - ruleSource: string, - promptBySource: Map -): PromptFile | undefined { - const normalized = normalizeRuleSource(ruleSource); - return promptBySource.get(normalized); -} - -function resolveTargetForTopLevel( - workspaceRoot: string, - targets: string[], - file?: string -): string { - if (file && file.trim().length > 0) { - const resolved = resolveWithinRoot(workspaceRoot, file); - return toRelativePathFromRoot(workspaceRoot, resolved); - } - if (targets.length > 0) { - return toRelativePathFromRoot(workspaceRoot, targets[0]!); - } - return '.'; -} - -function withReviewInstructionOverride(prompt: PromptFile, reviewInstruction?: string): PromptFile { - const instruction = reviewInstruction?.trim(); - if (!instruction) { - return prompt; - } - return { - ...prompt, - body: instruction, - }; -} - -function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { - const findings: AgentFinding[] = []; - - for (const event of events) { - if ( - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedInline && - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedTopLevel - ) { - continue; - } - - const payload = event.payload; - const file = payload.file ?? '.'; - const line = payload.line ?? 1; - findings.push({ - file, - line, - column: payload.column ?? 1, - severity: payload.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, - message: payload.message, - ruleId: payload.ruleId ?? payload.ruleSource, - ruleSource: payload.ruleSource, - ...('analysis' in payload && payload.analysis ? { analysis: payload.analysis } : {}), - ...('suggestion' in payload && payload.suggestion ? { suggestion: payload.suggestion } : {}), - ...('fix' in payload && payload.fix ? { fix: payload.fix } : {}), - ...('match' in payload && payload.match ? { match: payload.match } : {}), - }); - } - - return findings; -} - -// Build the concrete matched file-to-rule pairs that the agent should review for this run. -function buildFileRuleMatches( - relativeTargets: string[], - prompts: PromptFile[], - scanPaths: FilePatternConfig[] -): Array<{ file: string; ruleSource: string }> { - const resolver = new ScanPathResolver(); - const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); - const matches: Array<{ file: string; ruleSource: string }> = []; - - for (const relFile of relativeTargets) { - const resolution = - scanPaths.length > 0 - ? resolver.resolveConfiguration(relFile, scanPaths, availablePacks) - : { packs: availablePacks, overrides: {} }; - const matchedPrompts = prompts.filter((prompt) => { - if (prompt.pack === '') return true; - if (!prompt.pack) return false; - if (scanPaths.length > 0 && !resolution.packs.includes(prompt.pack)) return false; - if (!prompt.meta?.id) return true; - const disableKey = `${prompt.pack}.${prompt.meta.id}`; - const overrideValue = resolution.overrides[disableKey]; - return typeof overrideValue !== 'string' || overrideValue.toLowerCase() !== 'disabled'; - }); - - for (const prompt of matchedPrompts) { - matches.push({ file: relFile, ruleSource: normalizeRuleSource(prompt.fullPath) }); - } - } - - return matches; -} - -function summarizeToolOutput(toolName: AgentToolName, output: unknown): unknown { - if (typeof output !== 'object' || output === null) { - return output; - } - - const candidate = output as Record; - - switch (toolName) { - case 'read_file': { - const content = typeof candidate.content === 'string' ? candidate.content : ''; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - contentLength: content.length, - }; - } - case 'search_content': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'search_files': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'list_directory': { - const entries = Array.isArray(candidate.entries) ? candidate.entries : []; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - entryCount: entries.length, - }; - } - default: - return output; - } -} - -interface VisibleToolContext extends VisibleToolProgress { - signature: string; - progressFile?: string; -} - -function countLines(content: string): number { - if (content.length === 0) { - return 0; - } - return content.split('\n').filter((_, index, lines) => { - return !(index === lines.length - 1 && lines[index] === ''); - }).length; -} - -function tryResolveRelativePath(workspaceRoot: string, rawPath: string): string | undefined { - try { - return toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, rawPath)); - } catch { - return undefined; - } -} - -function resolveVisibleToolContext(params: { - toolName: AgentToolName; - input: unknown; - workspaceRoot: string; - promptBySource: Map; - targetFiles: Set; - currentProgressFile?: string; - defaultProgressFile?: string; -}): VisibleToolContext | undefined { - const { - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - currentProgressFile, - defaultProgressFile, - } = params; - - if (!VISIBLE_TOOL_NAMES.has(toolName as VisibleToolName)) { - return undefined; - } - - switch (toolName) { - case 'read_file': { - const parsed = READ_FILE_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'read_file', - path, - ...(progressFile ? { progressFile } : {}), - signature: `read_file:${path}`, - }; - } - case 'list_directory': { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = currentProgressFile ?? defaultProgressFile; - return { - toolName: 'list_directory', - path, - ...(progressFile ? { progressFile } : {}), - signature: `list_directory:${path}`, - }; - } - case 'lint': { - const parsed = LINT_TOOL_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const prompt = resolvePromptBySource(parsed.data.ruleSource, promptBySource); - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); - const path = resolvedPath ?? parsed.data.file; - const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'lint', - path, - ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), - ruleText, - ...(progressFile ? { progressFile } : {}), - signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, - }; - } - } -} - -function buildVisibleToolSuccessState( - context: VisibleToolContext, - output: unknown -): VisibleToolProgress { - switch (context.toolName) { - case 'read_file': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const content = typeof record.content === 'string' ? record.content : ''; - return { - ...context, - lineCount: countLines(content), - }; - } - case 'list_directory': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const entries = Array.isArray(record.entries) ? record.entries : []; - return { - ...context, - entryCount: entries.length, - }; - } - case 'lint': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - return { - ...context, - findingsCount: - typeof record.findingsRecorded === 'number' ? Math.max(0, Math.trunc(record.findingsRecorded)) : 0, - }; - } - } -} - -type FindingLikeViolation = { - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - description?: string; - analysis?: string; - message?: string; - suggestion?: string; - fix?: string; - confidence?: number; - checks?: GateChecks; -}; - -async function appendInlineFinding(params: { - violation: FindingLikeViolation; - result: PromptEvaluationResult; - content: string; - relFile: string; - prompt: PromptFile; - ruleSource: string; - store: Awaited>; -}): Promise { - const { violation, result, content, relFile, prompt, ruleSource, store } = params; - const filterDecision = computeFilterDecision(violation); - if (!filterDecision.surface) { - return false; - } - - const location = locateQuotedText( - content, - { - quoted_text: violation.quoted_text || '', - context_before: violation.context_before || '', - context_after: violation.context_after || '', - }, - 80, - violation.line - ); - - const line = location?.line ?? Math.max(1, Math.trunc(violation.line ?? 1)); - const column = location?.column ?? 1; - const match = location?.match ?? violation.quoted_text ?? ''; - const message = (violation.message || violation.description || fallbackMessage(result)).trim(); - - const finding: AgentFinding = { - file: relFile, - line, - column, - severity: severityFromPrompt(prompt), - message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(ruleSource), - ...(violation.analysis ? { analysis: violation.analysis } : {}), - ...(violation.suggestion ? { suggestion: violation.suggestion } : {}), - ...(violation.fix ? { fix: violation.fix } : {}), - ...(match ? { match } : {}), - }; - - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), - }, - }); - - return true; -} - -export async function runAgentExecutor(params: RunAgentExecutorParams): Promise { - const { - targets, - prompts, - provider, - workspaceRoot, - scanPaths, - sessionHomeDir = os.homedir(), - progressReporter, - maxSteps, - maxRetries, - maxParallelToolCalls, - userInstructions, - } = params; - - const promptBySource = new Map(); - for (const prompt of prompts) { - promptBySource.set(normalizeRuleSource(prompt.fullPath), prompt); - } - const validSources = Array.from(promptBySource.keys()).sort(); - - const store = await createReviewSessionStore({ homeDir: sessionHomeDir }); - let findingsCount = 0; - const relativeTargets = targets.map((target) => - toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, target)) - ); - const fileRuleMatches = buildFileRuleMatches(relativeTargets, prompts, scanPaths); - const defaultRuleName = String(prompts[0]?.meta.name || prompts[0]?.meta.id || 'Rule'); - const targetFiles = new Set(relativeTargets); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { - cwd: workspaceRoot, - targets: relativeTargets, - }, - }); - - let finalized = false; - let currentProgressFile: string | undefined; - const failedVisibleToolSignatures = new Set(); - let nestedToolUsage: TokenUsage | undefined; - - async function runTool( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ): Promise { - const visibleToolContext = resolveVisibleToolContext({ - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - ...(currentProgressFile ? { currentProgressFile } : {}), - ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), - }); - - if (visibleToolContext?.progressFile) { - const nextRuleName = visibleToolContext.ruleName ?? defaultRuleName; - if (currentProgressFile !== visibleToolContext.progressFile) { - progressReporter?.startFile(visibleToolContext.progressFile, nextRuleName); - currentProgressFile = visibleToolContext.progressFile; - } else if (visibleToolContext.ruleName) { - progressReporter?.updateRule(visibleToolContext.ruleName); - } - } - - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName, input }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolStart({ - ...visibleToolContext, - retrying: failedVisibleToolSignatures.has(visibleToolContext.signature), - }); - } - - try { - const output = await handler(input); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: true, - output: summarizeToolOutput(toolName, output), - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolSuccess(buildVisibleToolSuccessState(visibleToolContext, output)); - failedVisibleToolSignatures.delete(visibleToolContext.signature); - } - return output; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: false, - error: message, - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolError(visibleToolContext); - failedVisibleToolSignatures.add(visibleToolContext.signature); - } - throw error; - } - } - - async function lintToolHandler(input: unknown): Promise { - const parsed = LINT_TOOL_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - const promptForCall = withReviewInstructionOverride(prompt, parsed.reviewInstruction); - - const absoluteFile = resolveWithinRoot(workspaceRoot, parsed.file); - const relFile = toRelativePathFromRoot(workspaceRoot, absoluteFile); - const content = await readFile(absoluteFile, 'utf8'); - - const evaluator = createEvaluator( - promptForCall.meta.evaluator || Type.BASE, - provider, - promptForCall - ); - const result = await evaluator.evaluate(relFile, content); - nestedToolUsage = mergeTokenUsage(nestedToolUsage, result.usage); - - let findingsRecorded = 0; - const violations = isJudgeResult(result) - ? result.criteria.flatMap((criterion) => criterion.violations) - : result.violations; - for (const violation of violations) { - const wasRecorded = await appendInlineFinding({ - violation, - result, - content, - relFile, - prompt, - ruleSource: parsed.ruleSource, - store, - }); - if (wasRecorded) { - findingsCount += 1; - findingsRecorded += 1; - } - } - - return { - ok: true, - findingsRecorded, - schema: buildCheckLLMSchema().name, - }; - } - - async function reportFindingToolHandler(input: unknown): Promise { - const parsed = TOP_LEVEL_REPORT_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - - const references = parsed.references && parsed.references.length > 0 - ? parsed.references - : [{ file: resolveTargetForTopLevel(workspaceRoot, targets), startLine: 1, endLine: 1 }]; - - let findingsRecorded = 0; - for (const reference of references) { - const relFile = resolveTargetForTopLevel(workspaceRoot, targets, reference.file); - const finding: AgentFinding = { - file: relFile, - line: Math.max(1, Math.trunc(reference.startLine)), - column: 1, - severity: severityFromPrompt(prompt), - message: parsed.message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(parsed.ruleSource), - ...(parsed.suggestion ? { suggestion: parsed.suggestion } : {}), - }; - - findingsCount += 1; - findingsRecorded += 1; - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(parsed.references ? { references: parsed.references } : {}), - }, - }); - } - - return { ok: true, findingsRecorded }; - } - - async function readFileToolHandler(input: unknown): Promise { - const parsed = READ_FILE_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const content = await readFile(absolutePath, 'utf8'); - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - content, - }; - } - - async function searchFilesToolHandler(input: unknown): Promise { - const parsed = SEARCH_FILES_INPUT_SCHEMA.parse(input); - const scope = resolveGlobPatternWithinRoot(workspaceRoot, parsed.pattern); - const allMatches = await fg(scope.pattern, { - cwd: scope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const truncated = allMatches.length > MAX_SEARCH_FILE_RESULTS; - const matches = allMatches - .slice(0, MAX_SEARCH_FILE_RESULTS) - .map((match) => toRelativePathFromRoot(workspaceRoot, match)) - .sort((a, b) => a.localeCompare(b)); - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function listDirectoryToolHandler(input: unknown): Promise { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const entries = await readdir(absolutePath, { withFileTypes: true }); - - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - entries: entries - .map((entry) => ({ - name: entry.name, - type: entry.isDirectory() ? 'directory' : 'file', - })) - .sort((left, right) => left.name.localeCompare(right.name)), - }; - } - - async function searchContentToolHandler(input: unknown): Promise { - const parsed = SEARCH_CONTENT_INPUT_SCHEMA.parse(input); - const absoluteSearchRoot = resolveWithinRoot(workspaceRoot, parsed.path || '.'); - const globScope = resolveGlobPatternWithinRoot(absoluteSearchRoot, parsed.glob || '**/*'); - const files = await fg(globScope.pattern, { - cwd: globScope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const matches: Array<{ file: string; line: number; text: string }> = []; - let truncated = false; - - outer: for (const filePath of files) { - let content = ''; - try { - content = await readFile(filePath, 'utf8'); - } catch { - continue; - } - - const lines = content.split('\n'); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!; - if (line.includes(parsed.pattern)) { - if (matches.length >= MAX_CONTENT_MATCH_RESULTS) { - truncated = true; - break outer; - } - matches.push({ - file: toRelativePathFromRoot(workspaceRoot, filePath), - line: index + 1, - text: line, - }); - } - } - } - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function finalizeReviewToolHandler(input: unknown): Promise { - const parsed = FINALIZE_REVIEW_INPUT_SCHEMA.parse(input); - if (finalized) { - throw new AgentToolError( - 'finalize_review can only be called once per session.', - 'FINALIZE_REVIEW_ALREADY_CALLED' - ); - } - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { - totalFindings: findingsCount, - ...(parsed.summary ? { summary: parsed.summary } : {}), - }, - }); - finalized = true; - return { ok: true }; - } - - const tools = createAgentTools({ - runTool, - handlers: { - lint: lintToolHandler, - report_finding: reportFindingToolHandler, - read_file: readFileToolHandler, - search_files: searchFilesToolHandler, - list_directory: listDirectoryToolHandler, - search_content: searchContentToolHandler, - finalize_review: finalizeReviewToolHandler, - }, - }); - const availableTools = listAvailableTools(tools); - - let usage: AgentToolLoopResult['usage'] | undefined; - let requestFailures = 0; - let hadOperationalErrors = false; - let errorMessage: string | undefined; - - try { - const result = await provider.runAgentToolLoop({ - systemPrompt: buildAgentSystemPrompt({ - workspaceRoot, - fileRuleMatches, - availableTools, - ...(userInstructions ? { userInstructions } : {}), - }), - prompt: [ - `Workspace root: ${workspaceRoot}`, - `Targets: ${relativeTargets.join(', ')}`, - `Available ruleSources: ${validSources.join(', ')}`, - ].join('\n'), - tools, - ...(maxSteps !== undefined ? { maxSteps } : {}), - ...(maxRetries !== undefined ? { maxRetries } : {}), - ...(maxParallelToolCalls !== undefined ? { maxParallelToolCalls } : {}), - }); - usage = result.usage; - } catch (error) { - requestFailures += 1; - hadOperationalErrors = true; - errorMessage = error instanceof Error ? error.message : String(error); - } - - const events = await store.replay(); - const hasFinalizedEvent = events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - - if (!hasFinalizedEvent) { - hadOperationalErrors = true; - if (!errorMessage) { - errorMessage = 'Agent run ended without finalize_review.'; - } - } - - const findings = findingsFromEvents(events); - const aggregatedUsage = mergeTokenUsage(usage, nestedToolUsage); - progressReporter?.finishRun(hadOperationalErrors ? 'failed' : 'completed'); - - return { - findings, - events, - fileRuleMatches, - requestFailures, - hadOperationalErrors, - ...(errorMessage ? { errorMessage } : {}), - ...(aggregatedUsage ? { usage: aggregatedUsage } : {}), - }; -} diff --git a/src/agent/path-utils.ts b/src/agent/path-utils.ts deleted file mode 100644 index 55378366..00000000 --- a/src/agent/path-utils.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { realpathSync } from 'fs'; -import * as path from 'path'; -import { AgentToolError } from '../errors'; - -function isGlobSegment(segment: string): boolean { - return /[*?[\]{}()!+@]/.test(segment); -} - -export function resolveWithinRoot(root: string, inputPath: string): string { - const rootResolved = path.resolve(root); - const targetResolved = path.resolve(rootResolved, inputPath); - - // Lexical check: fast reject for obvious traversal (e.g. ../../etc) - const relative = path.relative(rootResolved, targetResolved); - const outsideRoot = relative.startsWith('..') || path.isAbsolute(relative); - if (outsideRoot) { - throw new AgentToolError( - `Path "${inputPath}" is outside workspace root bounds.`, - 'PATH_OUTSIDE_WORKSPACE_ROOT' - ); - } - - // Symlink check: resolve symlinks on both sides before comparing, so that - // a symlink inside the repo pointing to an external path is caught. - try { - const realRoot = realpathSync(rootResolved); - const realTarget = realpathSync(targetResolved); - const realRelative = path.relative(realRoot, realTarget); - if (realRelative.startsWith('..') || path.isAbsolute(realRelative)) { - throw new AgentToolError( - `Path "${inputPath}" escapes workspace root via symlink.`, - 'PATH_ESCAPES_WORKSPACE_ROOT_SYMLINK' - ); - } - } catch (err) { - // ENOENT means the path does not yet exist — no symlinks to follow, - // so the lexical check above is sufficient. Re-throw any other error. - if (!(err instanceof Error && 'code' in err && (err as { code: string }).code === 'ENOENT')) { - throw err; - } - } - - return targetResolved; -} - -export function resolveGlobPatternWithinRoot(root: string, pattern: string): { cwd: string; pattern: string } { - const normalizedPattern = pattern.replace(/\\/g, '/').trim(); - const segments = normalizedPattern.split('/').filter((segment) => segment.length > 0); - const firstGlobIndex = segments.findIndex((segment) => isGlobSegment(segment)); - - if (firstGlobIndex === -1) { - const literalPath = resolveWithinRoot(root, normalizedPattern || '.'); - return { - cwd: path.dirname(literalPath), - pattern: path.basename(literalPath), - }; - } - - const baseSegments = segments.slice(0, firstGlobIndex); - const globSegments = segments.slice(firstGlobIndex); - const basePath = baseSegments.length > 0 ? baseSegments.join('/') : '.'; - - return { - cwd: resolveWithinRoot(root, basePath), - pattern: globSegments.join('/'), - }; -} - -export function toRelativePathFromRoot(root: string, absolutePath: string): string { - const rootResolved = path.resolve(root); - const absoluteResolved = path.resolve(absolutePath); - return path.relative(rootResolved, absoluteResolved).replace(/\\/g, '/'); -} diff --git a/src/agent/progress.ts b/src/agent/progress.ts deleted file mode 100644 index dff0f64d..00000000 --- a/src/agent/progress.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { OutputFormat } from '../cli/types'; -import * as readline from 'node:readline'; -import ora, { type Ora } from 'ora'; - -const TOOL_PREFIX = ' └ '; -const MAX_TOOL_LINE_LENGTH = 140; - -export type VisibleToolName = 'read_file' | 'list_directory' | 'lint'; - -export interface VisibleToolProgress { - toolName: VisibleToolName; - path?: string; - ruleName?: string; - ruleText?: string; - lineCount?: number; - entryCount?: number; - findingsCount?: number; -} - -type RunStatus = 'completed' | 'failed'; - -export class AgentProgressReporter { - private readonly enabled: boolean; - private readonly spinner: Ora | undefined; - private readonly runStartedAt = Date.now(); - private activeFile: string | undefined; - private activeRuleName: string | undefined; - private activeLine = TOOL_PREFIX; - - constructor(enabled: boolean) { - this.enabled = enabled; - if (enabled) { - this.spinner = ora({ - spinner: 'dots', - stream: createOraStream(process.stderr), - isEnabled: true, - discardStdin: false, - color: false, - }); - } - } - - private writeLine(message: string): void { - if (!this.enabled) { - return; - } - process.stderr.write(`${message}\n`); - } - - private currentBlockText(): string { - return `Reviewing ${this.activeFile ?? ''} for ${this.activeRuleName ?? 'Rule'}\n${this.activeLine}`; - } - - private renderCurrent(): void { - if (!this.enabled || !this.activeFile || !this.spinner) { - return; - } - - const nextText = this.currentBlockText(); - if (!this.spinner.isSpinning) { - this.spinner.start(nextText); - return; - } - this.spinner.text = nextText; - this.spinner.render(); - } - - startFile(file: string, ruleName: string): void { - if (!this.enabled || !this.spinner) { - return; - } - - if (this.spinner.isSpinning && this.activeFile && this.activeFile !== file) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - - this.activeFile = file; - this.activeRuleName = ruleName; - this.activeLine = TOOL_PREFIX; - this.renderCurrent(); - } - - updateRule(ruleName: string): void { - if (!this.enabled || !this.activeFile || this.activeRuleName === ruleName) { - return; - } - - this.activeRuleName = ruleName; - this.renderCurrent(); - } - - showVisibleToolStart(params: VisibleToolProgress & { retrying?: boolean }): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = params.retrying - ? formatRetryingLine(params) - : formatInvocationLine(params); - this.renderCurrent(); - } - - showVisibleToolSuccess(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatSuccessLine(params); - this.renderCurrent(); - } - - showVisibleToolError(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatErrorLine(params); - this.renderCurrent(); - } - - finishRun(status: RunStatus = 'completed'): void { - if (!this.enabled) { - return; - } - - if (this.spinner?.isSpinning && this.activeFile) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - this.writeLine(formatRunFooter(status, this.runStartedAt)); - this.activeFile = undefined; - this.activeRuleName = undefined; - this.activeLine = TOOL_PREFIX; - } -} - -function formatInvocationLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Read(${sanitizePath(params.path)})`); - case 'list_directory': - return formatToolLine(`List(${sanitizePath(params.path)})`); - case 'lint': - return formatToolLine(`Lint("${formatRuleSnippet(params.ruleText)}")`); - } -} - -function formatRetryingLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Retrying Read(${sanitizePath(params.path)})...`); - case 'list_directory': - return formatToolLine(`Retrying List(${sanitizePath(params.path)})...`); - case 'lint': - return formatToolLine(`Retrying Lint("${formatRuleSnippet(params.ruleText)}")...`); - } -} - -function formatSuccessLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine( - `Read ${formatCount(params.lineCount ?? 0, 'line')} from ${sanitizePath(params.path)}` - ); - case 'list_directory': - return formatToolLine( - `Listed ${formatCount(params.entryCount ?? 0, 'entry')} in ${sanitizePath(params.path)}` - ); - case 'lint': - if ((params.findingsCount ?? 0) === 0) { - return formatToolLine(`Found no issues in ${sanitizePath(params.path)}`); - } - return formatToolLine( - `Found ${formatCount(params.findingsCount ?? 0, 'issue')} in ${sanitizePath(params.path)}` - ); - } -} - -function formatErrorLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Error reading ${sanitizePath(params.path)}`); - case 'list_directory': - return formatToolLine(`Error listing ${sanitizePath(params.path)}`); - case 'lint': - return formatToolLine(`Error linting ${sanitizePath(params.path)}`); - } -} - -function formatToolLine(message: string): string { - return truncate(`${TOOL_PREFIX}${message}`, MAX_TOOL_LINE_LENGTH); -} - -function formatRuleSnippet(ruleText: string | undefined): string { - const sanitized = sanitizeInline(ruleText || ''); - if (sanitized.length === 0) { - return '...'; - } - return truncate(`${sanitized}...`, 52); -} - -function sanitizePath(path: string | undefined): string { - const value = sanitizeInline(path || '.'); - return value.length > 0 ? value : '.'; -} - -function formatCount(count: number, noun: string): string { - return `${count} ${noun}${count === 1 ? '' : 's'}`; -} - -function sanitizeInline(value: string): string { - return value.replace(/\s+/g, ' ').trim(); -} - -function truncate(value: string, maxLength: number): string { - if (value.length <= maxLength) { - return value; - } - return `${value.slice(0, Math.max(0, maxLength - 3))}...`; -} - -function formatElapsed(startedAt: number): string { - const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); - const hours = Math.floor(elapsedSeconds / 3600); - const minutes = Math.floor((elapsedSeconds % 3600) / 60); - const seconds = elapsedSeconds % 60; - - if (hours > 0) { - return `${hours}h ${minutes}m ${seconds}s`; - } - if (minutes > 0) { - return `${minutes}m ${seconds}s`; - } - return `${seconds}s`; -} - -function formatRunFooter(status: RunStatus, startedAt: number): string { - const elapsed = formatElapsed(startedAt); - return status === 'failed' - ? `Review failed after ${elapsed}.` - : `Completed review in ${elapsed}.`; -} - -function createOraStream(stream: NodeJS.WriteStream): NodeJS.WriteStream { - return { - ...stream, - isTTY: stream.isTTY === true, - columns: stream.columns, - write: stream.write.bind(stream), - cursorTo: (x: number) => readline.cursorTo(stream, x), - clearLine: (dir: -1 | 0 | 1) => readline.clearLine(stream, dir), - moveCursor: (dx: number, dy: number) => readline.moveCursor(stream, dx, dy), - } as NodeJS.WriteStream; -} - -export function shouldEmitAgentProgress(params: { outputFormat: OutputFormat; printMode: boolean }): boolean { - const { outputFormat, printMode } = params; - if (printMode) { - return false; - } - if (outputFormat !== OutputFormat.Line) { - return false; - } - return process.stderr.isTTY === true; -} diff --git a/src/agent/prompt-builder.ts b/src/agent/prompt-builder.ts deleted file mode 100644 index 0d16870a..00000000 --- a/src/agent/prompt-builder.ts +++ /dev/null @@ -1,57 +0,0 @@ -export interface BuildAgentSystemPromptParams { - workspaceRoot: string; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - availableTools: Array<{ name: string; description: string }>; - userInstructions?: string; -} - -function formatBulletedList(values: string[]): string { - if (values.length === 0) { - return '- (none)'; - } - return values.map((value) => `- ${value}`).join('\n'); -} - -function formatFileRuleMatches( - matches: Array<{ file: string; ruleSource: string }> -): string { - if (matches.length === 0) { - return '- (none)'; - } - - const rulesByFile = new Map(); - for (const { file, ruleSource } of matches) { - const rules = rulesByFile.get(file) ?? []; - rules.push(ruleSource); - rulesByFile.set(file, rules); - } - - return Array.from(rulesByFile.entries()) - .map(([file, rules]) => `- ${file}\n${rules.map((rule) => ` - ${rule}`).join('\n')}`) - .join('\n'); -} - -export function buildAgentSystemPrompt(params: BuildAgentSystemPromptParams): string { - const date = new Date().toISOString().slice(0, 10); - const userInstructions = params.userInstructions?.trim(); - const fileRuleMatches = formatFileRuleMatches(params.fileRuleMatches); - - return `You are a senior technical writer. You review documentation files against source-backed rules to identify quality issues, inconsistencies, and violations. - -Your goal is to produce a thorough, complete review of every file against every matched rule. - -Workflow: -1. You are given matched file-rule pairs. Work through each file one at a time — complete every matched rule for a file before moving to the next. -2. For each file-rule pair, review the file against the rule. -3. After reviewing the file, read the rule. If the rule contains top-level review instructions — such as checking for documentation drift, verifying that certain files exist, or any other workspace-level check — carry them out and report any findings. -4. When every file has been reviewed against all of its matched rules, you MUST call the finalize_review tool. - -Available tools: -${formatBulletedList(params.availableTools.map((toolDef) => `${toolDef.name}: ${toolDef.description}`))} - -Review files and matched rules: -${fileRuleMatches}${userInstructions ? `\n\nUser Instructions (from VECTORLINT.md):\n${userInstructions}` : ''} - -Current date: ${date} -Workspace root: ${params.workspaceRoot}`; -} diff --git a/src/agent/review-session-store.ts b/src/agent/review-session-store.ts deleted file mode 100644 index d5fb7262..00000000 --- a/src/agent/review-session-store.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { randomUUID } from 'crypto'; -import { appendFile, mkdir, open, readFile } from 'fs/promises'; -import * as path from 'path'; -import { SESSION_EVENT_SCHEMA, SESSION_EVENT_TYPE, type SessionEvent } from './types'; - -type AppendableSessionEvent = Omit; - -export interface ReviewSessionStore { - sessionId: string; - sessionFilePath: string; - append: (event: AppendableSessionEvent) => Promise; - replay: () => Promise; - hasFinalizedEvent: () => Promise; -} - -function buildSessionFilePath(reviewsDir: string, sessionId: string): string { - return path.join(reviewsDir, `${sessionId}.jsonl`); -} - -async function createUniqueSessionFile(reviewsDir: string): Promise<{ sessionId: string; sessionFilePath: string }> { - while (true) { - const sessionId = randomUUID(); - const sessionFilePath = buildSessionFilePath(reviewsDir, sessionId); - - try { - const handle = await open(sessionFilePath, 'wx'); - await handle.close(); - return { sessionId, sessionFilePath }; - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'EEXIST') { - continue; - } - throw error; - } - } -} - -export async function createReviewSessionStore({ homeDir }: { homeDir: string }): Promise { - const reviewsDir = path.join(homeDir, '.vectorlint', 'reviews'); - await mkdir(reviewsDir, { recursive: true }); - - const { sessionId, sessionFilePath } = await createUniqueSessionFile(reviewsDir); - - async function append(event: AppendableSessionEvent): Promise { - const parsed = SESSION_EVENT_SCHEMA.parse({ - sessionId, - timestamp: new Date().toISOString(), - ...event, - }); - await appendFile(sessionFilePath, `${JSON.stringify(parsed)}\n`, 'utf8'); - } - - async function replay(): Promise { - let raw = ''; - try { - raw = await readFile(sessionFilePath, 'utf8'); - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'ENOENT') { - return []; - } - throw error; - } - - const events: SessionEvent[] = []; - const lines = raw.split('\n').filter((line) => line.trim().length > 0); - - for (const line of lines) { - try { - const candidate = JSON.parse(line) as unknown; - const parsed = SESSION_EVENT_SCHEMA.safeParse(candidate); - if (parsed.success) { - events.push(parsed.data); - } - } catch { - // Ignore malformed JSONL lines so replay can recover valid events. - } - } - - return events; - } - - async function hasFinalizedEvent(): Promise { - const events = await replay(); - return events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - } - - return { - sessionId, - sessionFilePath, - append, - replay, - hasFinalizedEvent, - }; -} diff --git a/src/agent/rule-id.ts b/src/agent/rule-id.ts deleted file mode 100644 index 2ee68813..00000000 --- a/src/agent/rule-id.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { PromptFile } from '../prompts/prompt-loader'; - -export function buildRuleId(prompt: PromptFile): string { - const pack = prompt.pack || 'Default'; - const rule = String(prompt.meta.id || prompt.filename || 'Rule'); - return `${pack}.${rule}`; -} - -export function normalizeRuleSource(ruleSource: string): string { - return ruleSource.replace(/\\/g, '/').replace(/^\.\//, ''); -} diff --git a/src/agent/tools-registry.ts b/src/agent/tools-registry.ts deleted file mode 100644 index 4bc49dea..00000000 --- a/src/agent/tools-registry.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { AgentToolDefinition } from '../providers/llm-provider'; -import { - FINALIZE_REVIEW_INPUT_SCHEMA, - LINT_TOOL_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, -} from './types'; - -export type AgentToolName = - | 'lint' - | 'report_finding' - | 'read_file' - | 'search_files' - | 'list_directory' - | 'search_content' - | 'finalize_review'; - -export type AgentToolHandler = (input: unknown) => Promise; -export type AgentToolHandlers = Record; - -export function createAgentTools(params: { - runTool: ( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ) => Promise; - handlers: AgentToolHandlers; -}): Record { - const { runTool, handlers } = params; - - return { - lint: { - description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.', - inputSchema: LINT_TOOL_INPUT_SCHEMA, - execute: (input) => runTool('lint', input, handlers.lint), - }, - report_finding: { - description: 'Record a top-level finding for the report.', - inputSchema: TOP_LEVEL_REPORT_INPUT_SCHEMA, - execute: (input) => runTool('report_finding', input, handlers.report_finding), - }, - read_file: { - description: 'Read a file inside the workspace root.', - inputSchema: READ_FILE_INPUT_SCHEMA, - execute: (input) => runTool('read_file', input, handlers.read_file), - }, - search_files: { - description: 'Find files in the workspace by glob pattern.', - inputSchema: SEARCH_FILES_INPUT_SCHEMA, - execute: (input) => runTool('search_files', input, handlers.search_files), - }, - list_directory: { - description: 'List files and directories inside a path in the workspace.', - inputSchema: LIST_DIRECTORY_INPUT_SCHEMA, - execute: (input) => runTool('list_directory', input, handlers.list_directory), - }, - search_content: { - description: 'Search workspace text content by substring and optional glob.', - inputSchema: SEARCH_CONTENT_INPUT_SCHEMA, - execute: (input) => runTool('search_content', input, handlers.search_content), - }, - finalize_review: { - description: 'REQUIRED: Call this tool when all matched file-rule pairs have been reviewed.', - inputSchema: FINALIZE_REVIEW_INPUT_SCHEMA, - execute: (input) => runTool('finalize_review', input, handlers.finalize_review), - }, - }; -} - -export function listAvailableTools( - tools: Record -): Array<{ name: string; description: string }> { - return Object.entries(tools).map(([name, definition]) => ({ - name, - description: definition.description, - })); -} diff --git a/src/agent/types.ts b/src/agent/types.ts deleted file mode 100644 index e1f2407d..00000000 --- a/src/agent/types.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { z } from 'zod'; - -export const SESSION_EVENT_TYPE = { - SessionStarted: 'session_started', - ToolCallStarted: 'tool_call_started', - ToolCallFinished: 'tool_call_finished', - FindingRecordedInline: 'finding_recorded_inline', - FindingRecordedTopLevel: 'finding_recorded_top_level', - SessionFinalized: 'session_finalized', -} as const; - -export const LINT_TOOL_INPUT_SCHEMA = z.object({ - file: z.string().min(1), - ruleSource: z.string().min(1), - reviewInstruction: z.string().min(1).optional(), -}); - -export const TOP_LEVEL_REFERENCE_SCHEMA = z.object({ - file: z.string().min(1), - startLine: z.number().int().positive(), - endLine: z.number().int().positive(), -}); - -export const TOP_LEVEL_REPORT_INPUT_SCHEMA = z.object({ - kind: z.literal('top-level'), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), -}); - -export const READ_FILE_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_FILES_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), -}); - -export const LIST_DIRECTORY_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_CONTENT_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), - path: z.string().optional(), - glob: z.string().optional(), -}); - -export const FINALIZE_REVIEW_INPUT_SCHEMA = z.object({ - summary: z.string().optional(), -}); - -const SESSION_EVENT_BASE_SCHEMA = z.object({ - sessionId: z.string().min(1), - timestamp: z.string().min(1), -}); - -const SESSION_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionStarted), - payload: z.object({ - cwd: z.string().min(1), - targets: z.array(z.string().min(1)), - }), -}); - -const TOOL_CALL_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallStarted), - payload: z.object({ - toolName: z.string().min(1), - input: z.unknown(), - }), -}); - -const TOOL_CALL_FINISHED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallFinished), - payload: z.object({ - toolName: z.string().min(1), - ok: z.boolean(), - output: z.unknown().optional(), - error: z.string().optional(), - }), -}); - -const FINDING_RECORDED_INLINE_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedInline), - payload: z.object({ - file: z.string().min(1), - line: z.number().int().positive(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - analysis: z.string().optional(), - suggestion: z.string().optional(), - fix: z.string().optional(), - match: z.string().optional(), - }), -}); - -const FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedTopLevel), - payload: z.object({ - file: z.string().min(1).optional(), - line: z.number().int().positive().optional(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), - }), -}); - -const SESSION_FINALIZED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionFinalized), - payload: z.object({ - totalFindings: z.number().int().nonnegative(), - summary: z.string().optional(), - }), -}); - -export const SESSION_EVENT_SCHEMA = z.discriminatedUnion('eventType', [ - SESSION_STARTED_EVENT_SCHEMA, - TOOL_CALL_STARTED_EVENT_SCHEMA, - TOOL_CALL_FINISHED_EVENT_SCHEMA, - FINDING_RECORDED_INLINE_EVENT_SCHEMA, - FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA, - SESSION_FINALIZED_EVENT_SCHEMA, -]); - -export type SessionEvent = z.infer; diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 50dbffc8..b2e7f0b7 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -18,7 +18,7 @@ import { resolveTargets } from '../scan/file-resolver'; import { parseCliOptions, parseEnvironment } from '../boundaries/index'; import { handleUnknownError } from '../errors/index'; import { evaluateFiles } from './orchestrator'; -import { DEFAULT_REVIEW_MODE, OUTPUT_FORMATS, OutputFormat } from './types'; +import { DEFAULT_REVIEW_MODEL_CALL, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; import { createNoopLogger } from '../logging/logger'; @@ -82,8 +82,8 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) - .option('-p, --print', 'Suppress interactive progress output in agent mode') + .option('--model-call ', 'Model call strategy: single (one structured call per rule), agent (bounded target-only paging), or auto (default)', DEFAULT_REVIEW_MODEL_CALL) + .option('--mode ', 'Removed: use --model-call instead') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') .action(async (paths: string[] = []) => { @@ -94,6 +94,15 @@ export function registerMainCommand(program: Command): void { return; } + // --mode is removed (audit Product Decision): the autonomous workspace-agent + // surface is gone. Fail clearly and point users at the replacement rather + // than silently mapping it onto --model-call. + const rawCliOpts = program.opts() as { mode?: unknown }; + if (rawCliOpts.mode !== undefined) { + console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); + process.exit(1); + } + // Parse and validate CLI options const cliOptions = await runOrExit('Parsing CLI options', () => parseCliOptions(program.opts())); @@ -192,6 +201,7 @@ export function registerMainCommand(program: Command): void { try { observability = await initializeObservability(env, runtimeLogger); + const requestBuilder = new DefaultRequestBuilder(directive, userInstructions.content || undefined); const provider = createProvider( env, { @@ -201,7 +211,7 @@ export function registerMainCommand(program: Command): void { logger: runtimeLogger, observability, }, - new DefaultRequestBuilder(directive, userInstructions.content || undefined) + requestBuilder ); // Create search provider if API key is available @@ -214,14 +224,14 @@ export function registerMainCommand(program: Command): void { prompts, rulesPath, provider, + requestBuilder, ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, - mode: cliOptions.mode, - printMode: cliOptions.print, + modelCall: cliOptions.modelCall, scanPaths: config.scanPaths, pricing: { inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 1e61a47d..205a2a80 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,41 +1,29 @@ import { readFileSync } from 'fs'; -import { readFile } from 'fs/promises'; import { randomUUID } from 'crypto'; import * as path from 'path'; -import * as os from 'os'; +import { pathToFileURL } from 'url'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { isJudgeResult } from '../prompts/schema'; -import { handleUnknownError, MissingDependencyError } from '../errors/index'; -import { processFindings } from '../findings'; -import { createEvaluator } from '../evaluators/index'; -import { Type, Severity } from '../evaluators/types'; +import { handleUnknownError } from '../errors/index'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from './types'; -import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '../agent/executor'; -import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; +import { OutputFormat } from './types'; +import { executorFor } from '../executors'; +import { chooseModelCall } from '../review/executor'; +import { buildReviewRequest } from '../review/request-builder'; +import type { ReviewResult, ReviewSeverity, ReviewTarget } from '../review/types'; import type { - EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessPromptResultParams, - RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, - RunPromptEvaluationResultSuccess + EvaluationOptions, EvaluationResult, ReportIssueParams, EvaluateFileResult, } from './types'; import { calculateCost, TokenUsageStats } from '../providers/token-usage'; -import { calculateCheckScore } from '../scoring'; -import { countWords } from '../chunking/utils'; -import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { - computeFilterDecision, - type FilterDecision, -} from "../evaluators/violation-filter"; -import { writeDebugRunArtifact } from "../debug/run-artifact"; +import { writeDebugRunArtifact } from '../debug/run-artifact'; +import { Severity } from '../evaluators/types'; function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } { const provider = process.env.LLM_PROVIDER; @@ -60,34 +48,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string return { ...(provider && { provider }), ...(name && { name }), ...(tag && { tag }) }; } - -/* - * Generic concurrency runner that executes workers in parallel up to a specified limit. - * Preserves result order matching input order. - */ -async function runWithConcurrency( - items: T[], - limit: number, - worker: (item: T, index: number) => Promise -): Promise { - const results: R[] = new Array(items.length); - let i = 0; - const workers = new Array(Math.min(limit, items.length)) - .fill(0) - .map(async () => { - while (true) { - const idx = i++; - if (idx >= items.length) break; - const item = items[idx]; - if (item !== undefined) { - results[idx] = await worker(item, idx); - } - } - }); - await Promise.all(workers); - return results; -} - /* * Reports an issue in either line or JSON format. */ @@ -154,259 +114,62 @@ function reportIssue(params: ReportIssueParams): void { } } -function getViolationFilterResults< - TViolation extends Parameters[0] ->( - violations: TViolation[] -): { - decisions: FilterDecision[]; - surfacedViolations: TViolation[]; -} { - const decisions = violations.map((v) => computeFilterDecision(v)); - const surfacedViolations = violations.filter( - (_v, i) => decisions[i]?.surface === true - ); - - return { decisions, surfacedViolations }; +function toSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; } -/* - * Routes an evaluation result through the shared finding processor. - * Check results are verified, filtered, scored, and reported; judge/rubric - * results are rejected (Phase 3) because subjective scoring is not a - * future-facing review type. - */ -function routePromptResult( - params: ProcessPromptResultParams -): ErrorTrackingResult { - const { - promptFile, - result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - debugJson, - } = params; - const meta = promptFile.meta; - const promptId = (meta.id || "").toString(); - - // Handle Check Result — routed through the shared finding processor - // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, - // filters, deduplicates, scores, and resolves severity; the orchestrator - // only feeds the returned ReviewResult to the existing formatter sinks. - if (!isJudgeResult(result)) { - const reviewResult = processFindings({ - pack: promptFile.pack, - ruleId: promptId, - ruleSource: promptFile.fullPath, - candidateFindings: result.violations, - wordCount: result.word_count, - promptMeta: { - ...(meta.severity !== undefined ? { severity: meta.severity } : {}), - ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - // Sanitize to the findings contract ({ id, name }): meta.criteria is - // PromptCriterionSpec[], which can carry legacy rubric weight/target. - ...(meta.criteria - ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } - : {}), - }, - targetContent: content, - }); - - // processFindings always returns exactly one score entry (processor contract). - const ruleScore = reviewResult.scores[0]!; - const severity = - ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - - // Report only verified findings through the existing line/json/rdjson/vale sink. - for (const finding of reviewResult.findings) { - reportIssue({ - file: relFile, - line: finding.line, - column: finding.column, - severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), - ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), - ...(finding.fix !== undefined ? { fix: finding.fix } : {}), - match: finding.match, - }); - } - - // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, - // consistent with prior operational reporting. They are warn-level and do not - // flag the run as operationally failed (audit Finding #6). - if (verbose) { - for (const diagnostic of reviewResult.diagnostics) { - console.warn(`[vectorlint] ${diagnostic.message}`); - } - } - - // Verified finding count drives the counts (audit Finding #6). - const findingCount = reviewResult.findings.length; - const totalErrors = severity === Severity.ERROR ? findingCount : 0; - const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; - - const scoreEntry: EvaluationSummary = { - id: ruleScore.ruleId, - scoreText: ruleScore.scoreText, - score: ruleScore.score, - }; - - if (debugJson) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - const runId = randomUUID(); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "check", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: decisions.map((d, i) => ({ - index: i, - surface: d.surface, - reasons: d.reasons, - })), - surfaced_violations: surfacedViolations, - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - - return { - errors: totalErrors, - warnings: totalWarnings, - hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, - hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, - scoreEntries: [scoreEntry], - }; - } +function contentTypeFor(file: string): string { + return path.extname(file).toLowerCase() === '.md' ? 'text/markdown' : 'text/plain'; +} - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a - // caller bypassed that boundary. Refuse it explicitly rather than - // misprojecting it as a check result. No adapter is added for judge metadata. - if (verbose) { - console.warn( - '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' - ); - } - return { - errors: 0, - warnings: 0, - hadOperationalErrors: true, - hadSeverityErrors: false, - scoreEntries: [], - }; +function emptyTokenUsage(): TokenUsageStats { + return { totalInputTokens: 0, totalOutputTokens: 0 }; } /* - * Runs a single prompt evaluation. - * BaseEvaluator auto-detects mode from criteria presence: - * - criteria defined → scored mode - * - no criteria → basic mode + * Writes a debug JSON artifact for a review run. The executor path reviews + * through the shared finding processor, so raw model output and per-candidate + * filter decisions are no longer reachable here; the artifact records the + * verified findings that surfaced and the review metadata instead. */ -async function runPromptEvaluation( - params: RunPromptEvaluationParams -): Promise { - const { promptFile, relFile, content, provider, searchProvider } = params; - +function writeReviewDebugArtifact(relFile: string, result: ReviewResult): void { + const runId = randomUUID(); + const model = getModelInfoFromEnv(); try { - const meta = promptFile.meta; - - const evaluatorType = String(meta.evaluator || Type.BASE); - const baseEvaluatorType = String(Type.BASE); - - // Specialized evaluators (e.g., technical-accuracy) require criteria - // BaseEvaluator handles both modes: scored (with criteria) and basic (without) - if (evaluatorType !== baseEvaluatorType) { - if ( - !meta || - !Array.isArray(meta.criteria) || - meta.criteria.length === 0 - ) { - throw new Error( - `Prompt ${promptFile.filename} has no criteria in frontmatter` - ); - } - } - const evaluator = createEvaluator( - evaluatorType, - provider, - promptFile, - searchProvider - ); - const result = await evaluator.evaluate(relFile, content); - - - const resultObj: RunPromptEvaluationResultSuccess = { ok: true, result }; - - return resultObj; - } catch (e: unknown) { - const err = handleUnknownError(e, `Running prompt ${promptFile.filename}`); - return { ok: false, error: err }; + const filePath = writeDebugRunArtifact(process.cwd(), runId, { + file: relFile, + ...(Object.keys(model).length > 0 ? { model } : {}), + ...(model.tag !== undefined ? { subdir: model.tag } : {}), + prompt: { + evaluation_type: 'check', + }, + raw_model_output: null, + filter_decisions: [], + surfaced_violations: result.findings, + }); + console.warn(`[vectorlint] Debug JSON written: ${filePath}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[vectorlint] Debug JSON write failed: ${message}`); } } /* - * Evaluates a single file with all applicable prompts. + * Determines the source-backed prompts that apply to a file, honoring the + * configured scan paths. When no rules match but a VECTORLINT.md user + * instruction guide exists, a synthetic rule is added so the reviewer model + * evaluates against the user instructions in the system prompt. */ -async function evaluateFile( - params: EvaluateFileParams -): Promise { - const { file, options, jsonFormatter } = params; - const { - prompts, - provider, - searchProvider, - concurrency, - scanPaths, - outputFormat = OutputFormat.Line, - verbose, - debugJson, - } = options; - - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let totalErrors = 0; - let totalWarnings = 0; - let requestFailures = 0; - let totalInputTokens = 0; - let totalOutputTokens = 0; - - const allScores = new Map(); - - const content = readFileSync(file, "utf-8"); - const relFile = path.relative(process.cwd(), file) || file; - - if (outputFormat === OutputFormat.Line) { - printFileHeader(relFile); - } - - // Determine applicable prompts for this file +function resolveApplicablePrompts( + relFile: string, + prompts: PromptFile[], + scanPaths: EvaluationOptions['scanPaths'], + userInstructionContent: string | undefined, +): PromptFile[] { const toRun: PromptFile[] = []; if (scanPaths && scanPaths.length > 0) { const resolver = new ScanPathResolver(); - // Extract available packs from loaded prompts const availablePacks = Array.from( new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p)) ); @@ -417,9 +180,7 @@ async function evaluateFile( availablePacks ); - // Filter prompts by active packs - only runs if explicitly in RunRules const activePrompts = prompts.filter((p) => { - // Prompts with empty pack (style guide only) always run if (p.pack === '') return true; if (!p.pack || !resolution.packs.includes(p.pack)) return false; if (!p.meta?.id) return true; @@ -433,13 +194,10 @@ async function evaluateFile( toRun.push(...activePrompts); } else { - // Fallback: When no scanPaths configured, run all prompts. toRun.push(...prompts); } - // If no rules matched but VECTORLINT.md exists, run an evaluation using it. - // The LLM will use the VECTORLINT.md content from the system prompt. - if (options.userInstructionContent) { + if (userInstructionContent) { toRun.push({ id: USER_INSTRUCTION_FILENAME, filename: USER_INSTRUCTION_FILENAME, @@ -454,346 +212,165 @@ async function evaluateFile( }); } - const results = await runWithConcurrency( - toRun, - concurrency, - async (prompt) => { - return runPromptEvaluation({ - promptFile: prompt, - relFile, - content, - provider, - ...(searchProvider !== undefined && { searchProvider }), - }); - } - ); + return toRun; +} - // Aggregate results from each prompt - for (let idx = 0; idx < toRun.length; idx++) { - const p = toRun[idx]; - const r = results[idx]; - if (!p || !r) continue; - - if (r.ok !== true) { - // Check if this is a missing dependency error - if so, skip gracefully - if (r.error instanceof MissingDependencyError) { - console.warn(`[vectorlint] Skipping ${p.filename}: ${r.error.message}`); - if (r.error.hint) { - console.warn(`[vectorlint] Hint: ${r.error.hint}`); - } - // Skip this evaluation entirely - don't count it as a failure - continue; - } +/* + * Evaluates a single file: builds a ReviewRequest from the applicable + * source-backed prompts, resolves the model-call strategy, dispatches through + * the selected ReviewExecutor, and routes the resulting ReviewResult to the + * existing line/json/vale output sinks. + */ +async function evaluateFile( + file: string, + options: EvaluationOptions, + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter, +): Promise { + const { + prompts, + scanPaths, + outputFormat = OutputFormat.Line, + verbose, + debugJson, + } = options; - // Other errors are actual failures - console.error(` Prompt failed: ${p.filename}`); - console.error(r.error); - hadOperationalErrors = true; - requestFailures += 1; - continue; - } + const allScores = new Map(); - // Accumulate token usage - if (r.result.usage) { - totalInputTokens += r.result.usage.inputTokens; - totalOutputTokens += r.result.usage.outputTokens; - } + const content = readFileSync(file, "utf-8"); + const relFile = path.relative(process.cwd(), file) || file; - const promptResult = routePromptResult({ - promptFile: p, - result: r.result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - ...(debugJson !== undefined ? { debugJson } : {}), - }); - totalErrors += promptResult.errors; - totalWarnings += promptResult.warnings; - hadOperationalErrors = - hadOperationalErrors || promptResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || promptResult.hadSeverityErrors; - - if (promptResult.scoreEntries && promptResult.scoreEntries.length > 0) { - const ruleName = (p.meta.id || p.filename).toString(); - allScores.set(ruleName, promptResult.scoreEntries); - } + if (outputFormat === OutputFormat.Line) { + printFileHeader(relFile); } - const tokenUsageStats: TokenUsageStats = { - totalInputTokens, - totalOutputTokens, - }; + const toRun = resolveApplicablePrompts( + relFile, + prompts, + scanPaths, + options.userInstructionContent, + ); - if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); - console.log(""); + // No applicable rules: nothing to review for this file. + if (toRun.length === 0) { + if (outputFormat === OutputFormat.Line) { + printEvaluationSummaries(allScores); + console.log(""); + } + return { + errors: 0, + warnings: 0, + requestFailures: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } - return { - errors: totalErrors, - warnings: totalWarnings, - requestFailures, - hadOperationalErrors, - hadSeverityErrors, - tokenUsage: tokenUsageStats + // Build the review request and dispatch through the executor selected by the + // resolved model-call strategy (audit Product Decision; Finding #2). + const target: ReviewTarget = { + uri: pathToFileURL(path.resolve(file)).href, + content, + contentType: contentTypeFor(file), + byteLength: Buffer.byteLength(content), }; -} - -function reportAgentFinding(params: { - finding: AgentFinding; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -}): void { - const { finding, outputFormat, jsonFormatter } = params; - - reportIssue({ - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), + const request = buildReviewRequest({ + target, + prompts: toRun, + config: { modelCall: options.modelCall }, + }); + const resolvedModelCall = chooseModelCall(request.modelCall, { + targetBytes: request.target.byteLength ?? Buffer.byteLength(content), + rules: request.rules.length, + }); + const executor = executorFor(resolvedModelCall, { + structuredModelClient: options.provider, + toolCallingModelClient: options.provider, + builder: options.requestBuilder, }); -} - -type AgentRuleScore = { - ruleId: string; - score: number; - scoreText: string; -}; - -async function getAgentFileWordCount( - file: string, - workspaceRoot: string, - cache: Map -): Promise { - const workspaceRelative = path.relative(workspaceRoot, path.resolve(workspaceRoot, file)) || file; - if (cache.has(workspaceRelative)) { - return cache.get(workspaceRelative)!; - } - const absolutePath = path.resolve(workspaceRoot, workspaceRelative); + let result: ReviewResult; try { - const content = await readFile(absolutePath, 'utf-8'); - const words = Math.max(1, countWords(content) || 1); - cache.set(workspaceRelative, words); - return words; - } catch { - cache.set(workspaceRelative, 1); - return 1; + result = await executor.run(request); + } catch (e: unknown) { + const err = handleUnknownError(e, `Reviewing ${relFile}`); + console.error(`Error reviewing file ${relFile}: ${err.message}`); + return { + errors: 0, + warnings: 0, + requestFailures: 1, + hadOperationalErrors: true, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } -} -async function buildAgentRuleScores( - findings: AgentFinding[], - prompts: PromptFile[], - fileRuleMatches: Array<{ file: string; ruleSource: string }>, - workspaceRoot: string -): Promise { - const fileWordCountCache = new Map(); - const findingsByRule = new Map(); - const filesByRuleSource = new Map>(); - - for (const finding of findings) { - const existing = findingsByRule.get(finding.ruleId) ?? []; - existing.push(finding); - findingsByRule.set(finding.ruleId, existing); - } - for (const match of fileRuleMatches) { - const files = filesByRuleSource.get(match.ruleSource) ?? new Set(); - files.add(match.file); - filesByRuleSource.set(match.ruleSource, files); + // Route the ReviewResult's verified findings through the existing sinks. + for (const finding of result.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity: toSeverity(finding.severity), + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); } - const results: AgentRuleScore[] = []; - for (const prompt of prompts) { - const ruleId = buildRuleId(prompt); - const ruleFindings = findingsByRule.get(ruleId) ?? []; - const matchedFiles = filesByRuleSource.get(normalizeRuleSource(prompt.fullPath)) ?? new Set(); - - if (matchedFiles.size === 0) { - results.push({ - ruleId, - score: 10, - scoreText: '10.0/10', - }); - continue; - } + for (const score of result.scores) { + allScores.set(score.ruleId, [ + { id: score.ruleId, scoreText: score.scoreText, score: score.score }, + ]); + } - let totalWords = 0; - for (const file of matchedFiles) { - totalWords += await getAgentFileWordCount(file, workspaceRoot, fileWordCountCache); + if (verbose) { + for (const diagnostic of result.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } - - const syntheticViolations = Array.from({ length: ruleFindings.length }, (_, index) => ({ - line: index + 1, - description: 'Agent finding', - analysis: 'Agent finding', - })); - - const scored = calculateCheckScore( - syntheticViolations, - Math.max(1, totalWords), - { - strictness: prompt.meta.strictness, - promptSeverity: prompt.meta.severity, - } - ); - - results.push({ - ruleId, - score: scored.final_score, - scoreText: `${scored.final_score.toFixed(1)}/10`, - }); } - return results; -} -// Retained in quarantine: unreachable from the CLI after --mode agent was -// deprecated (now falls back to standard evaluation). Removed in Phase 4. -// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine -async function evaluateFilesInAgentMode( - targets: string[], - options: EvaluationOptions, - outputFormat: OutputFormat, - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter -): Promise { - const workspaceRoot = inferAgentWorkspaceRoot(targets); - const progressReporter = new AgentProgressReporter( - shouldEmitAgentProgress({ - outputFormat, - printMode: options.printMode ?? false, - }) - ); + const totalErrors = result.findings.filter((f) => f.severity === 'error').length; + const totalWarnings = result.findings.filter((f) => f.severity === 'warning').length; - const agentResult: AgentExecutorResult = await runAgentExecutor({ - targets, - prompts: options.prompts, - provider: options.provider, - workspaceRoot, - scanPaths: options.scanPaths, - outputFormat, - printMode: options.printMode ?? false, - sessionHomeDir: os.homedir(), - progressReporter, - maxParallelToolCalls: 3, - maxRetries: options.agentMaxRetries ?? 10, - ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), - }); + const tokenUsage: TokenUsageStats = { + totalInputTokens: result.usage?.inputTokens ?? 0, + totalOutputTokens: result.usage?.outputTokens ?? 0, + }; - let totalErrors = 0; - let totalWarnings = 0; - const printedFileHeaders = new Set(); - for (const finding of agentResult.findings) { - if (outputFormat === OutputFormat.Line && !printedFileHeaders.has(finding.file)) { - printFileHeader(finding.file); - printedFileHeaders.add(finding.file); - } - reportAgentFinding({ finding, outputFormat, jsonFormatter }); - if (finding.severity === Severity.ERROR) { - totalErrors += 1; - } else { - totalWarnings += 1; - } + if (debugJson) { + writeReviewDebugArtifact(relFile, result); } if (outputFormat === OutputFormat.Line) { - const ruleScores = await buildAgentRuleScores( - agentResult.findings, - options.prompts, - agentResult.fileRuleMatches, - workspaceRoot - ); - const scoreSummary = new Map( - ruleScores.map((entry) => [ - entry.ruleId, - [{ id: 'overall', scoreText: entry.scoreText, score: entry.score }], - ]) - ); - printEvaluationSummaries(scoreSummary); - - if (agentResult.hadOperationalErrors) { - const message = agentResult.errorMessage ?? 'Agent run encountered an operational error.'; - console.error(`\n[agent] ${message}`); - } - } - - if ( - outputFormat === OutputFormat.Json || - outputFormat === OutputFormat.ValeJson || - outputFormat === OutputFormat.RdJson - ) { - console.log(jsonFormatter.toJson()); + printEvaluationSummaries(allScores); + console.log(""); } - const tokenUsage = { - totalInputTokens: agentResult.usage?.inputTokens ?? 0, - totalOutputTokens: agentResult.usage?.outputTokens ?? 0, - }; - return { - totalFiles: targets.length, - totalErrors, - totalWarnings, - requestFailures: agentResult.requestFailures, - hadOperationalErrors: agentResult.hadOperationalErrors, + errors: totalErrors, + warnings: totalWarnings, + requestFailures: 0, + hadOperationalErrors: result.hadOperationalErrors ?? false, hadSeverityErrors: totalErrors > 0, tokenUsage, }; } -function inferAgentWorkspaceRoot(targets: string[]): string { - if (targets.length === 0) { - return process.cwd(); - } - - const directories = targets.map((target) => path.dirname(path.resolve(target))); - let root = directories[0]!; - - for (const directory of directories.slice(1)) { - root = commonPathPrefix(root, directory); - } - - return root; -} - -function commonPathPrefix(left: string, right: string): string { - let candidate = path.resolve(left); - const target = path.resolve(right); - - while (true) { - const relative = path.relative(candidate, target); - const insideCandidate = !relative.startsWith('..') && !path.isAbsolute(relative); - if (insideCandidate) { - return candidate; - } - - const parent = path.dirname(candidate); - if (parent === candidate) { - return candidate; - } - candidate = parent; - } -} - /* - * Runs evaluations across all target files with configurable concurrency. - * Coordinates prompt-to-file mapping, evaluation execution, and result aggregation. - * Returns aggregated results for reporting. + * Runs reviews across all target files, dispatching each through the executor + * selected by the model-call strategy, and aggregates the results. */ export async function evaluateFiles( targets: string[], options: EvaluationOptions ): Promise { - const { outputFormat = OutputFormat.Line, mode = DEFAULT_REVIEW_MODE } = options; + const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -813,18 +390,10 @@ export async function evaluateFiles( jsonFormatter = new ValeJsonFormatter(); } - if (mode === AGENT_REVIEW_MODE) { - options.logger?.warn( - '--mode agent is deprecated and now falls back to standard mode. ' + - 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', - ); - // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. - } - for (const file of targets) { try { totalFiles += 1; - const fileResult = await evaluateFile({ file, options, jsonFormatter }); + const fileResult = await evaluateFile(file, options, jsonFormatter); totalErrors += fileResult.errors; totalWarnings += fileResult.warnings; requestFailures += fileResult.requestFailures; @@ -832,7 +401,6 @@ export async function evaluateFiles( hadOperationalErrors || fileResult.hadOperationalErrors; hadSeverityErrors = hadSeverityErrors || fileResult.hadSeverityErrors; - // Aggregate token usage if (fileResult.tokenUsage) { totalInputTokens += fileResult.tokenUsage.totalInputTokens; totalOutputTokens += fileResult.tokenUsage.totalOutputTokens; diff --git a/src/cli/types.ts b/src/cli/types.ts index a083f2e2..900b1037 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,15 +1,20 @@ import type { PromptFile } from '../prompts/prompt-loader'; -import type { LLMProvider } from '../providers/llm-provider'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; import type { SearchProvider } from '../providers/search-provider'; +import type { RequestBuilder } from '../providers/request-builder'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; +import type { ReviewModelCall } from '../review/types'; + +export { REVIEW_MODEL_CALLS } from '../review/executor'; +export type { ReviewModelCall } from '../review/types'; export enum OutputFormat { Line = "line", @@ -27,25 +32,26 @@ export const OUTPUT_FORMATS = [ export const DEFAULT_OUTPUT_FORMAT = OUTPUT_FORMATS[0]; -export const REVIEW_MODES = ['standard', 'agent'] as const; -export const DEFAULT_REVIEW_MODE = REVIEW_MODES[0]; -export const AGENT_REVIEW_MODE = REVIEW_MODES[1]; - -export type ReviewMode = (typeof REVIEW_MODES)[number]; +/** + * How the reviewer model is invoked for a review (audit Product Decision; + * Finding #2). `single` is one structured call per rule/chunk; `agent` is a + * bounded target-only paging run; `auto` resolves via `chooseModelCall`. + */ +export const DEFAULT_REVIEW_MODEL_CALL: ReviewModelCall = 'auto'; export interface EvaluationOptions { prompts: PromptFile[]; rulesPath: string | undefined; - provider: LLMProvider; + provider: StructuredModelClient & ToolCallingModelClient; + /** Retained for the search-provider capability; the executor path reviews structured output. */ searchProvider?: SearchProvider; + requestBuilder: RequestBuilder; concurrency: number; verbose: boolean; debugJson?: boolean; scanPaths: FilePatternConfig[]; outputFormat?: OutputFormat; - mode?: ReviewMode; - printMode?: boolean; - agentMaxRetries?: number; + modelCall: ReviewModelCall; pricing?: PricingConfig; userInstructionContent?: string; logger?: Logger; @@ -69,15 +75,6 @@ export interface ErrorTrackingResult { scoreEntries?: EvaluationSummary[]; } -export interface EvaluationContext { - content: string; - relFile: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - verbose?: boolean; - debugJson?: boolean; -} - export interface ReportIssueParams { file: string; line: number; @@ -94,34 +91,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessPromptResultParams extends EvaluationContext { - promptFile: PromptFile; - result: PromptEvaluationResult; -} - -export interface RunPromptEvaluationParams { - promptFile: PromptFile; - relFile: string; - content: string; - provider: LLMProvider; - searchProvider?: SearchProvider; -} - -export interface RunPromptEvaluationResultSuccess { - ok: true; - result: PromptEvaluationResult; -} - -export type RunPromptEvaluationResult = - | RunPromptEvaluationResultSuccess - | { ok: false; error: Error }; - -export interface EvaluateFileParams { - file: string; - options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -} - export interface EvaluateFileResult extends ErrorTrackingResult { requestFailures: number; tokenUsage?: TokenUsageStats; diff --git a/src/executors/index.ts b/src/executors/index.ts new file mode 100644 index 00000000..2b565a29 --- /dev/null +++ b/src/executors/index.ts @@ -0,0 +1,38 @@ +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { RequestBuilder } from '../providers/request-builder'; +import type { ReviewExecutor } from '../review/executor'; +import { SingleModelCallExecutor } from './single-model-call-executor'; +import { AgentModelCallExecutor } from './agent-model-call-executor'; + +/** + * The model-client dependencies every executor composes. Both capabilities are + * supplied because the resolved {@link ModelCall} is not known until + * {@link chooseModelCall} runs at review time (audit Finding #2). + */ +export interface ExecutorDeps { + structuredModelClient: StructuredModelClient; + toolCallingModelClient: ToolCallingModelClient; + builder: RequestBuilder; +} + +/** A resolved reviewer model-call strategy (after `chooseModelCall`). */ +export type ModelCall = 'single' | 'agent'; + +/** + * Selects the {@link ReviewExecutor} for a resolved model call. `single` maps + * to {@link SingleModelCallExecutor}; `agent` maps to + * {@link AgentModelCallExecutor}. Callers resolve `auto` via + * {@link chooseModelCall} before calling this factory. + */ +export function executorFor(modelCall: ModelCall, deps: ExecutorDeps): ReviewExecutor { + if (modelCall === 'agent') { + return new AgentModelCallExecutor(deps.toolCallingModelClient, deps.builder); + } + return new SingleModelCallExecutor(deps.structuredModelClient); +} + +export { SingleModelCallExecutor } from './single-model-call-executor'; +export { AgentModelCallExecutor } from './agent-model-call-executor'; +export { TargetReadCapability } from './target-read-capability-adapter'; +export { REVIEW_BUDGET_EXCEEDED_CODE, splitRuleId } from './shared'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index 635d5ecd..50b0a21a 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,40 +1,20 @@ -import type { TokenUsage } from './token-usage'; import type { StructuredModelClient } from './structured-model-client'; /** - * `LLMResult` now lives on the permanent structured-output capability. It is + * `LLMResult` lives on the permanent structured-output capability. It is * re-exported here so existing deep imports (`from './llm-provider'`) keep - * compiling until the legacy provider surface is removed in a later task. + * compiling. */ export type { LLMResult } from './structured-model-client'; -export interface AgentToolDefinition { - description: string; - inputSchema: unknown; - execute: (input: unknown) => Promise; -} - -export interface AgentToolLoopParams { - systemPrompt: string; - prompt: string; - tools: Record; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; -} - -export interface AgentToolLoopResult { - usage?: TokenUsage; -} - /** - * Temporary compile bridge, preserved only until later tasks remove the - * autonomous agent surface. It extends the permanent - * {@link StructuredModelClient} (single structured output) with the legacy - * `runAgentToolLoop` method still consumed by `src/agent/`. Do not expand this - * surface; `runAgentToolLoop` and the `Agent*` types above are deleted once the - * agent tree is removed (audit Finding #2; Product Decision). + * The provider capability historically named `LLMProvider`. The autonomous + * agent loop and the `Agent*` types that used to live here are removed (audit + * Finding #2; Product Decision): the provider surface is now the permanent + * {@link StructuredModelClient} structured-output capability, plus the bounded + * {@link ToolCallingModelClient} transport implemented by concrete providers. + * + * Kept as an alias so existing evaluator and factory imports continue to + * resolve the structured-output capability. */ -export interface LLMProvider extends StructuredModelClient { - runAgentToolLoop(params: AgentToolLoopParams): Promise; -} +export type LLMProvider = StructuredModelClient; diff --git a/src/providers/provider-factory.ts b/src/providers/provider-factory.ts index 3234087d..cbef40ef 100644 --- a/src/providers/provider-factory.ts +++ b/src/providers/provider-factory.ts @@ -4,7 +4,8 @@ import { createAnthropic } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; import type { LanguageModel } from 'ai'; -import { LLMProvider } from './llm-provider'; +import type { StructuredModelClient } from './structured-model-client'; +import type { ToolCallingModelClient } from './tool-calling-model-client'; import { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; import { RequestBuilder } from './request-builder'; import type { EnvConfig } from '../schemas/env-schemas'; @@ -38,7 +39,7 @@ export function createProvider( envConfig: EnvConfig, options: ProviderOptions = {}, builder?: RequestBuilder -): LLMProvider { +): StructuredModelClient & ToolCallingModelClient { let model: LanguageModel; let modelName: string; let temperature = 0.2; diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index ed314001..dba87369 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -2,7 +2,8 @@ import { generateText, Output, NoObjectGeneratedError, stepCountIs, tool } from import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; -import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { LLMProvider } from './llm-provider'; +import type { LLMResult } from './structured-model-client'; import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; @@ -252,70 +253,6 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { } } - runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { - const maxParallel = params.maxParallelToolCalls ?? 1; - const limit = pLimit(maxParallel); - - const mappedTools = Object.fromEntries( - Object.entries(params.tools).map(([name, definition]) => [ - name, - tool({ - description: definition.description, - inputSchema: definition.inputSchema as z.ZodType, - execute: (input: unknown) => limit(() => definition.execute(input)), - }), - ]) - ); - - const result = await generateText({ - model: this.config.model, - system: params.systemPrompt, - prompt: params.prompt, - ...(params.maxRetries !== undefined ? { maxRetries: params.maxRetries } : {}), - ...this.getObservabilityOptions({ - operation: 'agent-tool-loop', - provider: this.config.providerName ?? 'unknown', - model: this.config.modelName ?? 'unknown', - }), - stopWhen: stepCountIs(params.maxSteps ?? 1000), - providerOptions: { - openai: { - parallelToolCalls: maxParallel > 1, - }, - }, - tools: mappedTools, - }); - - if (this.config.debug) { - for (const [i, step] of result.steps.entries()) { - const toolNames = step.toolCalls.map((c) => c.toolName).join(', ') || '(none)'; - this.logger.debug( - `[agent] step ${i + 1}: finishReason=${step.finishReason} tools=[${toolNames}]` - ); - if (step.text) { - this.logger.debug( - `[agent] step ${i + 1} text: ${step.text.slice(0, 500)}${step.text.length > 500 ? '...' : ''}` - ); - } - } - this.logger.debug(`[agent] final finishReason=${result.finishReason} steps=${result.steps.length}`); - if (result.text) { - this.logger.debug( - `[agent] final text: ${result.text.slice(0, 500)}${result.text.length > 500 ? '...' : ''}` - ); - } - } - - return result.usage - ? { - usage: { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, - }, - } - : {}; - } - private getObservabilityOptions(context: AIExecutionContext): Record { if (!this.observability) { return {}; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 654f41df..468a0aca 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODE, OutputFormat, REVIEW_MODES } from '../cli/types'; +import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODEL_CALL, OutputFormat, REVIEW_MODEL_CALLS } from '../cli/types'; // CLI options schema for command line argument validation export const CLI_OPTIONS_SCHEMA = z.object({ @@ -8,8 +8,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.nativeEnum(OutputFormat).default(DEFAULT_OUTPUT_FORMAT), - mode: z.enum(REVIEW_MODES).default(DEFAULT_REVIEW_MODE), - print: z.boolean().default(false), + modelCall: z.enum(REVIEW_MODEL_CALLS).default(DEFAULT_REVIEW_MODEL_CALL), prompts: z.string().optional(), config: z.string().optional(), }); diff --git a/tests/agent/agent-executor.test.ts b/tests/agent/agent-executor.test.ts deleted file mode 100644 index 2f3d1ec5..00000000 --- a/tests/agent/agent-executor.test.ts +++ /dev/null @@ -1,841 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, describe, expect, it } from 'vitest'; -import type { PromptFile } from '../../src/prompts/prompt-loader'; -import type { LLMProvider } from '../../src/providers/llm-provider'; -import { Severity } from '../../src/evaluators/types'; -import { OutputFormat } from '../../src/cli/types'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; -import { AgentToolError } from '../../src/errors'; - -function makePrompt(): PromptFile { - return { - id: 'consistency', - filename: 'consistency.md', - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording.', - meta: { - id: 'Consistency', - name: 'Consistency', - type: 'check', - severity: Severity.WARNING, - }, - }; -} - -function makeProvider( - script: (params: Record) => Promise<{ usage?: { inputTokens: number; outputTokens: number } }> -): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: script as unknown as never, - } as unknown as LLMProvider; -} - -describe('agent executor', () => { - const tempDirs: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-agent-')); - tempDirs.push(repo); - return repo; - } - - afterEach(() => { - for (const dir of tempDirs.splice(0, tempDirs.length)) { - rmSync(dir, { recursive: true, force: true }); - } - }); - - it('exposes only non-mutating analysis tools plus finalize_review', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = (params.tools ?? {}) as Record; - const names = Object.keys(tools).sort(); - expect(names).toEqual([ - 'finalize_review', - 'lint', - 'list_directory', - 'read_file', - 'report_finding', - 'search_content', - 'search_files', - ]); - - const finalize = tools.finalize_review as { execute: (input: unknown) => Promise }; - await finalize.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('returns explicit tool error for unknown ruleSource with valid-source hints', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - let errorMessage = ''; - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - expect(error).toBeInstanceOf(AgentToolError); - errorMessage = error instanceof Error ? error.message : String(error); - } - expect(errorMessage).toContain('Unknown ruleSource'); - expect(errorMessage).toContain('Valid sources'); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('returns explicit tool error for unknown ruleSource in report_finding', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/does-not-exist.md', - message: 'Unknown source', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('Unknown ruleSource'); - expect(result.errorMessage).toContain('Valid sources'); - }); - - it('returns findings reconstructed from persisted inline and top-level session events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({ summary: 'done' }); - return { usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - const inlineEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedInline - ).length; - const topLevelEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedTopLevel - ).length; - const replayableFindingEvents = inlineEventCount + topLevelEventCount; - - expect(replayableFindingEvents).toBeGreaterThanOrEqual(2); - expect(result.findings.length).toBe(replayableFindingEvents); - expect(result.findings.some((finding: { line: number }) => finding.line > 1)).toBe(false); - expect( - result.findings.some( - (finding: { ruleId: string; ruleSource: string }) => - finding.ruleId === 'Default.Consistency' && - finding.ruleSource === 'packs/default/consistency.md' - ) - ).toBe(true); - expect(result.events.some((event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized)).toBe(true); - }); - - it('aggregates nested lint usage with agent loop usage', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [], - }, - usage: { - inputTokens: 7, - outputTokens: 3, - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - }; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.usage).toEqual({ - inputTokens: 18, - outputTokens: 8, - }); - }); - - it('falls back to matching all prompts when scanPaths is empty', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('records the required session event stream and preserves lifecycle ordering', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const eventTypes = result.events.map((event: { eventType: string }) => event.eventType); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallFinished); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionFinalized); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.SessionStarted)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.FindingRecordedInline)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - }); - - it('returns an operational error when finalize_review is called more than once', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.finalize_review.execute({ summary: 'first' }); - await tools.finalize_review.execute({ summary: 'second' }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - const finalizedCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized - ).length; - expect(finalizedCount).toBe(1); - }); - - it.each([ - { - toolName: 'read_file', - input: { path: '../outside.md' }, - }, - { - toolName: 'search_files', - input: { pattern: '../*.md' }, - }, - { - toolName: 'list_directory', - input: { path: '../' }, - }, - { - toolName: 'search_content', - input: { pattern: 'bad phrase', path: '../', glob: '**/*.md' }, - }, - ])( - 'returns explicit tool errors when $toolName is asked to access outside repository bounds', - async ({ toolName, input }) => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - let toolErrorValue: unknown; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools[toolName]!.execute(input); - } catch (error) { - toolErrorValue = error; - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(toolErrorValue).toBeInstanceOf(AgentToolError); - expect(toolError).toBeTruthy(); - expect(toolError).toMatch(/outside|workspace|root|bounds/i); - } - ); - - it('continues producing findings after a recoverable tool error and reports the tool diagnostic', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 2, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(toolError).toContain('Unknown ruleSource'); - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings.length).toBeGreaterThan(0); - expect( - result.events.some( - (event: { eventType: string; payload?: { ok?: boolean } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.ok === false - ) - ).toBe(true); - }); - - it('allows read-only search_content tool usage without requiring mutation capabilities', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\nanother line\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - const output = await tools.search_content.execute({ - pattern: 'bad phrase', - path: '.', - glob: '**/*.md', - }); - - expect(output).toBeTruthy(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('marks the run as operationally failed but preserves findings when finalize_review is missing', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - expect(result.findings.length).toBeGreaterThan(0); - }); - - it('uses reviewInstruction to override the prompt body for that lint invocation', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for wording consistency using the evidence you gathered.', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe( - 'Review this file for wording consistency using the evidence you gathered.' - ); - }); - - it('keeps lint prompt body unchanged when reviewInstruction is not provided', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe('Find inconsistent wording.'); - }); - - it('records judge-style violations as inline findings in agent mode', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const basePrompt = makePrompt(); - const judgePrompt: PromptFile = { - ...basePrompt, - body: 'Judge the document for clarity.', - meta: { - ...basePrompt.meta, - type: 'judge', - criteria: [{ id: 'Clarity', name: 'Clarity', weight: 1 }], - }, - }; - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - criteria: [ - { - name: 'Clarity', - score: 2, - summary: 'Needs work', - reasoning: 'The wording is unclear.', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Unclear wording', - analysis: 'This phrase is vague.', - message: 'Use clearer wording', - suggestion: 'Replace the vague phrase', - fix: 'better phrase', - rule_quote: 'Prefer precise language', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [judgePrompt], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - file: 'doc.md', - ruleId: 'Default.Consistency', - message: 'Use clearer wording', - }), - ]) - ); - }); - - it('redacts raw read_file content from persisted tool_call_finished events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'secret text\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const readFileEvent = result.events.find( - (event: { - eventType: string; - payload?: { toolName?: string; output?: { path?: string; contentLength?: number } }; - }) => event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.toolName === 'read_file' - ); - - expect(readFileEvent?.payload?.output).toEqual({ - path: 'doc.md', - contentLength: 'secret text\n'.length, - }); - }); - - it('records started and failed events for visible-tool path errors', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const started = result.events.find( - (event: { eventType: string; payload?: { toolName?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallStarted && event.payload?.toolName === 'read_file' - ); - const failed = result.events.find( - (event: { eventType: string; payload?: { toolName?: string; ok?: boolean; error?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished - && event.payload?.toolName === 'read_file' - && event.payload?.ok === false - ); - - expect(started).toBeDefined(); - expect(failed?.payload?.error).toContain('outside workspace root'); - }); -}); diff --git a/tests/agent/progress.test.ts b/tests/agent/progress.test.ts deleted file mode 100644 index df213ab4..00000000 --- a/tests/agent/progress.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { AgentProgressReporter } from '../../src/agent/progress'; - -describe('agent progress reporter', () => { - afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - it('animates spinner frames independently of content changes while active', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - reporter.showVisibleToolStart({ - toolName: 'lint', - path: 'README.md', - ruleName: 'Repetition', - ruleText: '# Repetition Flag any instance where the same wording repeats', - }); - - const initialWrites = writeSpy.mock.calls.length; - vi.advanceTimersByTime(250); - - expect(writeSpy.mock.calls.length).toBeGreaterThan(initialWrites); - - reporter.finishRun(); - }); - - it('is a no-op when disabled', () => { - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(false); - reporter.startFile('README.md', 'Repetition'); - reporter.finishRun(); - - expect(writeSpy).not.toHaveBeenCalled(); - }); - - it.each([ - { elapsedMs: 0, expected: 'Completed review in 0s.' }, - { elapsedMs: 60_000, expected: 'Completed review in 1m 0s.' }, - { elapsedMs: 3_600_000, expected: 'Completed review in 1h 0m 0s.' }, - ])('formats elapsed time boundaries for $expected', ({ elapsedMs, expected }) => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(elapsedMs); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain(expected); - }); - - it('formats the completion footer with elapsed time', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Completed review in 1m 25s.'); - }); - - it('formats a failed footer when the run ends with errors', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun('failed'); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Review failed after 1m 25s.'); - }); -}); diff --git a/tests/agent/prompt-builder.test.ts b/tests/agent/prompt-builder.test.ts deleted file mode 100644 index fe31074c..00000000 --- a/tests/agent/prompt-builder.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('agent prompt builder', () => { - it('builds a non-empty system prompt for valid inputs', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [ - { file: 'README.md', ruleSource: 'packs/default/ai-pattern.md' }, - { file: 'README.md', ruleSource: 'packs/default/consistency.md' }, - ], - availableTools: [ - { name: 'read_file', description: 'Read a file inside the workspace root.' }, - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - { name: 'finalize_review', description: 'Finalize review output and close the session.' }, - ], - }); - - expect(typeof prompt).toBe('string'); - expect(prompt.length).toBeGreaterThan(0); - expect(prompt).toContain('Workspace root: /workspace'); - expect(prompt).toContain('matched rules'); - }); - - it('supports optional user instructions without breaking prompt generation', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const withoutUserInstructions = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - }); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - userInstructions: 'Always enforce concise phrasing.', - }); - - expect(typeof withoutUserInstructions).toBe('string'); - expect(typeof prompt).toBe('string'); - expect(withoutUserInstructions.length).toBeGreaterThan(0); - expect(prompt.length).toBeGreaterThan(withoutUserInstructions.length); - }); -}); diff --git a/tests/agent/review-session-store.test.ts b/tests/agent/review-session-store.test.ts deleted file mode 100644 index 0258b561..00000000 --- a/tests/agent/review-session-store.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { appendFileSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -beforeEach(() => { - vi.resetModules(); -}); - -afterEach(() => { - vi.doUnmock('crypto'); - vi.resetModules(); -}); - -describe('review session store', () => { - it('creates session files in the VectorLint reviews directory', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(store.sessionFilePath).toContain( - `${path.sep}.vectorlint${path.sep}reviews${path.sep}` - ); - expect(path.basename(store.sessionFilePath)).toMatch(/\.jsonl$/); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); - - it('persists each appended event as a valid JSONL record', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const raw = readFileSync(store.sessionFilePath, 'utf8'); - const lines = raw.trim().split('\n'); - expect(lines).toHaveLength(2); - - const parsed = lines.map((line) => JSON.parse(line) as { eventType?: string }); - expect(parsed.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('reads persisted session events in order and reports completion state', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(await store.hasFinalizedEvent()).toBe(false); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - expect(await store.hasFinalizedEvent()).toBe(true); - }); - - it('recovers valid events when an existing session file contains malformed JSONL lines', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - - appendFileSync(store.sessionFilePath, 'not-json\n', 'utf8'); - appendFileSync(store.sessionFilePath, '{"eventType":"unknown"}\n', 'utf8'); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0 }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('creates a unique session file when an initial generated session id collides', async () => { - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const reviewsDir = path.join(home, '.vectorlint', 'reviews'); - mkdirSync(reviewsDir, { recursive: true }); - writeFileSync(path.join(reviewsDir, 'collision-id.jsonl'), '', 'utf8'); - - const randomUUIDMock = vi - .fn() - .mockReturnValueOnce('collision-id') - .mockReturnValueOnce('fresh-id'); - - vi.doMock('crypto', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - randomUUID: randomUUIDMock, - }; - }); - - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const store = await createReviewSessionStore({ homeDir: home }); - expect(path.basename(store.sessionFilePath)).toBe('fresh-id.jsonl'); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); -}); diff --git a/tests/agent/types-contract.test.ts b/tests/agent/types-contract.test.ts deleted file mode 100644 index 6363587a..00000000 --- a/tests/agent/types-contract.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -describe('agent contracts', () => { - it('accepts ruleSource-based inputs for lint and top-level findings', async () => { - const contracts = await import('../../src/agent/types'); - - const lintInput = contracts.LINT_TOOL_INPUT_SCHEMA.parse({ - file: 'docs/guide.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for consistency.', - }); - expect(lintInput.ruleSource).toBe('packs/default/consistency.md'); - - const topLevel = contracts.TOP_LEVEL_REPORT_INPUT_SCHEMA.parse({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file inconsistency', - references: [{ file: 'docs/guide.md', startLine: 2, endLine: 4 }], - }); - expect(topLevel.kind).toBe('top-level'); - expect(topLevel.ruleSource).toBe('packs/default/consistency.md'); - }); - - it('accepts finalized session events with completion metadata', async () => { - const contracts = await import('../../src/agent/types'); - - const event = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 1, summary: 'done' }, - }); - - expect(event.eventType).toBe(SESSION_EVENT_TYPE.SessionFinalized); - expect(event.payload.totalFindings).toBe(1); - }); - - it('accepts required tool and finding session event variants for deterministic replay', async () => { - const contracts = await import('../../src/agent/types'); - - const started = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName: 'lint', input: { file: 'docs/guide.md' } }, - }); - - const finished = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:01.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { toolName: 'lint', ok: true }, - }); - - const inlineFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:02.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: 'docs/guide.md', - line: 2, - message: 'Inconsistent term', - ruleSource: 'packs/default/consistency.md', - }, - }); - - const topLevelFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:03.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - message: 'Cross-file mismatch', - ruleSource: 'packs/default/consistency.md', - }, - }); - - expect(started.eventType).toBe(SESSION_EVENT_TYPE.ToolCallStarted); - expect(finished.eventType).toBe(SESSION_EVENT_TYPE.ToolCallFinished); - expect(inlineFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(topLevelFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - }); -}); diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 359da161..7d652c46 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -97,8 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'auto', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -142,8 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'auto', config: undefined, }); const observability = { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts deleted file mode 100644 index 0b61ea63..00000000 --- a/tests/orchestrator-agent-output.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; -import type { PromptFile } from '../src/prompts/prompt-loader'; -import type { LLMProvider } from '../src/providers/llm-provider'; -import type { Logger } from '../src/logging/logger'; -import { Severity } from '../src/evaluators/types'; - -function makePrompt(): PromptFile { - const id = 'consistency'; - const name = 'Consistency'; - return { - id, - filename: `${id}.md`, - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording', - meta: { - id: name, - name, - type: 'check', - severity: Severity.WARNING, - }, - }; -} - -type LoggerSpy = Logger & { warn: ReturnType }; - -function makeLogger(): LoggerSpy { - return { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - } as unknown as LoggerSpy; -} - -interface StandardProviderSpies { - provider: LLMProvider; - runPromptStructured: ReturnType; - runAgentToolLoop: ReturnType; -} - -function makeStandardProvider(): StandardProviderSpies { - const runPromptStructured = vi.fn().mockResolvedValue({ - data: { reasoning: 'ok', violations: [] }, - }); - const runAgentToolLoop = vi.fn().mockResolvedValue({ - usage: { inputTokens: 0, outputTokens: 0 }, - }); - const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; - return { provider, runPromptStructured, runAgentToolLoop }; -} - -// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no -// longer reaches the autonomous agent executor: it warns through the injected -// logger and falls back to standard evaluation. The retained agent executor -// code is covered directly by tests/agent/* and removed in Phase 4. -describe('agent mode deprecation', () => { - const tempRepos: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(process.cwd(), 'tmp-agent-orch-')); - tempRepos.push(repo); - return repo; - } - - beforeEach(() => { - vi.spyOn(console, 'log').mockImplementation(() => undefined); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); - vi.spyOn(console, 'error').mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - for (const repo of tempRepos.splice(0, tempRepos.length)) { - rmSync(repo, { recursive: true, force: true }); - } - }); - - it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('deprecated'), - ); - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), - ); - // Standard evaluation ran; the autonomous executor did not. - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); - - it('does not invoke the agent executor in line output mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - }); - - it('stays silent in standard mode without a logger', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runAgentToolLoop } = makeStandardProvider(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: DEFAULT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - }); - - // No deprecation warning and no executor invocation in standard mode. - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); -}); diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts deleted file mode 100644 index eaaa1529..00000000 --- a/tests/orchestrator-check-processor.test.ts +++ /dev/null @@ -1,356 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type CheckViolation = RawCheckResult["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -/** - * Builds a check violation that passes the confidence gate by default. Real - * model output always carries `message`; the helper sets it explicitly so the - * test reflects production behavior rather than the analysis fallback. - */ -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { - return { - line: 1, - analysis: "Issue 1", - message: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; - wordCount?: number; -}): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -describe("standard check evaluation via the shared finding processor", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("reports fully locatable findings, score, and counts unchanged", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ message: "First issue", analysis: "First issue" }), - makeCheckViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second issue", - analysis: "Second issue", - suggestion: "Suggestion 2", - fix: "Fix 2", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. - expect(run.totalWarnings).toBe(2); - expect(run.totalErrors).toBe(0); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("8.0/10"); - // The line summary prints the last segment of the Pack.Rule score id. - expect(scoreLine).toContain("CheckPrompt"); - }); - - it("counts only verified findings and excludes unanchored quotes from the score", async () => { - // One anchored + one unanchored quote. Both pass the confidence gate, so - // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CountingFixPrompt", - name: "Counting Fix Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ - line: 9, - quoted_text: "this quote is not anywhere in the content", - message: "Ghost issue", - analysis: "Ghost issue", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a - // finding; it contributes neither to the count nor the score. - expect(run.totalWarnings).toBe(1); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - }); - - it("does not flag severity errors when no finding can be verified", async () => { - // severity=error, but the only quote cannot anchor -> 0 verified findings -> - // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorNoVerifiedPrompt", - name: "Error No Verified Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - line: 9, - quoted_text: "nowhere to be found", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.hadSeverityErrors).toBe(false); - expect(run.hadOperationalErrors).toBe(false); - }); - - it("flags severity errors when verified error-severity findings exist", async () => { - // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> - // prompt severity error applies. - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorVerifiedPrompt", - name: "Error Verified Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [makeCheckViolation({ quoted_text: "Alpha text" })], - wordCount: 10, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(1); - expect(run.hadSeverityErrors).toBe(true); - }); - - it("emits verified findings with anchored location through the JSON sink", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ message: "First", analysis: "First" }), - makeCheckViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second", - analysis: "Second", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const issues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(issues).toHaveLength(2); - expect(issues[0]).toMatchObject({ - line: 1, - column: 1, - severity: Severity.WARNING, - message: "First", - rule: "TestPack.CheckJsonPrompt", - match: "Alpha text", - }); - expect(issues[1]).toMatchObject({ - line: 2, - message: "Second", - match: "Beta text", - }); - }); - - it("omits unanchored quotes from the Vale JSON output", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValePrompt", - name: "Check Vale Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ - line: 9, - quoted_text: "missing quote", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const issues = Object.values(parsed).flat(); - - expect(issues).toHaveLength(1); - expect(issues[0]).toMatchObject({ - Check: "TestPack.CheckValePrompt", - Line: 1, - Match: "Alpha text", - Severity: "warning", - }); - }); -}); diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts deleted file mode 100644 index b4d2cf89..00000000 --- a/tests/orchestrator-filtering.test.ts +++ /dev/null @@ -1,412 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { JudgeResult, RawCheckResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type CheckViolation = RawCheckResult["violations"][number]; -type JudgeViolation = JudgeResult["criteria"][number]["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-filtering-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -const EMPTY_CHECK_NOTES: JudgeViolation["check_notes"] = { - rule_supports_claim: "", - evidence_exact: "", - context_supports_violation: "", - plausible_non_violation: "", - fix_is_drop_in: "", - fix_preserves_meaning: "", -}; - -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { - return { - line: 1, - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeJudgeViolation( - overrides: Partial = {} -): JudgeViolation { - const { check_notes: checkNotesOverrides, ...rest } = overrides; - return { - line: 1, - quoted_text: "Alpha text", - context_before: "", - context_after: "", - description: "Issue 1", - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: { - ...EMPTY_CHECK_NOTES, - ...(checkNotesOverrides ?? {}), - }, - confidence: 0.9, - ...rest, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; - wordCount?: number; -}): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -function makeJudgeResult(violations: JudgeViolation[]): JudgeResult { - return { - type: EvaluationType.JUDGE, - final_score: 5, - criteria: [ - { - name: "Clarity", - weight: 1, - score: 2, - normalized_score: 5, - weighted_points: 1, - summary: "Needs work", - reasoning: "Reasoning", - violations, - }, - ], - }; -} - -describe("CLI violation filtering", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("filters low-confidence check violations from CLI counts by default", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation(), - makeCheckViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation(), - makeCheckViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); - }); - - it("does not mark severity error when no check violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckErrorPrompt", - name: "Check Error Prompt", - type: "check", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalErrors).toBe(0); - expect(defaultRun.hadSeverityErrors).toBe(false); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalErrors).toBe(1); - expect(zeroThresholdRun.hadSeverityErrors).toBe(true); - }); - - it("score reflects only surfaced violations, not filtered-out ones", async () => { - // 100-word file: 2 violations from model, 1 fails confidence gate - // With default threshold, only 1 violation surfaces - // Density: 1/100 * 100 * 10 = 10 penalty → score = 9.0 - // If bug were present (scoring all 2): 2/100 * 100 * 10 = 20 penalty → score = 8.0 - - const content = new Array(100).fill("word").join(" ") + "\n"; - const targetFile = createTempFile(content); - - const prompt = createPrompt({ - id: "ScorePrompt", - name: "Score Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ quoted_text: content.split(" ")[0] ?? "word" }), - makeCheckViolation({ - quoted_text: content.split(" ")[1] ?? "word", - confidence: 0.2, // fails confidence gate — should NOT affect score - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - // Score should reflect 1 surfaced violation, not 2 - const scoreLine = logCalls.find(l => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - expect(scoreLine).not.toContain("8.0/10"); - }); - - it("rejects judge/rubric results instead of projecting them as check findings", async () => { - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // orchestrator refuses a JudgeResult rather than running the old criterion - // extraction/reporting path: no findings are projected and the run is - // flagged with an operational error. - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "JudgePrompt", - name: "Judge Prompt", - type: "judge", - criteria: [{ id: "Clarity", name: "Clarity", weight: 1 }], - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - confidence: 0.2, - }), - ]) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.totalWarnings).toBe(0); - expect(run.hadOperationalErrors).toBe(true); - expect(run.hadSeverityErrors).toBe(false); - }); - - it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); - - it("does not emit dummy issues in Vale JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValeJsonPrompt", - name: "Check Vale JSON Prompt", - type: "check", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [ - makeCheckViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const allIssues = Object.values(parsed).flat(); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); -}); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts deleted file mode 100644 index dc0b1ad8..00000000 --- a/tests/orchestrator-finding-criteria.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; -import type { ReviewResult } from "../src/review/types"; - -const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), - PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), -})); - -// createEvaluator returns a controllable evaluator so evaluateFiles reaches -// routePromptResult -> processFindings without a real model call. -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -// Replace only processFindings (the findings boundary); keep every other -// findings export real so unrelated imports stay intact. -vi.mock("../src/findings", async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; -}); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -function makeCheckResult(): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: [], - word_count: 100, - }; -} - -describe("standard check orchestration sanitizes criteria at the findings boundary", () => { - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - PROCESS_FINDINGS_MOCK.mockReset(); - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("drops rubric weight/target from criteria before reaching processFindings", async () => { - const targetFile = createTempFile("Alpha text\n"); - // PromptCriterionSpec admits legacy rubric fields (weight, target). The - // orchestrator must strip them so only { id, name } crosses the boundary. - const prompt = createPrompt({ - id: "SanitizePrompt", - name: "Sanitize Prompt", - type: "check", - severity: Severity.WARNING, - criteria: [ - { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, - ], - }); - - EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); - PROCESS_FINDINGS_MOCK.mockReturnValue({ - findings: [], - scores: [ - { - ruleId: "TestPack.SanitizePrompt", - score: 10, - scoreText: "10.0/10", - severity: "warning", - }, - ], - diagnostics: [], - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); - const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { - promptMeta: { criteria?: Array> }; - }; - - // toEqual fails if any extra (weight/target) key survives the mapping. - expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); - for (const criterion of input.promptMeta.criteria ?? []) { - expect(criterion).not.toHaveProperty("weight"); - expect(criterion).not.toHaveProperty("target"); - } - }); -}); diff --git a/tests/providers/llm-provider-contract.test.ts b/tests/providers/llm-provider-contract.test.ts deleted file mode 100644 index 249854ca..00000000 --- a/tests/providers/llm-provider-contract.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { LanguageModel } from 'ai'; -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('LLMProvider agent contract', () => { - it('supports agent-mode execution via a provider loop interface', () => { - const provider = new VercelAIProvider({ - model: {} as unknown as LanguageModel, - }); - - const runAgentToolLoop = (provider as { runAgentToolLoop?: unknown }).runAgentToolLoop; - expect(typeof runAgentToolLoop).toBe('function'); - }); -}); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts index 7fa6b61d..45db05cc 100644 --- a/tests/providers/structured-model-client.test.ts +++ b/tests/providers/structured-model-client.test.ts @@ -45,13 +45,11 @@ describe('StructuredModelClient contract', () => { const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; expect(onlyRunPromptStructured).toBe(true); - // Runtime guard: a minimal implementation exposes only runPromptStructured. + // Runtime guard: a minimal implementation exposes exactly one capability. const client: StructuredModelClient = { runPromptStructured: () => Promise.resolve({ data: null }), }; - expect( - (client as unknown as Record).runAgentToolLoop, - ).toBeUndefined(); + expect(Object.keys(client)).toEqual(['runPromptStructured']); }); it('is satisfied by VercelAIProvider', () => { diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts index ccf84f68..a2239552 100644 --- a/tests/providers/tool-calling-model-client.test.ts +++ b/tests/providers/tool-calling-model-client.test.ts @@ -66,8 +66,6 @@ describe('ToolCallingModelClient contract', () => { 'read_file', 'search_content', 'report_finding', - 'reviewInstruction', - 'runAgentToolLoop', ]; for (const concept of forbidden) { expect(keys).not.toContain(concept); diff --git a/tests/providers/vercel-ai-provider-agent-loop.test.ts b/tests/providers/vercel-ai-provider-agent-loop.test.ts deleted file mode 100644 index b1bca660..00000000 --- a/tests/providers/vercel-ai-provider-agent-loop.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { z } from 'zod'; -import type { LanguageModel } from 'ai'; -import { createMockLogger } from '../utils'; - -const MOCK_GENERATE_TEXT = vi.hoisted(() => vi.fn()); -const MOCK_STEP_COUNT_IS = vi.hoisted(() => vi.fn(() => ({ type: 'stepCount' }))); -const MOCK_TOOL = vi.hoisted(() => - vi.fn((definition: Record) => definition) -); - -const ERROR_CLASSES = vi.hoisted(() => { - class NoObjectGeneratedError extends Error { - text: string; - - constructor(message: string, text: string) { - super(message); - this.name = 'NoObjectGeneratedError'; - this.text = text; - } - - static isInstance(error: unknown): error is NoObjectGeneratedError { - return error instanceof NoObjectGeneratedError; - } - } - - return { NoObjectGeneratedError }; -}); - -vi.mock('ai', () => { - const { NoObjectGeneratedError } = ERROR_CLASSES; - return { - generateText: MOCK_GENERATE_TEXT, - stepCountIs: MOCK_STEP_COUNT_IS, - tool: MOCK_TOOL, - Output: { - object: vi.fn((schema: unknown) => ({ - _outputType: 'object', - schema, - })), - }, - NoObjectGeneratedError, - }; -}); - -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('VercelAIProvider agent loop', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('applies configured retry and tool-concurrency limits during agent execution', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - const runAgentToolLoop = ( - provider as unknown as { - runAgentToolLoop?: (params: Record) => Promise; - } - ).runAgentToolLoop; - - expect(typeof runAgentToolLoop).toBe('function'); - if (typeof runAgentToolLoop !== 'function') { - throw new Error('runAgentToolLoop is not implemented'); - } - - await runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxRetries: 4, - maxSteps: 42, - maxParallelToolCalls: 1, - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - maxRetries?: number; - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - - expect(call.maxRetries).toBe(4); - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: false }, - }); - }); - - it('adds observability options to agent-loop generateText calls', async () => { - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => ({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.agent-tool-loop' }, - })), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(observability.decorateCall).toHaveBeenCalledWith({ - operation: 'agent-tool-loop', - provider: 'openai', - model: 'gpt-4o-mini', - }); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).toHaveProperty('experimental_telemetry'); - }); - - it('continues agent-loop AI calls when observability decoration fails', async () => { - const logger = createMockLogger(); - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => { - throw new Error('telemetry failed'); - }), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - logger, - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).not.toHaveProperty('experimental_telemetry'); - expect(logger.warn).toHaveBeenCalledWith( - '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', - expect.objectContaining({ error: 'telemetry failed', operation: 'agent-tool-loop' }) - ); - }); - - it('limits concurrent tool executes to maxParallelToolCalls', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxParallelToolCalls: 2, - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - tool_d: { description: 'd', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBeLessThanOrEqual(2); - expect(maxConcurrent).toBeGreaterThan(1); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: true }, - }); - }); - - it('defaults tool concurrency to 1 when maxParallelToolCalls is not supplied', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBe(1); - }); - - it('emits agent-loop debug output through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'final summary', - finishReason: 'stop', - steps: [ - { - finishReason: 'tool-calls', - text: 'step text', - toolCalls: [{ toolName: 'lint' }], - }, - ], - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const logger = createMockLogger(); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - debug: true, - logger, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(logger.debug).toHaveBeenCalled(); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] step 1: finishReason=tool-calls tools=[lint]') - ) - ).toBe(true); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] final finishReason=stop steps=1') - ) - ).toBe(true); - }); -}); From 6e2a04feda87a83ec9feb4c2c8259d12470c9fc8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:28:56 +0100 Subject: [PATCH 23/34] refactor(cli): reject legacy mode flag - Reject the removed --mode option without silently remapping it.\n- Cover single, agent, and auto executor dispatch plus ReviewResult routing.\n- Preserve the breaking CLI migration to --model-call single|agent|auto. --- src/cli/commands.ts | 3 +- tests/cli-mode-rejection.test.ts | 37 +++ tests/orchestrator-executor-dispatch.test.ts | 229 +++++++++++++++++++ 3 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 tests/cli-mode-rejection.test.ts create mode 100644 tests/orchestrator-executor-dispatch.test.ts diff --git a/src/cli/commands.ts b/src/cli/commands.ts index b2e7f0b7..9c0e362a 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -97,8 +97,7 @@ export function registerMainCommand(program: Command): void { // --mode is removed (audit Product Decision): the autonomous workspace-agent // surface is gone. Fail clearly and point users at the replacement rather // than silently mapping it onto --model-call. - const rawCliOpts = program.opts() as { mode?: unknown }; - if (rawCliOpts.mode !== undefined) { + if (program.opts().mode !== undefined) { console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); process.exit(1); } diff --git a/tests/cli-mode-rejection.test.ts b/tests/cli-mode-rejection.test.ts new file mode 100644 index 00000000..38c0f3db --- /dev/null +++ b/tests/cli-mode-rejection.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; + +describe('CLI --mode rejection', () => { + let exitSpy: ReturnType; + + beforeEach(() => { + vi.spyOn(console, 'error').mockImplementation(() => undefined); + exitSpy = vi.spyOn(process, 'exit').mockImplementation( + ((code?: string | number | null) => { + throw new Error(`exit:${code ?? ''}`); + }) as never, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('rejects --mode and points users at --model-call instead of mapping it', async () => { + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect( + program.parseAsync(['node', 'test', 'README.md', '--mode', 'agent']), + ).rejects.toThrow('exit:1'); + + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--mode is no longer supported'), + ); + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--model-call'), + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); +}); diff --git a/tests/orchestrator-executor-dispatch.test.ts b/tests/orchestrator-executor-dispatch.test.ts new file mode 100644 index 00000000..b4a1d093 --- /dev/null +++ b/tests/orchestrator-executor-dispatch.test.ts @@ -0,0 +1,229 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import * as path from 'path'; +import { evaluateFiles } from '../src/cli/orchestrator'; +import { OutputFormat, type EvaluationOptions } from '../src/cli/types'; +import { Severity } from '../src/evaluators/types'; +import type { PromptFile } from '../src/prompts/prompt-loader'; +import type { ReviewRequest, ReviewResult } from '../src/review/types'; + +const { EXECUTOR_FOR_MOCK, FAKE_EXECUTOR_RUN } = vi.hoisted(() => ({ + EXECUTOR_FOR_MOCK: vi.fn(), + FAKE_EXECUTOR_RUN: vi.fn(), +})); + +// Replace executorFor with a spy that returns a controllable executor so the +// orchestrator's wiring (ReviewRequest build -> chooseModelCall -> dispatch -> +// ReviewResult routing) is exercised without a real model call. +vi.mock('../src/executors', () => ({ + executorFor: EXECUTOR_FOR_MOCK, +})); + +function makePrompt(id: string): PromptFile { + return { + id, + filename: `${id}.md`, + fullPath: path.join(process.cwd(), 'prompts', `${id}.md`), + pack: 'TestPack', + body: `Rule body for ${id}`, + meta: { id, name: id, type: 'check', severity: Severity.WARNING }, + }; +} + +function makeOptions(prompts: PromptFile[], overrides: Partial = {}): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + requestBuilder: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Json, + modelCall: 'auto', + ...overrides, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), 'vectorlint-dispatch-')); + const filePath = path.join(dir, 'input.md'); + writeFileSync(filePath, content); + return filePath; +} + +function makeReviewResult(overrides: Partial = {}): ReviewResult { + return { + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: path.join(process.cwd(), 'prompts', 'CheckPrompt.md'), + severity: 'warning', + message: 'Vague advice found', + line: 1, + column: 1, + match: 'vague text', + suggestion: 'Be specific.', + }, + ], + scores: [ + { + ruleId: 'TestPack.CheckPrompt', + score: 8, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 1, + }, + ], + diagnostics: [], + hadOperationalErrors: false, + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5 }, + ...overrides, + }; +} + +describe('orchestrator executor dispatch', () => { + beforeEach(() => { + EXECUTOR_FOR_MOCK.mockReset(); + FAKE_EXECUTOR_RUN.mockReset(); + EXECUTOR_FOR_MOCK.mockReturnValue({ run: FAKE_EXECUTOR_RUN }); + FAKE_EXECUTOR_RUN.mockResolvedValue(makeReviewResult()); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('resolves auto to single for a normal-sized target and routes findings to JSON output', async () => { + const file = createTempFile('vague text here\n'); + const run = await evaluateFiles([file], makeOptions([makePrompt('CheckPrompt')])); + + // auto + small target + one rule resolves to the single executor. + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + + // The verified finding reaches the JSON sink. + const parsed = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) as { + files: Record> }>; + }; + const issues = Object.values(parsed.files).flatMap((f) => f.issues); + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + line: 1, + severity: Severity.WARNING, + message: 'Vague advice found', + rule: 'TestPack.CheckPrompt', + match: 'vague text', + }); + + expect(run.totalWarnings).toBe(1); + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it('resolves auto to agent for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await evaluateFiles([file], makeOptions([makePrompt('BigPrompt')])); + + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('honors an explicit single modelCall even for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await evaluateFiles([file], makeOptions([makePrompt('ForcedSingle')], { modelCall: 'single' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + }); + + it('honors an explicit agent modelCall even for a small target', async () => { + const file = createTempFile('small\n'); + await evaluateFiles([file], makeOptions([makePrompt('ForcedAgent')], { modelCall: 'agent' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('forwards the built ReviewRequest (target + rules + modelCall) to the executor', async () => { + const file = createTempFile('target content line one\n'); + await evaluateFiles([file], makeOptions([makePrompt('FwdPrompt')], { modelCall: 'agent' })); + + expect(FAKE_EXECUTOR_RUN).toHaveBeenCalledTimes(1); + const request = FAKE_EXECUTOR_RUN.mock.calls[0]![0] as ReviewRequest; + expect(request.target.content).toBe('target content line one\n'); + expect(request.rules).toHaveLength(1); + expect(request.rules[0]?.body).toBe('Rule body for FwdPrompt'); + expect(request.modelCall).toBe('agent'); + }); + + it('aggregates error-severity findings and flags hadSeverityErrors', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: 'src', + severity: 'error', + message: 'Severe issue', + line: 2, + column: 1, + match: 'vague text', + }, + ], + scores: [ + { ruleId: 'TestPack.CheckPrompt', score: 0, scoreText: '0.0/10', severity: 'error', findingCount: 1 }, + ], + }), + ); + const file = createTempFile('vague text\n'); + + const run = await evaluateFiles([file], makeOptions([makePrompt('ErrorPrompt')])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it('routes ReviewResult diagnostics to verbose console output', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [], + diagnostics: [{ level: 'warn', code: 'finding-evidence-not-locatable', message: 'could not anchor quote' }], + }), + ); + const file = createTempFile('vague text\n'); + + await evaluateFiles([file], makeOptions([makePrompt('DiagPrompt')], { verbose: true })); + + expect(vi.mocked(console.warn)).toHaveBeenCalledWith( + expect.stringContaining('could not anchor quote'), + ); + }); + + it('aggregates token usage from the ReviewResult', async () => { + const file = createTempFile('vague text\n'); + const run = await evaluateFiles([file], makeOptions([makePrompt('UsagePrompt')])); + + expect(run.tokenUsage?.totalInputTokens).toBe(10); + expect(run.tokenUsage?.totalOutputTokens).toBe(5); + }); + + it('skips the executor when no prompts apply', async () => { + const file = createTempFile('content\n'); + // A scan-path config that runs a pack none of the prompts belong to. + const run = await evaluateFiles( + [file], + makeOptions([makePrompt('OrphanPrompt')], { + scanPaths: [{ pattern: '**/*.md', runRules: ['OtherPack'], overrides: {} }], + }), + ); + + expect(EXECUTOR_FOR_MOCK).not.toHaveBeenCalled(); + expect(run.totalFiles).toBe(1); + expect(run.totalWarnings).toBe(0); + }); +}); From 64e434d88350204c20c2eb671d9fa4181059d7d6 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 12:39:52 +0100 Subject: [PATCH 24/34] refactor(cli): harden config and payload telemetry - create global config resources with restrictive permissions\n- propagate opt-in payload telemetry through both executors\n- cover CLI, package entrypoints, observability, and provider logging --- src/config/global-config.ts | 4 ++-- src/executors/agent-model-call-executor.ts | 6 +++++- src/executors/single-model-call-executor.ts | 5 ++++- src/observability/ai-observability.ts | 1 + src/observability/langfuse-observability.ts | 4 ++-- src/providers/request-builder.ts | 1 + src/providers/tool-calling-model-client.ts | 2 ++ src/providers/vercel-ai-provider.ts | 6 ++++++ tests/global-config.test.ts | 17 ++++++++++++++- tests/main-command-observability.test.ts | 4 ++-- .../langfuse-observability.test.ts | 21 ++++++++++++++++++- tests/package-entrypoint.test.ts | 21 +++++++++++++++++++ tests/vercel-ai-provider.test.ts | 1 + 13 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 tests/package-entrypoint.test.ts diff --git a/src/config/global-config.ts b/src/config/global-config.ts index 98b1a7fc..6a694122 100644 --- a/src/config/global-config.ts +++ b/src/config/global-config.ts @@ -74,11 +74,11 @@ export function ensureGlobalConfig(): string { const configDir = path.dirname(configPath); if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); + mkdirSync(configDir, { recursive: true, mode: 0o700 }); } if (!existsSync(configPath)) { - writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, 'utf-8'); + writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, { encoding: 'utf-8', mode: 0o600 }); } return configPath; diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts index 1b09f388..1679fecb 100644 --- a/src/executors/agent-model-call-executor.ts +++ b/src/executors/agent-model-call-executor.ts @@ -50,7 +50,10 @@ export class AgentModelCallExecutor implements ReviewExecutor { const capability = new TargetReadCapability(request.target.content); // Exactly one executor-owned tool is exposed: target-section paging. const tools = buildReadTargetSectionTool(capability); - const context = buildEvalContext(request.target.uri); + const context = { + ...buildEvalContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; @@ -136,6 +139,7 @@ export class AgentModelCallExecutor implements ReviewExecutor { // emitting its structured finding set. maxSteps: request.budget.maxChunksPerRule, maxParallelToolCalls: 1, + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, }, }); diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts index 3809cbb3..2bb87ec4 100644 --- a/src/executors/single-model-call-executor.ts +++ b/src/executors/single-model-call-executor.ts @@ -50,7 +50,10 @@ export class SingleModelCallExecutor implements ReviewExecutor { async run(request: ReviewRequest): Promise { const schema = buildCheckLLMSchema(); - const context = buildEvalContext(request.target.uri); + const context = { + ...buildEvalContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; const findings: ReviewFinding[] = []; const scores: ReviewScore[] = []; diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index e16a3c9d..88a76f26 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -4,6 +4,7 @@ export interface AIExecutionContext { model: string; evaluator?: string; rule?: string; + recordPayloadTelemetry?: boolean; } export interface AIObservability { diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index 25f4d32c..bae2d555 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -63,8 +63,8 @@ export class LangfuseObservability implements AIObservability { ...(context.evaluator ? { evaluator: context.evaluator } : {}), ...(context.rule ? { rule: context.rule } : {}), }, - recordInputs: true, - recordOutputs: true, + recordInputs: context.recordPayloadTelemetry === true, + recordOutputs: context.recordPayloadTelemetry === true, }, }; } diff --git a/src/providers/request-builder.ts b/src/providers/request-builder.ts index 806c53e8..9e0b239c 100644 --- a/src/providers/request-builder.ts +++ b/src/providers/request-builder.ts @@ -2,6 +2,7 @@ export interface EvalContext { fileType?: string; + recordPayloadTelemetry?: boolean; } export interface RequestBuilder { diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts index 2139c8f2..a038e7e5 100644 --- a/src/providers/tool-calling-model-client.ts +++ b/src/providers/tool-calling-model-client.ts @@ -21,6 +21,8 @@ export interface ToolCallDefinition { * transport honors them. It does not invent its own product-level loop. */ export interface ToolCallRunOptions { + /** Opt in to recording prompt and generated payloads in telemetry. */ + recordPayloadTelemetry?: boolean; /** Maximum number of model steps (tool-call rounds) before the run stops. */ maxSteps?: number; /** Provider retry budget for transient failures. */ diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index dba87369..634e08f8 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -91,6 +91,9 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { model: this.config.modelName ?? 'unknown', ...(evaluator ? { evaluator } : {}), ...(rule ? { rule } : {}), + ...(context?.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: context.recordPayloadTelemetry } + : {}), }); const result = await generateText({ @@ -197,6 +200,9 @@ export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { operation: 'tool-calling', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', + ...(options.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: options.recordPayloadTelemetry } + : {}), }), stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), providerOptions: { diff --git a/tests/global-config.test.ts b/tests/global-config.test.ts index 2a8d8c7b..acee276c 100644 --- a/tests/global-config.test.ts +++ b/tests/global-config.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import { loadGlobalConfig } from '../src/config/global-config'; +import { ensureGlobalConfig, loadGlobalConfig } from '../src/config/global-config'; import { GLOBAL_CONFIG_DIR, GLOBAL_CONFIG_FILE } from '../src/config/constants'; // Mock fs and os @@ -49,6 +49,21 @@ BOOL_KEY = true expect(fs.readFileSync).toHaveBeenCalledWith(mockConfigPath, 'utf-8'); }); + it('creates the global config directory and file with restrictive modes', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + + expect(ensureGlobalConfig()).toBe(mockConfigPath); + expect(fs.mkdirSync).toHaveBeenCalledWith( + path.dirname(mockConfigPath), + { recursive: true, mode: 0o700 }, + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + mockConfigPath, + expect.any(String), + { encoding: 'utf-8', mode: 0o600 }, + ); + }); + it('should NOT overwrite existing env vars', () => { const mockToml = ` [env] diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 7d652c46..d1619ad9 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -97,7 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - modelCall: 'auto', + modelCall: 'single', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -141,7 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - modelCall: 'auto', + modelCall: 'single', config: undefined, }); const observability = { diff --git a/tests/observability/langfuse-observability.test.ts b/tests/observability/langfuse-observability.test.ts index 56448489..ca0f73c4 100644 --- a/tests/observability/langfuse-observability.test.ts +++ b/tests/observability/langfuse-observability.test.ts @@ -26,7 +26,7 @@ describe('LangfuseObservability', () => { SHUTDOWN_MOCK.mockResolvedValue(undefined); }); - it('returns AI SDK telemetry options with full payload recording enabled', () => { + it('does not record payloads by default', () => { const subject = new LangfuseObservability({ publicKey: 'pk-lf-test', secretKey: 'sk-lf-test', @@ -49,6 +49,25 @@ describe('LangfuseObservability', () => { evaluator: 'clarity', rule: 'no-fluff', }, + recordInputs: false, + recordOutputs: false, + }, + }); + }); + + it('records payloads only when the review output policy opts in', () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + }); + + expect(subject.decorateCall({ + operation: 'structured-eval', + provider: 'openai', + model: 'gpt-4o', + recordPayloadTelemetry: true, + })).toMatchObject({ + experimental_telemetry: { recordInputs: true, recordOutputs: true, }, diff --git a/tests/package-entrypoint.test.ts b/tests/package-entrypoint.test.ts new file mode 100644 index 00000000..ff99ac77 --- /dev/null +++ b/tests/package-entrypoint.test.ts @@ -0,0 +1,21 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const PACKAGE_PATH = resolve(dirname(fileURLToPath(import.meta.url)), '../package.json'); + +describe('package entrypoints', () => { + it('keeps both published bin aliases pointed at the built CLI', () => { + const packageJson = JSON.parse(readFileSync(PACKAGE_PATH, 'utf8')) as { + main?: string; + bin?: Record; + }; + + expect(packageJson.main).toBe('dist/index.js'); + expect(packageJson.bin).toEqual({ + vectorlint: './dist/index.js', + veclint: './dist/index.js', + }); + }); +}); diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index ab258dd4..80bbaf5f 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -391,6 +391,7 @@ describe('VercelAIProvider', () => { }) as Record, }) ); + expect(logger.debug).not.toHaveBeenCalledWith('Test content'); }); it('does not log when debug is disabled', async () => { From 0c18c2dfbb58cb03854bfad19d40cc3f69a79c26 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:01:26 +0100 Subject: [PATCH 25/34] refactor(observability): remove legacy agent telemetry operation - Keep observability operation types aligned with the executor-only review surface --- src/observability/ai-observability.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 88a76f26..7ed80e03 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,5 +1,5 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'tool-calling' | 'agent-tool-loop'; + operation: 'structured-eval' | 'tool-calling'; provider: string; model: string; evaluator?: string; From 986e6e96b1907ec85c0568637fc6404914c3e05b Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:22:30 +0100 Subject: [PATCH 26/34] docs: document bounded harness architecture - Replace stale agent-mode guidance with single/agent/auto model-call docs. - Add the current harness architecture spec and migration guide. - Commit shared domain language and update AGENTS.md for the review harness. --- AGENTS.md | 55 ++++- CONTEXT.md | 151 +++++++++++++ README.md | 34 ++- ...0-vectorlint-harness-architecture-audit.md | 5 + docs/best-practices.mdx | 22 +- docs/cli-reference.mdx | 33 ++- docs/docs.json | 4 +- docs/how-it-works.mdx | 84 +++++--- docs/introduction.mdx | 2 +- ...3-31-agent-mode-implementation-plan.log.md | 8 + docs/migration-from-agent-mode.mdx | 51 +++++ docs/model-calls.mdx | 75 +++++++ docs/rubric-scoring.mdx | 23 +- docs/specs/2026-07-10-harness-architecture.md | 199 ++++++++++++++++++ docs/troubleshooting.mdx | 18 ++ docs/use-cases.mdx | 4 +- 16 files changed, 695 insertions(+), 73 deletions(-) create mode 100644 CONTEXT.md create mode 100644 docs/migration-from-agent-mode.mdx create mode 100644 docs/model-calls.mdx create mode 100644 docs/specs/2026-07-10-harness-architecture.md diff --git a/AGENTS.md b/AGENTS.md index 2d696a85..0604a5b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,46 @@ # Repository Guidelines -This repository implements VectorLint — a prompt‑driven, structured‑output content evaluator. Use this guide to navigate the codebase, run it locally, and contribute safely. +This repository implements VectorLint — a prompt‑driven, structured‑output content review harness. Use this guide to navigate the codebase, run it locally, and contribute safely. + +Use [`CONTEXT.md`](./CONTEXT.md) as the shared VectorLint domain language across code, docs, tests, and agent work. Prefer its terms when naming modules, writing tests, drafting docs, and describing architecture. + +## Agent Behavior Guidelines + +These guidelines reduce common LLM coding mistakes. They bias toward caution over speed; use judgment for trivial tasks. + +### Think Before Coding + +- State assumptions explicitly before implementing. +- If multiple interpretations exist, present them instead of picking silently. +- If a simpler approach exists, say so and push back when warranted. +- If something is unclear, stop, name the confusion, and ask. + +### Simplicity First + +- Write the minimum code that solves the problem. +- Do not add features beyond what was asked. +- Do not introduce abstractions for single-use code. +- Do not add flexibility or configurability that was not requested. +- Do not add error handling for impossible scenarios. +- If a change becomes larger than needed, simplify before finishing. + +### Surgical Changes + +- Touch only the files and lines required by the request. +- Do not improve adjacent code, comments, or formatting unless required. +- Match existing style, even when another style is tempting. +- If you notice unrelated dead code, mention it instead of deleting it. +- Remove imports, variables, functions, or files made unused by your own changes. +- Do not remove pre-existing dead code unless asked. + +### Goal-Driven Execution + +- Turn requests into verifiable goals before implementing. +- For bug fixes, write or identify a reproduction, then make it pass. +- For validation changes, cover invalid inputs, then make the checks pass. +- For refactors, verify behavior before and after the change. +- For multi-step tasks, state a brief plan with verification for each step. +- Loop until the success criteria are verified or a blocker is clear. ## Project Structure & Module Organization @@ -15,9 +55,12 @@ This repository implements VectorLint — a prompt‑driven, structured‑output - `output/` — TTY formatting (reporter, evidence location, line numbering) - `prompts/` — YAML frontmatter parsing, schema validation, directive loading - `providers/` — LLM abstractions (OpenAI, Anthropic, Azure, Gemini), request builder, provider factory + - `review/` — neutral review contract, boundary, budget, schemas, and model-call selection + - `executors/` — bounded `single` and `agent` model-call executors behind `ReviewExecutor` + - `findings/` — shared finding verification, filtering, scoring, diagnostics, and result assembly - `scan/` — file discovery (fast‑glob) honoring config and exclusions - `schemas/` — Zod schemas for all external data (API responses, config, CLI, env) - - `scoring/` — score calculation (density-based for check, rubric-based for judge) + - `scoring/` — score calculation for objective violation checks - `types/` — TypeScript type definitions - `presets/` — bundled rule packs (e.g., `VectorLint/`) - `tests/` — Vitest specs for config, scanning, evaluation, providers @@ -126,12 +169,14 @@ For documents >600 words, VectorLint automatically chunks content: ## Architecture & Principles - Boundary validation: all external data (files, CLI, env, APIs) validated at system boundaries using Zod schemas +- Bounded harness model: callers own exploration and context gathering; VectorLint owns constrained review through `single`/`agent`/`auto` model calls behind the `ReviewExecutor` contract +- On-page boundary: executors review target content plus explicit review context only; do not add workspace search, arbitrary file reads, model-authored rule overrides, or autonomous agent behavior - Type safety: strict TypeScript with no `any`; use `unknown` + schema validation for external data -- Dependency inversion: depend on `LLMProvider` and `SearchProvider` interfaces; keep providers thin (transport only) +- Dependency inversion: depend on `StructuredModelClient`, `ToolCallingModelClient`, and `SearchProvider` interfaces; keep providers thin (transport only) - Dependency injection: inject `RequestBuilder` via provider constructor to avoid coupling -- Separation of concerns: rules define rubric; schemas enforce structure; CLI orchestrates; evaluators process; reporters format +- Separation of concerns: rules define observable violation checks; schemas enforce structure; CLI orchestrates; executors run model calls; findings process results; reporters format output - Separation of concerns: when a file starts combining contracts, orchestration, and utility logic, extract shared helpers and types into focused modules -- Extensibility: add providers by implementing `LLMProvider` or `SearchProvider`; add evaluators via registry pattern +- Extensibility: add model providers by implementing the structured/tool-calling model client interfaces or search by implementing `SearchProvider` - Error handling: prefer the repository's custom error hierarchy over native `Error`; catch blocks use `unknown` type and extend existing custom error types before introducing raw exceptions - Shared domain constants: avoid magic strings for core runtime concepts; define shared constants, enums, or types and import them where needed - Naming: choose domain-accurate names that reflect the real abstraction level; avoid use-case-specific terminology in shared runtime code diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 00000000..ee0d542c --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,151 @@ +# VectorLint + +VectorLint is a content review harness. This language file defines the terms used to describe reviews, rules, findings, scoring, and execution so product, documentation, tests, and code can share the same vocabulary. + +## Language + +### Review Model + +**Content Review Harness**: +A system that reviews supplied target content against source-backed rules and returns structured review results. The harness does not own exploration, research, workspace search, or content rewriting. +_Avoid_: Autonomous agent, workspace agent, agent mode + +**Review**: +One evaluation of target content against one or more rules. A review produces findings, scores, diagnostics, usage metadata, and output. +_Avoid_: Lint run when referring to the domain process, agent session + +### Review Inputs + +**Caller**: +The person or system that invokes VectorLint and supplies the target, target content, and any allowed review context. A caller does not define rules or configuration as part of the review invocation. +_Avoid_: Agent, user agent, workspace owner + +**Target**: +The subject of a review. A target identifies what VectorLint is reviewing, regardless of whether it came from a file, memory, or another caller-provided source. +_Avoid_: File when the review subject may not be a file, page when the content may not be documentation + +**Target File**: +A file used as the source of a target. A target file is one way to provide target content, but not every target must come from a file. +_Avoid_: Target when the distinction between the file and the review subject matters + +**Target Content**: +The actual text or content body reviewed for a target. Findings and finding evidence are grounded in target content. +_Avoid_: Target file, page content + +**Review Context**: +Additional content explicitly supplied to VectorLint for a review. Review context is in scope only because it was allowed as part of the review input. +_Avoid_: Caller context, workspace context, discovered context, ambient context + +**On-Page Boundary**: +The rule that VectorLint reviews only target content and review context. VectorLint must not discover arbitrary workspace files or expand the review scope on its own. +_Avoid_: Workspace scope, project-wide scope, cross-file scope + +### Rules + +**Rule**: +A source-backed instruction that defines observable violation conditions VectorLint should look for. Rules exist before a review is invoked. +_Avoid_: Prompt when referring to the domain concept, model instruction when implying the model authored it + +**Via Negativa Review**: +A review approach that looks for evidence a rule was violated rather than evidence the content aligns with an ideal. VectorLint rules should be written so a model can answer whether an observable violation condition is present. +_Avoid_: Alignment review, subjective assessment, rubric judgment + +**Violation Condition**: +An observable yes/no condition that counts as a rule violation when present in target content. A violation condition should be specific enough to ground a finding in finding evidence. +_Avoid_: Criterion when it implies subjective grading, preference, guideline + +**Rule Pack**: +A named collection of related rules that can be applied together. +_Avoid_: Preset when describing the domain concept, folder + +**Review Configuration**: +Pre-existing settings that determine how VectorLint runs reviews, selects rules, and formats behavior. Configuration is not the same as target content or review context. +_Avoid_: Caller input when referring to review invocation, target configuration + +**Check Rule**: +A rule expressed through observable violation conditions. This is the only future-facing rule style in VectorLint. +_Avoid_: Direct rule, standard rule + +### Execution + +**Model Call**: +The call shape VectorLint uses to run a reviewer model against target content during a review. +_Avoid_: Execution strategy, content access, process mode, evaluator type, rule type, mode + +**Single Model Call**: +A model call where VectorLint supplies the review request and target content without giving the model a read tool. If VectorLint chunks a large target, each chunk is still reviewed through a single model call. +_Avoid_: Direct strategy, standard mode, check path + +**Agent Model Call**: +A bounded model call where the model may request sections of the target content through a single target-scoped read capability. This is for context management during large or context-sensitive reviews; it is not workspace exploration. +_Avoid_: Autonomous agent mode, workspace agent, file reader + +**Auto Model Call**: +The model call value where VectorLint deterministically chooses between single and agent. +_Avoid_: Smart mode, agent fallback + +**Target Read Capability**: +The bounded ability to read line ranges from target content. It cannot read arbitrary files, search the workspace, rewrite rules, or create top-level workspace findings. +_Avoid_: Tool suite, workspace tools, read_file + +### Findings + +**Candidate Finding**: +A potential issue raised before VectorLint has decided whether it should be reported. Candidate findings may be filtered out or become diagnostics. +_Avoid_: Violation when the issue has not been accepted, result + +**Verified Finding**: +A finding that VectorLint has accepted for reporting because it is grounded in the target content and passes review filters. +_Avoid_: Issue when finding-evidence status matters, raw finding + +**Finding**: +A reported content issue produced by a review. When precision matters, use candidate finding or verified finding. +_Avoid_: Violation as the default user-facing term, problem + +**Finding Evidence**: +The exact target-content text or surrounding context that supports a finding. +_Avoid_: Quote when the text may include surrounding context, match when referring to the concept rather than a located span + +**Finding Evidence Verification**: +The act of confirming that the finding evidence for a candidate finding can be located in the target content. Unverified finding evidence should not become a verified finding. +_Avoid_: Line fallback, model-provided location + +**Finding Processing**: +The review step that turns candidate findings into verified findings, diagnostics, scores, and final review output. +_Avoid_: Projection, result processing, output processing + +**Diagnostic**: +A structured note about review execution or finding processing, especially when something affects trust, completeness, or interpretation but is not itself a content finding. +_Avoid_: Finding, warning when referring to the structured domain object + +### Scoring And Output + +**Severity**: +The impact level attached to a finding or rule outcome. +_Avoid_: Priority, importance + +**Score**: +A normalized quality measurement for a rule or review. Scores come from verified finding count or density for objective violation checks. +_Avoid_: Grade when referring to the domain object, rating + +**Review Result**: +The structured outcome of a review: verified findings, scores, diagnostics, usage metadata, and operational status. +_Avoid_: Projection result, formatter result, raw model output + +**Output Format**: +The representation used to present a review result to a human or machine, such as line output or JSON. +_Avoid_: Review result, report type + +**Review Budget**: +The explicit limits for a review, such as model calls, target size, review context size, chunks per rule, and duration. A review budget limits work, not the number of verified findings emitted. +_Avoid_: Rate limit, timeout when referring to the full budget concept + +### Historical Terms + +**Autonomous Agent Mode**: +The old VectorLint direction where VectorLint exposed workspace tools to a model and let it explore beyond the target content. This is historical language only; current domain language should use single model call, agent model call, caller, and content review harness. +_Avoid_: Agent mode as a current feature, workspace-agent review + +**Judge Rule**: +The historical rubric-style assessment rule where a model graded content against criteria. This rule style is removed from the future-facing product model. +_Avoid_: Rubric mode, subjective check, evaluator kind diff --git a/README.md b/README.md index 662ac7f0..989399b3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # VectorLint: Prompt it, Lint it! [![npm version](https://img.shields.io/npm/v/vectorlint.svg)](https://www.npmjs.com/package/vectorlint) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -VectorLint is a command-line tool that evaluates and scores content using LLMs. It uses [LLM-as-a-Judge](https://en.wikipedia.org/wiki/LLM-as-a-Judge) to catch terminology, technical accuracy, and style issues that require contextual understanding. +VectorLint is a command-line content review harness that uses LLMs to find observable terminology, technical accuracy, and style violations that require contextual understanding. ![VectorLint Screenshot](./assets/VectorLint_screenshot.jpeg) @@ -144,16 +144,32 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode (under review) +## How VectorLint Runs Reviews -The `--mode agent` flag is under active rework. It currently enables an -autonomous cross-file review mode that is being **removed** in favor of a -bounded harness model. See -[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) -for the decision and the refactor plan. +VectorLint is a bounded review harness, not a workspace agent. Callers choose +the target content and any review context; VectorLint reviews that supplied +content against source-backed rules and returns findings, scores, diagnostics, +and usage metadata. -Until the refactor lands, `--mode agent` prints a deprecation warning and -falls back to standard mode. Do not build integrations against it. +Use `--model-call` to choose how the reviewer model is invoked: + +```bash +vectorlint doc.md --model-call single +vectorlint doc.md --model-call agent +vectorlint doc.md --model-call auto +``` + +- `single` runs structured model calls without tools. This is best for normal + documents and straightforward rule checks. +- `agent` runs a bounded target-only model call that can page through the + current target with `read_target_section`. It cannot search the workspace, + read arbitrary files, or override rules. +- `auto` is the default. VectorLint chooses `single` for normal inputs and + `agent` for large or multi-rule reviews that benefit from target paging. + +See the current +[`docs/specs/2026-07-10-harness-architecture.md`](docs/specs/2026-07-10-harness-architecture.md) +for the contract and boundary details. ## Contributing diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md index 51161ce0..356e1506 100644 --- a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -1,5 +1,10 @@ # VectorLint Harness Architecture Audit +> **HISTORICAL AUDIT.** This audit records the July 2026 decision that led to +> the bounded harness refactor. Some "current state" observations below +> describe the pre-refactor code. For the shipped architecture, see +> [`../specs/2026-07-10-harness-architecture.md`](../specs/2026-07-10-harness-architecture.md). + Date: 2026-07-10 Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. diff --git a/docs/best-practices.mdx b/docs/best-practices.mdx index 05c6ad84..a4e72760 100644 --- a/docs/best-practices.mdx +++ b/docs/best-practices.mdx @@ -51,23 +51,19 @@ LLMs evaluate content relative to an implied standard. Make that standard explic A grammar rule without context produces generic grammar findings. The same rule with a developer audience context produces findings calibrated to technical writing conventions. -## Use weights that reflect real priorities +## Write observable violation conditions -In judge rules, weights are the single most important configuration decision. They determine which criteria actually control the final score. Treat them as a statement of your content team's values — not arbitrary numbers. +VectorLint works best when each rule names what counts as a violation. Treat a rule as a checklist of conditions the reviewer can confirm from target evidence. -```yaml -criteria: - - name: Technical Accuracy - weight: 40 # Factual errors erode user trust — this matters most - - name: Clarity - weight: 30 # Unclear docs generate support tickets - - name: Tone - weight: 20 # Important but recoverable in editing - - name: SEO - weight: 10 # Nice to have, never at the expense of the above +```markdown +Flag a headline when it: +1. Omits the product or feature being discussed +2. Promises a benefit that the article does not support +3. Uses vague value words such as "seamless", "powerful", or "robust" +4. Exceeds 70 characters ``` -If everything has the same weight, nothing is prioritized. +Observable conditions make findings easier to verify and reduce subjective disagreements during review. ## Tier strictness by content type diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index febe11cc..128c7ffa 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -58,10 +58,35 @@ vectorlint --help ## Flags -| Flag | Description | -|------|-------------| -| `--help` | Print help and exit | -| `--version` | Print the installed VectorLint version and exit | +| Flag | Default | Description | +|------|---------|-------------| +| `--help` | — | Print help and exit | +| `--version` | — | Print the installed VectorLint version and exit | +| `--config ` | `.vectorlint.ini` | Path to a custom project config file | +| `--debug-json` | `false` | Write debug JSON artifacts for surfaced findings and review metadata | +| `--model-call ` | `auto` | Model-call strategy: `single`, `agent`, or `auto` | +| `--output ` | `line` | Output format: `line`, `json`, `vale-json`, or `rdjson` | +| `--show-prompt` | `false` | Print full prompt and injected content | +| `--show-prompt-trunc` | `false` | Print truncated prompt/content previews | +| `--verbose` | `false` | Enable verbose logging | + +### `--model-call` + +`--model-call` controls how VectorLint invokes the reviewer model: + +```bash +vectorlint doc.md --model-call single +vectorlint doc.md --model-call agent +vectorlint doc.md --model-call auto +``` + +| Value | Behavior | +| --- | --- | +| `single` | Runs structured model calls without tools | +| `agent` | Runs bounded target-only paging with `read_target_section` | +| `auto` | Default; chooses `single` or `agent` based on target size and rule count | + +See [Model calls](/model-calls) for usage guidance. ## Output diff --git a/docs/docs.json b/docs/docs.json index 5b7727c8..f8ee7c1b 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -20,7 +20,8 @@ "pages": [ "introduction", "use-cases", - "how-it-works" + "how-it-works", + "model-calls" ] }, { @@ -71,6 +72,7 @@ "configuration-schema", "env-variables", "frontmatter-fields", + "migration-from-agent-mode", "troubleshooting" ] } diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index 93e42fcf..d5870870 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -1,65 +1,83 @@ --- title: How it works -description: How VectorLint evaluates content using LLM-as-a-Judge, filtering, and two scoring methods. +description: How VectorLint reviews supplied content through a bounded harness. --- -VectorLint evaluates your content by sending it to a large language model (LLM) along with a structured prompt that defines your quality criteria. The LLM acts as a judge — reading your content, applying the criteria, and returning findings. VectorLint then filters those findings through a deterministic pipeline before surfacing violations in the CLI. +VectorLint is a content review harness. You give it target content, source-backed rules, provider credentials, and optional review context. VectorLint runs a bounded review and returns verified findings, scores, diagnostics, and usage metadata. -If you haven't run your first check yet, start with the [Quickstart](/quickstart). This page is for understanding the architecture behind what you're running. +If you haven't run your first check yet, start with the [Quickstart](/quickstart). This page explains the architecture behind that command. -## The evaluation pipeline +## The review pipeline -Every time you run `vectorlint doc.md`, three things happen in sequence. +Every time you run `vectorlint doc.md`, VectorLint follows the same high-level path. -### 1. Rule resolution +### 1. Resolve targets and rules -VectorLint reads your `.vectorlint.ini` to determine which rule packs apply to the file. It loads those rule files, prepends the contents of `VECTORLINT.md` (if present) to each rule's system prompt, and assembles the evaluation context. +VectorLint reads your project configuration to determine which rule packs apply to each target file. It loads rule files, applies `VECTORLINT.md` as global review context when present, and builds source-backed rules. -If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic "Style Guide Compliance" rule automatically. This is the zero-config path — the fastest way to get from nothing to a working evaluation without writing any rule pack files. +If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic style-guide rule automatically. This is the zero-config path. -### 2. LLM evaluation +### 2. Build a review request -VectorLint sends your content to the configured LLM provider with each rule's prompt. Depending on the rule type, the model returns either: +The CLI builds a `ReviewRequest` with: -- **A list of specific violations** (check rules) — for countable errors like grammar mistakes or banned terms -- **A scored rubric** (judge rules) — for quality dimensions like tone, clarity, or technical accuracy rated on a 1–4 scale +- target URI, content, content type, and byte length +- source-backed rules +- optional caller-supplied review context +- review budget +- output policy +- model-call strategy -Rules run concurrently up to the `Concurrency` limit set in `.vectorlint.ini`. +The review contract is intentionally neutral. It does not include workspace exploration tools, model-authored rule overrides, or subjective rubric scoring. -### 3. Filtering +### 3. Select a model call -Raw model output contains noise — potential violations that don't hold up under scrutiny. VectorLint reduces false positives through a two-phase filtering process: +VectorLint supports three model-call values: -1. **Candidate generation** — the model returns all potential violations, each tagged with required fields: rule support, exact evidence, context support, plausible non-violation, and fix quality. -2. **Deterministic surfacing** — VectorLint applies a strict filter and only surfaces violations that pass all required gates. +| Model call | Use it for | +| --- | --- | +| `single` | Normal reviews using structured model calls without tools | +| `agent` | Large or context-heavy reviews where the model needs target-section paging | +| `auto` | Default behavior; VectorLint chooses based on target size and rule count | -The CLI output is intentionally stricter than raw model candidates with fewer results and higher confidence. +`agent` is bounded to the current target. The only tool it can use is `read_target_section`, which reads line ranges from the target content already supplied to VectorLint. -You can tune how aggressively the filter operates with `CONFIDENCE_THRESHOLD` (default: `0.75`). Lower values surface more findings with higher recall; higher values surface fewer findings with higher precision. See [Configuring LLM providers](/llm-providers) for details. +See [Model calls](/model-calls) for CLI examples and selection details. -## Scoring +### 4. Process findings -VectorLint uses two scoring methods depending on the rule type. +Both model-call paths send raw candidate findings through the same finding-processing pipeline: -**Density-based scoring** (check rules) calculates error density, violations per 100 words. VectorLint weights a single error in a short document more heavily than the same error in a long one. This normalizes quality assessment across documents of any length. +1. filter candidates through the evidence gate +2. verify finding evidence against target content +3. deduplicate verified findings +4. score by verified finding count and density +5. resolve severity +6. assemble findings, scores, diagnostics, and usage metadata -**Rubric-based scoring** (judge rules) normalizes the LLM's 1–4 rating to a 1–10 scale and computes a weighted average across all criteria. Criteria weights reflect your real-world priorities — technical accuracy might carry a weight of 40 while SEO carries a weight of 10. +Unlocatable quoted evidence becomes a diagnostic and does not become a finding. -See [Quality scoring](/quality-scoring) for the full scoring reference. +### 5. Format output -## Two ways to define quality +The final `ReviewResult` is routed to the requested output format: -VectorLint gives you two complementary tools for expressing your content standards: +- `line` for human-readable terminal output +- `json` for VectorLint's native machine-readable output +- `vale-json` for Vale-compatible integrations +- `rdjson` for reviewdog-compatible integrations -- **`VECTORLINT.md`** — plain-language style instructions that apply globally to every evaluation. The fastest path to useful output: no rule files, no configuration syntax, just plain English instructions that the LLM uses as context for every check. -- **Rule pack files** — structured LLM prompts for specific, measurable checks with weighted scoring criteria. Use these when you need reproducible, auditable results on a particular dimension of quality. +## Boundaries -The two work together: `VECTORLINT.md` sets the baseline context, and rule pack files enforce specific criteria on top of it. +VectorLint reviews only the target content and explicit review context supplied to it. It does not search your workspace, read arbitrary files, or expand scope on its own. -See [Defining your style rules](/style-guide) for how to create both. +This makes VectorLint safe to call from external agents and CI systems: the caller owns exploration, and VectorLint owns constrained review. + +## More detail + +Read the current [harness architecture spec](https://github.com/TRocket-Labs/vectorlint/blob/main/docs/specs/2026-07-10-harness-architecture.md) for the code-level contract and removed surfaces. ## Next steps -- [Quickstart](/quickstart) — run your first evaluation in minutes -- [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules -- [Configuring a project](/project-config) — map files to rule packs \ No newline at end of file +- [Model calls](/model-calls) — choose `single`, `agent`, or `auto` +- [Defining your style rules](/style-guide) — create `VECTORLINT.md` and custom rules +- [CLI reference](/cli-reference) — confirm command syntax and flags diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 02197adf..0122713f 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -5,7 +5,7 @@ description: "VectorLint is a large language model (LLM)-based prose linter that ## What is VectorLint? -VectorLint is a command-line tool that evaluates and scores documentation using large language models (LLMs). Instead of regex patterns that can only catch surface-level issues, VectorLint uses an [LLM-as-a-Judge](https://en.wikipedia.org/wiki/LLM-as-a-Judge) approach to catch terminology misuse, technical inaccuracies, and style inconsistencies that require contextual understanding to detect. +VectorLint is a command-line content review harness that uses large language models (LLMs) to evaluate documentation. Instead of regex patterns that can only catch surface-level issues, VectorLint looks for observable terminology, technical accuracy, and style violations that require contextual understanding to detect. If you can write a prompt for it, you can lint it with VectorLint. diff --git a/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md b/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md index 4c19efb3..641d7801 100644 --- a/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md +++ b/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md @@ -1,5 +1,13 @@ # Execution Log +> **SUPERSEDED (2026-07-10).** This log describes the removed autonomous +> agent-mode implementation. VectorLint is now a bounded review harness with +> `single`/`agent`/`auto` model calls. See +> [`../specs/2026-07-10-harness-architecture.md`](../specs/2026-07-10-harness-architecture.md) +> for the current architecture and +> [`../migration-from-agent-mode.mdx`](../migration-from-agent-mode.mdx) +> for migration guidance. The details below are historical only. + - **Plan**: `docs/plans/2026-03-31-agent-mode-implementation-plan.md` - **Issue**: Not provided (executed from user directive in this session) - **Started**: 2026-03-31 diff --git a/docs/migration-from-agent-mode.mdx b/docs/migration-from-agent-mode.mdx new file mode 100644 index 00000000..4011c5fb --- /dev/null +++ b/docs/migration-from-agent-mode.mdx @@ -0,0 +1,51 @@ +--- +title: Migration from agent mode +description: Move integrations from the removed autonomous agent mode to bounded model calls. +--- + +VectorLint no longer supports autonomous workspace-agent mode. The current product is a bounded review harness: callers supply the target and any review context, and VectorLint reviews that supplied content against source-backed rules. + +## What changed + +- `--mode agent` was removed. Use `--model-call single`, `--model-call agent`, or `--model-call auto`. +- Workspace exploration moved out of VectorLint. External agents or scripts gather context before invoking VectorLint. +- Provider-level agent loops were removed. Providers expose structured-output and bounded tool-calling capabilities. +- Model-authored review instructions were removed. Rules remain source-backed. + +## Mapping + +| Before | After | +| --- | --- | +| `vectorlint doc.md --mode agent` | `vectorlint doc.md --model-call agent` for target paging, or `--model-call single` for structured calls | +| `--mode agent --output json` | `--model-call agent --output json` | +| `report_finding` for workspace findings | Move workspace-level findings to the caller; VectorLint reports target findings only | +| `search_content` or `read_file` workspace tools | Caller gathers context and passes it as review context | +| Model-supplied `reviewInstruction` | Edit the source-backed rule file | +| `runAgentToolLoop` on a provider | Implement `ReviewExecutor`, or use `StructuredModelClient` / `ToolCallingModelClient` behind an executor | + +## Why it changed + +The old direction mixed two ownership models: VectorLint as a linter and VectorLint as an autonomous workspace agent. That duplicated result contracts, weakened scope boundaries, and made findings less consistent across output formats. + +The harness model keeps responsibilities clear. The caller owns exploration. VectorLint owns bounded review. + +## What you get + +- Explicit budgets for target size, review context size, chunks per rule, model calls, and elapsed time. +- One finding-processing path for filtering, evidence verification, scoring, diagnostics, and formatting. +- The same result shape across `single`, `agent`, and `auto`. +- A stable contract external agents can call without granting VectorLint workspace exploration authority. + +## Migration steps + +1. Replace `--mode agent` with `--model-call agent` only when target paging is still useful. +2. Use `--model-call single` for normal reviews that do not need target-section paging. +3. Move workspace search, file selection, and context gathering into the caller. +4. Pass any needed context explicitly as review context when calling the review contract. +5. Rewrite subjective rubric rules as objective Via Negativa violation checks. + +## Related + +- [Model calls](/model-calls) +- [How it works](/how-it-works) +- [CLI reference](/cli-reference) diff --git a/docs/model-calls.mdx b/docs/model-calls.mdx new file mode 100644 index 00000000..dbe34c61 --- /dev/null +++ b/docs/model-calls.mdx @@ -0,0 +1,75 @@ +--- +title: Model calls +description: Choose single, agent, or auto model calls for VectorLint reviews. +--- + +VectorLint uses a model-call strategy to decide how the reviewer model sees target content. The strategy changes the invocation shape, not the rule semantics. + +## Choose a strategy + +| Value | Best for | What happens | +| --- | --- | --- | +| `single` | Normal documents and straightforward checks | VectorLint sends structured model calls without tools | +| `agent` | Large or context-heavy targets | VectorLint gives the model one target-only paging tool | +| `auto` | Default CLI behavior | VectorLint chooses `single` or `agent` from target size and rule count | + +Run with the default: + +```bash +vectorlint doc.md +``` + +Force structured calls: + +```bash +vectorlint doc.md --model-call single +``` + +Force target paging: + +```bash +vectorlint doc.md --model-call agent +``` + +## `single` + +`single` is the simplest model call. VectorLint supplies the rule prompt and target content, receives structured candidate findings, and sends them through shared finding processing. + +For large targets, the single path chunks content and merges candidate violations before verification and scoring. + +## `agent` + +`agent` is bounded target paging. The model can call `read_target_section` to read 1-based line ranges from the current target content. + +The agent model call cannot: + +- search the workspace +- read arbitrary files +- read files by URI +- modify content +- override rules +- report findings outside the target + +Use `agent` when a target is large enough that paging through sections helps the reviewer preserve context. + +## `auto` + +`auto` is the CLI default. VectorLint chooses `agent` when the target is larger than `600_000` bytes or when more than five rules apply. Otherwise, it chooses `single`. + +## Boundary + +VectorLint reviews only supplied target content and explicit review context. If an external agent or script wants VectorLint to consider other files, that caller must gather the relevant content and pass it as review context. VectorLint does not discover workspace context on its own. + +## Output + +All model-call strategies return the same review result shape: findings, scores, diagnostics, and usage metadata. Output format selection is separate: + +```bash +vectorlint doc.md --model-call agent --output json +``` + +## Related + +- [How it works](/how-it-works) +- [CLI reference](/cli-reference) +- [Migration from agent mode](/migration-from-agent-mode) diff --git a/docs/rubric-scoring.mdx b/docs/rubric-scoring.mdx index b5c52e1a..0e03ce1d 100644 --- a/docs/rubric-scoring.mdx +++ b/docs/rubric-scoring.mdx @@ -1,8 +1,21 @@ --- -title: Rubric Scoring -description: This page is coming soon. +title: Rubric scoring +description: Historical note about removed rubric-style judge rules. --- - - This page is coming soon. Check back for updates, or follow the [VectorLint GitHub repository](https://github.com/TRocket-Labs/vectorlint) for the latest news. - +Rubric-style `judge` rules are no longer part of VectorLint's current rule model. VectorLint now uses objective violation checks: rules describe observable conditions, and scoring is based on verified finding count and density. + +## What to use instead + +Write rules as Via Negativa checks. Describe what counts as a violation, require exact evidence from the target content, and provide concrete fix guidance. + +```markdown +Flag a section when it: +1. Makes a factual claim without a source +2. Uses a deprecated command name +3. Repeats the same setup step already shown earlier +``` + +## Migration + +If you have old rubric-style rules, rewrite each criterion as one or more observable violation conditions. See [Migration from agent mode](/migration-from-agent-mode) for related removed-surface guidance and [Defining your style rules](/style-guide) for current rule syntax. diff --git a/docs/specs/2026-07-10-harness-architecture.md b/docs/specs/2026-07-10-harness-architecture.md new file mode 100644 index 00000000..5498b9e4 --- /dev/null +++ b/docs/specs/2026-07-10-harness-architecture.md @@ -0,0 +1,199 @@ +# VectorLint Harness Architecture + +Status: Current as of the bounded harness refactor. + +Related audit: [`2026-07-10-vectorlint-harness-architecture-audit.md`](../audits/2026-07-10-vectorlint-harness-architecture-audit.md) + +## Purpose And Product Direction + +VectorLint is a bounded content review harness. External callers, such as Codex, Claude Code, CI jobs, scripts, or humans at the CLI, own exploration and decide what content to review. VectorLint owns constrained review of supplied target content against source-backed rules. + +This means VectorLint is not a workspace agent. It does not search arbitrary files, expand scope on its own, rewrite content, or let a model override the rule it is supposed to enforce. It receives target content, rules, optional caller-supplied review context, budgets, and output policy, then returns structured review results. + +Use the shared domain language in [`../../CONTEXT.md`](../../CONTEXT.md) when changing code, docs, tests, or agent handoffs. + +## Review Contract + +The review contract lives in [`src/review/types.ts`](../../src/review/types.ts), with boundary schemas in [`src/review/schemas.ts`](../../src/review/schemas.ts). + +`ReviewRequest` contains: + +- `target`: the target URI, content, content type, and optional byte length. +- `rules`: source-backed rules with `id`, `source`, `body`, optional `name`, optional `severity`, and optional Via Negativa violation conditions. +- `context`: optional caller-supplied review context. +- `budget`: explicit bounds for one review. +- `outputPolicy`: usage and payload telemetry policy. +- `modelCall`: `single`, `agent`, or `auto`. + +Every executor implements: + +```ts +interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} +``` + +`ReviewResult` contains: + +- `findings`: verified findings anchored in target content. +- `scores`: per-rule scores. +- `diagnostics`: operational and finding-processing notes. +- `usage`: optional model-call and token metadata. +- `hadOperationalErrors`: optional flag for partial-result runs with operational errors. + +The contract deliberately does not expose `evaluator`, `judge`, subjective rubric scoring, model-authored rule overrides, or autonomous workspace tools. + +## Rule Model + +VectorLint rules are objective Via Negativa checks. A rule should name observable violation conditions and ask the reviewer to find evidence that those conditions are present. The reviewer should not grade broad alignment against an ideal. + +Good rules answer yes/no questions: + +- Is this sentence using a banned phrase? +- Does this claim omit required evidence? +- Does this section repeat information already stated nearby? + +Future-facing rule authors should avoid subjective judge/rubric designs. Historical rubric-style language remains only where it describes removed behavior or migration guidance. + +## Model Calls + +The model-call strategy is selected with `--model-call single|agent|auto`. The allowed values and default come from [`src/review/executor.ts`](../../src/review/executor.ts), [`src/cli/types.ts`](../../src/cli/types.ts), and [`src/schemas/cli-schemas.ts`](../../src/schemas/cli-schemas.ts). + +### `single` + +`single` uses [`SingleModelCallExecutor`](../../src/executors/single-model-call-executor.ts). It performs one structured model call per rule/chunk through `StructuredModelClient`. For targets above 600 words, it chunks line-numbered content and merges violations before shared finding processing. + +The single executor owns no tool surface. + +### `agent` + +`agent` uses [`AgentModelCallExecutor`](../../src/executors/agent-model-call-executor.ts). It performs one bounded tool-calling run per rule through `ToolCallingModelClient`. + +The only executor-owned tool is `read_target_section`, defined in [`src/executors/target-read-capability-adapter.ts`](../../src/executors/target-read-capability-adapter.ts). It reads 1-based line ranges from in-memory `request.target.content` and returns model-visible errors for invalid ranges. + +The agent model call cannot: + +- search the workspace; +- read arbitrary files; +- read URIs outside the target content; +- rewrite rules; +- create top-level workspace findings; +- override source-backed rule prompts. + +### `auto` + +`auto` resolves through `chooseModelCall` in [`src/review/executor.ts`](../../src/review/executor.ts). It selects `agent` when target content is larger than `AGENT_MODEL_CALL_BYTE_THRESHOLD` (`600_000` bytes) or when more than five rules apply. Otherwise, it selects `single`. + +The CLI default is `auto`. + +## On-Page Boundary + +The on-page boundary is implemented in [`src/review/boundary.ts`](../../src/review/boundary.ts) and enforced by executor design. + +Target content is always in scope. Caller-supplied review context is in scope only because the caller explicitly provided it. Workspace files are out of scope unless the caller passes their content as review context. Rule bodies are source-backed and loaded before the review request is built. + +The target-read adapter performs no filesystem reads. It slices the target content already present in memory. + +## Budgets + +Default budgets live in [`src/review/budget.ts`](../../src/review/budget.ts): + +| Field | Default | Meaning | +| --- | ---: | --- | +| `maxTargetBytes` | `1_000_000` | maximum target content size | +| `maxCallerContextBytes` | `500_000` | maximum caller-supplied context size | +| `maxChunksPerRule` | `20` | maximum chunks/tool steps per rule | +| `maxModelCallsPerReview` | `50` | maximum model calls in one review | +| `maxWallClockMs` | `300_000` | maximum elapsed review time, in milliseconds | + +Budgets limit work, not output. There is intentionally no `maxFindingsPerRule`. There is also no headless retry budget because VectorLint no longer has a headless autonomous executor. + +`maxWallClockMs` is the review timeout. Executors check model-call count and elapsed time before model calls and surface budget exhaustion as diagnostics when partial results can be returned. + +## Finding Processing + +Both model-call strategies use the same finding-processing pipeline in [`src/findings/`](../../src/findings/): + +1. Filter candidate findings through the evidence gate. +2. Verify finding evidence against target content. +3. Deduplicate verified findings. +4. Score by verified finding count and density. +5. Resolve severity. +6. Assemble findings, scores, diagnostics, and operational status. + +Unlocatable quoted evidence becomes a `finding-evidence-not-locatable` diagnostic and emits no finding. VectorLint does not fall back to model-provided line numbers for unverified evidence. + +Diagnostics describe operational or finding-processing conditions such as unlocatable evidence, budget exhaustion, schema parse failures, and provider failures. They are not content findings. + +## Provider And Model Capabilities + +Providers are transport capabilities, not product owners. + +[`StructuredModelClient`](../../src/providers/structured-model-client.ts) performs one structured model call and returns validated output. + +[`ToolCallingModelClient`](../../src/providers/tool-calling-model-client.ts) performs one bounded tool-calling generation using caller-supplied tools and returns structured output. The provider does not define product tools. The executor supplies the tool map and run bounds. + +Neither provider capability is an autonomous agent loop. + +## Architecture Diagram + +```text +CLI / External Caller + | + v +ReviewRequest + - target + - source-backed rules + - caller-supplied context + - budget + - output policy + - modelCall + | + v +chooseModelCall(auto?) ──> single ──> SingleModelCallExecutor + └─> agent ──> AgentModelCallExecutor + | + v + read_target_section + target content only + | + v +Shared Finding Processing + - filter + - verify evidence + - dedupe + - score + - diagnostics + | + v +ReviewResult + | + v +Formatters + - line + - json + - vale-json + - rdjson +``` + +## Removal Notes + +Use `--model-call`, not `--mode agent`. The CLI still rejects `--mode` with a clear migration error so old integrations fail loudly. + +Autonomous workspace-agent behavior is removed. If an integration needs project-wide exploration, it should do that before invoking VectorLint and pass the selected target content and review context explicitly. + +Subjective `judge`/rubric rules are removed from the future-facing architecture rather than migrated. New rules should be written as objective Via Negativa checks with observable violation conditions. + +## References + +- [`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](../audits/2026-07-10-vectorlint-harness-architecture-audit.md) +- [`src/review/types.ts`](../../src/review/types.ts) +- [`src/review/executor.ts`](../../src/review/executor.ts) +- [`src/review/budget.ts`](../../src/review/budget.ts) +- [`src/review/boundary.ts`](../../src/review/boundary.ts) +- [`src/executors/single-model-call-executor.ts`](../../src/executors/single-model-call-executor.ts) +- [`src/executors/agent-model-call-executor.ts`](../../src/executors/agent-model-call-executor.ts) +- [`src/executors/target-read-capability-adapter.ts`](../../src/executors/target-read-capability-adapter.ts) +- [`src/findings/processor.ts`](../../src/findings/processor.ts) +- [`src/providers/structured-model-client.ts`](../../src/providers/structured-model-client.ts) +- [`src/providers/tool-calling-model-client.ts`](../../src/providers/tool-calling-model-client.ts) diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 677502d6..9b30d49a 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -156,6 +156,24 @@ Without a search provider, `technical-accuracy` evaluators cannot verify facts a --- +### `--mode agent` no longer works + +**Symptom:** VectorLint exits with an error that `--mode` is no longer supported. + +**Cause:** Autonomous agent mode was removed. VectorLint now uses `--model-call` to choose how the reviewer model is invoked. + +**Fix:** Replace the removed flag with one of the supported model-call values: + +```bash +vectorlint doc.md --model-call single +vectorlint doc.md --model-call agent +vectorlint doc.md --model-call auto +``` + +Use `agent` only when target-section paging helps with large or context-heavy content. It is bounded to the current target and cannot search the workspace. See [Migration from agent mode](/migration-from-agent-mode) for integration guidance. + +--- + ## CI issues ### CI pipeline fails immediately with no findings output diff --git a/docs/use-cases.mdx b/docs/use-cases.mdx index 84bddf70..1d7f7d98 100644 --- a/docs/use-cases.mdx +++ b/docs/use-cases.mdx @@ -11,7 +11,7 @@ VectorLint fits anywhere a team publishes written content at scale and needs con **The path:** Write your standards as plain-language instructions in `VECTORLINT.md`. VectorLint converts them into a "Style Guide Compliance" rule and runs it against every file, every time — no regex, no word lists, no manual review for comma usage or passive voice. -If your team has a detailed guide, see [Defining your style rules](/style-guide) for an extraction prompt that converts it into a `VECTORLINT.md`-optimized file. For more targeted enforcement, write rule pack files that score specific criteria on a weighted rubric. +If your team has a detailed guide, see [Defining your style rules](/style-guide) for an extraction prompt that converts it into a `VECTORLINT.md`-optimized file. For more targeted enforcement, write rule pack files that describe specific, observable violation conditions. **Who does this:** Technical writing teams, content operations teams, developer advocacy teams. @@ -88,4 +88,4 @@ See [Configuring a project](/project-config) for the full configuration referenc - [Quickstart](/quickstart) — run your first evaluation in under ten minutes - [How it works](/how-it-works) — understand the evaluation pipeline, PAT filtering, and scoring -- [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules \ No newline at end of file +- [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules From 9c122f81ff2f60bbae4ad02dd861c705b0b82e17 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 13:34:45 +0100 Subject: [PATCH 27/34] refactor(errors): drop orphaned AgentToolError class - AgentToolError's only consumers were src/agent/executor.ts, src/agent/path-utils.ts, and tests/agent/agent-executor.test.ts, all deleted with the autonomous workspace-agent surface - Remove the now-unused class so no agent-tied symbol lingers in the error hierarchy Refs: docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md (Product Decision; Finding #2) --- src/errors/index.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/errors/index.ts b/src/errors/index.ts index 86e257ba..ca196d95 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -30,14 +30,6 @@ export class ProcessingError extends VectorlintError { } } -// Agent tool/runtime error for agent-mode operational failures -export class AgentToolError extends VectorlintError { - constructor(message: string, code = 'AGENT_TOOL_ERROR') { - super(message, code); - this.name = 'AgentToolError'; - } -} - // Missing dependency error for when required dependencies are not available export class MissingDependencyError extends VectorlintError { constructor( From 7e32601eb4e71d5dac25c4ee3722b88e164272dc Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 14:45:11 +0100 Subject: [PATCH 28/34] docs: reframe internal agent-mode cleanup - Remove public migration and deprecation framing for unreleased agent-mode paths. - Drop the migration page, navigation entry, and troubleshooting guidance that implied external users. - Reword historical audit and architecture notes around internal implementation cleanup. --- ...0-vectorlint-harness-architecture-audit.md | 8 +-- docs/docs.json | 1 - ...3-31-agent-mode-implementation-plan.log.md | 4 +- docs/migration-from-agent-mode.mdx | 51 ------------------- docs/model-calls.mdx | 1 - docs/rubric-scoring.mdx | 6 +-- docs/specs/2026-07-10-harness-architecture.md | 10 ++-- docs/troubleshooting.mdx | 18 ------- 8 files changed, 14 insertions(+), 85 deletions(-) delete mode 100644 docs/migration-from-agent-mode.mdx diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md index 356e1506..ab661ec0 100644 --- a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -1,8 +1,10 @@ # VectorLint Harness Architecture Audit > **HISTORICAL AUDIT.** This audit records the July 2026 decision that led to -> the bounded harness refactor. Some "current state" observations below -> describe the pre-refactor code. For the shipped architecture, see +> the bounded harness refactor. Agent mode was an unreleased internal +> implementation path with no public users; this audit records the decision to +> remove it before shipping it as a public contract. Some "current state" +> observations below describe the pre-refactor code. For the shipped architecture, see > [`../specs/2026-07-10-harness-architecture.md`](../specs/2026-07-10-harness-architecture.md). Date: 2026-07-10 @@ -380,6 +382,6 @@ Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on bra - `docs/research/**` excluded from linting. - `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. -**Current state.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the autonomous workspace-agent surface is deprecated at the CLI boundary: `--mode agent` emits a deprecation warning and falls back to standard evaluation. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. The deprecation notice and pointer to this audit live in the README `## Agent Mode` section and the `--mode` help text. +**Current state at audit time.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the unreleased autonomous workspace-agent surface is quarantined at the CLI boundary: `--mode` is blocked so internal agent-mode wiring cannot be reached from the CLI. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. A pointer to this audit lives in the README `## Agent Mode` section and the `--mode` help text. This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. diff --git a/docs/docs.json b/docs/docs.json index f8ee7c1b..d88e89c8 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -72,7 +72,6 @@ "configuration-schema", "env-variables", "frontmatter-fields", - "migration-from-agent-mode", "troubleshooting" ] } diff --git a/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md b/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md index 641d7801..233c16b2 100644 --- a/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md +++ b/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md @@ -4,9 +4,7 @@ > agent-mode implementation. VectorLint is now a bounded review harness with > `single`/`agent`/`auto` model calls. See > [`../specs/2026-07-10-harness-architecture.md`](../specs/2026-07-10-harness-architecture.md) -> for the current architecture and -> [`../migration-from-agent-mode.mdx`](../migration-from-agent-mode.mdx) -> for migration guidance. The details below are historical only. +> for the current architecture. The details below are historical only. - **Plan**: `docs/plans/2026-03-31-agent-mode-implementation-plan.md` - **Issue**: Not provided (executed from user directive in this session) diff --git a/docs/migration-from-agent-mode.mdx b/docs/migration-from-agent-mode.mdx deleted file mode 100644 index 4011c5fb..00000000 --- a/docs/migration-from-agent-mode.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Migration from agent mode -description: Move integrations from the removed autonomous agent mode to bounded model calls. ---- - -VectorLint no longer supports autonomous workspace-agent mode. The current product is a bounded review harness: callers supply the target and any review context, and VectorLint reviews that supplied content against source-backed rules. - -## What changed - -- `--mode agent` was removed. Use `--model-call single`, `--model-call agent`, or `--model-call auto`. -- Workspace exploration moved out of VectorLint. External agents or scripts gather context before invoking VectorLint. -- Provider-level agent loops were removed. Providers expose structured-output and bounded tool-calling capabilities. -- Model-authored review instructions were removed. Rules remain source-backed. - -## Mapping - -| Before | After | -| --- | --- | -| `vectorlint doc.md --mode agent` | `vectorlint doc.md --model-call agent` for target paging, or `--model-call single` for structured calls | -| `--mode agent --output json` | `--model-call agent --output json` | -| `report_finding` for workspace findings | Move workspace-level findings to the caller; VectorLint reports target findings only | -| `search_content` or `read_file` workspace tools | Caller gathers context and passes it as review context | -| Model-supplied `reviewInstruction` | Edit the source-backed rule file | -| `runAgentToolLoop` on a provider | Implement `ReviewExecutor`, or use `StructuredModelClient` / `ToolCallingModelClient` behind an executor | - -## Why it changed - -The old direction mixed two ownership models: VectorLint as a linter and VectorLint as an autonomous workspace agent. That duplicated result contracts, weakened scope boundaries, and made findings less consistent across output formats. - -The harness model keeps responsibilities clear. The caller owns exploration. VectorLint owns bounded review. - -## What you get - -- Explicit budgets for target size, review context size, chunks per rule, model calls, and elapsed time. -- One finding-processing path for filtering, evidence verification, scoring, diagnostics, and formatting. -- The same result shape across `single`, `agent`, and `auto`. -- A stable contract external agents can call without granting VectorLint workspace exploration authority. - -## Migration steps - -1. Replace `--mode agent` with `--model-call agent` only when target paging is still useful. -2. Use `--model-call single` for normal reviews that do not need target-section paging. -3. Move workspace search, file selection, and context gathering into the caller. -4. Pass any needed context explicitly as review context when calling the review contract. -5. Rewrite subjective rubric rules as objective Via Negativa violation checks. - -## Related - -- [Model calls](/model-calls) -- [How it works](/how-it-works) -- [CLI reference](/cli-reference) diff --git a/docs/model-calls.mdx b/docs/model-calls.mdx index dbe34c61..9f52a6b9 100644 --- a/docs/model-calls.mdx +++ b/docs/model-calls.mdx @@ -72,4 +72,3 @@ vectorlint doc.md --model-call agent --output json - [How it works](/how-it-works) - [CLI reference](/cli-reference) -- [Migration from agent mode](/migration-from-agent-mode) diff --git a/docs/rubric-scoring.mdx b/docs/rubric-scoring.mdx index 0e03ce1d..e6cb0d9d 100644 --- a/docs/rubric-scoring.mdx +++ b/docs/rubric-scoring.mdx @@ -12,10 +12,10 @@ Write rules as Via Negativa checks. Describe what counts as a violation, require ```markdown Flag a section when it: 1. Makes a factual claim without a source -2. Uses a deprecated command name +2. Uses an obsolete command name 3. Repeats the same setup step already shown earlier ``` -## Migration +## Updating old rules -If you have old rubric-style rules, rewrite each criterion as one or more observable violation conditions. See [Migration from agent mode](/migration-from-agent-mode) for related removed-surface guidance and [Defining your style rules](/style-guide) for current rule syntax. +If you have old rubric-style rules, rewrite each criterion as one or more observable violation conditions. See [Defining your style rules](/style-guide) for current rule syntax. diff --git a/docs/specs/2026-07-10-harness-architecture.md b/docs/specs/2026-07-10-harness-architecture.md index 5498b9e4..52125a33 100644 --- a/docs/specs/2026-07-10-harness-architecture.md +++ b/docs/specs/2026-07-10-harness-architecture.md @@ -53,7 +53,7 @@ Good rules answer yes/no questions: - Does this claim omit required evidence? - Does this section repeat information already stated nearby? -Future-facing rule authors should avoid subjective judge/rubric designs. Historical rubric-style language remains only where it describes removed behavior or migration guidance. +Future-facing rule authors should avoid subjective judge/rubric designs. Historical rubric-style language remains only where it describes removed behavior. ## Model Calls @@ -176,13 +176,13 @@ Formatters - rdjson ``` -## Removal Notes +## Internal Implementation Notes -Use `--model-call`, not `--mode agent`. The CLI still rejects `--mode` with a clear migration error so old integrations fail loudly. +`--model-call` is the documented CLI surface. The internal `--mode` flag is rejected at the CLI boundary so unreleased agent-mode wiring cannot be used accidentally. -Autonomous workspace-agent behavior is removed. If an integration needs project-wide exploration, it should do that before invoking VectorLint and pass the selected target content and review context explicitly. +The unreleased autonomous workspace implementation path is removed. External callers that need project-wide exploration should gather context before invoking VectorLint and pass selected content explicitly. -Subjective `judge`/rubric rules are removed from the future-facing architecture rather than migrated. New rules should be written as objective Via Negativa checks with observable violation conditions. +Subjective `judge`/rubric rules are not part of the future-facing architecture. New rules should be written as objective Via Negativa checks with observable violation conditions. ## References diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 9b30d49a..677502d6 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -156,24 +156,6 @@ Without a search provider, `technical-accuracy` evaluators cannot verify facts a --- -### `--mode agent` no longer works - -**Symptom:** VectorLint exits with an error that `--mode` is no longer supported. - -**Cause:** Autonomous agent mode was removed. VectorLint now uses `--model-call` to choose how the reviewer model is invoked. - -**Fix:** Replace the removed flag with one of the supported model-call values: - -```bash -vectorlint doc.md --model-call single -vectorlint doc.md --model-call agent -vectorlint doc.md --model-call auto -``` - -Use `agent` only when target-section paging helps with large or context-heavy content. It is bounded to the current target and cannot search the workspace. See [Migration from agent mode](/migration-from-agent-mode) for integration guidance. - ---- - ## CI issues ### CI pipeline fails immediately with no findings output From 6dde8f597cb53bf0fdf3cd4ba564483365e68bcc Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 14:47:59 +0100 Subject: [PATCH 29/34] docs: clarify historical audit framing - Replace remaining public transition wording in the historical audit with internal removal language. - Keep the audit historical while aligning it to unreleased agent-mode scope. --- .../2026-07-10-vectorlint-harness-architecture-audit.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md index ab661ec0..205e0c6d 100644 --- a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -261,7 +261,7 @@ Future contributors will optimize toward the wrong architecture and trust contra Recommendation: -Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and migration plan. +Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and internal removal plan for unreleased agent-mode paths. ### 9. Secrets And Sensitive Content Need Better Handling @@ -338,7 +338,7 @@ Results: 1. Mark the agentic capabilities spec superseded. 2. Write a new harness architecture spec. 3. Update README and CLI docs. -4. Document migration from autonomous workspace-agent mode to direct/reader/auto execution. +4. Document the bounded harness architecture that replaces the unreleased autonomous workspace-agent implementation path. ## Suggested Target Architecture From ab9b8e1215ff003cb66fd8204a51fb06fcd78a11 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 14:54:17 +0100 Subject: [PATCH 30/34] docs: enforce documentation artifact boundaries - Remove tracked audit, run-note, and spec artifacts from product docs. - Replace product-doc links to the removed spec with current usage docs. - Add AGENTS guidance that keeps coordination artifacts in ignored workspace state. --- AGENTS.md | 8 + README.md | 4 +- ...0-vectorlint-harness-architecture-audit.md | 387 ------------------ docs/how-it-works.mdx | 4 - ...3-31-agent-mode-implementation-plan.log.md | 28 -- docs/specs/2026-07-10-harness-architecture.md | 199 --------- 6 files changed, 9 insertions(+), 621 deletions(-) delete mode 100644 docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md delete mode 100644 docs/logs/2026-03-31-agent-mode-implementation-plan.log.md delete mode 100644 docs/specs/2026-07-10-harness-architecture.md diff --git a/AGENTS.md b/AGENTS.md index 0604a5b6..9f685e00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,14 @@ These guidelines reduce common LLM coding mistakes. They bias toward caution ove - For multi-step tasks, state a brief plan with verification for each step. - Loop until the success criteria are verified or a blocker is clear. +### Documentation Artifact Boundaries + +- Do not commit raw planning or investigation artifacts to the product codebase. +- Keep audits, plans, specs, run notes, and similar coordination artifacts out of tracked repo paths such as `docs/audits/`, `docs/plans/`, `docs/specs/`, `audits/`, `plans/`, and `specs/`. +- Store coordination artifacts in `.agent-runs/` or another ignored workspace location. +- If a durable architectural decision must be committed, write it as an ADR. ADRs are the only allowed committed decision/planning artifacts. +- Product documentation may describe shipped behavior, configuration, and usage, but it must not preserve internal audit/plan/spec documents as reviewed product docs. + ## Project Structure & Module Organization - `src/` diff --git a/README.md b/README.md index 989399b3..113378f2 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,7 @@ vectorlint doc.md --model-call auto - `auto` is the default. VectorLint chooses `single` for normal inputs and `agent` for large or multi-rule reviews that benefit from target paging. -See the current -[`docs/specs/2026-07-10-harness-architecture.md`](docs/specs/2026-07-10-harness-architecture.md) -for the contract and boundary details. +See [Model calls](docs/model-calls.mdx) for more details. ## Contributing diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md deleted file mode 100644 index 205e0c6d..00000000 --- a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md +++ /dev/null @@ -1,387 +0,0 @@ -# VectorLint Harness Architecture Audit - -> **HISTORICAL AUDIT.** This audit records the July 2026 decision that led to -> the bounded harness refactor. Agent mode was an unreleased internal -> implementation path with no public users; this audit records the decision to -> remove it before shipping it as a public contract. Some "current state" -> observations below describe the pre-refactor code. For the shipped architecture, see -> [`../specs/2026-07-10-harness-architecture.md`](../specs/2026-07-10-harness-architecture.md). - -Date: 2026-07-10 - -Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. - -## Product Decision - -VectorLint should not keep the current autonomous workspace-agent mode. - -The retained agent-like capability should be a constrained reader executor: a model or headless agent can read the target content in sections when that is useful for context management, but it should not search the workspace, inspect arbitrary files, perform top-level workspace checks, or rewrite the rule being evaluated. - -The intended execution model is: - -- `direct`: send target content and rule in one structured review call. -- `reader`: give the executor a read-section capability over the target content only. -- `auto`: choose `direct` for normal-sized inputs and `reader` for large inputs or explicit context-management needs. - -This is an execution strategy decision, not a return to VectorLint-as-agent. - -## Product Direction Under Review - -VectorLint should become a programmable, bounded, on-page content review harness that another caller can invoke. - -The caller can be Codex, Claude Code, another coding agent, CI, a local CLI wrapper, or a future service. That caller owns exploration, cross-page reasoning, research, and context gathering. VectorLint owns the constrained review of target content against source-backed rules and returns structured findings, scores, diagnostics, and usage metadata. - -In practical terms: - -- VectorLint reviews the target page or explicitly supplied content. -- External context is caller-supplied, not discovered by VectorLint. -- Rules define review behavior, granularity, severity, and scoring constraints. -- Executors can be API models, headless local agents, or constrained reader executors, but they receive the same review request contract. -- VectorLint should not expose a model-controlled workspace tool loop as its core product surface. - -## Objective System Audit Summary - -Separate from product direction, the current codebase has several system-level liabilities: - -- Build health is weak: `npx tsc --noEmit`, `npm run lint`, and parts of `npm run test:run` fail in the current workspace. -- Runtime contracts are duplicated across hand-written TypeScript, JSON-schema objects, Zod schemas, and formatter-specific shapes. -- Standard mode and agent mode project the same underlying findings differently. -- CLI orchestration owns too many responsibilities: config, file matching, evaluation, scoring, output formatting, debug artifacts, and agent routing. -- Evidence verification and counting are inconsistent enough to affect exit behavior. -- Observability and debug paths can record full prompts, content, and outputs. -- Documentation and implementation disagree on the current agent-mode topology and tool contracts. - -## Architecture Verdict - -The current codebase contains two execution models at once: - -1. The older evaluator path: one structured model call per rule or chunk. -2. The newer autonomous workspace-agent path: VectorLint gives a model tools to read/search the workspace and call lint as a nested tool. - -The second path should not continue as-is. The useful piece is not "agent mode" broadly; it is target-aware reading for context management. The current implementation also duplicates result projection, scoring, evidence validation, output formatting, and configuration behavior from the first path. That is why the architecture feels harder to maintain: the code is not just messy; it is serving two different ownership models. - -The highest-leverage change is to define a neutral review-domain contract and make every executor implement that contract. - -## Proposed Core Contract - -The contract should be centered on review, not providers or agents. - -```ts -interface ReviewRequest { - target: ReviewTarget; - rules: ReviewRule[]; - context?: ReviewContext[]; - budget: ReviewBudget; - outputPolicy: ReviewOutputPolicy; -} - -interface ReviewExecutor { - run(request: ReviewRequest): Promise; -} - -interface ReviewResult { - findings: ReviewFinding[]; - scores: ReviewScore[]; - diagnostics: ReviewDiagnostic[]; - usage?: ReviewUsage; -} -``` - -API models, reasoning models, and headless CLI agents become adapter implementations behind this contract. The CLI becomes orchestration around request creation, result formatting, and exit behavior. - -## Major Findings - -### 1. The Current Agent Surface Is Broader Than The Decided Execution Model - -Evidence: - -- `README.md:148` presents `--mode agent` as autonomous cross-file review mode. -- `src/agent/prompt-builder.ts:43` instructs the model to perform workspace-level checks. -- `src/agent/executor.ts:670` exposes `read_file`. -- `src/agent/executor.ts:715` exposes `search_content`. - -Impact: - -VectorLint still behaves like it owns exploration. That puts it in competition with external coding agents instead of becoming a harness those agents can use. It also overshoots the decided reader-executor model, which only needs target-scoped reading for context management. - -Recommendation: - -Remove the current autonomous workspace-agent surface. Preserve only a constrained reader executor if needed: - -- It may read sections of the target content. -- It may not search the workspace. -- It may not read arbitrary files. -- It may not perform top-level workspace findings. -- It may not override source-backed rules. - -### 2. The Provider Interface Mixes Structured Review With Autonomous Tool Loops - -Evidence: - -- `src/providers/llm-provider.ts:28` requires both `runPromptStructured` and `runAgentToolLoop`. -- `src/providers/vercel-ai-provider.ts:136` implements the agent loop on the same provider. -- `src/agent/executor.ts:796` depends on the provider-level agent loop. - -Impact: - -Every future executor has to pretend to be both a structured model provider and an autonomous agent runner. That is the wrong abstraction for headless CLI support and for a constrained reader executor. - -Recommendation: - -Split the contracts: - -- `StructuredModelClient` for API structured output. -- `HeadlessReviewExecutor` for local CLI agents. -- `ReaderReviewExecutor` for target-scoped read-section execution. -- `ReviewExecutor` as the stable domain-level interface. -- `TelemetrySink` or observability decoration as an optional cross-cutting concern. - -### 3. Runtime Type Contracts Are Not Strong Enough For A Harness - -Evidence: - -- `src/providers/vercel-ai-provider.ts:119` returns `output as T`. -- `src/providers/llm-provider.ts:9` defines tool input and output as `unknown`. -- `src/agent/tools-registry.ts:21` exposes tools without typed output contracts. -- `npx tsc --noEmit` currently fails in core files. - -Impact: - -The code presents types as guarantees, but several are only assertions. A programmable harness needs reliable runtime validation because external callers and headless adapters will depend on these contracts. - -Recommendation: - -Tie structured execution to concrete Zod schemas or canonical parsers. Introduce typed tool contracts only if tools remain. Add a `typecheck` script and wire it into CI and build. - -### 4. Standard Mode And Agent Mode Project Results Differently - -Evidence: - -- `src/cli/orchestrator.ts:626` derives standard check severity from scoring. -- `src/agent/executor.ts:431` stamps agent findings with prompt severity. -- `src/agent/executor.ts:598` flattens judge criteria before recording findings. -- `src/cli/orchestrator.ts:1216` builds agent scores only for line output. -- `src/cli/orchestrator.ts:1237` emits structured output without agent scores or diagnostics. - -Impact: - -The same underlying review can produce different severity, rule identity, score data, JSON shape, and exit behavior depending on mode. That makes VectorLint hard to trust as a machine-facing gate. - -Recommendation: - -Extract one shared result projection pipeline: - -```text -PromptEvaluationResult - -> verified findings - -> rule and criterion identity - -> severity and score - -> diagnostics - -> output formatter -``` - -Every executor should return into this pipeline. - -### 5. The On-Page Boundary Is Not Enforced - -Evidence: - -- `src/agent/executor.ts:585` lets `lint` resolve any file inside `workspaceRoot`. -- `src/agent/executor.ts:670` lets `read_file` read any workspace file. -- `src/agent/executor.ts:715` lets `search_content` scan workspace content. -- `src/agent/executor.ts:134` allows model-supplied `reviewInstruction`. -- `src/agent/executor.ts:583` replaces the source-backed prompt body with the model-supplied override. - -Impact: - -Prompt-injected page content can indirectly steer the model to read non-target files or rewrite the rule being evaluated. That breaks deterministic review semantics and creates privacy and security risk. - -Recommendation: - -For the new harness, enforce a target/context allowlist: - -- Target content is always in scope. -- Caller-supplied context is in scope. -- Workspace reads are out of scope unless the caller explicitly includes those files as context. -- Reader execution can page through target content only. -- Rule bodies are source-backed and caller-authored, not model-authored. - -### 6. Evidence Handling Can Misreport Findings - -Evidence: - -- Standard mode skips unverifiable quotes but can still count original surfaced violation totals in some paths. -- `src/agent/executor.ts:415` attempts to locate evidence. -- `src/agent/executor.ts:426` falls back to the model-provided line when location fails. - -Impact: - -Standard mode can fail a run without printing the issue that caused the failure. Agent mode can surface hallucinated evidence as if it were verified. - -Recommendation: - -Use one evidence verifier. Count only emitted, verified findings. If evidence cannot be located, return a diagnostic and either skip the finding or mark the run operationally failed. - -### 7. Cost And Work Are Not Bounded Enough - -Evidence: - -- `src/providers/vercel-ai-provider.ts:161` defaults agent loops to 1000 steps. -- `src/agent/executor.ts:577` allows repeated nested `lint` calls. -- `src/evaluators/base-evaluator.ts:213` runs one model call per chunk. -- `src/agent/executor.ts:680` and `src/agent/executor.ts:715` glob before applying result caps. - -Impact: - -A single review can multiply cost through agent steps, nested lint calls, rules, chunks, and workspace search. Even a constrained reader executor will be slower than a direct call, so `auto` must have clear thresholds and budgets. - -Recommendation: - -Add explicit budgets: - -- Max target bytes. -- Max caller context bytes. -- Max chunks per rule. -- Max model calls per review. -- Max findings per rule. -- Max wall-clock duration. -- Max headless executor retries. - -### 8. Documentation Still Points At The Old Product - -Evidence: - -- `README.md:148` documents autonomous workspace-agent mode. -- `docs/specs/2026-03-17-agentic-capabilities-design.md:24` frames the roadmap around cross-document agent mode. -- The same spec describes implementation details that are not true now: one agent per rule, final `AgentFindingSchema`, AbortSignal propagation, ripgrep JSON, and Vale JSON omission of agent findings. - -Impact: - -Future contributors will optimize toward the wrong architecture and trust contracts that the code does not implement. - -Recommendation: - -Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and internal removal plan for unreleased agent-mode paths. - -### 9. Secrets And Sensitive Content Need Better Handling - -Evidence: - -- `src/config/global-config.ts:13` says the global config stores API keys. -- `src/config/global-config.ts:76` creates the config directory with default permissions. -- `src/config/global-config.ts:81` writes the config file with default permissions. -- `src/observability/langfuse-observability.ts:66` records inputs. -- `src/observability/langfuse-observability.ts:67` records outputs. -- `src/providers/vercel-ai-provider.ts:59` can log full prompts and content. - -Impact: - -Provider keys, unreleased docs, caller context, and model outputs can leak through filesystem permissions, telemetry, debug logs, or persisted artifacts. - -Recommendation: - -Use private file modes for config and review artifacts. Make payload telemetry a separate opt-in from metadata telemetry. Add redaction or safe debug modes. - -## Verification Results - -Commands run from `/Users/klinsmann/Projects/TinyRocketLabs/vectorlint`: - -```bash -npm run lint -npm run test:run -npx tsc --noEmit -``` - -Results: - -- `npm run lint` failed before linting because typed ESLint rules were applied to a `.cjs` research script without parser type information. -- `npm run test:run` passed most tests but failed 4 suites during module resolution for `ora` and `@langfuse/otel`. -- `npx tsc --noEmit` failed with contract errors in agent, orchestrator, observability, logging, and provider modules. - -## Recommended Refactor Sequence - -### Phase 1: Stop The Bleeding - -1. Remove or hide the current autonomous workspace-agent mode. -2. Fix `tsc --noEmit`. -3. Fix test module resolution. -4. Fix lint configuration for non-TypeScript research scripts. -5. Add `typecheck` to CI and build validation. - -### Phase 2: Define The Harness Contract - -1. Create a neutral review-domain module. -2. Define `ReviewRequest`, `ReviewRule`, `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, and `ReviewDiagnostic`. -3. Define target/context boundary rules. -4. Define structured output shape once. -5. Define execution strategy: `direct`, `reader`, and `auto`. - -### Phase 3: Share Result Projection - -1. Extract evidence location and verification. -2. Extract filtering. -3. Extract severity and scoring. -4. Extract output formatter inputs. -5. Make standard mode and any executor mode use the same projection path. - -### Phase 4: Replace The Agent Loop With Executors - -1. Keep API model execution as direct execution. -2. Add target-scoped reader execution as another executor. -3. Add headless CLI execution behind the same review contract if still useful. -4. Pass the same review request to each executor. -5. Remove model-controlled rule overrides. -6. Remove workspace read/search tools from core review. - -### Phase 5: Update Documentation - -1. Mark the agentic capabilities spec superseded. -2. Write a new harness architecture spec. -3. Update README and CLI docs. -4. Document the bounded harness architecture that replaces the unreleased autonomous workspace-agent implementation path. - -## Suggested Target Architecture - -```text -CLI / API / External Agent - -> ReviewRequestBuilder - -> ReviewExecutor - -> ApiModelExecutor - -> ReaderExecutor - -> HeadlessCliExecutor - -> ResultProjection - -> EvidenceVerifier - -> ViolationFilter - -> ScoreCalculator - -> Diagnostics - -> OutputFormatter - -> line - -> json - -> rdjson - -> vale-json -``` - -The important inversion is that VectorLint no longer asks, "What tools should this agent use?" It asks, "Given this target, these rules, this context, this execution strategy, and this budget, what findings can be verified?" - -## Final Recommendation - -The decided direction is cleaner than the current architecture. The current pain is not a sign that VectorLint is too complex; it is a sign that the codebase is carrying two ownership models and several duplicated runtime contracts. - -Make VectorLint the harness. Let external agents be agents. Keep reader execution only as a bounded way to manage target-content context. - ---- - -## Appendix: Phase 1 — Completed (2026-07-13) - -Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on branch `codex/ci/harness-stop-the-bleeding`. It shifts the verification baseline recorded in "Verification Results" above and addresses Finding 1 at the CLI boundary. - -**Shifted baseline.** The 2026-07-13 pre-work capture differed from the original verification results: `npm run lint` and `npm run test:run` already passed, so the earlier lint failure and the four `ora` / `@langfuse/otel` module-resolution failures were stale. Only `npx tsc --noEmit` was still failing. Phase 1 cleared those remaining type errors (narrowed at boundaries; no strict compiler options relaxed) and added durable gates so the baseline cannot silently regress: - -- `npm run typecheck` (`tsc --noEmit`) and `npm run verify` (typecheck + lint + test:run) scripts. -- `vitest.config.ts` for stable test module resolution. -- `docs/research/**` excluded from linting. -- `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. - -**Current state at audit time.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the unreleased autonomous workspace-agent surface is quarantined at the CLI boundary: `--mode` is blocked so internal agent-mode wiring cannot be reached from the CLI. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. A pointer to this audit lives in the README `## Agent Mode` section and the `--mode` help text. - -This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index d5870870..9bcc1ac5 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -72,10 +72,6 @@ VectorLint reviews only the target content and explicit review context supplied This makes VectorLint safe to call from external agents and CI systems: the caller owns exploration, and VectorLint owns constrained review. -## More detail - -Read the current [harness architecture spec](https://github.com/TRocket-Labs/vectorlint/blob/main/docs/specs/2026-07-10-harness-architecture.md) for the code-level contract and removed surfaces. - ## Next steps - [Model calls](/model-calls) — choose `single`, `agent`, or `auto` diff --git a/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md b/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md deleted file mode 100644 index 233c16b2..00000000 --- a/docs/logs/2026-03-31-agent-mode-implementation-plan.log.md +++ /dev/null @@ -1,28 +0,0 @@ -# Execution Log - -> **SUPERSEDED (2026-07-10).** This log describes the removed autonomous -> agent-mode implementation. VectorLint is now a bounded review harness with -> `single`/`agent`/`auto` model calls. See -> [`../specs/2026-07-10-harness-architecture.md`](../specs/2026-07-10-harness-architecture.md) -> for the current architecture. The details below are historical only. - -- **Plan**: `docs/plans/2026-03-31-agent-mode-implementation-plan.md` -- **Issue**: Not provided (executed from user directive in this session) -- **Started**: 2026-03-31 -- **Status**: completed - ---- - -## Tasks - -### Task: Implement agent mode runtime, wiring, and contracts from red tests -- **Status**: completed -- **What was done**: Implemented the provider agent-loop contract, built the new agent runtime modules (types, session store, path safety, progress, executor), wired CLI/orchestrator `--mode agent` and `--print`, and updated README agent-mode documentation. -- **Files changed**: `src/providers/llm-provider.ts`, `src/providers/vercel-ai-provider.ts`, `src/agent/types.ts`, `src/agent/review-session-store.ts`, `src/agent/path-utils.ts`, `src/agent/progress.ts`, `src/agent/executor.ts`, `src/cli/types.ts`, `src/schemas/cli-schemas.ts`, `src/cli/commands.ts`, `src/cli/orchestrator.ts`, `tests/providers/vercel-ai-provider-agent-loop.test.ts`, `README.md` -- **Tried**: Initial agent-mode wiring used `process.cwd()` as repository root, which broke tool-relative file resolution in orchestrator tests; switched to inferred common root across targets for agent execution. - -### Task: Wire lint context, user instructions, and request-failure attribution -- **Status**: completed -- **What was done**: Added per-call lint context prompt augmentation, passed `userInstructionContent` through agent-mode orchestrator into executor system prompt construction, and split request-failure counting from operational finalize/workflow errors in agent mode. -- **Files changed**: `src/agent/executor.ts`, `src/agent/prompt-builder.ts`, `src/cli/orchestrator.ts`, `tests/agent/agent-executor.test.ts`, `tests/orchestrator-agent-output.test.ts`, `tests/agent/prompt-builder.test.ts` -- **Tried**: Initial test assertions for prompt-builder expected older section headings (`Role:`/`Operating Policy`); updated assertions to match current builder copy while preserving behavior checks. diff --git a/docs/specs/2026-07-10-harness-architecture.md b/docs/specs/2026-07-10-harness-architecture.md deleted file mode 100644 index 52125a33..00000000 --- a/docs/specs/2026-07-10-harness-architecture.md +++ /dev/null @@ -1,199 +0,0 @@ -# VectorLint Harness Architecture - -Status: Current as of the bounded harness refactor. - -Related audit: [`2026-07-10-vectorlint-harness-architecture-audit.md`](../audits/2026-07-10-vectorlint-harness-architecture-audit.md) - -## Purpose And Product Direction - -VectorLint is a bounded content review harness. External callers, such as Codex, Claude Code, CI jobs, scripts, or humans at the CLI, own exploration and decide what content to review. VectorLint owns constrained review of supplied target content against source-backed rules. - -This means VectorLint is not a workspace agent. It does not search arbitrary files, expand scope on its own, rewrite content, or let a model override the rule it is supposed to enforce. It receives target content, rules, optional caller-supplied review context, budgets, and output policy, then returns structured review results. - -Use the shared domain language in [`../../CONTEXT.md`](../../CONTEXT.md) when changing code, docs, tests, or agent handoffs. - -## Review Contract - -The review contract lives in [`src/review/types.ts`](../../src/review/types.ts), with boundary schemas in [`src/review/schemas.ts`](../../src/review/schemas.ts). - -`ReviewRequest` contains: - -- `target`: the target URI, content, content type, and optional byte length. -- `rules`: source-backed rules with `id`, `source`, `body`, optional `name`, optional `severity`, and optional Via Negativa violation conditions. -- `context`: optional caller-supplied review context. -- `budget`: explicit bounds for one review. -- `outputPolicy`: usage and payload telemetry policy. -- `modelCall`: `single`, `agent`, or `auto`. - -Every executor implements: - -```ts -interface ReviewExecutor { - run(request: ReviewRequest): Promise; -} -``` - -`ReviewResult` contains: - -- `findings`: verified findings anchored in target content. -- `scores`: per-rule scores. -- `diagnostics`: operational and finding-processing notes. -- `usage`: optional model-call and token metadata. -- `hadOperationalErrors`: optional flag for partial-result runs with operational errors. - -The contract deliberately does not expose `evaluator`, `judge`, subjective rubric scoring, model-authored rule overrides, or autonomous workspace tools. - -## Rule Model - -VectorLint rules are objective Via Negativa checks. A rule should name observable violation conditions and ask the reviewer to find evidence that those conditions are present. The reviewer should not grade broad alignment against an ideal. - -Good rules answer yes/no questions: - -- Is this sentence using a banned phrase? -- Does this claim omit required evidence? -- Does this section repeat information already stated nearby? - -Future-facing rule authors should avoid subjective judge/rubric designs. Historical rubric-style language remains only where it describes removed behavior. - -## Model Calls - -The model-call strategy is selected with `--model-call single|agent|auto`. The allowed values and default come from [`src/review/executor.ts`](../../src/review/executor.ts), [`src/cli/types.ts`](../../src/cli/types.ts), and [`src/schemas/cli-schemas.ts`](../../src/schemas/cli-schemas.ts). - -### `single` - -`single` uses [`SingleModelCallExecutor`](../../src/executors/single-model-call-executor.ts). It performs one structured model call per rule/chunk through `StructuredModelClient`. For targets above 600 words, it chunks line-numbered content and merges violations before shared finding processing. - -The single executor owns no tool surface. - -### `agent` - -`agent` uses [`AgentModelCallExecutor`](../../src/executors/agent-model-call-executor.ts). It performs one bounded tool-calling run per rule through `ToolCallingModelClient`. - -The only executor-owned tool is `read_target_section`, defined in [`src/executors/target-read-capability-adapter.ts`](../../src/executors/target-read-capability-adapter.ts). It reads 1-based line ranges from in-memory `request.target.content` and returns model-visible errors for invalid ranges. - -The agent model call cannot: - -- search the workspace; -- read arbitrary files; -- read URIs outside the target content; -- rewrite rules; -- create top-level workspace findings; -- override source-backed rule prompts. - -### `auto` - -`auto` resolves through `chooseModelCall` in [`src/review/executor.ts`](../../src/review/executor.ts). It selects `agent` when target content is larger than `AGENT_MODEL_CALL_BYTE_THRESHOLD` (`600_000` bytes) or when more than five rules apply. Otherwise, it selects `single`. - -The CLI default is `auto`. - -## On-Page Boundary - -The on-page boundary is implemented in [`src/review/boundary.ts`](../../src/review/boundary.ts) and enforced by executor design. - -Target content is always in scope. Caller-supplied review context is in scope only because the caller explicitly provided it. Workspace files are out of scope unless the caller passes their content as review context. Rule bodies are source-backed and loaded before the review request is built. - -The target-read adapter performs no filesystem reads. It slices the target content already present in memory. - -## Budgets - -Default budgets live in [`src/review/budget.ts`](../../src/review/budget.ts): - -| Field | Default | Meaning | -| --- | ---: | --- | -| `maxTargetBytes` | `1_000_000` | maximum target content size | -| `maxCallerContextBytes` | `500_000` | maximum caller-supplied context size | -| `maxChunksPerRule` | `20` | maximum chunks/tool steps per rule | -| `maxModelCallsPerReview` | `50` | maximum model calls in one review | -| `maxWallClockMs` | `300_000` | maximum elapsed review time, in milliseconds | - -Budgets limit work, not output. There is intentionally no `maxFindingsPerRule`. There is also no headless retry budget because VectorLint no longer has a headless autonomous executor. - -`maxWallClockMs` is the review timeout. Executors check model-call count and elapsed time before model calls and surface budget exhaustion as diagnostics when partial results can be returned. - -## Finding Processing - -Both model-call strategies use the same finding-processing pipeline in [`src/findings/`](../../src/findings/): - -1. Filter candidate findings through the evidence gate. -2. Verify finding evidence against target content. -3. Deduplicate verified findings. -4. Score by verified finding count and density. -5. Resolve severity. -6. Assemble findings, scores, diagnostics, and operational status. - -Unlocatable quoted evidence becomes a `finding-evidence-not-locatable` diagnostic and emits no finding. VectorLint does not fall back to model-provided line numbers for unverified evidence. - -Diagnostics describe operational or finding-processing conditions such as unlocatable evidence, budget exhaustion, schema parse failures, and provider failures. They are not content findings. - -## Provider And Model Capabilities - -Providers are transport capabilities, not product owners. - -[`StructuredModelClient`](../../src/providers/structured-model-client.ts) performs one structured model call and returns validated output. - -[`ToolCallingModelClient`](../../src/providers/tool-calling-model-client.ts) performs one bounded tool-calling generation using caller-supplied tools and returns structured output. The provider does not define product tools. The executor supplies the tool map and run bounds. - -Neither provider capability is an autonomous agent loop. - -## Architecture Diagram - -```text -CLI / External Caller - | - v -ReviewRequest - - target - - source-backed rules - - caller-supplied context - - budget - - output policy - - modelCall - | - v -chooseModelCall(auto?) ──> single ──> SingleModelCallExecutor - └─> agent ──> AgentModelCallExecutor - | - v - read_target_section - target content only - | - v -Shared Finding Processing - - filter - - verify evidence - - dedupe - - score - - diagnostics - | - v -ReviewResult - | - v -Formatters - - line - - json - - vale-json - - rdjson -``` - -## Internal Implementation Notes - -`--model-call` is the documented CLI surface. The internal `--mode` flag is rejected at the CLI boundary so unreleased agent-mode wiring cannot be used accidentally. - -The unreleased autonomous workspace implementation path is removed. External callers that need project-wide exploration should gather context before invoking VectorLint and pass selected content explicitly. - -Subjective `judge`/rubric rules are not part of the future-facing architecture. New rules should be written as objective Via Negativa checks with observable violation conditions. - -## References - -- [`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](../audits/2026-07-10-vectorlint-harness-architecture-audit.md) -- [`src/review/types.ts`](../../src/review/types.ts) -- [`src/review/executor.ts`](../../src/review/executor.ts) -- [`src/review/budget.ts`](../../src/review/budget.ts) -- [`src/review/boundary.ts`](../../src/review/boundary.ts) -- [`src/executors/single-model-call-executor.ts`](../../src/executors/single-model-call-executor.ts) -- [`src/executors/agent-model-call-executor.ts`](../../src/executors/agent-model-call-executor.ts) -- [`src/executors/target-read-capability-adapter.ts`](../../src/executors/target-read-capability-adapter.ts) -- [`src/findings/processor.ts`](../../src/findings/processor.ts) -- [`src/providers/structured-model-client.ts`](../../src/providers/structured-model-client.ts) -- [`src/providers/tool-calling-model-client.ts`](../../src/providers/tool-calling-model-client.ts) From 3b962987bd162896634f3509e3918a72c3c7891d Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Fri, 24 Jul 2026 07:10:50 +0100 Subject: [PATCH 31/34] docs: align harness language with reviews - Restore main's review terminology and current project structure. - Remove stale evaluator, rubric, and technical-accuracy concepts. - Preserve the domain-language and documentation-boundary guidance. --- AGENTS.md | 84 +++++++++++++++++++++++------------------ CONTEXT.md | 6 +-- README.md | 2 +- docs/best-practices.mdx | 10 ++--- docs/how-it-works.mdx | 2 +- docs/introduction.mdx | 12 ++---- 6 files changed, 61 insertions(+), 55 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9b854236..b5848117 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,9 +1,31 @@ # Repository Guidelines -This repository implements VectorLint — a prompt‑driven, structured‑output content review harness. Use this guide to navigate the codebase, run it locally, and contribute safely. +This repository implements VectorLint — a prompt‑driven, structured‑output content reviewer. Use this guide to navigate the codebase, run it locally, and contribute safely. Use [`CONTEXT.md`](./CONTEXT.md) as the shared VectorLint domain language across code, docs, tests, and agent work. Prefer its terms when naming modules, writing tests, drafting docs, and describing architecture. +## Project Structure & Module Organization + +- `src/` + - `index.ts` — CLI entry; orchestrates config, discovery, review, reporting + - `boundaries/` — external data validation (config, CLI args, env vars, YAML, API responses) + - `chunking/` — content chunking for large documents (recursive chunker, merger, utilities) + - `cli/` — command definitions and CLI orchestration + - `config/` — configuration loading and management + - `errors/` — custom error types and validation errors + - `executors/` — bounded single-call and target-paging review strategies + - `findings/` — finding verification, filtering, severity, and scoring + - `output/` — TTY formatting (reporter, evidence location, line numbering) + - `prompts/` — YAML frontmatter parsing, schema validation, directive loading + - `providers/` — LLM abstractions (OpenAI, Anthropic, Azure, Gemini), request builder, provider factory + - `scan/` — file discovery (fast‑glob) honoring config and exclusions + - `review/` — shared review contracts, budgets, boundaries, and request building + - `schemas/` — Zod schemas for all external data (API responses, config, CLI, env) + - `scoring/` — density-based score calculation + - `types/` — TypeScript type definitions +- `presets/` — bundled rule packs (e.g., `VectorLint/`) +- `tests/` — Vitest specs for config, scanning, review, providers + ## Agent Behavior Guidelines These guidelines reduce common LLM coding mistakes. They bias toward caution over speed; use judgment for trivial tasks. @@ -42,36 +64,28 @@ These guidelines reduce common LLM coding mistakes. They bias toward caution ove - For multi-step tasks, state a brief plan with verification for each step. - Loop until the success criteria are verified or a blocker is clear. +### Error Organization + +- Put domain error classes in focused error files and import them into the modules that throw them. +- Reuse the repository's custom error hierarchy instead of throwing native `Error` directly. +- Catch errors as `unknown`; narrow or convert them with the existing error utilities. +- Introduce error handling only for real failure modes, not speculative or impossible scenarios. + +### Comments + +- Write comments only when they explain non-obvious current behavior, invariants, or constraints. +- Do not preserve planning discussions, rejected alternatives, absent fields, or speculative future architecture in production comments. +- Do not use comments to restate code that is already clear from names and types. +- Keep security and scope-boundary comments when they explain constraints that the implementation must preserve. +- Test supported behavior and current invariants; do not add negative tests solely to memorialize rejected designs. + ### Documentation Artifact Boundaries - Do not commit raw planning or investigation artifacts to the product codebase. - Keep audits, plans, specs, run notes, and similar coordination artifacts out of tracked repo paths such as `docs/audits/`, `docs/plans/`, `docs/specs/`, `audits/`, `plans/`, and `specs/`. - Store coordination artifacts in `.agent-runs/` or another ignored workspace location. - If a durable architectural decision must be committed, write it as an ADR. ADRs are the only allowed committed decision/planning artifacts. -- Product documentation may describe shipped behavior, configuration, and usage, but it must not preserve internal audit/plan/spec documents as reviewed product docs. - -## Project Structure & Module Organization - -- `src/` - - `index.ts` — CLI entry; orchestrates config, discovery, evaluation, reporting - - `boundaries/` — external data validation (config, CLI args, env vars, YAML, API responses) - - `chunking/` — content chunking for large documents (recursive chunker, merger, utilities) - - `cli/` — command definitions and CLI orchestration - - `config/` — configuration loading and management - - `errors/` — custom error types and validation errors - - `evaluators/` — evaluation logic (base evaluator, registry, specific evaluators) - - `output/` — TTY formatting (reporter, evidence location, line numbering) - - `prompts/` — YAML frontmatter parsing, schema validation, directive loading - - `providers/` — LLM abstractions (OpenAI, Anthropic, Azure, Gemini), request builder, provider factory - - `review/` — neutral review contract, boundary, budget, schemas, and model-call selection - - `executors/` — bounded `single` and `agent` model-call executors behind `ReviewExecutor` - - `findings/` — shared finding verification, filtering, scoring, diagnostics, and result assembly - - `scan/` — file discovery (fast‑glob) honoring config and exclusions - - `schemas/` — Zod schemas for all external data (API responses, config, CLI, env) - - `scoring/` — score calculation for objective violation checks - - `types/` — TypeScript type definitions -- `presets/` — bundled rule packs (e.g., `VectorLint/`) -- `tests/` — Vitest specs for config, scanning, evaluation, providers +- Product documentation may describe shipped behavior, configuration, and usage, but it must not preserve internal audit, plan, or spec documents as reviewed product docs. ## Configuration System @@ -110,7 +124,7 @@ Rules must be organized into subdirectories (packs) within `RulesPath`. ### Zero-Config Mode -If you just want to evaluate against a user instruction guide without specific rules: +If you just want to review against a user instruction guide without specific rules: 1. Create a `VECTORLINT.md` file with your user instruction content 2. Run `vectorlint doc.md` — VectorLint creates a synthetic rule from your user instructions @@ -129,19 +143,18 @@ If you just want to evaluate against a user instruction guide without specific r VectorLint assembles prompts in this order: -1. **Directive** (`src/prompts/directive-loader.ts`) — Role definition, task, and evaluation instructions +1. **Directive** (`src/prompts/directive-loader.ts`) — Role definition, task, and review instructions 2. **User Instructions** (`VECTORLINT.md`) — Optional global style context -3. **Rule** (the prompt body from the rule file) — Specific evaluation criteria +3. **Rule** (the prompt body from the rule file) — Specific review criteria -The content to evaluate is sent as a **user message** with line numbers prepended. +The content to review is sent as a **user message** with line numbers prepended. ### Chunking For documents >600 words, VectorLint automatically chunks content: - Uses recursive splitting (paragraphs → lines → sentences → words) -- Each chunk is evaluated separately +- Each chunk is reviewed separately - Results are merged and deduplicated -- Disable with `evaluateAs: document` in rule frontmatter ## Coding Style & Naming Conventions @@ -177,15 +190,12 @@ For documents >600 words, VectorLint automatically chunks content: ## Architecture & Principles - Boundary validation: all external data (files, CLI, env, APIs) validated at system boundaries using Zod schemas -- Bounded harness model: callers own exploration and context gathering; VectorLint owns constrained review through `single`/`agent`/`auto` model calls behind the `ReviewExecutor` contract -- On-page boundary: executors review target content plus explicit review context only; do not add workspace search, arbitrary file reads, model-authored rule overrides, or autonomous agent behavior - Type safety: strict TypeScript with no `any`; use `unknown` + schema validation for external data -- Dependency inversion: depend on `StructuredModelClient`, `ToolCallingModelClient`, and `SearchProvider` interfaces; keep providers thin (transport only) +- Dependency inversion: depend on `StructuredModelClient` and `ToolCallingModelClient`; keep providers thin (transport only) - Dependency injection: inject `RequestBuilder` via provider constructor to avoid coupling -- Separation of concerns: rules define observable violation checks; schemas enforce structure; CLI orchestrates; executors run model calls; findings process results; reporters format output +- Separation of concerns: rules define criteria; schemas enforce structure; CLI orchestrates; executors review; finding processors verify; reporters format - Separation of concerns: when a file starts combining contracts, orchestration, and utility logic, extract shared helpers and types into focused modules -- Extensibility: add model providers by implementing the structured/tool-calling model client interfaces -- Error handling: prefer the repository's custom error hierarchy over native `Error`; catch blocks use `unknown` type and extend existing custom error types before introducing raw exceptions +- Extensibility: add providers by implementing the model-client contracts; add review strategies behind `ReviewExecutor` - Shared domain constants: avoid magic strings for core runtime concepts; define shared constants, enums, or types and import them where needed - Naming: choose domain-accurate names that reflect the real abstraction level; avoid use-case-specific terminology in shared runtime code - Logging: route runtime logging through an injected logger interface; keep concrete logger implementations behind the abstraction diff --git a/CONTEXT.md b/CONTEXT.md index 7a245ff6..34b48e9a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -11,7 +11,7 @@ A system that reviews supplied target content against source-backed rules and re _Avoid_: Autonomous agent, workspace agent, agent mode **Review**: -One evaluation of target content against one or more rules. A review produces findings, scores, diagnostics, usage metadata, and output. +A review of target content against one or more rules. It produces findings, scores, diagnostics, usage metadata, and output. _Avoid_: Lint run when referring to the domain process, agent session ### Review Inputs @@ -48,7 +48,7 @@ _Avoid_: Prompt when referring to the domain concept, model instruction when imp **Via Negativa Review**: A review approach that looks for evidence a rule was violated rather than evidence the content aligns with an ideal. VectorLint rules should be written so a model can answer whether an observable violation condition is present. -_Avoid_: Alignment review, subjective assessment, rubric judgment +_Avoid_: Alignment review, subjective assessment **Violation Condition**: An observable yes/no condition that counts as a rule violation when present in target content. A violation condition should be specific enough to ground a finding in finding evidence. @@ -70,7 +70,7 @@ _Avoid_: Direct rule, standard rule **Model Call**: The call shape VectorLint uses to run a reviewer model against target content during a review. -_Avoid_: Execution strategy, content access, process mode, evaluator type, rule type, mode +_Avoid_: Execution strategy, content access, process mode, rule type, mode **Single Model Call**: A model call where VectorLint supplies the review request and target content without giving the model a read tool. If VectorLint chunks a large target, each chunk is still reviewed through a single model call. diff --git a/README.md b/README.md index 113378f2..d72fbf8b 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ VectorLint scores your content using error density, enabling you to measure qual ## How VectorLint Reduces False Positives -VectorLint uses a PAT (Pay A Tax) evaluation approach: +VectorLint uses a PAT (Pay A Tax) review approach: 1. **Candidate generation:** the model returns all potential violations with required gate-check fields (rule support, exact evidence, context support, plausible non-violation, and fix quality). 2. **Deterministic surfacing:** VectorLint applies a strict filter and only surfaces violations that pass all required gates. diff --git a/docs/best-practices.mdx b/docs/best-practices.mdx index a4e72760..7b3bb2c1 100644 --- a/docs/best-practices.mdx +++ b/docs/best-practices.mdx @@ -13,7 +13,7 @@ After you run `VECTORLINT.md` against your content library and complete your fir ## Keep VECTORLINT.md under 800 tokens -VectorLint emits a warning at 4,000 tokens, but a practical target is *much* lower. Under 800 tokens leaves headroom for rule-specific prompts to add context without the combined system prompt becoming unwieldy. Long context degrades LLM precision and increases API costs on every evaluation. +VectorLint emits a warning at 4,000 tokens, but a practical target is *much* lower. Under 800 tokens leaves headroom for rule-specific prompts to add context without the combined system prompt becoming unwieldy. Long context degrades LLM precision and increases API costs on every review. If your `VECTORLINT.md` is growing, that's usually a sign that some rules are specific enough to belong in a dedicated rule pack file instead. @@ -28,7 +28,7 @@ Check if the writing is clear. Write: ``` -You are a clarity evaluator for developer documentation. Flag sentences that: +You are a clarity reviewer for developer documentation. Flag sentences that: 1. Exceed 25 words 2. Use passive voice where active voice is possible 3. Contain filler phrases: "it is important to note", "please be aware", "in order to" @@ -38,7 +38,7 @@ The second prompt gives the LLM a defined audience, measurable criteria, and spe ## Give rules domain context -LLMs evaluate content relative to an implied standard. Make that standard explicit with a context block in your rule prompt. Tell the model who the audience is, what they value, and what good looks like for your specific content type. +LLMs review content relative to an implied standard. Make that standard explicit with a context block in your rule prompt. Tell the model who the audience is, what they value, and what good looks like for your specific content type. ```markdown ## CONTEXT BANK @@ -99,7 +99,7 @@ The goal is a workflow where every finding is worth reading. That takes iteratio ## Set a higher confidence threshold in CI than locally -In CI, a false positive blocks a merge. Set `CONFIDENCE_THRESHOLD` higher in your CI environment than in local development so only the highest-confidence findings gate a merge. Lower-confidence candidates still surface locally where a writer can evaluate them in context. +In CI, a false positive blocks a merge. Set `CONFIDENCE_THRESHOLD` higher in your CI environment than in local development so only the highest-confidence findings gate a merge. Lower-confidence candidates still surface locally where a writer can review them in context. ```bash # Local development — catch more, review in context @@ -131,6 +131,6 @@ Skipping this step leads to rules that look correct on paper but produce noise i ## Next steps -- [Tuning evaluation precision](/false-positive-tuning) — detailed guidance on CONFIDENCE_THRESHOLD and strictness +- [Tuning review precision](/false-positive-tuning) — detailed guidance on CONFIDENCE_THRESHOLD and strictness - [CI Integration](/ci-integration) — set up content quality gates in your pipeline - [Customize style rules](/customize-style-rules) — write effective prompts for rule pack files diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index 9bcc1ac5..5e7ad395 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -28,7 +28,7 @@ The CLI builds a `ReviewRequest` with: - output policy - model-call strategy -The review contract is intentionally neutral. It does not include workspace exploration tools, model-authored rule overrides, or subjective rubric scoring. +The review contract contains only the inputs needed for a bounded content review. ### 3. Select a model call diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 0122713f..505970d1 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -5,13 +5,13 @@ description: "VectorLint is a large language model (LLM)-based prose linter that ## What is VectorLint? -VectorLint is a command-line content review harness that uses large language models (LLMs) to evaluate documentation. Instead of regex patterns that can only catch surface-level issues, VectorLint looks for observable terminology, technical accuracy, and style violations that require contextual understanding to detect. +VectorLint is a command-line tool that reviews and scores documentation using large language models (LLMs). Unlike regex patterns that catch only surface-level issues, VectorLint can find terminology misuse, unsupported claims, and style inconsistencies that require contextual understanding. If you can write a prompt for it, you can lint it with VectorLint. ## Why VectorLint exists -Traditional prose linters like Vale work by matching text against fixed regex patterns and word lists. They catch what you explicitly tell them to catch — but they can't reason about meaning, context, or technical accuracy. +Traditional prose linters like Vale work by matching text against fixed regex patterns and word lists. They catch what you explicitly tell them to catch — but they can't reason about meaning, context, or internal consistency. VectorLint fills that gap. You define a rule once as a Markdown prompt, and the LLM applies it across your entire content library — scoring each document, surfacing specific violations, and explaining why each violation matters. @@ -36,11 +36,7 @@ This gives documentation teams something they haven't had before: a **shared, me ## How scoring works -VectorLint uses two scoring methods depending on the rule type: - -**Density-based scoring** is used for rules that count discrete violations (like a grammar checker). VectorLint calculates scores based on error density — errors per 100 words — so results are comparable across documents of any length. - -**Rubric-based scoring** is used for rules that measure quality on a spectrum (like tone or completeness). The LLM scores each criterion on a 1–4 scale, which VectorLint normalizes to a 1–10 scale for consistent reporting. +VectorLint calculates scores from error density—verified violations per 100 words—so results are comparable across documents of different lengths. ## How false positives are reduced @@ -49,7 +45,7 @@ VectorLint filters raw LLM candidates through a series of gate checks before sur 1. **Candidate generation** — the LLM returns all potential violations, each with required gate-check fields: rule support, exact evidence, context support, plausible non-violation, and fix quality. 2. **Deterministic filtering** — VectorLint applies a strict filter and only surfaces violations that pass all required gates. -VectorLint's output is intentionally stricter than raw model candidates; it reports only findings that pass all gates. You can tune how aggressively the pipeline filters findings to match your content workflow. See [Tuning evaluation precision](/false-positive-tuning). +VectorLint's output is intentionally stricter than raw model candidates; it reports only findings that pass all gates. You can tune how aggressively the pipeline filters findings to match your content workflow. See [Tuning review precision](/false-positive-tuning). ## Next steps From 210822de447b8178686066f32748763a256be0f3 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Fri, 24 Jul 2026 08:33:15 +0100 Subject: [PATCH 32/34] docs: align public docs with measurable content quality - Define VectorLint through observable standards and measurable feedback - Remove internal review contracts and processing details from user docs - Add terminology boundaries for contributor and public documentation --- AGENTS.md | 4 +++ README.md | 77 +++++++++++++++++------------------------- docs/cli-reference.mdx | 14 ++++---- docs/docs.json | 4 +-- docs/how-it-works.mdx | 56 +++++++----------------------- docs/model-calls.mdx | 61 +++++++++++---------------------- 6 files changed, 77 insertions(+), 139 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b5848117..413eb997 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,6 +86,10 @@ These guidelines reduce common LLM coding mistakes. They bias toward caution ove - Store coordination artifacts in `.agent-runs/` or another ignored workspace location. - If a durable architectural decision must be committed, write it as an ADR. ADRs are the only allowed committed decision/planning artifacts. - Product documentation may describe shipped behavior, configuration, and usage, but it must not preserve internal audit, plan, or spec documents as reviewed product docs. +- `CONTEXT.md` is the project’s ubiquitous language. It may define internal domain concepts needed by contributors, provided those concepts use the same terminology users encounter. +- `AGENTS.md` should require consistent terminology across code, tests, and documentation. +- User-facing `/docs` should use only the subset of that language needed for user goals. +- Internal contracts, processing stages, tool names, architecture constraints, and rejected alternatives should not leak into `/docs` merely because they exist in `CONTEXT.md`. ## Configuration System diff --git a/README.md b/README.md index d72fbf8b..42643505 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -# VectorLint: Prompt it, Lint it! [![npm version](https://img.shields.io/npm/v/vectorlint.svg)](https://www.npmjs.com/package/vectorlint) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +# VectorLint [![npm version](https://img.shields.io/npm/v/vectorlint.svg)](https://www.npmjs.com/package/vectorlint) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -VectorLint is a command-line content review harness that uses LLMs to find observable terminology, technical accuracy, and style violations that require contextual understanding. +VectorLint is a content review harness that turns observable quality standards into measurable feedback for agents. + +VectorLint reviews content against rules that describe observable traits. It returns structured, source-grounded findings and quality scores, giving agents repeatable signals they can use to revise content and review it again. ![VectorLint Screenshot](./assets/VectorLint_screenshot.jpeg) @@ -30,35 +32,27 @@ Run VectorLint without installing: npx vectorlint path/to/article.md ``` -## Enforce Your Style Guide +## Define Your Quality Standards -Define rules as Markdown files with YAML frontmatter to enforce your specific content standards: +Define rules as Markdown files with YAML frontmatter. Each rule describes the observable traits that indicate content does not meet one of your standards. -- **Check SEO Optimization** - Verify content follows SEO best practices -- **Detect AI-Generated Content** - Identify artificial writing patterns -- **Verify Technical Accuracy** - Catch outdated or incorrect technical information -- **Ensure Tone & Voice Consistency** - Match content to appropriate tone for your audience +Rules can identify prohibited terminology, unsupported claims, repetitive explanations, vague guidance, or other quality issues specific to your content. -If you can write a prompt for it, you can lint it with VectorLint. +VectorLint works best when each rule states what evidence counts as a finding instead of asking for a general judgment of whether the content is good. 👉 **[Learn how to create custom rules →](./CREATING_RULES.md)** ## Quality Scores -VectorLint scores your content using error density, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. - -- **Density-Based Scoring:** For errors that can be counted, scores are calculated based on **error density (errors per 100 words)**, making quality assessment fair across documents of any length. - -## How VectorLint Reduces False Positives +VectorLint turns review results into comparable quality signals. Agents can use those scores to measure whether revisions improve content across repeated review cycles. -VectorLint uses a PAT (Pay A Tax) review approach: +For countable findings, VectorLint calculates scores from error density, or findings per 100 words. This makes results comparable across content of different lengths. -1. **Candidate generation:** the model returns all potential violations with required gate-check fields (rule support, exact evidence, context support, plausible non-violation, and fix quality). -2. **Deterministic surfacing:** VectorLint applies a strict filter and only surfaces violations that pass all required gates. +## Grounded Findings -This means CLI output is intentionally stricter than raw model candidates, reducing noisy findings and improving precision. +Every reported finding includes evidence from the reviewed content. VectorLint confirms that evidence can be located in the source and omits findings that cannot be grounded there. -The confidence gate is user-configurable via: +Adjust finding sensitivity with: ```bash CONFIDENCE_THRESHOLD=0.75 @@ -72,15 +66,15 @@ CONFIDENCE_THRESHOLD=0.75 ### 1. Zero-Config Mode (Fastest) -If you just want to check your content against a style guide: +If you want to review content against a single set of quality standards: ```bash vectorlint init --quick ``` -This creates a `VECTORLINT.md` file where you can paste your style guide. +This creates a `VECTORLINT.md` file where you can define your quality standards. -> **Note:** You must set up your credentials in `~/.vectorlint/config.toml` (see Step 3) before running checks. +> **Note:** You must set up your credentials in `~/.vectorlint/config.toml` (see Step 3) before running a review. Then run: @@ -97,14 +91,15 @@ vectorlint init ``` This creates: + - **VectorLint Config** (`.vectorlint.ini`): Project-specific settings. -- **App Config** (`~/.vectorlint/config.toml`): LLM provider API keys. +- **App Config** (`~/.vectorlint/config.toml`): Model provider API keys. 👉 **[Full configuration reference →](./CONFIGURATION.md)** ### 3. Configure API Keys -Open your global **App Config** (`~/.vectorlint/config.toml`) and uncomment the section for your preferred LLM provider (OpenAI, Anthropic, Gemini, or Azure). +Open your global **App Config** (`~/.vectorlint/config.toml`) and uncomment the section for your preferred model provider (OpenAI, Anthropic, Gemini, or Azure). ```toml [env] @@ -114,7 +109,7 @@ OPENAI_API_KEY = "sk-..." > *Note: You can also use a local `.env` file in your project, which takes precedence over the global config.* -**Run a check:** +**Run a review:** ```bash vectorlint doc.md @@ -124,9 +119,9 @@ VectorLint is bundled with a `VectorLint` preset containing rules for AI pattern 👉 **[Learn how to create custom rules →](./CREATING_RULES.md)** -### 4. Optional: Enable Langfuse observability +### 4. Optional: Configure Langfuse observability -VectorLint can emit AI execution telemetry through Langfuse without hardcoding Langfuse into the provider layer. This is best-effort instrumentation for the Vercel AI SDK calls used by `VercelAIProvider`. +VectorLint can send model execution telemetry to Langfuse. Add these environment variables to your global config or local `.env` file: @@ -144,30 +139,20 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## How VectorLint Runs Reviews - -VectorLint is a bounded review harness, not a workspace agent. Callers choose -the target content and any review context; VectorLint reviews that supplied -content against source-backed rules and returns findings, scores, diagnostics, -and usage metadata. +## Choose a Review Approach -Use `--model-call` to choose how the reviewer model is invoked: +VectorLint chooses a review approach automatically. The default works for most +content: ```bash -vectorlint doc.md --model-call single -vectorlint doc.md --model-call agent -vectorlint doc.md --model-call auto +vectorlint doc.md ``` -- `single` runs structured model calls without tools. This is best for normal - documents and straightforward rule checks. -- `agent` runs a bounded target-only model call that can page through the - current target with `read_target_section`. It cannot search the workspace, - read arbitrary files, or override rules. -- `auto` is the default. VectorLint chooses `single` for normal inputs and - `agent` for large or multi-rule reviews that benefit from target paging. +Use `--model-call` when you need to override that choice for a particular +review. Choose `single` for normal, self-contained documents or `agent` for +large documents whose relevant context spans multiple sections. -See [Model calls](docs/model-calls.mdx) for more details. +See [Model calls](docs/model-calls.mdx) for selection guidance and examples. ## Contributing @@ -175,5 +160,5 @@ We welcome your contributions! Whether it's adding new rules, fixing bugs, or im ## Resources -- **[Creating Custom Rules](./CREATING_RULES.md)** - Write your own quality checks in Markdown +- **[Creating Custom Rules](./CREATING_RULES.md)** - Define observable quality standards in Markdown - **[Configuration Guide](./CONFIGURATION.md)** - Complete reference for `.vectorlint.ini` diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index f71d07e2..2755e19d 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -63,8 +63,8 @@ vectorlint --help | `--help` | — | Print help and exit | | `--version` | — | Print the installed VectorLint version and exit | | `--config ` | `.vectorlint.ini` | Path to a custom project config file | -| `--debug-json` | `false` | Write debug JSON artifacts for surfaced findings and review metadata | -| `--model-call ` | `auto` | Model-call strategy: `single`, `agent`, or `auto` | +| `--debug-json` | `false` | Write debug JSON artifacts under `.vectorlint/runs/` | +| `--model-call ` | `auto` | Review approach: `single`, `agent`, or `auto` | | `--output ` | `line` | Output format: `line`, `json`, `vale-json`, or `rdjson` | | `--show-prompt` | `false` | Print full prompt and injected content | | `--show-prompt-trunc` | `false` | Print truncated prompt/content previews | @@ -72,7 +72,7 @@ vectorlint --help ### `--model-call` -`--model-call` controls how VectorLint invokes the reviewer model: +`--model-call` selects the review approach. Leave the default, `auto`, enabled for most reviews. ```bash vectorlint doc.md --model-call single @@ -80,11 +80,11 @@ vectorlint doc.md --model-call agent vectorlint doc.md --model-call auto ``` -| Value | Behavior | +| Value | Use it for | | --- | --- | -| `single` | Runs structured model calls without tools | -| `agent` | Runs bounded target-only paging with `read_target_section` | -| `auto` | Default; chooses `single` or `agent` based on target size and rule count | +| `auto` | Most reviews; VectorLint selects an approach for the current document | +| `single` | Normal, self-contained documents and focused rules | +| `agent` | Large documents or reviews that compare context across sections | See [Model calls](/model-calls) for usage guidance. diff --git a/docs/docs.json b/docs/docs.json index 1c827adb..0f99546d 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -20,8 +20,7 @@ "pages": [ "introduction", "use-cases", - "how-it-works", - "model-calls" + "how-it-works" ] }, { @@ -68,6 +67,7 @@ "group": "Reference", "pages": [ "cli-reference", + "model-calls", "configuration-schema", "env-variables", "frontmatter-fields", diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index 5e7ad395..a6fc51c8 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -1,11 +1,11 @@ --- title: How it works -description: How VectorLint reviews supplied content through a bounded harness. +description: How VectorLint applies rules, verifies findings, and reports results. --- -VectorLint is a content review harness. You give it target content, source-backed rules, provider credentials, and optional review context. VectorLint runs a bounded review and returns verified findings, scores, diagnostics, and usage metadata. +VectorLint reviews target files against your content rules and reports specific, evidence-backed findings. -If you haven't run your first check yet, start with the [Quickstart](/quickstart). This page explains the architecture behind that command. +If you haven't run your first review yet, start with the [Quickstart](/quickstart). This page explains what happens when you run that command. ## The review pipeline @@ -17,61 +17,31 @@ VectorLint reads your project configuration to determine which rule packs apply If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic style-guide rule automatically. This is the zero-config path. -### 2. Build a review request +### 2. Choose a review approach -The CLI builds a `ReviewRequest` with: +VectorLint uses `auto` by default and selects an approach based on the content being reviewed. You can override the default with `--model-call` when a particular document needs a different approach. -- target URI, content, content type, and byte length -- source-backed rules -- optional caller-supplied review context -- review budget -- output policy -- model-call strategy +See [Model calls](/model-calls) for selection guidance and CLI examples. -The review contract contains only the inputs needed for a bounded content review. +### 3. Review the content -### 3. Select a model call +VectorLint reviews the target against each applicable rule. For long documents, it divides the content into manageable sections and combines the results. -VectorLint supports three model-call values: +### 4. Validate and score findings -| Model call | Use it for | -| --- | --- | -| `single` | Normal reviews using structured model calls without tools | -| `agent` | Large or context-heavy reviews where the model needs target-section paging | -| `auto` | Default behavior; VectorLint chooses based on target size and rule count | +Before reporting a finding, VectorLint confirms that its quoted evidence appears in the target content. It removes duplicate findings and calculates a score from the verified violations. -`agent` is bounded to the current target. The only tool it can use is `read_target_section`, which reads line ranges from the target content already supplied to VectorLint. +Findings without locatable evidence are omitted from the reported violations. -See [Model calls](/model-calls) for CLI examples and selection details. +### 5. Present the results -### 4. Process findings - -Both model-call paths send raw candidate findings through the same finding-processing pipeline: - -1. filter candidates through the evidence gate -2. verify finding evidence against target content -3. deduplicate verified findings -4. score by verified finding count and density -5. resolve severity -6. assemble findings, scores, diagnostics, and usage metadata - -Unlocatable quoted evidence becomes a diagnostic and does not become a finding. - -### 5. Format output - -The final `ReviewResult` is routed to the requested output format: +VectorLint presents the verified findings using the requested output format: - `line` for human-readable terminal output - `json` for VectorLint's native machine-readable output - `vale-json` for Vale-compatible integrations - `rdjson` for reviewdog-compatible integrations -## Boundaries - -VectorLint reviews only the target content and explicit review context supplied to it. It does not search your workspace, read arbitrary files, or expand scope on its own. - -This makes VectorLint safe to call from external agents and CI systems: the caller owns exploration, and VectorLint owns constrained review. - ## Next steps - [Model calls](/model-calls) — choose `single`, `agent`, or `auto` diff --git a/docs/model-calls.mdx b/docs/model-calls.mdx index 9f52a6b9..e55f49ea 100644 --- a/docs/model-calls.mdx +++ b/docs/model-calls.mdx @@ -3,66 +3,45 @@ title: Model calls description: Choose single, agent, or auto model calls for VectorLint reviews. --- -VectorLint uses a model-call strategy to decide how the reviewer model sees target content. The strategy changes the invocation shape, not the rule semantics. +VectorLint supports three review approaches through `--model-call`. Leave the default, `auto`, enabled for most reviews. Override it when the size or structure of a particular document calls for a different approach. -## Choose a strategy +## Choose an approach -| Value | Best for | What happens | -| --- | --- | --- | -| `single` | Normal documents and straightforward checks | VectorLint sends structured model calls without tools | -| `agent` | Large or context-heavy targets | VectorLint gives the model one target-only paging tool | -| `auto` | Default CLI behavior | VectorLint chooses `single` or `agent` from target size and rule count | +| Value | Use it for | +| --- | --- | +| `auto` | Most reviews. VectorLint selects an approach for the current document. | +| `single` | Normal, self-contained documents and focused rules. | +| `agent` | Large documents or reviews that need to compare context across multiple sections. | -Run with the default: +## Use the default ```bash vectorlint doc.md ``` -Force structured calls: +This is equivalent to: ```bash -vectorlint doc.md --model-call single +vectorlint doc.md --model-call auto ``` -Force target paging: +## Override the default + +Choose `single` for a normal, self-contained document: ```bash -vectorlint doc.md --model-call agent +vectorlint doc.md --model-call single ``` -## `single` - -`single` is the simplest model call. VectorLint supplies the rule prompt and target content, receives structured candidate findings, and sends them through shared finding processing. - -For large targets, the single path chunks content and merges candidate violations before verification and scoring. - -## `agent` - -`agent` is bounded target paging. The model can call `read_target_section` to read 1-based line ranges from the current target content. - -The agent model call cannot: +Choose `agent` for a large document or when a rule needs context from sections that are far apart: -- search the workspace -- read arbitrary files -- read files by URI -- modify content -- override rules -- report findings outside the target - -Use `agent` when a target is large enough that paging through sections helps the reviewer preserve context. - -## `auto` - -`auto` is the CLI default. VectorLint chooses `agent` when the target is larger than `600_000` bytes or when more than five rules apply. Otherwise, it chooses `single`. - -## Boundary - -VectorLint reviews only supplied target content and explicit review context. If an external agent or script wants VectorLint to consider other files, that caller must gather the relevant content and pass it as review context. VectorLint does not discover workspace context on its own. +```bash +vectorlint doc.md --model-call agent +``` -## Output +Changing `--model-call` does not change which rules run, how findings are scored, or which output formats are available. -All model-call strategies return the same review result shape: findings, scores, diagnostics, and usage metadata. Output format selection is separate: +## Combine with an output format ```bash vectorlint doc.md --model-call agent --output json From 8508c04ce9618aae34c5fc3706a27547303f9a20 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Fri, 24 Jul 2026 08:48:01 +0100 Subject: [PATCH 33/34] docs: standardize review strategy terminology - Use strategy consistently for model-call selection - Simplify the rules resource label --- README.md | 8 ++++---- docs/cli-reference.mdx | 6 +++--- docs/how-it-works.mdx | 4 ++-- docs/model-calls.mdx | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 42643505..9d47d5e2 100644 --- a/README.md +++ b/README.md @@ -139,16 +139,16 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Choose a Review Approach +## Choose a Review Strategy -VectorLint chooses a review approach automatically. The default works for most +VectorLint chooses a review strategy automatically. The default works for most content: ```bash vectorlint doc.md ``` -Use `--model-call` when you need to override that choice for a particular +Use `--model-call` when you need to override that strategy for a particular review. Choose `single` for normal, self-contained documents or `agent` for large documents whose relevant context spans multiple sections. @@ -160,5 +160,5 @@ We welcome your contributions! Whether it's adding new rules, fixing bugs, or im ## Resources -- **[Creating Custom Rules](./CREATING_RULES.md)** - Define observable quality standards in Markdown +- **[Creating Rules](./CREATING_RULES.md)** - Define observable quality standards in Markdown - **[Configuration Guide](./CONFIGURATION.md)** - Complete reference for `.vectorlint.ini` diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index 2755e19d..3aecd2a1 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -64,7 +64,7 @@ vectorlint --help | `--version` | — | Print the installed VectorLint version and exit | | `--config ` | `.vectorlint.ini` | Path to a custom project config file | | `--debug-json` | `false` | Write debug JSON artifacts under `.vectorlint/runs/` | -| `--model-call ` | `auto` | Review approach: `single`, `agent`, or `auto` | +| `--model-call ` | `auto` | Review strategy: `single`, `agent`, or `auto` | | `--output ` | `line` | Output format: `line`, `json`, `vale-json`, or `rdjson` | | `--show-prompt` | `false` | Print full prompt and injected content | | `--show-prompt-trunc` | `false` | Print truncated prompt/content previews | @@ -72,7 +72,7 @@ vectorlint --help ### `--model-call` -`--model-call` selects the review approach. Leave the default, `auto`, enabled for most reviews. +`--model-call` selects the review strategy. Leave the default, `auto`, enabled for most reviews. ```bash vectorlint doc.md --model-call single @@ -82,7 +82,7 @@ vectorlint doc.md --model-call auto | Value | Use it for | | --- | --- | -| `auto` | Most reviews; VectorLint selects an approach for the current document | +| `auto` | Most reviews; VectorLint selects a strategy for the current document | | `single` | Normal, self-contained documents and focused rules | | `agent` | Large documents or reviews that compare context across sections | diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index a6fc51c8..b2adb6db 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -17,9 +17,9 @@ VectorLint reads your project configuration to determine which rule packs apply If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic style-guide rule automatically. This is the zero-config path. -### 2. Choose a review approach +### 2. Choose a review strategy -VectorLint uses `auto` by default and selects an approach based on the content being reviewed. You can override the default with `--model-call` when a particular document needs a different approach. +VectorLint uses `auto` by default and selects a strategy based on the content being reviewed. You can override the default with `--model-call` when a particular document needs a different strategy. See [Model calls](/model-calls) for selection guidance and CLI examples. diff --git a/docs/model-calls.mdx b/docs/model-calls.mdx index e55f49ea..ea09f5e3 100644 --- a/docs/model-calls.mdx +++ b/docs/model-calls.mdx @@ -3,13 +3,13 @@ title: Model calls description: Choose single, agent, or auto model calls for VectorLint reviews. --- -VectorLint supports three review approaches through `--model-call`. Leave the default, `auto`, enabled for most reviews. Override it when the size or structure of a particular document calls for a different approach. +VectorLint supports three review strategies through `--model-call`. Leave the default, `auto`, enabled for most reviews. Override it when the size or structure of a particular document calls for a different strategy. -## Choose an approach +## Choose a strategy | Value | Use it for | | --- | --- | -| `auto` | Most reviews. VectorLint selects an approach for the current document. | +| `auto` | Most reviews. VectorLint selects a strategy for the current document. | | `single` | Normal, self-contained documents and focused rules. | | `agent` | Large documents or reviews that need to compare context across multiple sections. | From f27b5b6604e8d7ee06b74426481cdedfb1d6d43c Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Fri, 24 Jul 2026 10:17:15 +0100 Subject: [PATCH 34/34] docs: clarify credential configuration options - Recognize both global config and local environment files in setup --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d47d5e2..bd2e4339 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ vectorlint init --quick This creates a `VECTORLINT.md` file where you can define your quality standards. -> **Note:** You must set up your credentials in `~/.vectorlint/config.toml` (see Step 3) before running a review. +> **Note:** Before running a review, set up your credentials in either `~/.vectorlint/config.toml` or a local `.env` file (see Step 3). Then run: