diff --git a/AGENTS.md b/AGENTS.md index 0373fad3..bf199f48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,26 +1,28 @@ # Repository Guidelines -This repository implements VectorLint — a prompt‑driven, structured‑output content evaluator. Use this guide to navigate the codebase, run it locally, and contribute safely. +This repository implements VectorLint — a prompt‑driven, structured‑output content reviewer. Use this guide to navigate the codebase, run it locally, and contribute safely. ## Project Structure & Module Organization - `src/` - - `index.ts` — CLI entry; orchestrates config, discovery, evaluation, reporting + - `index.ts` — CLI entry; orchestrates config, discovery, review, reporting - `boundaries/` — external data validation (config, CLI args, env vars, YAML, API responses) - `chunking/` — content chunking for large documents (recursive chunker, merger, utilities) - `cli/` — command definitions and CLI orchestration - `config/` — configuration loading and management - `errors/` — custom error types and validation errors - - `evaluators/` — evaluation logic (base evaluator, registry, specific evaluators) + - `executors/` — bounded single-call and target-paging review strategies + - `findings/` — finding verification, filtering, severity, and scoring - `output/` — TTY formatting (reporter, evidence location, line numbering) - `prompts/` — YAML frontmatter parsing, schema validation, directive loading - `providers/` — LLM abstractions (OpenAI, Anthropic, Azure, Gemini), request builder, provider factory - `scan/` — file discovery (fast‑glob) honoring config and exclusions + - `review/` — shared review contracts, budgets, boundaries, and request building - `schemas/` — Zod schemas for all external data (API responses, config, CLI, env) - `scoring/` — density-based score calculation - `types/` — TypeScript type definitions - `presets/` — bundled rule packs (e.g., `VectorLint/`) -- `tests/` — Vitest specs for config, scanning, evaluation, providers +- `tests/` — Vitest specs for config, scanning, review, providers ## Agent Behavior Guidelines @@ -112,7 +114,7 @@ Rules must be organized into subdirectories (packs) within `RulesPath`. ### Zero-Config Mode -If you just want to evaluate against a user instruction guide without specific rules: +If you just want to review against a user instruction guide without specific rules: 1. Create a `VECTORLINT.md` file with your user instruction content 2. Run `vectorlint doc.md` — VectorLint creates a synthetic rule from your user instructions @@ -131,19 +133,18 @@ If you just want to evaluate against a user instruction guide without specific r VectorLint assembles prompts in this order: -1. **Directive** (`src/prompts/directive-loader.ts`) — Role definition, task, and evaluation instructions +1. **Directive** (`src/prompts/directive-loader.ts`) — Role definition, task, and review instructions 2. **User Instructions** (`VECTORLINT.md`) — Optional global style context -3. **Rule** (the prompt body from the rule file) — Specific evaluation criteria +3. **Rule** (the prompt body from the rule file) — Specific review criteria -The content to evaluate is sent as a **user message** with line numbers prepended. +The content to review is sent as a **user message** with line numbers prepended. ### Chunking For documents >600 words, VectorLint automatically chunks content: - Uses recursive splitting (paragraphs → lines → sentences → words) -- Each chunk is evaluated separately +- Each chunk is reviewed separately - Results are merged and deduplicated -- Disable with `evaluateAs: document` in rule frontmatter ## Coding Style & Naming Conventions @@ -180,11 +181,11 @@ For documents >600 words, VectorLint automatically chunks content: - Boundary validation: all external data (files, CLI, env, APIs) validated at system boundaries using Zod schemas - Type safety: strict TypeScript with no `any`; use `unknown` + schema validation for external data -- Dependency inversion: depend on `LLMProvider` and `SearchProvider` interfaces; keep providers thin (transport only) +- Dependency inversion: depend on `StructuredModelClient` and `ToolCallingModelClient`; keep providers thin (transport only) - Dependency injection: inject `RequestBuilder` via provider constructor to avoid coupling -- Separation of concerns: rules define criteria; schemas enforce structure; CLI orchestrates; evaluators process; reporters format +- Separation of concerns: rules define criteria; schemas enforce structure; CLI orchestrates; executors review; finding processors verify; reporters format - Separation of concerns: when a file starts combining contracts, orchestration, and utility logic, extract shared helpers and types into focused modules -- Extensibility: add providers by implementing `LLMProvider` or `SearchProvider`; add evaluators via registry pattern +- Extensibility: add providers by implementing the model-client contracts; add review strategies behind `ReviewExecutor` - Shared domain constants: avoid magic strings for core runtime concepts; define shared constants, enums, or types and import them where needed - Naming: choose domain-accurate names that reflect the real abstraction level; avoid use-case-specific terminology in shared runtime code - Logging: route runtime logging through an injected logger interface; keep concrete logger implementations behind the abstraction @@ -206,7 +207,3 @@ VectorLint supports multiple output formats via the `--output` flag: - Anthropic: Claude models (Opus, Sonnet, Haiku) - Azure OpenAI: Azure-hosted OpenAI models - Google Gemini: Gemini Pro and other Gemini models - -### Search Providers - -- Perplexity: Sonar models with web search capabilities (used by technical-accuracy evaluator) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index ea7f8506..a6432375 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -15,7 +15,7 @@ VectorLint is configured via a `.vectorlint.ini` file in the root of your projec # Optional: Path to custom rules directory # If omitted, only preset rules (from RunRules) are used # RulesPath=.github/rules -# Number of concurrent evaluations (Default: 4) +# Number of concurrent reviews (Default: 4) Concurrency=4 # Default severity for violations (Default: warning) DefaultSeverity=warning @@ -34,8 +34,6 @@ GrammarChecker.strictness=7 RunRules=Acme # Higher strictness for docs GrammarChecker.strictness=9 -# Require a higher score for technical accuracy -TechnicalAccuracy.threshold=8.0 # Marketing content - run "TechCorp" pack [content/marketing/**/*.md] @@ -56,7 +54,7 @@ These settings control the application's core behavior. | Setting | Type | Default | Description | | ----------------- | ------- | ------------ | ------------------------------------------------------------------ | | `RulesPath` | string | (none) | Root directory for custom rule packs. If omitted, only presets are used. | -| `Concurrency` | integer | `4` | Number of concurrent evaluations to run. | +| `Concurrency` | integer | `4` | Number of concurrent reviews to run. | | `DefaultSeverity` | string | `warning` | Default severity level (`warning` or `error`) for reported issues. | --- @@ -69,36 +67,23 @@ You can place a `VECTORLINT.md` file in your project root to define global style If no `.vectorlint.ini` exists, VectorLint will automatically: 1. Detect `VECTORLINT.md` 2. Create a synthetic "Style Guide Compliance" rule -3. Evaluate your contents against it +3. Review your contents against it ### Combined Mode -If you have configured rules (via `.vectorlint.ini`), the content of `VECTORLINT.md` is **prepended** to the system prompt for every evaluation. This ensures your global style preferences (tone, terminology) are respected across all specific rules. +If you have configured rules (via `.vectorlint.ini`), the content of `VECTORLINT.md` is **prepended** to the system prompt for every review. This ensures your global style preferences (tone, terminology) are respected across all specific rules. > **Note:** Keep `VECTORLINT.md` concise. VectorLint will emit a warning if the file exceeds ~4,000 tokens, as very large contexts can degrade performance and increase costs. --- -## LLM & Search Providers +## LLM Providers -VectorLint relies on LLM and Search providers. These are configured globally in `~/.vectorlint/config.toml`, or project scope using a `.env` file (which takes precedence). +VectorLint relies on an LLM provider. Configure it globally in `~/.vectorlint/config.toml`, or at project scope using a `.env` file (which takes precedence). You can generate these files using the `vectorlint init` command. -### LLM Providers - VectorLint supports multiple LLM providers. Set `LLM_PROVIDER` to your desired provider (e.g., `openai`, `anthropic`, `gemini`) and provide the corresponding API key. -### Search Provider - -Some evaluators, such as **TechnicalAccuracy**, require access to current external knowledge to verify facts. VectorLint supports search providers to fetch this information. - -**Example configuration for Perplexity:** - -```bash -SEARCH_PROVIDER=perplexity -PERPLEXITY_API_KEY=pplx-... -``` - ### Observability VectorLint can optionally emit AI execution telemetry to Langfuse. The first implementation is scoped to the Vercel AI SDK calls made through `VercelAIProvider`. @@ -224,7 +209,7 @@ You can use named levels or direct numeric multipliers: [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=strict -TechnicalAccuracy.strictness=20 +Terminology.strictness=20 ``` --- diff --git a/CREATING_RULES.md b/CREATING_RULES.md index d9c60cdf..961f4883 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -1,6 +1,6 @@ # Creating Rules for VectorLint -A comprehensive guide to creating powerful, reusable content evaluations using VectorLint's prompt system. +A comprehensive guide to creating powerful, reusable content reviews using VectorLint's prompt system. ## Table of Contents @@ -36,9 +36,8 @@ Every rule is a Markdown file with two parts: ```markdown --- # YAML Frontmatter (Configuration) -id: MyEval -name: My Content Evaluator -evaluator: base +id: MyRule +name: My Content Reviewer severity: error --- @@ -57,9 +56,7 @@ project/ │ └── rules/ │ ├── Acme/ ← Company style guide pack │ │ ├── grammar-checker.md -│ │ ├── headline-evaluator.md -│ │ └── Technical/ ← Nested organization supported -│ │ └── technical-accuracy.md +│ │ └── headline-reviewer.md │ └── TechCorp/ ← Another company's pack │ └── brand-voice.md └── .vectorlint.ini @@ -78,7 +75,6 @@ score from the verified findings. ```markdown --- -evaluator: base id: GrammarChecker name: Grammar Checker severity: error @@ -117,7 +113,7 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes The `target` field allows you to: -1. **Specify which part** of content to evaluate (via regex) +1. **Specify which part** of content to review (via regex) 2. **Require certain content** to exist (e.g., "must have an H1 headline") 3. **Provide helpful suggestions** when content is missing @@ -136,13 +132,13 @@ target: **When `required: true`:** -- If content matches → Evaluation proceeds normally +- If content matches → Review proceeds normally - If no match → Immediate `error` with the suggestion message **When `required: false` or omitted:** -- If content matches → Evaluate the matched content -- If no match → Evaluate entire content +- If content matches → Review the matched content +- If no match → Review entire content --- @@ -153,12 +149,10 @@ target: | Field | Type | Required | Description | | ------------- | ------------- | -------- | -------------------------------------------------------------- | | `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | -| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | | `id` | string | **Yes** | Unique identifier (used in error reporting) | | `name` | string | **Yes** | Human-readable name | | `severity` | string | No | `error` or `warning` (default: `warning`) | | `strictness` | number/string | No | Density penalty: a positive number, `lenient`, `standard`, or `strict` | -| `evaluateAs` | string | No | `document` or `chunk` (default: `chunk`) | | `target` | object | No | Content matching specification | --- @@ -178,7 +172,7 @@ Check if the headline is good. ✅ **Good:** ```markdown -You are a headline evaluator for developer blog posts. Assess whether the headline: +You are a headline reviewer for developer blog posts. Assess whether the headline: 1. Clearly communicates a specific benefit 2. Uses natural, conversational language (avoid buzzwords) @@ -208,7 +202,6 @@ Help the LLM understand your domain: ```markdown --- -evaluator: base id: GrammarChecker name: Grammar Checker severity: error @@ -226,4 +219,4 @@ Report any errors found with specific examples. --- -**Happy evaluating! 🚀** +**Happy reviewing! 🚀** diff --git a/README.md b/README.md index bdbef88b..10016f7b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # VectorLint: Prompt it, Lint it! [![npm version](https://img.shields.io/npm/v/vectorlint.svg)](https://www.npmjs.com/package/vectorlint) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -VectorLint is a command-line tool that uses LLMs to find and score terminology, technical accuracy, and style issues that require contextual understanding. +VectorLint is a command-line tool that uses LLMs to find and score terminology, consistency, and style issues that require contextual understanding. ![VectorLint Screenshot](./assets/VectorLint_screenshot.jpeg) @@ -51,7 +51,7 @@ VectorLint scores your content using error density, enabling you to measure qual ## How VectorLint Reduces False Positives -VectorLint uses a PAT (Pay A Tax) evaluation approach: +VectorLint uses a PAT (Pay A Tax) review approach: 1. **Candidate generation:** the model returns all potential violations with required gate-check fields (rule support, exact evidence, context support, plausible non-violation, and fix quality). 2. **Deterministic surfacing:** VectorLint applies a strict filter and only surfaces violations that pass all required gates. diff --git a/docs/VECTORLINT.md b/docs/VECTORLINT.md index 9fe79f70..49a2a514 100644 --- a/docs/VECTORLINT.md +++ b/docs/VECTORLINT.md @@ -1,7 +1,7 @@ # VectorLint example styling rules -{/* This file is prepended to the system prompt for every VectorLint evaluation - This file defines global style instructions for all VectorLint evaluations. +{/* This file is prepended to the system prompt for every VectorLint review + This file defines global style instructions for all VectorLint reviews. VectorLint prepends its contents to the system prompt for every rule it runs. Keep this file under ~800 tokens to avoid performance and cost issues. Adapt the rules below to match your team's style guide. diff --git a/docs/best-practices.mdx b/docs/best-practices.mdx index 9eab45e4..ed807cde 100644 --- a/docs/best-practices.mdx +++ b/docs/best-practices.mdx @@ -13,7 +13,7 @@ After you run `VECTORLINT.md` against your content library and complete your fir ## Keep VECTORLINT.md under 800 tokens -VectorLint emits a warning at 4,000 tokens, but a practical target is *much* lower. Under 800 tokens leaves headroom for rule-specific prompts to add context without the combined system prompt becoming unwieldy. Long context degrades LLM precision and increases API costs on every evaluation. +VectorLint emits a warning at 4,000 tokens, but a practical target is *much* lower. Under 800 tokens leaves headroom for rule-specific prompts to add context without the combined system prompt becoming unwieldy. Long context degrades LLM precision and increases API costs on every review. If your `VECTORLINT.md` is growing, that's usually a sign that some rules are specific enough to belong in a dedicated rule pack file instead. @@ -28,7 +28,7 @@ Check if the writing is clear. Write: ``` -You are a clarity evaluator for developer documentation. Flag sentences that: +You are a clarity reviewer for developer documentation. Flag sentences that: 1. Exceed 25 words 2. Use passive voice where active voice is possible 3. Contain filler phrases: "it is important to note", "please be aware", "in order to" @@ -38,7 +38,7 @@ The second prompt gives the LLM a defined audience, measurable criteria, and spe ## Give rules domain context -LLMs evaluate content relative to an implied standard. Make that standard explicit with a context block in your rule prompt. Tell the model who the audience is, what they value, and what good looks like for your specific content type. +LLMs review content relative to an implied standard. Make that standard explicit with a context block in your rule prompt. Tell the model who the audience is, what they value, and what good looks like for your specific content type. ```markdown ## CONTEXT BANK @@ -85,7 +85,7 @@ The goal is a workflow where every finding is worth reading. That takes iteratio ## Set a higher confidence threshold in CI than locally -In CI, a false positive blocks a merge. Set `CONFIDENCE_THRESHOLD` higher in your CI environment than in local development so only the highest-confidence findings gate a merge. Lower-confidence candidates still surface locally where a writer can evaluate them in context. +In CI, a false positive blocks a merge. Set `CONFIDENCE_THRESHOLD` higher in your CI environment than in local development so only the highest-confidence findings gate a merge. Lower-confidence candidates still surface locally where a writer can review them in context. ```bash # Local development — catch more, review in context @@ -117,6 +117,6 @@ Skipping this step leads to rules that look correct on paper but produce noise i ## Next steps -- [Tuning evaluation precision](/false-positive-tuning) — detailed guidance on CONFIDENCE_THRESHOLD and strictness +- [Tuning review precision](/false-positive-tuning) — detailed guidance on CONFIDENCE_THRESHOLD and strictness - [CI Integration](/ci-integration) — set up content quality gates in your pipeline - [Customize style rules](/customize-style-rules) — write effective prompts for rule pack files diff --git a/docs/ci-integration.mdx b/docs/ci-integration.mdx index 0b34eec1..d02c481e 100644 --- a/docs/ci-integration.mdx +++ b/docs/ci-integration.mdx @@ -155,6 +155,6 @@ Store API keys in your CI system's secret or environment variable manager — ne ## Next steps -- [Tuning evaluation precision](/false-positive-tuning) — set `CONFIDENCE_THRESHOLD` and strictness for CI +- [Tuning review precision](/false-positive-tuning) — set `CONFIDENCE_THRESHOLD` and strictness for CI - [Project Configuration](/project-config) — configure which files CI checks - [CLI reference](/cli-reference) — full command and exit code reference diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index febe11cc..b140f0b8 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -89,8 +89,6 @@ VectorLint reads configuration from environment variables in addition to config | `ANTHROPIC_API_KEY` | — | API key for Anthropic | | `GEMINI_API_KEY` | — | API key for Google Gemini | | `AZURE_OPENAI_API_KEY` | — | API key for Azure OpenAI | -| `SEARCH_PROVIDER` | — | Search provider for `technical-accuracy` evaluator: `perplexity` | -| `PERPLEXITY_API_KEY` | — | API key for Perplexity search | | `CONFIDENCE_THRESHOLD` | `0.75` | PAT pipeline confidence gate. Lower surfaces more findings; higher surfaces fewer. | For the full environment variable reference see [Environment variables](/env-variables). @@ -99,5 +97,5 @@ For the full environment variable reference see [Environment variables](/env-var - [Installation](/installation) — install VectorLint globally or run with npx - [Configuration](/configuration) — configure VectorLint for your project -- [Tuning evaluation precision](/false-positive-tuning) — adjust `CONFIDENCE_THRESHOLD` and strictness +- [Tuning review precision](/false-positive-tuning) — adjust `CONFIDENCE_THRESHOLD` and strictness - [CI Integration](/ci-integration) — use exit codes to gate merges on content quality diff --git a/docs/configuration-schema.mdx b/docs/configuration-schema.mdx index 3ce62cb3..98fe19aa 100644 --- a/docs/configuration-schema.mdx +++ b/docs/configuration-schema.mdx @@ -14,7 +14,7 @@ These settings go at the top of `.vectorlint.ini`, outside any file pattern sect | Setting | Default | Description | |---|---|---| | `RulesPath` | _(none)_ | Path to your custom rule packs directory. Subdirectories inside this path become available pack names. If omitted, only the bundled `VectorLint` preset is available. | -| `Concurrency` | `4` | Number of rule evaluations to run in parallel. Reduce this if you hit API rate limits. | +| `Concurrency` | `4` | Number of rule reviews to run in parallel. Reduce this if you hit API rate limits. | | `DefaultSeverity` | `warning` | Default severity for reported violations: `warning` or `error`. Individual rules can override this with their own `severity` frontmatter field. | ```ini @@ -27,7 +27,7 @@ DefaultSeverity=warning ## File pattern sections -File pattern sections map glob patterns to rule packs. VectorLint evaluates a file only if at least one pattern matches it. +File pattern sections map glob patterns to rule packs. VectorLint reviews a file only if at least one pattern matches it. ```ini [**/*.md] @@ -91,7 +91,7 @@ Set strictness per rule, per pattern: [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.strictness=strict +Clarity.strictness=strict ``` ### Strictness levels @@ -111,9 +111,9 @@ GrammarChecker.strictness=strict ## Global style context (VECTORLINT.md) -Place a `VECTORLINT.md` file in your project root to define style instructions that apply to every evaluation. VectorLint prepends its contents to the system prompt for every rule, automatically applying your tone, terminology, and baseline standards across all checks. +Place a `VECTORLINT.md` file in your project root to define style instructions that apply to every review. VectorLint prepends its contents to the system prompt for every rule, automatically applying your tone, terminology, and baseline standards across all checks. -Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds 4,000 tokens, as large context blocks degrade evaluation performance and increase costs. +Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds 4,000 tokens, as large context blocks degrade review performance and increase costs. ### Zero-config mode @@ -170,7 +170,7 @@ GrammarChecker.strictness=7 [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.threshold=9 +Clarity.threshold=9 # Marketing content — brand voice rules [content/marketing/**/*.md] diff --git a/docs/configuration.mdx b/docs/configuration.mdx index 4567a758..6a610136 100644 --- a/docs/configuration.mdx +++ b/docs/configuration.mdx @@ -7,7 +7,7 @@ VectorLint is configured through four files. Running `vectorlint init` creates t | File | Location | Purpose | Required | |------|----------|---------|----------| -| `VECTORLINT.md` | Project root | Global style instructions in plain language — prepended to every evaluation | Required for zero-config mode | +| `VECTORLINT.md` | Project root | Global style instructions in plain language — prepended to every review | Required for zero-config mode | | `.vectorlint.ini` | Project root | File patterns, rule packs, strictness overrides | Required for rule packs and file-specific configuration | | `config.toml` | `~/.vectorlint/` | LLM provider API keys — applies globally across projects | Always required | | Rule pack files | `RulesPath` directory | Targeted LLM prompts for specific checks — for advanced content quality workflows | Optional | @@ -29,7 +29,7 @@ If you want to check content against a style guide with no additional setup: vectorlint init --quick ``` -This creates a `VECTORLINT.md` file in your project root where you paste your style guide. VectorLint automatically creates a "Style Guide Compliance" rule from it and evaluates your content against it. +This creates a `VECTORLINT.md` file in your project root where you paste your style guide. VectorLint automatically creates a "Style Guide Compliance" rule from it and reviews your content against it. Run a check: @@ -53,7 +53,7 @@ This creates both `.vectorlint.ini` and `~/.vectorlint/config.toml`. Edit them m Each file has its own reference page with complete field definitions, examples, and behavior details. -- [LLM Providers](/llm-providers) — configure your LLM provider, search provider, and false-positive filtering threshold +- [LLM Providers](/llm-providers) — configure your LLM provider and false-positive filtering threshold - [Project Configuration](/project-config) — map file patterns to rule packs, set strictness overrides, and understand cascading behavior - [Defining your style rules](/style-guide) — create `VECTORLINT.md` and write targeted rule pack files @@ -75,10 +75,10 @@ VectorLint ships with a built-in preset containing rules for AI pattern detectio - Configure the model and API keys that power your evaluations. + Configure the model and API keys that power your reviews. - Map files to rule packs and tune evaluation behavior. + Map files to rule packs and tune review behavior. Create VECTORLINT.md and write custom rule pack files. diff --git a/docs/customize-style-rules.mdx b/docs/customize-style-rules.mdx index e19760c4..75521561 100644 --- a/docs/customize-style-rules.mdx +++ b/docs/customize-style-rules.mdx @@ -27,7 +27,6 @@ Create a new file at `.github/rules/MyTeam/grammar-checker.md`: --- id: GrammarChecker name: Grammar Checker -evaluator: base severity: error --- @@ -74,7 +73,7 @@ See [Project Configuration](/project-config) for the full strictness reference. ## Writing effective prompts -The Markdown body of your rule file is the prompt sent to the LLM. Specificity here directly determines evaluation quality — a vague prompt produces vague findings. +The Markdown body of your rule file is the prompt sent to the LLM. Specificity here directly determines review quality — a vague prompt produces vague findings. **Be explicit about what you're looking for:** @@ -83,7 +82,7 @@ The Markdown body of your rule file is the prompt sent to the LLM. Specificity h Check if the headline is good. # ✅ Specific -You are a headline evaluator for developer blog posts. Assess whether the headline: +You are a headline reviewer for developer blog posts. Assess whether the headline: 1. Clearly communicates a specific benefit to the reader 2. Uses natural, conversational language (avoid buzzwords like "leverage" or "unlock") diff --git a/docs/env-variables.mdx b/docs/env-variables.mdx index 20aced06..cb7e8fa6 100644 --- a/docs/env-variables.mdx +++ b/docs/env-variables.mdx @@ -26,22 +26,13 @@ In CI, pass them directly as pipeline environment variables or secrets. See [CI | `AZURE_OPENAI_ENDPOINT` | If using Azure | Your Azure OpenAI resource endpoint URL. | | `AZURE_OPENAI_DEPLOYMENT` | If using Azure | Your Azure OpenAI deployment name. | -## Search provider - -Required only for rules that use the `technical-accuracy` evaluator. If not set, rules that depend on external lookup return reduced-confidence results. - -| Variable | Required | Description | -|----------|----------|-------------| -| `SEARCH_PROVIDER` | If using technical-accuracy | Search provider to use. Accepted value: `perplexity`. | -| `PERPLEXITY_API_KEY` | If using Perplexity | API key for Perplexity search. | - -## Evaluation behavior +## Review behavior | Variable | Default | Description | |----------|---------|-------------| | `CONFIDENCE_THRESHOLD` | `0.75` | PAT pipeline confidence gate. Controls how strictly raw model candidates are filtered before surfacing violations. Accepted range: `0`–`1`. Invalid values fall back to `0.75`. | -Lower values surface more findings (higher recall, more noise). Higher values surface fewer findings (higher precision, fewer false positives). See [Tuning evaluation precision](/false-positive-tuning) for guidance on when to adjust this. +Lower values surface more findings (higher recall, more noise). Higher values surface fewer findings (higher precision, fewer false positives). See [Tuning review precision](/false-positive-tuning) for guidance on when to adjust this. ## Example configurations @@ -60,16 +51,6 @@ LLM_PROVIDER=anthropic ANTHROPIC_API_KEY=sk-ant-... ``` -### With search provider - -```toml -[env] -LLM_PROVIDER = "openai" -OPENAI_API_KEY = "sk-..." -SEARCH_PROVIDER = "perplexity" -PERPLEXITY_API_KEY = "pplx-..." -``` - ### CI environment with higher confidence threshold ```bash @@ -90,4 +71,4 @@ When the same variable is set in multiple places, VectorLint resolves it in this - [LLM Providers](/llm-providers) — configure your provider with full setup examples - [CI Integration](/ci-integration) — pass environment variables securely in pipelines -- [Tuning evaluation precision](/false-positive-tuning) — when and how to adjust `CONFIDENCE_THRESHOLD` +- [Tuning review precision](/false-positive-tuning) — when and how to adjust `CONFIDENCE_THRESHOLD` diff --git a/docs/false-positive-tuning.mdx b/docs/false-positive-tuning.mdx index 50e00f0e..8c044a36 100644 --- a/docs/false-positive-tuning.mdx +++ b/docs/false-positive-tuning.mdx @@ -1,11 +1,11 @@ --- -title: Tuning evaluation precision +title: Tuning review precision description: Use `CONFIDENCE_THRESHOLD` and strictness overrides together to reduce noise and get accurate findings across different content types. --- -VectorLint gives you two complementary levers for controlling evaluation precision. Used together, they let you calibrate how aggressively VectorLint surfaces findings — globally across your project, and per content type. +VectorLint gives you two complementary levers for controlling review precision. Used together, they let you calibrate how aggressively VectorLint surfaces findings — globally across your project, and per content type. -- **`CONFIDENCE_THRESHOLD`** — controls how strictly the PAT pipeline filters raw model candidates before surfacing them. A global setting that applies to every evaluation. +- **`CONFIDENCE_THRESHOLD`** — controls how strictly the PAT pipeline filters raw model candidates before surfacing them. A global setting that applies to every review. - **Strictness overrides** — controls how harshly rules score error density for specific file patterns. Set per content type in `.vectorlint.ini`. Understanding when to reach for each one is the key to a low-noise, high-signal workflow. @@ -47,7 +47,7 @@ Different content types warrant different quality bars. A draft circulated inter [content/docs/**/*.md] RunRules=TechDocs GrammarChecker.strictness=strict -TechnicalAccuracy.strictness=9 +Clarity.strictness=9 # Blog posts — standard. Quality matters, but tone is flexible. [content/blog/**/*.md] @@ -75,7 +75,7 @@ RunRules= In CI, false positives block merges. A finding that a writer might reasonably dismiss becomes a pipeline failure that needs explaining. Two adjustments help: -**Raise `CONFIDENCE_THRESHOLD` in CI.** Set it higher in your CI environment's `.env` than in local development. This means only the highest-confidence findings block a merge — lower-confidence candidates still get caught locally where a writer can evaluate them in context. +**Raise `CONFIDENCE_THRESHOLD` in CI.** Set it higher in your CI environment's `.env` than in local development. This means only the highest-confidence findings block a merge — lower-confidence candidates still get caught locally where a writer can review them in context. ```bash # .env in CI environment diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index 31ea9267..a07a35fd 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -3,7 +3,7 @@ title: Frontmatter field reference description: Complete reference for YAML frontmatter fields in VectorLint rule files. --- -Every VectorLint rule file begins with a YAML frontmatter block that controls how VectorLint runs the evaluation, handles the result, and reports violations. The Markdown body that follows is the prompt sent to the LLM. +Every VectorLint rule file begins with a YAML frontmatter block that controls how VectorLint runs the review, handles the result, and reports violations. The Markdown body that follows is the prompt sent to the LLM. ```markdown --- @@ -27,20 +27,18 @@ Your LLM prompt goes here. | Field | Default | Description | |---|---|---| | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | -| `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | -| `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | -| `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | +| `target` | _(none)_ | Regex specification to extract and review a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | ## Target fields -The `target` field narrows evaluation to a specific part of the document. It is an object with the following sub-fields: +The `target` field narrows review to a specific part of the document. It is an object with the following sub-fields: | Field | Type | Required | Description | |---|---|---|---| | `regex` | string | Yes | Regular expression pattern to match the target content. | | `flags` | string | No | Regex flags (e.g., `"mu"` for multiline and Unicode). | | `group` | number | No | Capture group index to extract. If omitted, the full match is used. | -| `required` | boolean | No | When `true`, a missing match immediately reports an `error` with the `suggestion` text and skips LLM evaluation. When `false` or omitted, a missing match causes VectorLint to evaluate the full document instead. | +| `required` | boolean | No | When `true`, a missing match immediately reports an `error` with the `suggestion` text and skips LLM review. When `false` or omitted, a missing match causes VectorLint to review the full document instead. | | `suggestion` | string | No | Message shown when `required: true` and the pattern does not match. | **Example — targeting only the H1 headline:** @@ -56,15 +54,13 @@ target: ## Complete example -The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before evaluation runs. +The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before review runs. ```markdown --- -evaluator: base id: HeadlineQuality name: Headline Quality severity: error -evaluateAs: document target: regex: '^#\s+(.+)$' flags: "mu" @@ -73,7 +69,7 @@ target: suggestion: Add an H1 headline for the article. --- -You are a headline reviewer for developer blog posts. Evaluate the H1 headline +You are a headline reviewer for developer blog posts. Review the H1 headline and report specific, fixable violations — for example: a vague benefit, a missing subject, or passive voice. For each violation, quote the exact text and suggest a concrete rewrite. diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index dd1bc7c9..dbede090 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -1,23 +1,23 @@ --- title: How it works -description: How VectorLint evaluates content, filters findings, and calculates scores. +description: How VectorLint reviews content, filters findings, and calculates scores. --- VectorLint sends your content to a large language model (LLM) with a structured prompt that defines your quality criteria. The model returns candidate findings, which VectorLint filters through a deterministic pipeline before surfacing violations in the CLI. If you haven't run your first check yet, start with the [Quickstart](/quickstart). This page is for understanding the architecture behind what you're running. -## The evaluation pipeline +## The review pipeline Every time you run `vectorlint doc.md`, three things happen in sequence. ### 1. Rule resolution -VectorLint reads your `.vectorlint.ini` to determine which rule packs apply to the file. It loads those rule files, prepends the contents of `VECTORLINT.md` (if present) to each rule's system prompt, and assembles the evaluation context. +VectorLint reads your `.vectorlint.ini` to determine which rule packs apply to the file. It loads those rule files, prepends the contents of `VECTORLINT.md` (if present) to each rule's system prompt, and assembles the review context. -If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic "Style Guide Compliance" rule automatically. This is the zero-config path — the fastest way to get from nothing to a working evaluation without writing any rule pack files. +If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a synthetic "Style Guide Compliance" rule automatically. This is the zero-config path — the fastest way to get from nothing to a working review without writing any rule pack files. -### 2. LLM evaluation +### 2. LLM review VectorLint sends your content to the configured LLM provider with each rule's prompt. The model returns a list of specific violations with supporting evidence. @@ -44,7 +44,7 @@ See [Quality scoring](/quality-scoring) for the full scoring reference. VectorLint gives you two complementary tools for expressing your content standards: -- **`VECTORLINT.md`** — plain-language style instructions that apply globally to every evaluation. The fastest path to useful output: no rule files, no configuration syntax, just plain English instructions that the LLM uses as context for every check. +- **`VECTORLINT.md`** — plain-language style instructions that apply globally to every review. The fastest path to useful output: no rule files, no configuration syntax, just plain English instructions that the LLM uses as context for every check. - **Rule pack files** — structured LLM prompts for specific, measurable standards. Use these when you need reproducible, auditable results on a particular dimension of quality. The two work together: `VECTORLINT.md` sets the baseline context, and rule pack files enforce specific criteria on top of it. @@ -53,6 +53,6 @@ See [Defining your style rules](/style-guide) for how to create both. ## Next steps -- [Quickstart](/quickstart) — run your first evaluation in minutes +- [Quickstart](/quickstart) — run your first review in minutes - [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules - [Configuring a project](/project-config) — map files to rule packs diff --git a/docs/initial-style-check.mdx b/docs/initial-style-check.mdx index ededf0fe..14eb7b01 100644 --- a/docs/initial-style-check.mdx +++ b/docs/initial-style-check.mdx @@ -1,9 +1,9 @@ --- title: Running your first style check -description: Use VECTORLINT.md to run your first content evaluation without writing any rules. +description: Use VECTORLINT.md to run your first content review without writing any rules. --- -The fastest way to get value from VectorLint is to create a `VECTORLINT.md` file in your project root. This is the zero-config path — no rule pack files, no `.vectorlint.ini` required. VectorLint detects the file automatically, creates a "Style Guide Compliance" rule from its contents, and evaluates your content against it. +The fastest way to get value from VectorLint is to create a `VECTORLINT.md` file in your project root. This is the zero-config path — no rule pack files, no `.vectorlint.ini` required. VectorLint detects the file automatically, creates a "Style Guide Compliance" rule from its contents, and reviews your content against it. ## Before you start @@ -11,7 +11,7 @@ You'll need a working VectorLint installation with an LLM provider configured. I ## Step 1: Create your VECTORLINT.md -Create a file called `VECTORLINT.md` in your project root. Write your style standards as plain-language instructions — short imperative rules grouped under headings. VectorLint passes this content directly to the LLM as context for every evaluation. +Create a file called `VECTORLINT.md` in your project root. Write your style standards as plain-language instructions — short imperative rules grouped under headings. VectorLint passes this content directly to the LLM as context for every review. Here's a minimal example to start from: diff --git a/docs/installation.mdx b/docs/installation.mdx index 961bb1bf..2a92e77a 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -8,7 +8,7 @@ description: Install VectorLint globally or run it with npx. Covers Node.js requ VectorLint requires Node.js 18 or later. Node.js 22 LTS is recommended. If you need to install or update Node.js, use [nvm](https://github.com/nvm-sh/nvm) (recommended) or download directly from [nodejs.org](https://nodejs.org). -Installing VectorLint makes the CLI available, but the tool requires a configured LLM provider to evaluate content. If you run `vectorlint` against a file before completing provider configuration, you get no output and no error. After installing, complete [Configuring LLM providers](/llm-providers) before running your first check. +Installing VectorLint makes the CLI available, but the tool requires a configured LLM provider to review content. If you run `vectorlint` against a file before completing provider configuration, you get no output and no error. After installing, complete [Configuring LLM providers](/llm-providers) before running your first check. --- @@ -55,4 +55,4 @@ npx vectorlint@2.5.0 path/to/doc.md - [Quickstart](/quickstart) — configure your LLM provider and run your first check end to end - [Configuring LLM providers](/llm-providers) — set up your API key for OpenAI, Anthropic, Gemini, Azure OpenAI, or Amazon Bedrock -- [Using VECTORLINT.md](/initial-style-check) — start evaluating content without writing any rule pack files \ No newline at end of file +- [Using VECTORLINT.md](/initial-style-check) — start reviewing content without writing any rule pack files \ No newline at end of file diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 1041f912..505970d1 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -5,13 +5,13 @@ description: "VectorLint is a large language model (LLM)-based prose linter that ## What is VectorLint? -VectorLint is a command-line tool that evaluates and scores documentation using large language models (LLMs). Unlike regex patterns that catch only surface-level issues, VectorLint can find terminology misuse, technical inaccuracies, and style inconsistencies that require contextual understanding. +VectorLint is a command-line tool that reviews and scores documentation using large language models (LLMs). Unlike regex patterns that catch only surface-level issues, VectorLint can find terminology misuse, unsupported claims, and style inconsistencies that require contextual understanding. If you can write a prompt for it, you can lint it with VectorLint. ## Why VectorLint exists -Traditional prose linters like Vale work by matching text against fixed regex patterns and word lists. They catch what you explicitly tell them to catch — but they can't reason about meaning, context, or technical accuracy. +Traditional prose linters like Vale work by matching text against fixed regex patterns and word lists. They catch what you explicitly tell them to catch — but they can't reason about meaning, context, or internal consistency. VectorLint fills that gap. You define a rule once as a Markdown prompt, and the LLM applies it across your entire content library — scoring each document, surfacing specific violations, and explaining why each violation matters. @@ -45,7 +45,7 @@ VectorLint filters raw LLM candidates through a series of gate checks before sur 1. **Candidate generation** — the LLM returns all potential violations, each with required gate-check fields: rule support, exact evidence, context support, plausible non-violation, and fix quality. 2. **Deterministic filtering** — VectorLint applies a strict filter and only surfaces violations that pass all required gates. -VectorLint's output is intentionally stricter than raw model candidates; it reports only findings that pass all gates. You can tune how aggressively the pipeline filters findings to match your content workflow. See [Tuning evaluation precision](/false-positive-tuning). +VectorLint's output is intentionally stricter than raw model candidates; it reports only findings that pass all gates. You can tune how aggressively the pipeline filters findings to match your content workflow. See [Tuning review precision](/false-positive-tuning). ## Next steps diff --git a/docs/llm-providers.mdx b/docs/llm-providers.mdx index 90eecf0b..e2578572 100644 --- a/docs/llm-providers.mdx +++ b/docs/llm-providers.mdx @@ -3,7 +3,7 @@ title: Configuring LLM providers description: Connect VectorLint to OpenAI, Anthropic, Azure OpenAI, Google Gemini, or Amazon Bedrock. --- -VectorLint sends your content to an LLM (Large Language Model) for evaluation. Configure a provider and supply credentials before running your first check. +VectorLint sends your content to an LLM (Large Language Model) for review. Configure a provider and supply credentials before running your first check. ## Supported providers @@ -199,27 +199,6 @@ export AWS_SECRET_ACCESS_KEY=... If your environment already provides AWS credentials through an IAM role or `~/.aws/credentials`, you can omit AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. -## Search provider (optional) - -The `technical-accuracy` evaluator verifies factual claims against live web search. You need a separate search provider credential. VectorLint currently supports [Perplexity](https://www.perplexity.ai/) for this purpose. - -**Global config** (`~/.vectorlint/config.toml`) - -```toml -[env] -SEARCH_PROVIDER = "perplexity" -PERPLEXITY_API_KEY = "pplx-..." -``` - -**Project `.env` file** - -```bash -SEARCH_PROVIDER=perplexity -PERPLEXITY_API_KEY=pplx-... -``` - -You don't need a search provider for standard `base` evaluator rules. - ## Tracking LLM costs You can configure per-token pricing so VectorLint calculates and reports estimated costs after each run. Check your provider's pricing page for current values. diff --git a/docs/presets.mdx b/docs/presets.mdx index bb9b77d2..0fe880fa 100644 --- a/docs/presets.mdx +++ b/docs/presets.mdx @@ -1,6 +1,6 @@ --- title: Presets -description: Use VectorLint's built-in rule presets to start evaluating content without writing custom rules. +description: Use VectorLint's built-in rule presets to start reviewing content without writing custom rules. --- VectorLint ships with a built-in rule preset that activates automatically when you run `vectorlint init`. Presets give you immediate, useful output without requiring you to write any rules first. diff --git a/docs/project-config.mdx b/docs/project-config.mdx index e78d00b7..e42797d5 100644 --- a/docs/project-config.mdx +++ b/docs/project-config.mdx @@ -1,6 +1,6 @@ --- title: Configuring a project -description: Map files to rule packs and tune evaluation behavior with .vectorlint.ini. +description: Map files to rule packs and tune review behavior with .vectorlint.ini. --- `.vectorlint.ini` is VectorLint's project configuration file. It lives in your project root and controls three things: global behavior settings, where VectorLint looks for custom rules, and which rule packs run on which files. @@ -9,12 +9,12 @@ Run `vectorlint init` to generate a starter file. You can edit it manually at an ## Global Settings -These settings apply to all evaluations in the project. +These settings apply to all reviews in the project. | Setting | Type | Default | Description | |---------|------|---------|-------------| | `RulesPath` | string | _(none)_ | Root directory for custom rule packs. If omitted, only built-in presets are used. | -| `Concurrency` | integer | `4` | Number of evaluations to run in parallel. | +| `Concurrency` | integer | `4` | Number of reviews to run in parallel. | | `DefaultSeverity` | string | `warning` | Severity level for reported violations: `warning` or `error`. | ```ini @@ -36,7 +36,7 @@ RunRules=Acme [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.threshold=8.0 +Clarity.threshold=8.0 [content/marketing/**/*.md] RunRules=TechCorp @@ -116,7 +116,7 @@ You can use named levels (`lenient`, `standard`, `strict`) or direct numeric mul [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=strict -TechnicalAccuracy.strictness=20 +Clarity.strictness=20 ``` ## Complete Example @@ -137,7 +137,7 @@ GrammarChecker.strictness=5 [content/docs/**/*.md] RunRules=Acme GrammarChecker.strictness=9 -TechnicalAccuracy.threshold=8.0 +Clarity.threshold=8.0 # Marketing — brand voice rules [content/marketing/**/*.md] @@ -153,15 +153,15 @@ RunRules= In addition to `.vectorlint.ini`, you can place a `VECTORLINT.md` file in your project root to define global style instructions in plain language. -**Zero-config mode:** If no `.vectorlint.ini` exists, VectorLint automatically detects `VECTORLINT.md`, creates a synthetic "Style Guide Compliance" rule, and evaluates content against it. +**Zero-config mode:** If no `.vectorlint.ini` exists, VectorLint automatically detects `VECTORLINT.md`, creates a synthetic "Style Guide Compliance" rule, and reviews content against it. -**Combined mode:** When `.vectorlint.ini` is present, the contents of `VECTORLINT.md` are prepended to the system prompt for every evaluation — making your global tone and terminology preferences active across all rules. +**Combined mode:** When `.vectorlint.ini` is present, the contents of `VECTORLINT.md` are prepended to the system prompt for every review — making your global tone and terminology preferences active across all rules. -Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds approximately 4,000 tokens, as very large contexts can degrade evaluation quality and increase API costs. +Keep `VECTORLINT.md` concise. VectorLint emits a warning if the file exceeds approximately 4,000 tokens, as very large contexts can degrade review quality and increase API costs. ## Next Steps -- [LLM Providers](/llm-providers) — configure the model that runs evaluations +- [LLM Providers](/llm-providers) — configure the model that runs reviews - [Style Guide](/style-guide) — write custom rules for your content standards diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index c6775b23..bd77f8ac 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -3,13 +3,13 @@ title: Quickstart description: Install VectorLint, configure your LLM provider, and run your first content check. --- -Set up VectorLint and run your first content evaluation in under ten minutes. You install VectorLint, add your style instructions, connect an LLM (Large Language Model) provider, and see real output on a file you own. +Set up VectorLint and run your first content review in under ten minutes. You install VectorLint, add your style instructions, connect an LLM (Large Language Model) provider, and see real output on a file you own. ## Before you begin VectorLint requires Node.js 18 or later. Node.js 22 LTS is recommended. If you need to install or update Node.js, use [nvm](https://github.com/nvm-sh/nvm) or download directly from [nodejs.org](https://nodejs.org). -You will also need an API key for a supported LLM provider. VectorLint sends your content to the provider you configure for evaluation. Supported providers include: OpenAI, Anthropic, Google Gemini, Azure OpenAI, Amazon Bedrock. +You will also need an API key for a supported LLM provider. VectorLint sends your content to the provider you configure for review. Supported providers include: OpenAI, Anthropic, Google Gemini, Azure OpenAI, Amazon Bedrock. VectorLint requires a configured LLM provider to produce output. Installing the package alone is not enough — if you skip provider configuration, running `vectorlint` against a file will return no results and no error. Complete Step 2 before running a check. @@ -61,7 +61,7 @@ You can also set provider values in a `.env` file in your project root. Project- ## Step 3: Create your VECTORLINT.md -After you initialize VectorLint in your repository, the application generates a VECTORLINT.md file in your project root. After you copy your style standards as plain-language instructions into the file, VectorLint builds a "Style Guide Compliance" rule and applies it to your VectorLint evaluations. +After you initialize VectorLint in your repository, the application generates a VECTORLINT.md file in your project root. After you copy your style standards as plain-language instructions into the file, VectorLint builds a "Style Guide Compliance" rule and applies it to your VectorLint reviews. Run the quick-init command to create the file: @@ -103,7 +103,7 @@ Point VectorLint at any Markdown file in your project: vectorlint path/to/doc.md ``` -VectorLint reads your `VECTORLINT.md`, evaluates the file against it, and prints findings in your terminal. If the file is clean, VectorLint produces no output and exits with status `0`. If the file has violations, VectorLint prints each finding with its location and a suggested fix. +VectorLint reads your `VECTORLINT.md`, reviews the file against it, and prints findings in your terminal. If the file is clean, VectorLint produces no output and exits with status `0`. If the file has violations, VectorLint prints each finding with its location and a suggested fix. --- diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index bf29b1cd..20cf7430 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -1,19 +1,19 @@ --- title: Defining your style rules -description: Set global style instructions and create targeted LLM prompts to evaluate content against your style guide, terminology, and content standards. +description: Set global style instructions and create targeted LLM prompts to review content against your style guide, terminology, and content standards. --- -VectorLint gives you two ways to bring your style guide into evaluations. Understanding which to use and when to combine them is the starting point for any content quality workflow: +VectorLint gives you two ways to bring your style guide into reviews. Understanding which to use and when to combine them is the starting point for any content quality workflow: -- **`VECTORLINT.md` file**: the global style instructions in plain language. Create this file from your style guide and place it in your project root. VectorLint prepends its contents to the system prompt for every evaluation, making your tone, terminology, and baseline standards apply across all rules automatically. Use it for broad, always-applicable guidance. See [Project Configuration](/project-config) for details. +- **`VECTORLINT.md` file**: the global style instructions in plain language. Create this file from your style guide and place it in your project root. VectorLint prepends its contents to the system prompt for every review, making your tone, terminology, and baseline standards apply across all rules automatically. Use it for broad, always-applicable guidance. See [Project Configuration](/project-config) for details. -- **Rule pack files**: the targeted LLM prompts for specific checks. Create these as separate Markdown files as described below. Each file is a structured prompt that instructs the LLM to evaluate content against one specific standard: grammar, headline quality, AI pattern detection, technical accuracy, and so on. Rules are organized into packs and mapped to file patterns in `.vectorlint.ini`. Rule pack files live in subdirectories under `RulesPath` — see [File Structure Reference](#file-structure-reference) below. +- **Rule pack files**: the targeted LLM prompts for specific checks. Create these as separate Markdown files as described below. Each file is a structured prompt that instructs the LLM to review content against one specific standard: grammar, headline quality, AI pattern detection, unsupported claims, and so on. Rules are organized into packs and mapped to file patterns in `.vectorlint.ini`. Rule pack files live in subdirectories under `RulesPath` — see [File Structure Reference](#file-structure-reference) below. If you can write a prompt for it, you can lint for it with VectorLint. ## Creating your VECTORLINT.md -Place a `VECTORLINT.md` file in your project root to define global style instructions that apply to every evaluation. VectorLint prepends its contents to the system prompt for every rule it runs. +Place a `VECTORLINT.md` file in your project root to define global style instructions that apply to every review. VectorLint prepends its contents to the system prompt for every rule it runs. Keep this file concise. VectorLint emits a warning if the file exceeds approximately 4,000 tokens, as very large contexts can degrade performance and increase API costs. @@ -32,7 +32,7 @@ Rules for extraction: - Remove all tables, comparison columns, governing references, and checklists — extract only the actionable rules they contain. - Remove any rule that is structural (document architecture, heading - levels) rather than evaluable at the sentence or paragraph level. + levels) rather than reviewable at the sentence or paragraph level. - Use plain markdown bullets. No bold, no nested lists beyond one level. - Target output: under 600 words / ~800 tokens. - Do not include a preamble or closing statement. @@ -49,7 +49,7 @@ To help you get started, the VectorLint docs repository includes an [example VEC ## Creating rule pack files -Rule pack files are optional Markdown files, each containing a targeted LLM prompt for a specific check. Unlike `VECTORLINT.md`, which sets broad context for every evaluation, rule pack files enforce precise, measurable criteria — grammar, headline quality, AI pattern detection, technical accuracy, and so on. You can create as many as your content workflow requires and organize them into packs mapped to specific file patterns in `.vectorlint.ini`. +Rule pack files are optional Markdown files, each containing a targeted LLM prompt for a specific check. Unlike `VECTORLINT.md`, which sets broad context for every review, rule pack files enforce precise, measurable criteria — grammar, headline quality, AI pattern detection, unsupported claims, and so on. You can create as many as your content workflow requires and organize them into packs mapped to specific file patterns in `.vectorlint.ini`. Each rule file has two parts: YAML frontmatter that configures how VectorLint handles the result, and a Markdown body that is the actual prompt sent to the LLM. @@ -57,9 +57,8 @@ Each rule file has two parts: YAML frontmatter that configures how VectorLint ha ```markdown --- -id: MyEval -name: My Content Evaluator -evaluator: base +id: MyRule +name: My Content Reviewer severity: error --- @@ -77,9 +76,7 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| -| `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | -| `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | @@ -89,7 +86,6 @@ The LLM returns a list of specific violations. VectorLint scores the result usin ```markdown --- -evaluator: base id: GrammarChecker name: Grammar Checker severity: error @@ -111,7 +107,7 @@ Report any errors found with specific examples. ## Targeting Specific Content -The `target` field lets you evaluate a specific portion of a document — for example, only the H1 headline — rather than the full content. +The `target` field lets you review a specific portion of a document — for example, only the H1 headline — rather than the full content. ```yaml target: @@ -122,9 +118,9 @@ target: suggestion: Add an H1 headline for the article. ``` -**When `required: true`:** If the pattern doesn't match, VectorLint reports an immediate `error` with the `suggestion` text, and skips LLM evaluation. +**When `required: true`:** If the pattern doesn't match, VectorLint reports an immediate `error` with the `suggestion` text, and skips LLM review. -**When `required: false` (or omitted):** If the pattern doesn't match, VectorLint evaluates the full document instead. +**When `required: false` (or omitted):** If the pattern doesn't match, VectorLint reviews the full document instead. ## File Structure Reference @@ -134,9 +130,7 @@ project/ │ └── rules/ ← RulesPath in .vectorlint.ini │ ├── Acme/ ← Pack: "Acme" │ │ ├── grammar-checker.md -│ │ ├── headline-evaluator.md -│ │ └── Technical/ ← Nested subdirectory (supported) -│ │ └── technical-accuracy.md +│ │ └── headline-reviewer.md │ └── TechCorp/ ← Pack: "TechCorp" │ └── brand-voice.md └── .vectorlint.ini @@ -155,7 +149,6 @@ RunRules=Acme ```markdown --- -evaluator: base id: AIPatterns name: AI Pattern Detector severity: warning @@ -173,4 +166,4 @@ human-sounding rewrite. - [Customize style rules](/customize-style-rules) — learn how to write effective LLM prompts for your rule pack files - [Project Configuration](/project-config) — assign rule packs to file patterns -- [LLM Providers](/llm-providers) — configure the model used for evaluations +- [LLM Providers](/llm-providers) — configure the model used for reviews diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index 677502d6..faa50747 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -1,6 +1,6 @@ --- title: Troubleshooting -description: Diagnose and fix common VectorLint issues with installation, configuration, API keys, and evaluation output. +description: Diagnose and fix common VectorLint issues with installation, configuration, API keys, and review output. --- @@ -99,24 +99,6 @@ See [Environment variables](/env-variables) for the full variable reference. --- -### Search provider not working for TechnicalAccuracy rules - -**Symptom:** Rules using `evaluator: technical-accuracy` produce no findings or return an error about search. - -**Cause:** No search provider is configured. - -**Fix:** Add a search provider to your config: - -```toml -[env] -SEARCH_PROVIDER = "perplexity" -PERPLEXITY_API_KEY = "pplx-..." -``` - -Without a search provider, `technical-accuracy` evaluators cannot verify facts against external sources and will return reduced-confidence or no results. - ---- - ## Configuration issues ### Rules not running on a file diff --git a/docs/use-cases.mdx b/docs/use-cases.mdx index 75091090..d0275ade 100644 --- a/docs/use-cases.mdx +++ b/docs/use-cases.mdx @@ -29,18 +29,6 @@ For content-specific tuning, see [Customizing style rules](/customize-style-rule --- -## Verify technical accuracy - -**The problem:** Outdated version numbers, deprecated API references, and incorrect command syntax are difficult category of errors to catch in manual review, because catching them requires domain expertise your reviewers may not have. - -**The path:** The `technical-accuracy` evaluator uses a search provider to check factual claims in your content against current external sources. Point it at your API reference pages or CLI documentation and it surfaces stale information before it ships. - -See [Defining your style rules](/style-guide) for how to configure a `technical-accuracy` rule. - -**Who does this:** Developer documentation teams, API reference authors, teams maintaining large technical content libraries. - ---- - ## Gate content quality in CI **The problem:** Teams can skip manual quality checks because the pipeline doesn't enforce it. @@ -86,6 +74,6 @@ See [Configuring a project](/project-config) for the full configuration referenc ## Next steps -- [Quickstart](/quickstart) — run your first evaluation in under ten minutes -- [How it works](/how-it-works) — understand the evaluation pipeline, PAT filtering, and scoring +- [Quickstart](/quickstart) — run your first review in under ten minutes +- [How it works](/how-it-works) — understand the review pipeline, PAT filtering, and scoring - [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules diff --git a/package-lock.json b/package-lock.json index 49e625dd..0c065650 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@ai-sdk/azure": "^3.0.34", "@ai-sdk/google": "^3.0.31", "@ai-sdk/openai": "^3.0.33", - "@ai-sdk/perplexity": "^1.0.0", "@langfuse/otel": "^5.1.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/sdk-node": "^0.214.0", @@ -334,51 +333,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@ai-sdk/perplexity": { - "version": "1.1.9", - "resolved": "https://registry.npmjs.org/@ai-sdk/perplexity/-/perplexity-1.1.9.tgz", - "integrity": "sha512-Ytolh/v2XupXbTvjE18EFBrHLoNMH0Ueji3lfSPhCoRUfkwrgZ2D9jlNxvCNCCRiGJG5kfinSHvzrH5vGDklYA==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "1.1.3", - "@ai-sdk/provider-utils": "2.2.8" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.0.0" - } - }, - "node_modules/@ai-sdk/provider": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", - "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@ai-sdk/provider-utils": { - "version": "2.2.8", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", - "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "1.1.3", - "nanoid": "^3.3.8", - "secure-json-parse": "^2.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.23.8" - } - }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -5417,6 +5371,7 @@ "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, "funding": [ { "type": "github", @@ -6113,12 +6068,6 @@ "node": ">=10" } }, - "node_modules/secure-json-parse": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", - "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", - "license": "BSD-3-Clause" - }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", diff --git a/package.json b/package.json index 3a73ad4d..14aaa18b 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,6 @@ "@ai-sdk/azure": "^3.0.34", "@ai-sdk/google": "^3.0.31", "@ai-sdk/openai": "^3.0.33", - "@ai-sdk/perplexity": "^1.0.0", "@langfuse/otel": "^5.1.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/sdk-node": "^0.214.0", diff --git a/presets/VectorLint/ai-pattern.md b/presets/VectorLint/ai-pattern.md index 9c0ec2ec..ca7e1f06 100644 --- a/presets/VectorLint/ai-pattern.md +++ b/presets/VectorLint/ai-pattern.md @@ -1,10 +1,8 @@ --- specVersion: 2.0.0 -evaluator: base id: AIPattern name: AI Pattern severity: warning -evaluateAs: document --- # AI Pattern diff --git a/presets/VectorLint/capitalization.md b/presets/VectorLint/capitalization.md index e454d51a..5c18dffd 100644 --- a/presets/VectorLint/capitalization.md +++ b/presets/VectorLint/capitalization.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Capitalization name: Capitalization severity: warning -evaluateAs: document --- # Capitalization diff --git a/presets/VectorLint/consistency.md b/presets/VectorLint/consistency.md index fb17a50d..e7ee8a5e 100644 --- a/presets/VectorLint/consistency.md +++ b/presets/VectorLint/consistency.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Consistency name: Consistency severity: warning -evaluateAs: document --- # Consistency diff --git a/presets/VectorLint/inclusivity.md b/presets/VectorLint/inclusivity.md index fab769ce..dc645519 100644 --- a/presets/VectorLint/inclusivity.md +++ b/presets/VectorLint/inclusivity.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Inclusivity name: Inclusivity severity: warning -evaluateAs: document --- # Inclusivity diff --git a/presets/VectorLint/passive-voice.md b/presets/VectorLint/passive-voice.md index e71c93d0..46e3f33b 100644 --- a/presets/VectorLint/passive-voice.md +++ b/presets/VectorLint/passive-voice.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: PassiveVoice name: Passive Voice severity: warning -evaluateAs: document --- # Passive Voice diff --git a/presets/VectorLint/repetition.md b/presets/VectorLint/repetition.md index 6849b9c1..ddb9c909 100644 --- a/presets/VectorLint/repetition.md +++ b/presets/VectorLint/repetition.md @@ -1,10 +1,8 @@ --- specVersion: 2.0.0 -evaluator: base id: Repetition name: Repetition severity: warning -evaluateAs: document --- # Repetition diff --git a/presets/VectorLint/unsupported-claims.md b/presets/VectorLint/unsupported-claims.md index c2146a38..af20a30f 100644 --- a/presets/VectorLint/unsupported-claims.md +++ b/presets/VectorLint/unsupported-claims.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: UnsupportedClaims name: Unsupported Claims severity: warning -evaluateAs: document --- # Unsupported Claims diff --git a/presets/VectorLint/wordiness.md b/presets/VectorLint/wordiness.md index 87d386f3..56203e4c 100644 --- a/presets/VectorLint/wordiness.md +++ b/presets/VectorLint/wordiness.md @@ -1,10 +1,8 @@ --- specVersion: 1.0.0 -evaluator: base id: Wordiness name: Wordiness severity: warning -evaluateAs: document --- # Wordiness diff --git a/src/agent/executor.ts b/src/agent/executor.ts deleted file mode 100644 index eda9f0e4..00000000 --- a/src/agent/executor.ts +++ /dev/null @@ -1,840 +0,0 @@ -import { readdir, readFile } from 'fs/promises'; -import * as os from 'os'; -import fg from 'fast-glob'; -import { buildEvaluationLLMSchema, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; -import type { PromptFile } from '../prompts/prompt-loader'; -import { Type, Severity } from '../evaluators/types'; -import { computeFilterDecision } from '../evaluators/violation-filter'; -import { locateQuotedText } from '../output/location'; -import type { AgentToolLoopResult, LLMProvider } from '../providers/llm-provider'; -import type { TokenUsage } from '../providers/token-usage'; -import type { OutputFormat } from '../cli/types'; -import { createEvaluator } from '../evaluators'; -import { createReviewSessionStore } from './review-session-store'; -import { buildAgentSystemPrompt } from './prompt-builder'; -import { AgentToolError } from '../errors'; -import { ScanPathResolver } from '../boundaries/scan-path-resolver'; -import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import { - createAgentTools, - listAvailableTools, - type AgentToolHandler, - type AgentToolName, -} from './tools-registry'; -import { - LINT_TOOL_INPUT_SCHEMA, - FINALIZE_REVIEW_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, - SESSION_EVENT_TYPE, - type SessionEvent, -} from './types'; -import { resolveGlobPatternWithinRoot, resolveWithinRoot, toRelativePathFromRoot } from './path-utils'; -import type { AgentProgressReporter, VisibleToolName, VisibleToolProgress } from './progress'; -import { buildRuleId, normalizeRuleSource } from './rule-id'; -export interface AgentFinding { - file: string; - line: number; - column: number; - severity: Severity; - message: string; - ruleId: string; - ruleSource: string; - analysis?: string; - suggestion?: string; - fix?: string; - match?: string; -} - -export interface RunAgentExecutorParams { - targets: string[]; - prompts: PromptFile[]; - provider: LLMProvider; - workspaceRoot: string; - scanPaths: FilePatternConfig[]; - outputFormat: OutputFormat; - printMode: boolean; - sessionHomeDir?: string; - progressReporter?: AgentProgressReporter; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; - userInstructions?: string; -} - -export interface AgentExecutorResult { - findings: AgentFinding[]; - events: SessionEvent[]; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - requestFailures: number; - hadOperationalErrors: boolean; - errorMessage?: string; - usage?: AgentToolLoopResult['usage']; -} - -const MAX_SEARCH_FILE_RESULTS = 500; -const MAX_CONTENT_MATCH_RESULTS = 200; -const VISIBLE_TOOL_NAMES = new Set(['read_file', 'list_directory', 'lint']); - -function mergeTokenUsage(left?: TokenUsage, right?: TokenUsage): TokenUsage | undefined { - if (!left && !right) { - return undefined; - } - - return { - inputTokens: (left?.inputTokens ?? 0) + (right?.inputTokens ?? 0), - outputTokens: (left?.outputTokens ?? 0) + (right?.outputTokens ?? 0), - }; -} - -function buildUnknownRuleSourceError(ruleSource: string, validSources: string[]): Error { - const validHint = validSources.length > 0 ? validSources.join(', ') : '(none)'; - return new AgentToolError( - `Unknown ruleSource "${ruleSource}". Valid sources: ${validHint}`, - 'UNKNOWN_RULE_SOURCE' - ); -} - -function fallbackMessage(result: PromptEvaluationResult): string { - if (result.reasoning) { - return result.reasoning; - } - return 'Potential issue detected'; -} - -function severityFromPrompt(prompt: PromptFile): Severity { - return prompt.meta.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING; -} - -function resolvePromptBySource( - ruleSource: string, - promptBySource: Map -): PromptFile | undefined { - const normalized = normalizeRuleSource(ruleSource); - return promptBySource.get(normalized); -} - -function resolveTargetForTopLevel( - workspaceRoot: string, - targets: string[], - file?: string -): string { - if (file && file.trim().length > 0) { - const resolved = resolveWithinRoot(workspaceRoot, file); - return toRelativePathFromRoot(workspaceRoot, resolved); - } - if (targets.length > 0) { - return toRelativePathFromRoot(workspaceRoot, targets[0]!); - } - return '.'; -} - -function withReviewInstructionOverride(prompt: PromptFile, reviewInstruction?: string): PromptFile { - const instruction = reviewInstruction?.trim(); - if (!instruction) { - return prompt; - } - return { - ...prompt, - body: instruction, - }; -} - -function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { - const findings: AgentFinding[] = []; - - for (const event of events) { - if ( - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedInline && - event.eventType !== SESSION_EVENT_TYPE.FindingRecordedTopLevel - ) { - continue; - } - - const payload = event.payload; - const file = payload.file ?? '.'; - const line = payload.line ?? 1; - findings.push({ - file, - line, - column: payload.column ?? 1, - severity: payload.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, - message: payload.message, - ruleId: payload.ruleId ?? payload.ruleSource, - ruleSource: payload.ruleSource, - ...('analysis' in payload && payload.analysis ? { analysis: payload.analysis } : {}), - ...('suggestion' in payload && payload.suggestion ? { suggestion: payload.suggestion } : {}), - ...('fix' in payload && payload.fix ? { fix: payload.fix } : {}), - ...('match' in payload && payload.match ? { match: payload.match } : {}), - }); - } - - return findings; -} - -function buildFileRuleMatches( - relativeTargets: string[], - prompts: PromptFile[], - scanPaths: FilePatternConfig[] -): Array<{ file: string; ruleSource: string }> { - const resolver = new ScanPathResolver(); - const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); - const matches: Array<{ file: string; ruleSource: string }> = []; - - for (const relFile of relativeTargets) { - const resolution = - scanPaths.length > 0 - ? resolver.resolveConfiguration(relFile, scanPaths, availablePacks) - : { packs: availablePacks, overrides: {} }; - const matchedPrompts = prompts.filter((prompt) => { - if (prompt.pack === '') return true; - if (!prompt.pack) return false; - if (scanPaths.length > 0 && !resolution.packs.includes(prompt.pack)) return false; - if (!prompt.meta?.id) return true; - const disableKey = `${prompt.pack}.${prompt.meta.id}`; - const overrideValue = resolution.overrides[disableKey]; - return typeof overrideValue !== 'string' || overrideValue.toLowerCase() !== 'disabled'; - }); - - for (const prompt of matchedPrompts) { - matches.push({ file: relFile, ruleSource: normalizeRuleSource(prompt.fullPath) }); - } - } - - return matches; -} - -function summarizeToolOutput(toolName: AgentToolName, output: unknown): unknown { - if (typeof output !== 'object' || output === null) { - return output; - } - - const candidate = output as Record; - - switch (toolName) { - case 'read_file': { - const content = typeof candidate.content === 'string' ? candidate.content : ''; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - contentLength: content.length, - }; - } - case 'search_content': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'search_files': { - const matches = Array.isArray(candidate.matches) ? candidate.matches : []; - return { - matchCount: matches.length, - ...(candidate.truncated === true ? { truncated: true } : {}), - }; - } - case 'list_directory': { - const entries = Array.isArray(candidate.entries) ? candidate.entries : []; - return { - ...(typeof candidate.path === 'string' ? { path: candidate.path } : {}), - entryCount: entries.length, - }; - } - default: - return output; - } -} - -interface VisibleToolContext extends VisibleToolProgress { - signature: string; - progressFile?: string; -} - -function countLines(content: string): number { - if (content.length === 0) { - return 0; - } - return content.split('\n').filter((_, index, lines) => { - return !(index === lines.length - 1 && lines[index] === ''); - }).length; -} - -function tryResolveRelativePath(workspaceRoot: string, rawPath: string): string | undefined { - try { - return toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, rawPath)); - } catch { - return undefined; - } -} - -function resolveVisibleToolContext(params: { - toolName: AgentToolName; - input: unknown; - workspaceRoot: string; - promptBySource: Map; - targetFiles: Set; - currentProgressFile?: string; - defaultProgressFile?: string; -}): VisibleToolContext | undefined { - const { - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - currentProgressFile, - defaultProgressFile, - } = params; - - if (!VISIBLE_TOOL_NAMES.has(toolName as VisibleToolName)) { - return undefined; - } - - switch (toolName) { - case 'read_file': { - const parsed = READ_FILE_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'read_file', - path, - ...(progressFile ? { progressFile } : {}), - signature: `read_file:${path}`, - }; - } - case 'list_directory': { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); - const path = resolvedPath ?? parsed.data.path; - const progressFile = currentProgressFile ?? defaultProgressFile; - return { - toolName: 'list_directory', - path, - ...(progressFile ? { progressFile } : {}), - signature: `list_directory:${path}`, - }; - } - case 'lint': { - const parsed = LINT_TOOL_INPUT_SCHEMA.safeParse(input); - if (!parsed.success) { - return undefined; - } - const prompt = resolvePromptBySource(parsed.data.ruleSource, promptBySource); - const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); - const path = resolvedPath ?? parsed.data.file; - const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; - const progressFile = resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile); - return { - toolName: 'lint', - path, - ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), - ruleText, - ...(progressFile ? { progressFile } : {}), - signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, - }; - } - } -} - -function buildVisibleToolSuccessState( - context: VisibleToolContext, - output: unknown -): VisibleToolProgress { - switch (context.toolName) { - case 'read_file': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const content = typeof record.content === 'string' ? record.content : ''; - return { - ...context, - lineCount: countLines(content), - }; - } - case 'list_directory': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - const entries = Array.isArray(record.entries) ? record.entries : []; - return { - ...context, - entryCount: entries.length, - }; - } - case 'lint': { - const record = typeof output === 'object' && output !== null ? (output as Record) : {}; - return { - ...context, - findingsCount: - typeof record.findingsRecorded === 'number' ? Math.max(0, Math.trunc(record.findingsRecorded)) : 0, - }; - } - } -} - -type FindingLikeViolation = { - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - description?: string; - analysis?: string; - message?: string; - suggestion?: string; - fix?: string; - confidence?: number; - checks?: GateChecks; -}; - -async function appendInlineFinding(params: { - violation: FindingLikeViolation; - result: PromptEvaluationResult; - content: string; - relFile: string; - prompt: PromptFile; - ruleSource: string; - store: Awaited>; -}): Promise { - const { violation, result, content, relFile, prompt, ruleSource, store } = params; - const filterDecision = computeFilterDecision(violation); - if (!filterDecision.surface) { - return false; - } - - const location = locateQuotedText( - content, - { - quoted_text: violation.quoted_text || '', - context_before: violation.context_before || '', - context_after: violation.context_after || '', - }, - 80, - violation.line - ); - - const line = location?.line ?? Math.max(1, Math.trunc(violation.line ?? 1)); - const column = location?.column ?? 1; - const match = location?.match ?? violation.quoted_text ?? ''; - const message = (violation.message || violation.description || fallbackMessage(result)).trim(); - - const finding: AgentFinding = { - file: relFile, - line, - column, - severity: severityFromPrompt(prompt), - message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(ruleSource), - ...(violation.analysis ? { analysis: violation.analysis } : {}), - ...(violation.suggestion ? { suggestion: violation.suggestion } : {}), - ...(violation.fix ? { fix: violation.fix } : {}), - ...(match ? { match } : {}), - }; - - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), - }, - }); - - return true; -} - -export async function runAgentExecutor(params: RunAgentExecutorParams): Promise { - const { - targets, - prompts, - provider, - workspaceRoot, - scanPaths, - sessionHomeDir = os.homedir(), - progressReporter, - maxSteps, - maxRetries, - maxParallelToolCalls, - userInstructions, - } = params; - - const promptBySource = new Map(); - for (const prompt of prompts) { - promptBySource.set(normalizeRuleSource(prompt.fullPath), prompt); - } - const validSources = Array.from(promptBySource.keys()).sort(); - - const store = await createReviewSessionStore({ homeDir: sessionHomeDir }); - let findingsCount = 0; - const relativeTargets = targets.map((target) => - toRelativePathFromRoot(workspaceRoot, resolveWithinRoot(workspaceRoot, target)) - ); - const fileRuleMatches = buildFileRuleMatches(relativeTargets, prompts, scanPaths); - const defaultRuleName = String(prompts[0]?.meta.name || prompts[0]?.meta.id || 'Rule'); - const targetFiles = new Set(relativeTargets); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { - cwd: workspaceRoot, - targets: relativeTargets, - }, - }); - - let finalized = false; - let currentProgressFile: string | undefined; - const failedVisibleToolSignatures = new Set(); - let nestedToolUsage: TokenUsage | undefined; - - async function runTool( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ): Promise { - const visibleToolContext = resolveVisibleToolContext({ - toolName, - input, - workspaceRoot, - promptBySource, - targetFiles, - ...(currentProgressFile ? { currentProgressFile } : {}), - ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), - }); - - if (visibleToolContext?.progressFile) { - const nextRuleName = visibleToolContext.ruleName ?? defaultRuleName; - if (currentProgressFile !== visibleToolContext.progressFile) { - progressReporter?.startFile(visibleToolContext.progressFile, nextRuleName); - currentProgressFile = visibleToolContext.progressFile; - } else if (visibleToolContext.ruleName) { - progressReporter?.updateRule(visibleToolContext.ruleName); - } - } - - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName, input }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolStart({ - ...visibleToolContext, - retrying: failedVisibleToolSignatures.has(visibleToolContext.signature), - }); - } - - try { - const output = await handler(input); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: true, - output: summarizeToolOutput(toolName, output), - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolSuccess(buildVisibleToolSuccessState(visibleToolContext, output)); - failedVisibleToolSignatures.delete(visibleToolContext.signature); - } - return output; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - await store.append({ - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { - toolName, - ok: false, - error: message, - }, - }); - if (visibleToolContext) { - progressReporter?.showVisibleToolError(visibleToolContext); - failedVisibleToolSignatures.add(visibleToolContext.signature); - } - throw error; - } - } - - async function lintToolHandler(input: unknown): Promise { - const parsed = LINT_TOOL_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - const promptForCall = withReviewInstructionOverride(prompt, parsed.reviewInstruction); - - const absoluteFile = resolveWithinRoot(workspaceRoot, parsed.file); - const relFile = toRelativePathFromRoot(workspaceRoot, absoluteFile); - const content = await readFile(absoluteFile, 'utf8'); - - const evaluator = createEvaluator( - promptForCall.meta.evaluator || Type.BASE, - provider, - promptForCall - ); - const result = await evaluator.evaluate(relFile, content); - nestedToolUsage = mergeTokenUsage(nestedToolUsage, result.usage); - - let findingsRecorded = 0; - const violations = result.violations; - for (const violation of violations) { - const wasRecorded = await appendInlineFinding({ - violation, - result, - content, - relFile, - prompt, - ruleSource: parsed.ruleSource, - store, - }); - if (wasRecorded) { - findingsCount += 1; - findingsRecorded += 1; - } - } - - return { - ok: true, - findingsRecorded, - schema: buildEvaluationLLMSchema().name, - }; - } - - async function reportFindingToolHandler(input: unknown): Promise { - const parsed = TOP_LEVEL_REPORT_INPUT_SCHEMA.parse(input); - const prompt = resolvePromptBySource(parsed.ruleSource, promptBySource); - if (!prompt) { - throw buildUnknownRuleSourceError(parsed.ruleSource, validSources); - } - - const references = parsed.references && parsed.references.length > 0 - ? parsed.references - : [{ file: resolveTargetForTopLevel(workspaceRoot, targets), startLine: 1, endLine: 1 }]; - - let findingsRecorded = 0; - for (const reference of references) { - const relFile = resolveTargetForTopLevel(workspaceRoot, targets, reference.file); - const finding: AgentFinding = { - file: relFile, - line: Math.max(1, Math.trunc(reference.startLine)), - column: 1, - severity: severityFromPrompt(prompt), - message: parsed.message, - ruleId: buildRuleId(prompt), - ruleSource: normalizeRuleSource(parsed.ruleSource), - ...(parsed.suggestion ? { suggestion: parsed.suggestion } : {}), - }; - - findingsCount += 1; - findingsRecorded += 1; - await store.append({ - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - ruleId: finding.ruleId, - ruleSource: finding.ruleSource, - message: finding.message, - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(parsed.references ? { references: parsed.references } : {}), - }, - }); - } - - return { ok: true, findingsRecorded }; - } - - async function readFileToolHandler(input: unknown): Promise { - const parsed = READ_FILE_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const content = await readFile(absolutePath, 'utf8'); - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - content, - }; - } - - async function searchFilesToolHandler(input: unknown): Promise { - const parsed = SEARCH_FILES_INPUT_SCHEMA.parse(input); - const scope = resolveGlobPatternWithinRoot(workspaceRoot, parsed.pattern); - const allMatches = await fg(scope.pattern, { - cwd: scope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const truncated = allMatches.length > MAX_SEARCH_FILE_RESULTS; - const matches = allMatches - .slice(0, MAX_SEARCH_FILE_RESULTS) - .map((match) => toRelativePathFromRoot(workspaceRoot, match)) - .sort((a, b) => a.localeCompare(b)); - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function listDirectoryToolHandler(input: unknown): Promise { - const parsed = LIST_DIRECTORY_INPUT_SCHEMA.parse(input); - const absolutePath = resolveWithinRoot(workspaceRoot, parsed.path); - const entries = await readdir(absolutePath, { withFileTypes: true }); - - return { - path: toRelativePathFromRoot(workspaceRoot, absolutePath), - entries: entries - .map((entry) => ({ - name: entry.name, - type: entry.isDirectory() ? 'directory' : 'file', - })) - .sort((left, right) => left.name.localeCompare(right.name)), - }; - } - - async function searchContentToolHandler(input: unknown): Promise { - const parsed = SEARCH_CONTENT_INPUT_SCHEMA.parse(input); - const absoluteSearchRoot = resolveWithinRoot(workspaceRoot, parsed.path || '.'); - const globScope = resolveGlobPatternWithinRoot(absoluteSearchRoot, parsed.glob || '**/*'); - const files = await fg(globScope.pattern, { - cwd: globScope.cwd, - dot: false, - onlyFiles: true, - absolute: true, - }); - - const matches: Array<{ file: string; line: number; text: string }> = []; - let truncated = false; - - outer: for (const filePath of files) { - let content = ''; - try { - content = await readFile(filePath, 'utf8'); - } catch { - continue; - } - - const lines = content.split('\n'); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!; - if (line.includes(parsed.pattern)) { - if (matches.length >= MAX_CONTENT_MATCH_RESULTS) { - truncated = true; - break outer; - } - matches.push({ - file: toRelativePathFromRoot(workspaceRoot, filePath), - line: index + 1, - text: line, - }); - } - } - } - - return { matches, ...(truncated ? { truncated: true } : {}) }; - } - - async function finalizeReviewToolHandler(input: unknown): Promise { - const parsed = FINALIZE_REVIEW_INPUT_SCHEMA.parse(input); - if (finalized) { - throw new AgentToolError( - 'finalize_review can only be called once per session.', - 'FINALIZE_REVIEW_ALREADY_CALLED' - ); - } - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { - totalFindings: findingsCount, - ...(parsed.summary ? { summary: parsed.summary } : {}), - }, - }); - finalized = true; - return { ok: true }; - } - - const tools = createAgentTools({ - runTool, - handlers: { - lint: lintToolHandler, - report_finding: reportFindingToolHandler, - read_file: readFileToolHandler, - search_files: searchFilesToolHandler, - list_directory: listDirectoryToolHandler, - search_content: searchContentToolHandler, - finalize_review: finalizeReviewToolHandler, - }, - }); - const availableTools = listAvailableTools(tools); - - let usage: AgentToolLoopResult['usage'] | undefined; - let requestFailures = 0; - let hadOperationalErrors = false; - let errorMessage: string | undefined; - - try { - const result = await provider.runAgentToolLoop({ - systemPrompt: buildAgentSystemPrompt({ - workspaceRoot, - fileRuleMatches, - availableTools, - ...(userInstructions ? { userInstructions } : {}), - }), - prompt: [ - `Workspace root: ${workspaceRoot}`, - `Targets: ${relativeTargets.join(', ')}`, - `Available ruleSources: ${validSources.join(', ')}`, - ].join('\n'), - tools, - ...(maxSteps !== undefined ? { maxSteps } : {}), - ...(maxRetries !== undefined ? { maxRetries } : {}), - ...(maxParallelToolCalls !== undefined ? { maxParallelToolCalls } : {}), - }); - usage = result.usage; - } catch (error) { - requestFailures += 1; - hadOperationalErrors = true; - errorMessage = error instanceof Error ? error.message : String(error); - } - - const events = await store.replay(); - const hasFinalizedEvent = events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - - if (!hasFinalizedEvent) { - hadOperationalErrors = true; - if (!errorMessage) { - errorMessage = 'Agent run ended without finalize_review.'; - } - } - - const findings = findingsFromEvents(events); - const aggregatedUsage = mergeTokenUsage(usage, nestedToolUsage); - progressReporter?.finishRun(hadOperationalErrors ? 'failed' : 'completed'); - - return { - findings, - events, - fileRuleMatches, - requestFailures, - hadOperationalErrors, - ...(errorMessage ? { errorMessage } : {}), - ...(aggregatedUsage ? { usage: aggregatedUsage } : {}), - }; -} diff --git a/src/agent/path-utils.ts b/src/agent/path-utils.ts deleted file mode 100644 index 55378366..00000000 --- a/src/agent/path-utils.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { realpathSync } from 'fs'; -import * as path from 'path'; -import { AgentToolError } from '../errors'; - -function isGlobSegment(segment: string): boolean { - return /[*?[\]{}()!+@]/.test(segment); -} - -export function resolveWithinRoot(root: string, inputPath: string): string { - const rootResolved = path.resolve(root); - const targetResolved = path.resolve(rootResolved, inputPath); - - // Lexical check: fast reject for obvious traversal (e.g. ../../etc) - const relative = path.relative(rootResolved, targetResolved); - const outsideRoot = relative.startsWith('..') || path.isAbsolute(relative); - if (outsideRoot) { - throw new AgentToolError( - `Path "${inputPath}" is outside workspace root bounds.`, - 'PATH_OUTSIDE_WORKSPACE_ROOT' - ); - } - - // Symlink check: resolve symlinks on both sides before comparing, so that - // a symlink inside the repo pointing to an external path is caught. - try { - const realRoot = realpathSync(rootResolved); - const realTarget = realpathSync(targetResolved); - const realRelative = path.relative(realRoot, realTarget); - if (realRelative.startsWith('..') || path.isAbsolute(realRelative)) { - throw new AgentToolError( - `Path "${inputPath}" escapes workspace root via symlink.`, - 'PATH_ESCAPES_WORKSPACE_ROOT_SYMLINK' - ); - } - } catch (err) { - // ENOENT means the path does not yet exist — no symlinks to follow, - // so the lexical check above is sufficient. Re-throw any other error. - if (!(err instanceof Error && 'code' in err && (err as { code: string }).code === 'ENOENT')) { - throw err; - } - } - - return targetResolved; -} - -export function resolveGlobPatternWithinRoot(root: string, pattern: string): { cwd: string; pattern: string } { - const normalizedPattern = pattern.replace(/\\/g, '/').trim(); - const segments = normalizedPattern.split('/').filter((segment) => segment.length > 0); - const firstGlobIndex = segments.findIndex((segment) => isGlobSegment(segment)); - - if (firstGlobIndex === -1) { - const literalPath = resolveWithinRoot(root, normalizedPattern || '.'); - return { - cwd: path.dirname(literalPath), - pattern: path.basename(literalPath), - }; - } - - const baseSegments = segments.slice(0, firstGlobIndex); - const globSegments = segments.slice(firstGlobIndex); - const basePath = baseSegments.length > 0 ? baseSegments.join('/') : '.'; - - return { - cwd: resolveWithinRoot(root, basePath), - pattern: globSegments.join('/'), - }; -} - -export function toRelativePathFromRoot(root: string, absolutePath: string): string { - const rootResolved = path.resolve(root); - const absoluteResolved = path.resolve(absolutePath); - return path.relative(rootResolved, absoluteResolved).replace(/\\/g, '/'); -} diff --git a/src/agent/progress.ts b/src/agent/progress.ts deleted file mode 100644 index dff0f64d..00000000 --- a/src/agent/progress.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { OutputFormat } from '../cli/types'; -import * as readline from 'node:readline'; -import ora, { type Ora } from 'ora'; - -const TOOL_PREFIX = ' └ '; -const MAX_TOOL_LINE_LENGTH = 140; - -export type VisibleToolName = 'read_file' | 'list_directory' | 'lint'; - -export interface VisibleToolProgress { - toolName: VisibleToolName; - path?: string; - ruleName?: string; - ruleText?: string; - lineCount?: number; - entryCount?: number; - findingsCount?: number; -} - -type RunStatus = 'completed' | 'failed'; - -export class AgentProgressReporter { - private readonly enabled: boolean; - private readonly spinner: Ora | undefined; - private readonly runStartedAt = Date.now(); - private activeFile: string | undefined; - private activeRuleName: string | undefined; - private activeLine = TOOL_PREFIX; - - constructor(enabled: boolean) { - this.enabled = enabled; - if (enabled) { - this.spinner = ora({ - spinner: 'dots', - stream: createOraStream(process.stderr), - isEnabled: true, - discardStdin: false, - color: false, - }); - } - } - - private writeLine(message: string): void { - if (!this.enabled) { - return; - } - process.stderr.write(`${message}\n`); - } - - private currentBlockText(): string { - return `Reviewing ${this.activeFile ?? ''} for ${this.activeRuleName ?? 'Rule'}\n${this.activeLine}`; - } - - private renderCurrent(): void { - if (!this.enabled || !this.activeFile || !this.spinner) { - return; - } - - const nextText = this.currentBlockText(); - if (!this.spinner.isSpinning) { - this.spinner.start(nextText); - return; - } - this.spinner.text = nextText; - this.spinner.render(); - } - - startFile(file: string, ruleName: string): void { - if (!this.enabled || !this.spinner) { - return; - } - - if (this.spinner.isSpinning && this.activeFile && this.activeFile !== file) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - - this.activeFile = file; - this.activeRuleName = ruleName; - this.activeLine = TOOL_PREFIX; - this.renderCurrent(); - } - - updateRule(ruleName: string): void { - if (!this.enabled || !this.activeFile || this.activeRuleName === ruleName) { - return; - } - - this.activeRuleName = ruleName; - this.renderCurrent(); - } - - showVisibleToolStart(params: VisibleToolProgress & { retrying?: boolean }): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = params.retrying - ? formatRetryingLine(params) - : formatInvocationLine(params); - this.renderCurrent(); - } - - showVisibleToolSuccess(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatSuccessLine(params); - this.renderCurrent(); - } - - showVisibleToolError(params: VisibleToolProgress): void { - if (!this.enabled || !this.activeFile) { - return; - } - - if (params.ruleName) { - this.activeRuleName = params.ruleName; - } - - this.activeLine = formatErrorLine(params); - this.renderCurrent(); - } - - finishRun(status: RunStatus = 'completed'): void { - if (!this.enabled) { - return; - } - - if (this.spinner?.isSpinning && this.activeFile) { - this.spinner.stopAndPersist({ - symbol: '', - text: this.currentBlockText(), - }); - } - this.writeLine(formatRunFooter(status, this.runStartedAt)); - this.activeFile = undefined; - this.activeRuleName = undefined; - this.activeLine = TOOL_PREFIX; - } -} - -function formatInvocationLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Read(${sanitizePath(params.path)})`); - case 'list_directory': - return formatToolLine(`List(${sanitizePath(params.path)})`); - case 'lint': - return formatToolLine(`Lint("${formatRuleSnippet(params.ruleText)}")`); - } -} - -function formatRetryingLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Retrying Read(${sanitizePath(params.path)})...`); - case 'list_directory': - return formatToolLine(`Retrying List(${sanitizePath(params.path)})...`); - case 'lint': - return formatToolLine(`Retrying Lint("${formatRuleSnippet(params.ruleText)}")...`); - } -} - -function formatSuccessLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine( - `Read ${formatCount(params.lineCount ?? 0, 'line')} from ${sanitizePath(params.path)}` - ); - case 'list_directory': - return formatToolLine( - `Listed ${formatCount(params.entryCount ?? 0, 'entry')} in ${sanitizePath(params.path)}` - ); - case 'lint': - if ((params.findingsCount ?? 0) === 0) { - return formatToolLine(`Found no issues in ${sanitizePath(params.path)}`); - } - return formatToolLine( - `Found ${formatCount(params.findingsCount ?? 0, 'issue')} in ${sanitizePath(params.path)}` - ); - } -} - -function formatErrorLine(params: VisibleToolProgress): string { - switch (params.toolName) { - case 'read_file': - return formatToolLine(`Error reading ${sanitizePath(params.path)}`); - case 'list_directory': - return formatToolLine(`Error listing ${sanitizePath(params.path)}`); - case 'lint': - return formatToolLine(`Error linting ${sanitizePath(params.path)}`); - } -} - -function formatToolLine(message: string): string { - return truncate(`${TOOL_PREFIX}${message}`, MAX_TOOL_LINE_LENGTH); -} - -function formatRuleSnippet(ruleText: string | undefined): string { - const sanitized = sanitizeInline(ruleText || ''); - if (sanitized.length === 0) { - return '...'; - } - return truncate(`${sanitized}...`, 52); -} - -function sanitizePath(path: string | undefined): string { - const value = sanitizeInline(path || '.'); - return value.length > 0 ? value : '.'; -} - -function formatCount(count: number, noun: string): string { - return `${count} ${noun}${count === 1 ? '' : 's'}`; -} - -function sanitizeInline(value: string): string { - return value.replace(/\s+/g, ' ').trim(); -} - -function truncate(value: string, maxLength: number): string { - if (value.length <= maxLength) { - return value; - } - return `${value.slice(0, Math.max(0, maxLength - 3))}...`; -} - -function formatElapsed(startedAt: number): string { - const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); - const hours = Math.floor(elapsedSeconds / 3600); - const minutes = Math.floor((elapsedSeconds % 3600) / 60); - const seconds = elapsedSeconds % 60; - - if (hours > 0) { - return `${hours}h ${minutes}m ${seconds}s`; - } - if (minutes > 0) { - return `${minutes}m ${seconds}s`; - } - return `${seconds}s`; -} - -function formatRunFooter(status: RunStatus, startedAt: number): string { - const elapsed = formatElapsed(startedAt); - return status === 'failed' - ? `Review failed after ${elapsed}.` - : `Completed review in ${elapsed}.`; -} - -function createOraStream(stream: NodeJS.WriteStream): NodeJS.WriteStream { - return { - ...stream, - isTTY: stream.isTTY === true, - columns: stream.columns, - write: stream.write.bind(stream), - cursorTo: (x: number) => readline.cursorTo(stream, x), - clearLine: (dir: -1 | 0 | 1) => readline.clearLine(stream, dir), - moveCursor: (dx: number, dy: number) => readline.moveCursor(stream, dx, dy), - } as NodeJS.WriteStream; -} - -export function shouldEmitAgentProgress(params: { outputFormat: OutputFormat; printMode: boolean }): boolean { - const { outputFormat, printMode } = params; - if (printMode) { - return false; - } - if (outputFormat !== OutputFormat.Line) { - return false; - } - return process.stderr.isTTY === true; -} diff --git a/src/agent/prompt-builder.ts b/src/agent/prompt-builder.ts deleted file mode 100644 index 0d16870a..00000000 --- a/src/agent/prompt-builder.ts +++ /dev/null @@ -1,57 +0,0 @@ -export interface BuildAgentSystemPromptParams { - workspaceRoot: string; - fileRuleMatches: Array<{ file: string; ruleSource: string }>; - availableTools: Array<{ name: string; description: string }>; - userInstructions?: string; -} - -function formatBulletedList(values: string[]): string { - if (values.length === 0) { - return '- (none)'; - } - return values.map((value) => `- ${value}`).join('\n'); -} - -function formatFileRuleMatches( - matches: Array<{ file: string; ruleSource: string }> -): string { - if (matches.length === 0) { - return '- (none)'; - } - - const rulesByFile = new Map(); - for (const { file, ruleSource } of matches) { - const rules = rulesByFile.get(file) ?? []; - rules.push(ruleSource); - rulesByFile.set(file, rules); - } - - return Array.from(rulesByFile.entries()) - .map(([file, rules]) => `- ${file}\n${rules.map((rule) => ` - ${rule}`).join('\n')}`) - .join('\n'); -} - -export function buildAgentSystemPrompt(params: BuildAgentSystemPromptParams): string { - const date = new Date().toISOString().slice(0, 10); - const userInstructions = params.userInstructions?.trim(); - const fileRuleMatches = formatFileRuleMatches(params.fileRuleMatches); - - return `You are a senior technical writer. You review documentation files against source-backed rules to identify quality issues, inconsistencies, and violations. - -Your goal is to produce a thorough, complete review of every file against every matched rule. - -Workflow: -1. You are given matched file-rule pairs. Work through each file one at a time — complete every matched rule for a file before moving to the next. -2. For each file-rule pair, review the file against the rule. -3. After reviewing the file, read the rule. If the rule contains top-level review instructions — such as checking for documentation drift, verifying that certain files exist, or any other workspace-level check — carry them out and report any findings. -4. When every file has been reviewed against all of its matched rules, you MUST call the finalize_review tool. - -Available tools: -${formatBulletedList(params.availableTools.map((toolDef) => `${toolDef.name}: ${toolDef.description}`))} - -Review files and matched rules: -${fileRuleMatches}${userInstructions ? `\n\nUser Instructions (from VECTORLINT.md):\n${userInstructions}` : ''} - -Current date: ${date} -Workspace root: ${params.workspaceRoot}`; -} diff --git a/src/agent/review-session-store.ts b/src/agent/review-session-store.ts deleted file mode 100644 index d5fb7262..00000000 --- a/src/agent/review-session-store.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { randomUUID } from 'crypto'; -import { appendFile, mkdir, open, readFile } from 'fs/promises'; -import * as path from 'path'; -import { SESSION_EVENT_SCHEMA, SESSION_EVENT_TYPE, type SessionEvent } from './types'; - -type AppendableSessionEvent = Omit; - -export interface ReviewSessionStore { - sessionId: string; - sessionFilePath: string; - append: (event: AppendableSessionEvent) => Promise; - replay: () => Promise; - hasFinalizedEvent: () => Promise; -} - -function buildSessionFilePath(reviewsDir: string, sessionId: string): string { - return path.join(reviewsDir, `${sessionId}.jsonl`); -} - -async function createUniqueSessionFile(reviewsDir: string): Promise<{ sessionId: string; sessionFilePath: string }> { - while (true) { - const sessionId = randomUUID(); - const sessionFilePath = buildSessionFilePath(reviewsDir, sessionId); - - try { - const handle = await open(sessionFilePath, 'wx'); - await handle.close(); - return { sessionId, sessionFilePath }; - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'EEXIST') { - continue; - } - throw error; - } - } -} - -export async function createReviewSessionStore({ homeDir }: { homeDir: string }): Promise { - const reviewsDir = path.join(homeDir, '.vectorlint', 'reviews'); - await mkdir(reviewsDir, { recursive: true }); - - const { sessionId, sessionFilePath } = await createUniqueSessionFile(reviewsDir); - - async function append(event: AppendableSessionEvent): Promise { - const parsed = SESSION_EVENT_SCHEMA.parse({ - sessionId, - timestamp: new Date().toISOString(), - ...event, - }); - await appendFile(sessionFilePath, `${JSON.stringify(parsed)}\n`, 'utf8'); - } - - async function replay(): Promise { - let raw = ''; - try { - raw = await readFile(sessionFilePath, 'utf8'); - } catch (error) { - if (error instanceof Error && 'code' in error && (error as { code: string }).code === 'ENOENT') { - return []; - } - throw error; - } - - const events: SessionEvent[] = []; - const lines = raw.split('\n').filter((line) => line.trim().length > 0); - - for (const line of lines) { - try { - const candidate = JSON.parse(line) as unknown; - const parsed = SESSION_EVENT_SCHEMA.safeParse(candidate); - if (parsed.success) { - events.push(parsed.data); - } - } catch { - // Ignore malformed JSONL lines so replay can recover valid events. - } - } - - return events; - } - - async function hasFinalizedEvent(): Promise { - const events = await replay(); - return events.some((event) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized); - } - - return { - sessionId, - sessionFilePath, - append, - replay, - hasFinalizedEvent, - }; -} diff --git a/src/agent/rule-id.ts b/src/agent/rule-id.ts deleted file mode 100644 index 2ee68813..00000000 --- a/src/agent/rule-id.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { PromptFile } from '../prompts/prompt-loader'; - -export function buildRuleId(prompt: PromptFile): string { - const pack = prompt.pack || 'Default'; - const rule = String(prompt.meta.id || prompt.filename || 'Rule'); - return `${pack}.${rule}`; -} - -export function normalizeRuleSource(ruleSource: string): string { - return ruleSource.replace(/\\/g, '/').replace(/^\.\//, ''); -} diff --git a/src/agent/tools-registry.ts b/src/agent/tools-registry.ts deleted file mode 100644 index 4bc49dea..00000000 --- a/src/agent/tools-registry.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { AgentToolDefinition } from '../providers/llm-provider'; -import { - FINALIZE_REVIEW_INPUT_SCHEMA, - LINT_TOOL_INPUT_SCHEMA, - LIST_DIRECTORY_INPUT_SCHEMA, - READ_FILE_INPUT_SCHEMA, - SEARCH_CONTENT_INPUT_SCHEMA, - SEARCH_FILES_INPUT_SCHEMA, - TOP_LEVEL_REPORT_INPUT_SCHEMA, -} from './types'; - -export type AgentToolName = - | 'lint' - | 'report_finding' - | 'read_file' - | 'search_files' - | 'list_directory' - | 'search_content' - | 'finalize_review'; - -export type AgentToolHandler = (input: unknown) => Promise; -export type AgentToolHandlers = Record; - -export function createAgentTools(params: { - runTool: ( - toolName: AgentToolName, - input: unknown, - handler: AgentToolHandler - ) => Promise; - handlers: AgentToolHandlers; -}): Record { - const { runTool, handlers } = params; - - return { - lint: { - description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.', - inputSchema: LINT_TOOL_INPUT_SCHEMA, - execute: (input) => runTool('lint', input, handlers.lint), - }, - report_finding: { - description: 'Record a top-level finding for the report.', - inputSchema: TOP_LEVEL_REPORT_INPUT_SCHEMA, - execute: (input) => runTool('report_finding', input, handlers.report_finding), - }, - read_file: { - description: 'Read a file inside the workspace root.', - inputSchema: READ_FILE_INPUT_SCHEMA, - execute: (input) => runTool('read_file', input, handlers.read_file), - }, - search_files: { - description: 'Find files in the workspace by glob pattern.', - inputSchema: SEARCH_FILES_INPUT_SCHEMA, - execute: (input) => runTool('search_files', input, handlers.search_files), - }, - list_directory: { - description: 'List files and directories inside a path in the workspace.', - inputSchema: LIST_DIRECTORY_INPUT_SCHEMA, - execute: (input) => runTool('list_directory', input, handlers.list_directory), - }, - search_content: { - description: 'Search workspace text content by substring and optional glob.', - inputSchema: SEARCH_CONTENT_INPUT_SCHEMA, - execute: (input) => runTool('search_content', input, handlers.search_content), - }, - finalize_review: { - description: 'REQUIRED: Call this tool when all matched file-rule pairs have been reviewed.', - inputSchema: FINALIZE_REVIEW_INPUT_SCHEMA, - execute: (input) => runTool('finalize_review', input, handlers.finalize_review), - }, - }; -} - -export function listAvailableTools( - tools: Record -): Array<{ name: string; description: string }> { - return Object.entries(tools).map(([name, definition]) => ({ - name, - description: definition.description, - })); -} diff --git a/src/agent/types.ts b/src/agent/types.ts deleted file mode 100644 index e1f2407d..00000000 --- a/src/agent/types.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { z } from 'zod'; - -export const SESSION_EVENT_TYPE = { - SessionStarted: 'session_started', - ToolCallStarted: 'tool_call_started', - ToolCallFinished: 'tool_call_finished', - FindingRecordedInline: 'finding_recorded_inline', - FindingRecordedTopLevel: 'finding_recorded_top_level', - SessionFinalized: 'session_finalized', -} as const; - -export const LINT_TOOL_INPUT_SCHEMA = z.object({ - file: z.string().min(1), - ruleSource: z.string().min(1), - reviewInstruction: z.string().min(1).optional(), -}); - -export const TOP_LEVEL_REFERENCE_SCHEMA = z.object({ - file: z.string().min(1), - startLine: z.number().int().positive(), - endLine: z.number().int().positive(), -}); - -export const TOP_LEVEL_REPORT_INPUT_SCHEMA = z.object({ - kind: z.literal('top-level'), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), -}); - -export const READ_FILE_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_FILES_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), -}); - -export const LIST_DIRECTORY_INPUT_SCHEMA = z.object({ - path: z.string().min(1), -}); - -export const SEARCH_CONTENT_INPUT_SCHEMA = z.object({ - pattern: z.string().min(1), - path: z.string().optional(), - glob: z.string().optional(), -}); - -export const FINALIZE_REVIEW_INPUT_SCHEMA = z.object({ - summary: z.string().optional(), -}); - -const SESSION_EVENT_BASE_SCHEMA = z.object({ - sessionId: z.string().min(1), - timestamp: z.string().min(1), -}); - -const SESSION_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionStarted), - payload: z.object({ - cwd: z.string().min(1), - targets: z.array(z.string().min(1)), - }), -}); - -const TOOL_CALL_STARTED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallStarted), - payload: z.object({ - toolName: z.string().min(1), - input: z.unknown(), - }), -}); - -const TOOL_CALL_FINISHED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.ToolCallFinished), - payload: z.object({ - toolName: z.string().min(1), - ok: z.boolean(), - output: z.unknown().optional(), - error: z.string().optional(), - }), -}); - -const FINDING_RECORDED_INLINE_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedInline), - payload: z.object({ - file: z.string().min(1), - line: z.number().int().positive(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - analysis: z.string().optional(), - suggestion: z.string().optional(), - fix: z.string().optional(), - match: z.string().optional(), - }), -}); - -const FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.FindingRecordedTopLevel), - payload: z.object({ - file: z.string().min(1).optional(), - line: z.number().int().positive().optional(), - column: z.number().int().positive().optional(), - severity: z.enum(['error', 'warning']).optional(), - ruleId: z.string().min(1).optional(), - ruleSource: z.string().min(1), - message: z.string().min(1), - suggestion: z.string().optional(), - references: z.array(TOP_LEVEL_REFERENCE_SCHEMA).optional(), - }), -}); - -const SESSION_FINALIZED_EVENT_SCHEMA = SESSION_EVENT_BASE_SCHEMA.extend({ - eventType: z.literal(SESSION_EVENT_TYPE.SessionFinalized), - payload: z.object({ - totalFindings: z.number().int().nonnegative(), - summary: z.string().optional(), - }), -}); - -export const SESSION_EVENT_SCHEMA = z.discriminatedUnion('eventType', [ - SESSION_STARTED_EVENT_SCHEMA, - TOOL_CALL_STARTED_EVENT_SCHEMA, - TOOL_CALL_FINISHED_EVENT_SCHEMA, - FINDING_RECORDED_INLINE_EVENT_SCHEMA, - FINDING_RECORDED_TOP_LEVEL_EVENT_SCHEMA, - SESSION_FINALIZED_EVENT_SCHEMA, -]); - -export type SessionEvent = z.infer; diff --git a/src/boundaries/rule-pack-loader.ts b/src/boundaries/rule-pack-loader.ts index 6769b1a6..a899c41b 100644 --- a/src/boundaries/rule-pack-loader.ts +++ b/src/boundaries/rule-pack-loader.ts @@ -64,9 +64,9 @@ export class RulePackLoader { } /** - * Recursively finds all evaluation files in a pack directory. - * @param packRoot The root directory of the eval pack - * @returns A list of absolute file paths to evaluation files + * Recursively finds all rule files in a pack directory. + * @param packRoot The root directory of the rule pack + * @returns A list of absolute file paths to rule files */ async findRuleFiles(packPath: string): Promise { const rules: string[] = []; diff --git a/src/chunking/merger.ts b/src/chunking/merger.ts index c76e17ff..c65f8aea 100644 --- a/src/chunking/merger.ts +++ b/src/chunking/merger.ts @@ -1,8 +1,8 @@ -import type { EvaluationItem } from "../prompts/schema"; +import type { ReviewItem } from "../prompts/schema"; export function mergeViolations( - chunkViolations: EvaluationItem[][] -): EvaluationItem[] { + chunkViolations: ReviewItem[][] +): ReviewItem[] { const all = chunkViolations.flat(); // Deduplicate using composite key (quoted_text + description + analysis) diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 079a9b29..e6258970 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -4,8 +4,6 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { dirname } from 'path'; import { createProvider } from '../providers/provider-factory'; -import { PerplexitySearchProvider } from '../providers/perplexity-provider'; -import type { SearchProvider } from '../providers/search-provider'; import { loadConfig } from '../boundaries/config-loader'; import { loadUserInstructions } from '../boundaries/user-instruction-loader'; import { loadRuleFile, type PromptFile } from '../prompts/prompt-loader'; @@ -17,8 +15,8 @@ import { loadDirective } from '../prompts/directive-loader'; import { resolveTargets } from '../scan/file-resolver'; import { parseCliOptions, parseEnvironment } from '../boundaries/index'; import { handleUnknownError } from '../errors/index'; -import { evaluateFiles } from './orchestrator'; -import { DEFAULT_REVIEW_MODE, OUTPUT_FORMATS, OutputFormat } from './types'; +import { reviewFiles } from './orchestrator'; +import { DEFAULT_REVIEW_MODEL_CALL, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; import { createNoopLogger } from '../logging/logger'; @@ -71,10 +69,7 @@ async function runOrExit( } } -/* - * Registers the main evaluation command with Commander. - * This is the default command that runs content evaluations against target files. - */ +/** Registers the main content-review command with Commander. */ export function registerMainCommand(program: Command): void { program .option('-v, --verbose', 'Enable verbose logging') @@ -82,8 +77,8 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default).', DEFAULT_REVIEW_MODE) - .option('-p, --print', 'Suppress interactive progress output') + .option('--model-call ', 'Model call strategy: single (one structured call per rule), agent (bounded target-only paging), or auto (default)', DEFAULT_REVIEW_MODEL_CALL) + .option('--mode ', 'Removed: use --model-call instead') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') .action(async (paths: string[] = []) => { @@ -94,6 +89,11 @@ export function registerMainCommand(program: Command): void { return; } + if (program.opts().mode !== undefined) { + console.error('Error: --mode is no longer supported. Use --model-call single|agent|auto instead.'); + process.exit(1); + } + // Parse and validate CLI options const cliOptions = await runOrExit('Parsing CLI options', () => parseCliOptions(program.opts())); @@ -177,7 +177,7 @@ export function registerMainCommand(program: Command): void { } if (targets.length === 0) { - console.error('Error: no target files found to evaluate.'); + console.error('Error: no target files found to review.'); process.exit(1); } @@ -192,6 +192,7 @@ export function registerMainCommand(program: Command): void { try { observability = await initializeObservability(env, runtimeLogger); + const requestBuilder = new DefaultRequestBuilder(directive, userInstructions.content || undefined); const provider = createProvider( env, { @@ -201,27 +202,21 @@ export function registerMainCommand(program: Command): void { logger: runtimeLogger, observability, }, - new DefaultRequestBuilder(directive, userInstructions.content || undefined) + requestBuilder ); - // Create search provider if API key is available - const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY - ? new PerplexitySearchProvider({ logger: runtimeLogger }) - : undefined; - - // Run evaluations via orchestrator - const result = await evaluateFiles(targets, { + // Run reviews via the orchestrator. + const result = await reviewFiles(targets, { prompts, rulesPath, provider, - ...(searchProvider ? { searchProvider } : {}), + requestBuilder, concurrency: config.concurrency, verbose: cliOptions.verbose, logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, - mode: cliOptions.mode, - printMode: cliOptions.print, + modelCall: cliOptions.modelCall, scanPaths: config.scanPaths, pricing: { inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, diff --git a/src/cli/init-command.ts b/src/cli/init-command.ts index 66998623..14a428ac 100644 --- a/src/cli/init-command.ts +++ b/src/cli/init-command.ts @@ -20,7 +20,7 @@ RunRules=VectorLint const USER_INSTRUCTION_TEMPLATE = `# User Instructions @@ -117,7 +117,7 @@ export function registerInitCommand(program: Command): void { console.log(`Next steps:`); console.log(` 1. Open ${getGlobalConfigPath()} and configure your API keys (e.g., OPENAI_API_KEY)`); if (createUserInstructions) { - console.log(` 2. Edit ${USER_INSTRUCTION_FILENAME} to define your instructions for content evaluation`); + console.log(` 2. Edit ${USER_INSTRUCTION_FILENAME} to define your content-review instructions`); } console.log(` ${createUserInstructions ? '3' : '2'}. Run 'vectorlint ' to start linting your content`); if (createConfig) { diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 5b36ea40..cc853ad0 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,40 +1,29 @@ import { readFileSync } from 'fs'; -import { readFile } from 'fs/promises'; import { randomUUID } from 'crypto'; import * as path from 'path'; -import * as os from 'os'; +import { pathToFileURL } from 'url'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { handleUnknownError, MissingDependencyError } from '../errors/index'; -import { processFindings } from '../findings'; -import { createEvaluator } from '../evaluators/index'; -import { Type, Severity } from '../evaluators/types'; +import { printFileHeader, printIssueRow, printReviewSummaries, type ReviewSummary } from '../output/reporter'; +import { handleUnknownError } from '../errors/index'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; -import { OutputFormat, DEFAULT_REVIEW_MODE, AGENT_REVIEW_MODE } from './types'; -import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '../agent/executor'; -import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; +import { OutputFormat } from './types'; +import { executorFor } from '../executors'; +import { chooseModelCall } from '../review/executor'; +import { buildReviewRequest } from '../review/request-builder'; +import type { ReviewResult, ReviewSeverity, ReviewTarget } from '../review/types'; import type { - EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessPromptResultParams, - RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, - RunPromptEvaluationResultSuccess + ReviewOptions, ReviewRunResult, ReportIssueParams, ReviewFileResult, } from './types'; import { calculateCost, TokenUsageStats } from '../providers/token-usage'; -import { calculateScore } from '../scoring'; -import { countWords } from '../chunking/utils'; -import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { - computeFilterDecision, - type FilterDecision, -} from "../evaluators/violation-filter"; -import { writeDebugRunArtifact } from "../debug/run-artifact"; +import { writeDebugRunArtifact } from '../debug/run-artifact'; +import { Severity } from '../review/severity'; function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } { const provider = process.env.LLM_PROVIDER; @@ -59,34 +48,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string return { ...(provider && { provider }), ...(name && { name }), ...(tag && { tag }) }; } - -/* - * Generic concurrency runner that executes workers in parallel up to a specified limit. - * Preserves result order matching input order. - */ -async function runWithConcurrency( - items: T[], - limit: number, - worker: (item: T, index: number) => Promise -): Promise { - const results: R[] = new Array(items.length); - let i = 0; - const workers = new Array(Math.min(limit, items.length)) - .fill(0) - .map(async () => { - while (true) { - const idx = i++; - if (idx >= items.length) break; - const item = items[idx]; - if (item !== undefined) { - results[idx] = await worker(item, idx); - } - } - }); - await Promise.all(workers); - return results; -} - /* * Reports an issue in either line or JSON format. */ @@ -153,216 +114,60 @@ function reportIssue(params: ReportIssueParams): void { } } -function getViolationFilterResults< - TViolation extends Parameters[0] ->( - violations: TViolation[] -): { - decisions: FilterDecision[]; - surfacedViolations: TViolation[]; -} { - const decisions = violations.map((v) => computeFilterDecision(v)); - const surfacedViolations = violations.filter( - (_v, i) => decisions[i]?.surface === true - ); - - return { decisions, surfacedViolations }; +function toSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; } -/** Routes an evaluation result through the shared finding processor. */ -function routePromptResult( - params: ProcessPromptResultParams -): ErrorTrackingResult { - const { - promptFile, - result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - debugJson, - } = params; - const meta = promptFile.meta; - const promptId = (meta.id || "").toString(); - - const reviewResult = processFindings({ - pack: promptFile.pack, - ruleId: promptId, - ruleSource: promptFile.fullPath, - candidateFindings: result.violations, - wordCount: result.word_count, - promptMeta: { - ...(meta.severity !== undefined ? { severity: meta.severity } : {}), - ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - ...(meta.criteria - ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } - : {}), - }, - targetContent: content, - }); - - const ruleScore = reviewResult.scores[0]!; - const severity = - ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - - for (const finding of reviewResult.findings) { - reportIssue({ - file: relFile, - line: finding.line, - column: finding.column, - severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), - ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), - ...(finding.fix !== undefined ? { fix: finding.fix } : {}), - match: finding.match, - }); - } - - if (verbose) { - for (const diagnostic of reviewResult.diagnostics) { - console.warn(`[vectorlint] ${diagnostic.message}`); - } - } - - const findingCount = reviewResult.findings.length; - const totalErrors = severity === Severity.ERROR ? findingCount : 0; - const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; - - const scoreEntry: EvaluationSummary = { - id: ruleScore.ruleId, - scoreText: ruleScore.scoreText, - score: ruleScore.score, - }; - - if (debugJson) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - const runId = randomUUID(); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: decisions.map((d, i) => ({ - index: i, - surface: d.surface, - reasons: d.reasons, - })), - surfaced_violations: surfacedViolations, - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - - return { - errors: totalErrors, - warnings: totalWarnings, - hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, - hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, - scoreEntries: [scoreEntry], - }; +function contentTypeFor(file: string): string { + return path.extname(file).toLowerCase() === '.md' ? 'text/markdown' : 'text/plain'; } -/** Runs a single prompt evaluation. */ -async function runPromptEvaluation( - params: RunPromptEvaluationParams -): Promise { - const { promptFile, relFile, content, provider, searchProvider } = params; +function emptyTokenUsage(): TokenUsageStats { + return { totalInputTokens: 0, totalOutputTokens: 0 }; +} +/* + * Writes a debug JSON artifact for a review run. The executor path reviews + * through the shared finding processor, so raw model output and per-candidate + * filter decisions are no longer reachable here; the artifact records the + * verified findings that surfaced and the review metadata instead. + */ +function writeReviewDebugArtifact(relFile: string, result: ReviewResult): void { + const runId = randomUUID(); + const model = getModelInfoFromEnv(); try { - const meta = promptFile.meta; - - const evaluatorType = String(meta.evaluator || Type.BASE); - const baseEvaluatorType = String(Type.BASE); - - // Specialized evaluators (e.g., technical-accuracy) require criteria. - if (evaluatorType !== baseEvaluatorType) { - if ( - !meta || - !Array.isArray(meta.criteria) || - meta.criteria.length === 0 - ) { - throw new Error( - `Prompt ${promptFile.filename} has no criteria in frontmatter` - ); - } - } - const evaluator = createEvaluator( - evaluatorType, - provider, - promptFile, - searchProvider - ); - const result = await evaluator.evaluate(relFile, content); - - - const resultObj: RunPromptEvaluationResultSuccess = { ok: true, result }; - - return resultObj; - } catch (e: unknown) { - const err = handleUnknownError(e, `Running prompt ${promptFile.filename}`); - return { ok: false, error: err }; + const filePath = writeDebugRunArtifact(process.cwd(), runId, { + file: relFile, + ...(Object.keys(model).length > 0 ? { model } : {}), + ...(model.tag !== undefined ? { subdir: model.tag } : {}), + prompt: {}, + raw_model_output: null, + filter_decisions: [], + surfaced_violations: result.findings, + }); + console.warn(`[vectorlint] Debug JSON written: ${filePath}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[vectorlint] Debug JSON write failed: ${message}`); } } /* - * Evaluates a single file with all applicable prompts. + * Determines the source-backed prompts that apply to a file, honoring the + * configured scan paths. When no rules match but a VECTORLINT.md user + * instruction guide exists, a synthetic rule is added so the reviewer model + * reviews against the user instructions in the system prompt. */ -async function evaluateFile( - params: EvaluateFileParams -): Promise { - const { file, options, jsonFormatter } = params; - const { - prompts, - provider, - searchProvider, - concurrency, - scanPaths, - outputFormat = OutputFormat.Line, - verbose, - debugJson, - } = options; - - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let totalErrors = 0; - let totalWarnings = 0; - let requestFailures = 0; - let totalInputTokens = 0; - let totalOutputTokens = 0; - - const allScores = new Map(); - - const content = readFileSync(file, "utf-8"); - const relFile = path.relative(process.cwd(), file) || file; - - if (outputFormat === OutputFormat.Line) { - printFileHeader(relFile); - } - - // Determine applicable prompts for this file +function resolveApplicablePrompts( + relFile: string, + prompts: PromptFile[], + scanPaths: ReviewOptions['scanPaths'], + userInstructionContent: string | undefined, +): PromptFile[] { const toRun: PromptFile[] = []; if (scanPaths && scanPaths.length > 0) { const resolver = new ScanPathResolver(); - // Extract available packs from loaded prompts const availablePacks = Array.from( new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p)) ); @@ -373,9 +178,7 @@ async function evaluateFile( availablePacks ); - // Filter prompts by active packs - only runs if explicitly in RunRules const activePrompts = prompts.filter((p) => { - // Prompts with empty pack (style guide only) always run if (p.pack === '') return true; if (!p.pack || !resolution.packs.includes(p.pack)) return false; if (!p.meta?.id) return true; @@ -389,13 +192,10 @@ async function evaluateFile( toRun.push(...activePrompts); } else { - // Fallback: When no scanPaths configured, run all prompts. toRun.push(...prompts); } - // If no rules matched but VECTORLINT.md exists, run an evaluation using it. - // The LLM will use the VECTORLINT.md content from the system prompt. - if (options.userInstructionContent) { + if (userInstructionContent) { toRun.push({ id: USER_INSTRUCTION_FILENAME, filename: USER_INSTRUCTION_FILENAME, @@ -410,344 +210,163 @@ async function evaluateFile( }); } - const results = await runWithConcurrency( - toRun, - concurrency, - async (prompt) => { - return runPromptEvaluation({ - promptFile: prompt, - relFile, - content, - provider, - ...(searchProvider !== undefined && { searchProvider }), - }); - } - ); + return toRun; +} - // Aggregate results from each prompt - for (let idx = 0; idx < toRun.length; idx++) { - const p = toRun[idx]; - const r = results[idx]; - if (!p || !r) continue; - - if (r.ok !== true) { - // Check if this is a missing dependency error - if so, skip gracefully - if (r.error instanceof MissingDependencyError) { - console.warn(`[vectorlint] Skipping ${p.filename}: ${r.error.message}`); - if (r.error.hint) { - console.warn(`[vectorlint] Hint: ${r.error.hint}`); - } - // Skip this evaluation entirely - don't count it as a failure - continue; - } +/* + * Reviews a single file: builds a ReviewRequest from the applicable + * source-backed prompts, resolves the model-call strategy, dispatches through + * the selected ReviewExecutor, and routes the resulting ReviewResult to the + * existing line/json/vale output sinks. + */ +async function reviewFile( + file: string, + options: ReviewOptions, + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter, +): Promise { + const { + prompts, + scanPaths, + outputFormat = OutputFormat.Line, + verbose, + debugJson, + } = options; - // Other errors are actual failures - console.error(` Prompt failed: ${p.filename}`); - console.error(r.error); - hadOperationalErrors = true; - requestFailures += 1; - continue; - } + const allScores = new Map(); - // Accumulate token usage - if (r.result.usage) { - totalInputTokens += r.result.usage.inputTokens; - totalOutputTokens += r.result.usage.outputTokens; - } + const content = readFileSync(file, "utf-8"); + const relFile = path.relative(process.cwd(), file) || file; - const promptResult = routePromptResult({ - promptFile: p, - result: r.result, - content, - relFile, - outputFormat, - jsonFormatter, - verbose, - ...(debugJson !== undefined ? { debugJson } : {}), - }); - totalErrors += promptResult.errors; - totalWarnings += promptResult.warnings; - hadOperationalErrors = - hadOperationalErrors || promptResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || promptResult.hadSeverityErrors; - - if (promptResult.scoreEntries && promptResult.scoreEntries.length > 0) { - const ruleName = (p.meta.id || p.filename).toString(); - allScores.set(ruleName, promptResult.scoreEntries); - } + if (outputFormat === OutputFormat.Line) { + printFileHeader(relFile); } - const tokenUsageStats: TokenUsageStats = { - totalInputTokens, - totalOutputTokens, - }; + const toRun = resolveApplicablePrompts( + relFile, + prompts, + scanPaths, + options.userInstructionContent, + ); - if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); - console.log(""); + // No applicable rules: nothing to review for this file. + if (toRun.length === 0) { + if (outputFormat === OutputFormat.Line) { + printReviewSummaries(allScores); + console.log(""); + } + return { + errors: 0, + warnings: 0, + requestFailures: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } - return { - errors: totalErrors, - warnings: totalWarnings, - requestFailures, - hadOperationalErrors, - hadSeverityErrors, - tokenUsage: tokenUsageStats + const target: ReviewTarget = { + uri: pathToFileURL(path.resolve(file)).href, + content, + contentType: contentTypeFor(file), + byteLength: Buffer.byteLength(content), }; -} - -function reportAgentFinding(params: { - finding: AgentFinding; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -}): void { - const { finding, outputFormat, jsonFormatter } = params; - - reportIssue({ - file: finding.file, - line: finding.line, - column: finding.column, - severity: finding.severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis ? { analysis: finding.analysis } : {}), - ...(finding.suggestion ? { suggestion: finding.suggestion } : {}), - ...(finding.fix ? { fix: finding.fix } : {}), - ...(finding.match ? { match: finding.match } : {}), + const request = buildReviewRequest({ + target, + prompts: toRun, + config: { modelCall: options.modelCall }, + }); + const resolvedModelCall = chooseModelCall(request.modelCall, { + targetBytes: request.target.byteLength ?? Buffer.byteLength(content), + rules: request.rules.length, + }); + const executor = executorFor(resolvedModelCall, { + structuredModelClient: options.provider, + toolCallingModelClient: options.provider, + builder: options.requestBuilder, }); -} - -type AgentRuleScore = { - ruleId: string; - score: number; - scoreText: string; -}; - -async function getAgentFileWordCount( - file: string, - workspaceRoot: string, - cache: Map -): Promise { - const workspaceRelative = path.relative(workspaceRoot, path.resolve(workspaceRoot, file)) || file; - if (cache.has(workspaceRelative)) { - return cache.get(workspaceRelative)!; - } - const absolutePath = path.resolve(workspaceRoot, workspaceRelative); + let result: ReviewResult; try { - const content = await readFile(absolutePath, 'utf-8'); - const words = Math.max(1, countWords(content) || 1); - cache.set(workspaceRelative, words); - return words; - } catch { - cache.set(workspaceRelative, 1); - return 1; + result = await executor.run(request); + } catch (e: unknown) { + const err = handleUnknownError(e, `Reviewing ${relFile}`); + console.error(`Error reviewing file ${relFile}: ${err.message}`); + return { + errors: 0, + warnings: 0, + requestFailures: 1, + hadOperationalErrors: true, + hadSeverityErrors: false, + tokenUsage: emptyTokenUsage(), + }; } -} -async function buildAgentRuleScores( - findings: AgentFinding[], - prompts: PromptFile[], - fileRuleMatches: Array<{ file: string; ruleSource: string }>, - workspaceRoot: string -): Promise { - const fileWordCountCache = new Map(); - const findingsByRule = new Map(); - const filesByRuleSource = new Map>(); - - for (const finding of findings) { - const existing = findingsByRule.get(finding.ruleId) ?? []; - existing.push(finding); - findingsByRule.set(finding.ruleId, existing); - } - for (const match of fileRuleMatches) { - const files = filesByRuleSource.get(match.ruleSource) ?? new Set(); - files.add(match.file); - filesByRuleSource.set(match.ruleSource, files); + // Route the ReviewResult's verified findings through the existing sinks. + for (const finding of result.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity: toSeverity(finding.severity), + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); } - const results: AgentRuleScore[] = []; - for (const prompt of prompts) { - const ruleId = buildRuleId(prompt); - const ruleFindings = findingsByRule.get(ruleId) ?? []; - const matchedFiles = filesByRuleSource.get(normalizeRuleSource(prompt.fullPath)) ?? new Set(); - - if (matchedFiles.size === 0) { - results.push({ - ruleId, - score: 10, - scoreText: '10.0/10', - }); - continue; - } + for (const score of result.scores) { + allScores.set(score.ruleId, [ + { id: score.ruleId, scoreText: score.scoreText, score: score.score }, + ]); + } - let totalWords = 0; - for (const file of matchedFiles) { - totalWords += await getAgentFileWordCount(file, workspaceRoot, fileWordCountCache); + if (verbose) { + for (const diagnostic of result.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } - - const syntheticViolations = Array.from({ length: ruleFindings.length }, (_, index) => ({ - line: index + 1, - description: 'Agent finding', - analysis: 'Agent finding', - })); - - const scored = calculateScore( - syntheticViolations, - Math.max(1, totalWords), - { - strictness: prompt.meta.strictness, - promptSeverity: prompt.meta.severity, - } - ); - - results.push({ - ruleId, - score: scored.final_score, - scoreText: `${scored.final_score.toFixed(1)}/10`, - }); } - return results; -} -// eslint-disable-next-line @typescript-eslint/no-unused-vars -async function evaluateFilesInAgentMode( - targets: string[], - options: EvaluationOptions, - outputFormat: OutputFormat, - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter -): Promise { - const workspaceRoot = inferAgentWorkspaceRoot(targets); - const progressReporter = new AgentProgressReporter( - shouldEmitAgentProgress({ - outputFormat, - printMode: options.printMode ?? false, - }) - ); + const totalErrors = result.findings.filter((f) => f.severity === 'error').length; + const totalWarnings = result.findings.filter((f) => f.severity === 'warning').length; - const agentResult: AgentExecutorResult = await runAgentExecutor({ - targets, - prompts: options.prompts, - provider: options.provider, - workspaceRoot, - scanPaths: options.scanPaths, - outputFormat, - printMode: options.printMode ?? false, - sessionHomeDir: os.homedir(), - progressReporter, - maxParallelToolCalls: 3, - maxRetries: options.agentMaxRetries ?? 10, - ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), - }); + const tokenUsage: TokenUsageStats = { + totalInputTokens: result.usage?.inputTokens ?? 0, + totalOutputTokens: result.usage?.outputTokens ?? 0, + }; - let totalErrors = 0; - let totalWarnings = 0; - const printedFileHeaders = new Set(); - for (const finding of agentResult.findings) { - if (outputFormat === OutputFormat.Line && !printedFileHeaders.has(finding.file)) { - printFileHeader(finding.file); - printedFileHeaders.add(finding.file); - } - reportAgentFinding({ finding, outputFormat, jsonFormatter }); - if (finding.severity === Severity.ERROR) { - totalErrors += 1; - } else { - totalWarnings += 1; - } + if (debugJson) { + writeReviewDebugArtifact(relFile, result); } if (outputFormat === OutputFormat.Line) { - const ruleScores = await buildAgentRuleScores( - agentResult.findings, - options.prompts, - agentResult.fileRuleMatches, - workspaceRoot - ); - const scoreSummary = new Map( - ruleScores.map((entry) => [ - entry.ruleId, - [{ id: 'overall', scoreText: entry.scoreText, score: entry.score }], - ]) - ); - printEvaluationSummaries(scoreSummary); - - if (agentResult.hadOperationalErrors) { - const message = agentResult.errorMessage ?? 'Agent run encountered an operational error.'; - console.error(`\n[agent] ${message}`); - } - } - - if ( - outputFormat === OutputFormat.Json || - outputFormat === OutputFormat.ValeJson || - outputFormat === OutputFormat.RdJson - ) { - console.log(jsonFormatter.toJson()); + printReviewSummaries(allScores); + console.log(""); } - const tokenUsage = { - totalInputTokens: agentResult.usage?.inputTokens ?? 0, - totalOutputTokens: agentResult.usage?.outputTokens ?? 0, - }; - return { - totalFiles: targets.length, - totalErrors, - totalWarnings, - requestFailures: agentResult.requestFailures, - hadOperationalErrors: agentResult.hadOperationalErrors, + errors: totalErrors, + warnings: totalWarnings, + requestFailures: 0, + hadOperationalErrors: result.hadOperationalErrors ?? false, hadSeverityErrors: totalErrors > 0, tokenUsage, }; } -function inferAgentWorkspaceRoot(targets: string[]): string { - if (targets.length === 0) { - return process.cwd(); - } - - const directories = targets.map((target) => path.dirname(path.resolve(target))); - let root = directories[0]!; - - for (const directory of directories.slice(1)) { - root = commonPathPrefix(root, directory); - } - - return root; -} - -function commonPathPrefix(left: string, right: string): string { - let candidate = path.resolve(left); - const target = path.resolve(right); - - while (true) { - const relative = path.relative(candidate, target); - const insideCandidate = !relative.startsWith('..') && !path.isAbsolute(relative); - if (insideCandidate) { - return candidate; - } - - const parent = path.dirname(candidate); - if (parent === candidate) { - return candidate; - } - candidate = parent; - } -} - /* - * Runs evaluations across all target files with configurable concurrency. - * Coordinates prompt-to-file mapping, evaluation execution, and result aggregation. - * Returns aggregated results for reporting. + * Runs reviews across all target files, dispatching each through the executor + * selected by the model-call strategy, and aggregates the results. */ -export async function evaluateFiles( +export async function reviewFiles( targets: string[], - options: EvaluationOptions -): Promise { - const { outputFormat = OutputFormat.Line, mode = DEFAULT_REVIEW_MODE } = options; + options: ReviewOptions +): Promise { + const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -767,16 +386,10 @@ export async function evaluateFiles( jsonFormatter = new ValeJsonFormatter(); } - if (mode === AGENT_REVIEW_MODE) { - options.logger?.warn( - '--mode agent falls back to standard evaluation.', - ); - } - for (const file of targets) { try { totalFiles += 1; - const fileResult = await evaluateFile({ file, options, jsonFormatter }); + const fileResult = await reviewFile(file, options, jsonFormatter); totalErrors += fileResult.errors; totalWarnings += fileResult.warnings; requestFailures += fileResult.requestFailures; @@ -784,7 +397,6 @@ export async function evaluateFiles( hadOperationalErrors || fileResult.hadOperationalErrors; hadSeverityErrors = hadSeverityErrors || fileResult.hadSeverityErrors; - // Aggregate token usage if (fileResult.tokenUsage) { totalInputTokens += fileResult.tokenUsage.totalInputTokens; totalOutputTokens += fileResult.tokenUsage.totalOutputTokens; diff --git a/src/cli/types.ts b/src/cli/types.ts index a083f2e2..15741a05 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,15 +1,19 @@ import type { PromptFile } from '../prompts/prompt-loader'; -import type { LLMProvider } from '../providers/llm-provider'; -import type { SearchProvider } from '../providers/search-provider'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { RequestBuilder } from '../providers/request-builder'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; -import type { EvaluationSummary } from '../output/reporter'; +import type { ReviewSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult } from '../prompts/schema'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; +import type { ReviewModelCall } from '../review/types'; + +export { REVIEW_MODEL_CALLS } from '../review/executor'; +export type { ReviewModelCall } from '../review/types'; export enum OutputFormat { Line = "line", @@ -27,31 +31,26 @@ export const OUTPUT_FORMATS = [ export const DEFAULT_OUTPUT_FORMAT = OUTPUT_FORMATS[0]; -export const REVIEW_MODES = ['standard', 'agent'] as const; -export const DEFAULT_REVIEW_MODE = REVIEW_MODES[0]; -export const AGENT_REVIEW_MODE = REVIEW_MODES[1]; - -export type ReviewMode = (typeof REVIEW_MODES)[number]; +/** Default reviewer model-call strategy. */ +export const DEFAULT_REVIEW_MODEL_CALL: ReviewModelCall = 'auto'; -export interface EvaluationOptions { +export interface ReviewOptions { prompts: PromptFile[]; rulesPath: string | undefined; - provider: LLMProvider; - searchProvider?: SearchProvider; + provider: StructuredModelClient & ToolCallingModelClient; + requestBuilder: RequestBuilder; concurrency: number; verbose: boolean; debugJson?: boolean; scanPaths: FilePatternConfig[]; outputFormat?: OutputFormat; - mode?: ReviewMode; - printMode?: boolean; - agentMaxRetries?: number; + modelCall: ReviewModelCall; pricing?: PricingConfig; userInstructionContent?: string; logger?: Logger; } -export interface EvaluationResult { +export interface ReviewRunResult { totalFiles: number; totalErrors: number; totalWarnings: number; @@ -66,16 +65,7 @@ export interface ErrorTrackingResult { warnings: number; hadOperationalErrors: boolean; hadSeverityErrors: boolean; - scoreEntries?: EvaluationSummary[]; -} - -export interface EvaluationContext { - content: string; - relFile: string; - outputFormat: OutputFormat; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - verbose?: boolean; - debugJson?: boolean; + scoreEntries?: ReviewSummary[]; } export interface ReportIssueParams { @@ -94,35 +84,7 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessPromptResultParams extends EvaluationContext { - promptFile: PromptFile; - result: PromptEvaluationResult; -} - -export interface RunPromptEvaluationParams { - promptFile: PromptFile; - relFile: string; - content: string; - provider: LLMProvider; - searchProvider?: SearchProvider; -} - -export interface RunPromptEvaluationResultSuccess { - ok: true; - result: PromptEvaluationResult; -} - -export type RunPromptEvaluationResult = - | RunPromptEvaluationResultSuccess - | { ok: false; error: Error }; - -export interface EvaluateFileParams { - file: string; - options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; -} - -export interface EvaluateFileResult extends ErrorTrackingResult { +export interface ReviewFileResult extends ErrorTrackingResult { requestFailures: number; tokenUsage?: TokenUsageStats; } diff --git a/src/cli/validate-command.ts b/src/cli/validate-command.ts index c05b6d64..56471959 100644 --- a/src/cli/validate-command.ts +++ b/src/cli/validate-command.ts @@ -19,7 +19,7 @@ const __dirname = dirname(__filename); /* * Registers the 'validate' command with Commander. - * This command validates prompt configuration files without running evaluations. + * This command validates prompt configuration files without running reviews. * It checks YAML frontmatter structure, schema compliance, and prompt completeness. * * Note: process.exit is intentional in CLI commands to set proper exit codes. diff --git a/src/config/global-config.ts b/src/config/global-config.ts index 98b1a7fc..439b99cd 100644 --- a/src/config/global-config.ts +++ b/src/config/global-config.ts @@ -56,13 +56,6 @@ const DEFAULT_GLOBAL_CONFIG_TEMPLATE = `# VectorLint Environment Configuration # BEDROCK_MODEL = "global.anthropic.claude-sonnet-4-5-20250929-v1:0" # BEDROCK_TEMPERATURE = "0.2" -# ============================================ -# Search Provider Configuration (Optional) -# Enables technical accuracy verification -# ============================================ - -# SEARCH_PROVIDER = "perplexity" -# PERPLEXITY_API_KEY = "pplx-0000000000000000" `; /** @@ -74,11 +67,11 @@ export function ensureGlobalConfig(): string { const configDir = path.dirname(configPath); if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); + mkdirSync(configDir, { recursive: true, mode: 0o700 }); } if (!existsSync(configPath)) { - writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, 'utf-8'); + writeFileSync(configPath, DEFAULT_GLOBAL_CONFIG_TEMPLATE, { encoding: 'utf-8', mode: 0o600 }); } return configPath; diff --git a/src/debug/violation-filter.ts b/src/debug/violation-filter.ts index 2a93d35d..762bf9ae 100644 --- a/src/debug/violation-filter.ts +++ b/src/debug/violation-filter.ts @@ -1,4 +1,4 @@ export { computeFilterDecision, type FilterDecision, -} from "../evaluators/violation-filter"; +} from "../findings/filter-decision"; diff --git a/src/errors/index.ts b/src/errors/index.ts index 3e982bf3..ca196d95 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -30,13 +30,6 @@ export class ProcessingError extends VectorlintError { } } -export class AgentToolError extends VectorlintError { - constructor(message: string, code = 'AGENT_TOOL_ERROR') { - super(message, code); - this.name = 'AgentToolError'; - } -} - // Missing dependency error for when required dependencies are not available export class MissingDependencyError extends VectorlintError { constructor( diff --git a/src/evaluators/accuracy-evaluator.ts b/src/evaluators/accuracy-evaluator.ts deleted file mode 100644 index 2634dffe..00000000 --- a/src/evaluators/accuracy-evaluator.ts +++ /dev/null @@ -1,254 +0,0 @@ -import { BaseEvaluator } from "./base-evaluator"; -import { registerEvaluator } from "./evaluator-registry"; -import type { LLMProvider } from "../providers/llm-provider"; -import type { SearchProvider } from "../providers/search-provider"; -import type { PromptFile } from "../schemas/prompt-schemas"; -import type { PromptEvaluationResult } from "../prompts/schema"; -import type { TokenUsage } from "../providers/token-usage"; -import { renderTemplate } from "../prompts/template-renderer"; -import { getPrompt } from "./prompt-loader"; -import { z } from "zod"; -import { Type, type Severity } from "./types"; -import { MissingDependencyError } from "../errors/index"; -import { countWords } from "../chunking"; - -// Schema for claim extraction response -const CLAIM_EXTRACTION_SCHEMA = z.object({ - claims: z.array(z.string()), -}); - -// Schema for search result -const SEARCH_RESULT_SCHEMA = z.object({ - snippet: z.string(), - url: z.string(), - title: z.string().optional(), -}); - -type SearchResult = z.infer; - -interface ClaimExtractionResult { - claims: string[]; - usage?: TokenUsage; -} - -/** - * Technical Accuracy Evaluator - Acts as an orchestrator only. - * - Evaluator (this class): Orchestrates data gathering (claims, search evidence) - * - Eval (prompt): Contains all evaluation logic via templates - * - * Architecture: - * 1. Extract claims from content (via LLM with claim-extraction prompt) - * 2. Search for evidence for each claim (via SearchProvider) - * 3. Pass content, claims, and evidence to the main eval prompt (via templates) - * 4. Return the structured evaluation result - * - * Evaluators should NOT contain evaluation logic - all evaluation is done by the prompt. - */ -export class TechnicalAccuracyEvaluator extends BaseEvaluator { - private static readonly CLAIM_EXTRACTION_PROMPT_KEY = "claim-extraction"; - - constructor( - llmProvider: LLMProvider, - prompt: PromptFile, - private searchProvider: SearchProvider, - defaultSeverity?: Severity - ) { - super(llmProvider, prompt, defaultSeverity); - } - - async evaluate(_file: string, content: string): Promise { - // Step 1: Extract factual claims from the content - const { claims, usage: claimUsage } = await this.extractClaims(content); - - // If no claims found, return success (empty items array, perfect score) - // Use the scoring module to calculate result - if (claims.length === 0) { - const wordCount = countWords(content) || 1; - const raw: PromptEvaluationResult = { - violations: [], - word_count: wordCount, - ...(claimUsage && { usage: claimUsage }), - }; - return raw; - } - - // Step 2: Search for evidence for each claim - const searchResults = await this.searchForEvidence(claims); - - // Step 3: Prepare template variables - const templateVars = { - content: content, - claims: this.formatClaimsForTemplate(claims), - searchResults: this.formatSearchResultsForTemplate(searchResults), - }; - - // Step 4: Render the prompt with template variables - const renderedPrompt = renderTemplate(this.getPromptBody(), templateVars); - - // Step 5: Create enriched prompt with rendered body - const enrichedPrompt: PromptFile = { - ...this.prompt, - body: renderedPrompt, - }; - - // Step 6: Use parent's evaluation logic with enriched prompt - const evaluator = new BaseEvaluator( - this.llmProvider, - enrichedPrompt, - this.defaultSeverity - ); - const result = await evaluator.evaluate(_file, content); - - // Aggregate token usage from claim extraction + evaluation - if (claimUsage) { - const totalUsage: TokenUsage = { - inputTokens: claimUsage.inputTokens + (result.usage?.inputTokens || 0), - outputTokens: - claimUsage.outputTokens + (result.usage?.outputTokens || 0), - }; - result.usage = totalUsage; - } - - return result; - } - - /** - * Extract factual claims from content using the claim extraction prompt. - */ - private async extractClaims(content: string): Promise { - try { - const claimSchema = { - name: "ClaimExtraction", - schema: { - type: "object", - properties: { - claims: { - type: "array", - items: { type: "string" }, - }, - }, - required: ["claims"], - }, - }; - - const claimExtractionPrompt = getPrompt( - TechnicalAccuracyEvaluator.CLAIM_EXTRACTION_PROMPT_KEY - ); - - const { data: claimData, usage } = - await this.llmProvider.runPromptStructured( - content, - claimExtractionPrompt, - claimSchema - ); - - // Validate the response with Zod schema - const claimResult = CLAIM_EXTRACTION_SCHEMA.parse(claimData); - - return { claims: claimResult.claims, ...(usage && { usage }) }; - } catch (e: unknown) { - const err = e instanceof Error ? e : new Error(String(e)); - console.warn(`[vectorlint] Claim extraction failed: ${err.message}`); - return { claims: [] }; - } - } - - /** - * Search for evidence for each claim. - * Returns a map of claim index to search results. - */ - private async searchForEvidence( - claims: string[] - ): Promise> { - const resultsMap = new Map(); - - for (let i = 0; i < claims.length; i++) { - const claim = claims[i]; - // Skip if claim is undefined (shouldn't happen, but TypeScript requires the check) - if (!claim) { - resultsMap.set(i, []); - continue; - } - - try { - const snippetsRaw: unknown = await this.searchProvider.search(claim); - - // Validate search results - const SEARCH_RESULTS_ARRAY_SCHEMA = z.array(SEARCH_RESULT_SCHEMA); - const snippets = SEARCH_RESULTS_ARRAY_SCHEMA.parse(snippetsRaw); - - resultsMap.set(i, snippets); - } catch (e: unknown) { - const err = e instanceof Error ? e : new Error(String(e)); - console.warn( - `[vectorlint] Search failed for claim "${claim}": ${err.message}` - ); - resultsMap.set(i, []); - } - } - - return resultsMap; - } - - /** - * Format claims as a numbered list for the template. - */ - private formatClaimsForTemplate(claims: string[]): string { - return claims.map((claim, index) => `${index + 1}. ${claim}`).join("\n"); - } - - /** - * Format search results grouped by claim for the template. - */ - private formatSearchResultsForTemplate( - resultsMap: Map - ): string { - const formatted: string[] = []; - - for (const [index, results] of resultsMap.entries()) { - const claimNum = index + 1; - formatted.push(`\n### Claim ${claimNum} Evidence:`); - - if (results.length === 0) { - formatted.push("No search results found."); - } else { - results.forEach((result, i) => { - formatted.push(`[${i + 1}] ${result.snippet} (${result.url})`); - }); - } - } - - return formatted.join("\n"); - } - - /** - * Get the prompt body, ensuring it's defined. - * Throws an error if the prompt body is missing. - */ - private getPromptBody(): string { - if (!this.prompt.body) { - throw new Error("Prompt body is empty or undefined"); - } - return this.prompt.body; - } -} - -// Self-register on module load using registerEvaluator directly -registerEvaluator( - Type.TECHNICAL_ACCURACY, - (llmProvider, prompt, searchProvider, defaultSeverity) => { - if (!searchProvider) { - throw new MissingDependencyError( - "technical-accuracy evaluator requires a search provider", - "search-provider", - "Configure TAVILY_API_KEY or PERPLEXITY_API_KEY in .env, or remove this eval" - ); - } - return new TechnicalAccuracyEvaluator( - llmProvider, - prompt, - searchProvider, - defaultSeverity - ); - } -); diff --git a/src/evaluators/base-evaluator.ts b/src/evaluators/base-evaluator.ts deleted file mode 100644 index 2b71ae5f..00000000 --- a/src/evaluators/base-evaluator.ts +++ /dev/null @@ -1,128 +0,0 @@ -import path from "path"; -import type { LLMProvider } from "../providers/llm-provider"; -import type { EvalContext } from "../providers/request-builder"; -import type { PromptFile } from "../schemas/prompt-schemas"; -import type { TokenUsage } from "../providers/token-usage"; -import { - buildEvaluationLLMSchema, - type EvaluationLLMResult, - type PromptEvaluationResult, -} from "../prompts/schema"; -import { registerEvaluator } from "./evaluator-registry"; -import type { Evaluator } from "./evaluator"; -import { Type, Severity } from "./types"; -import { - mergeViolations, - RecursiveChunker, - countWords, - type Chunk, -} from "../chunking"; -import { prependLineNumbers } from "../output/line-numbering"; - -const CHUNKING_THRESHOLD = 600; // Word count threshold for enabling chunking -const MAX_CHUNK_SIZE = 500; // Maximum words per chunk - -/** Evaluates rule violations, chunking large documents when needed. */ -export class BaseEvaluator implements Evaluator { - constructor( - protected llmProvider: LLMProvider, - protected prompt: PromptFile, - protected defaultSeverity?: Severity - ) { } - - async evaluate(file: string, content: string): Promise { - const ext = path.extname(file); - const context: EvalContext = ext ? { fileType: ext } : {}; - return this.runEvaluation(content, context); - } - - protected chunkContent(content: string): Chunk[] { - const wordCount = countWords(content) || 1; - - const chunkingEnabled = this.prompt.meta.evaluateAs !== "document"; - - if (!chunkingEnabled || wordCount <= CHUNKING_THRESHOLD) { - // Chunking disabled or content is small enough - return as single chunk - return [ - { - content, - index: 0, - }, - ]; - } - - const chunker = new RecursiveChunker(); - return chunker.chunk(content, { maxChunkSize: MAX_CHUNK_SIZE }); - } - - /** - * Aggregates token usage from multiple LLM calls. - */ - protected aggregateUsage( - usages: (TokenUsage | undefined)[] - ): TokenUsage | undefined { - const validUsages = usages.filter((u): u is TokenUsage => u !== undefined); - if (validUsages.length === 0) return undefined; - - return validUsages.reduce( - (acc, u) => ({ - inputTokens: acc.inputTokens + u.inputTokens, - outputTokens: acc.outputTokens + u.outputTokens, - }), - { inputTokens: 0, outputTokens: 0 } - ); - } - - protected async runEvaluation( - content: string, - context?: EvalContext - ): Promise { - const schema = buildEvaluationLLMSchema(); - - // Prepend line numbers for deterministic line reporting - const numberedContent = prependLineNumbers(content); - const chunks = this.chunkContent(numberedContent); - const totalWordCount = countWords(content) || 1; - - // Collect all violations from all chunks - const allChunkViolations: EvaluationLLMResult["violations"][] = []; - const rawChunkOutputs: EvaluationLLMResult[] = []; - const chunkReasonings: string[] = []; - const usages: (TokenUsage | undefined)[] = []; - - for (const chunk of chunks) { - const { data: llmResult, usage } = - await this.llmProvider.runPromptStructured( - chunk.content, - this.prompt.body, - schema, - context - ); - allChunkViolations.push(llmResult.violations); - rawChunkOutputs.push(llmResult); - if (llmResult.reasoning) chunkReasonings.push(llmResult.reasoning); - usages.push(usage); - } - - // Merge and deduplicate violations - const mergedViolations = mergeViolations(allChunkViolations); - - const aggregatedUsage = this.aggregateUsage(usages); - const reasoning = chunkReasonings.join(" ").trim() || undefined; - - return { - violations: mergedViolations, - word_count: totalWordCount, - ...(reasoning && { reasoning }), - raw_model_output: rawChunkOutputs.length === 1 ? rawChunkOutputs[0] : rawChunkOutputs, - ...(aggregatedUsage && { usage: aggregatedUsage }), - }; - } -} - -registerEvaluator( - Type.BASE, - (llmProvider, prompt, _searchProvider, defaultSeverity) => { - return new BaseEvaluator(llmProvider, prompt, defaultSeverity); - } -); diff --git a/src/evaluators/evaluator-registry.ts b/src/evaluators/evaluator-registry.ts deleted file mode 100644 index 9b135e9f..00000000 --- a/src/evaluators/evaluator-registry.ts +++ /dev/null @@ -1,81 +0,0 @@ -import type { Evaluator } from './evaluator'; -import type { LLMProvider } from '../providers/llm-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import type { PromptFile } from '../schemas/prompt-schemas'; - -/* - * Factory function signature for creating evaluators. - * Evaluators can optionally depend on search providers for fact verification. - */ -import type { Severity } from './types'; - -/* - * Factory function signature for creating evaluators. - * Evaluators can optionally depend on search providers for fact verification. - */ -export type EvaluatorFactory = ( - llmProvider: LLMProvider, - prompt: PromptFile, - searchProvider?: SearchProvider, - defaultSeverity?: Severity -) => Evaluator; - -/* - * EvaluatorRegistry manages evaluator type registration and instantiation. - * Evaluators self-register by calling registerEvaluator() in their module. - */ -class EvaluatorRegistry { - private registry = new Map(); - - register(type: string, factory: EvaluatorFactory): void { - if (this.registry.has(type)) { - throw new Error(`Evaluator type '${type}' is already registered`); - } - this.registry.set(type, factory); - } - - create( - type: string, - llmProvider: LLMProvider, - prompt: PromptFile, - searchProvider?: SearchProvider, - defaultSeverity?: Severity - ): Evaluator { - const factory = this.registry.get(type); - - if (!factory) { - const available = Array.from(this.registry.keys()).join(', '); - throw new Error( - `Unknown evaluator type: '${type}'. Available types: ${available || 'none'}` - ); - } - - return factory(llmProvider, prompt, searchProvider, defaultSeverity); - } - - getRegisteredTypes(): string[] { - return Array.from(this.registry.keys()); - } -} - -// Singleton instance -const REGISTRY = new EvaluatorRegistry(); - -// Public API -export function registerEvaluator(type: string, factory: EvaluatorFactory): void { - REGISTRY.register(type, factory); -} - -export function createEvaluator( - type: string, - llmProvider: LLMProvider, - prompt: PromptFile, - searchProvider?: SearchProvider, - defaultSeverity?: Severity -): Evaluator { - return REGISTRY.create(type, llmProvider, prompt, searchProvider, defaultSeverity); -} - -export function getRegisteredEvaluatorTypes(): string[] { - return REGISTRY.getRegisteredTypes(); -} diff --git a/src/evaluators/evaluator.ts b/src/evaluators/evaluator.ts deleted file mode 100644 index df0ebba1..00000000 --- a/src/evaluators/evaluator.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { PromptEvaluationResult } from '../prompts/schema'; - -/* - * Core evaluator interface for content evaluation. - * Implementations receive a file path and content, returning structured evaluation results. - */ -export interface Evaluator { - evaluate(file: string, content: string): Promise; -} diff --git a/src/evaluators/index.ts b/src/evaluators/index.ts deleted file mode 100644 index 35e0a1c8..00000000 --- a/src/evaluators/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Evaluators module - exports evaluator interface, base class, and registry. - * - * Import this module to: - * - Access the Evaluator interface for type definitions - * - Use BaseEvaluator as a base class for custom evaluators - * - Use registry functions to create and register evaluators - * - * Importing this module also triggers self-registration of all built-in evaluators. - */ - -// Core interface -export type { Evaluator } from './evaluator'; - -// Base evaluator class (also triggers 'base' registration on import) -export { BaseEvaluator } from './base-evaluator'; - -// Registry functions -export { - registerEvaluator, - createEvaluator, - getRegisteredEvaluatorTypes, - type EvaluatorFactory, -} from './evaluator-registry'; - -// Prompt loader for evaluator-specific prompts -export { getPrompt } from './prompt-loader'; - -// Import specialized evaluators to trigger their self-registration -// These must be imported after base-evaluator to ensure registry is ready -import './accuracy-evaluator'; diff --git a/src/evaluators/prompt-loader.ts b/src/evaluators/prompt-loader.ts deleted file mode 100644 index 16d437fc..00000000 --- a/src/evaluators/prompt-loader.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { z } from "zod"; -import promptsData from "./prompts.json"; - -/** - * Schema for evaluator prompts JSON file. - * Simple key-value mapping: prompt key -> prompt content string. - */ -const PROMPTS_SCHEMA = z.record(z.string(), z.string()); - -const PROMPTS = PROMPTS_SCHEMA.parse(promptsData); - -/** - * Get an evaluator prompt by key. - * Evaluators call this with their known prompt key to retrieve the prompt content. - * - * @param key - The prompt key (e.g., "claim-extraction") - * @returns The prompt content string - * @throws Error if the prompt key is not found - */ -export function getPrompt(key: string): string { - const prompt = PROMPTS[key]; - if (!prompt) { - const available = Object.keys(PROMPTS).join(", "); - throw new Error( - `Prompt '${key}' not found. Available prompts: ${available || "none"}` - ); - } - return prompt; -} diff --git a/src/evaluators/prompts.json b/src/evaluators/prompts.json deleted file mode 100644 index a436fe3f..00000000 --- a/src/evaluators/prompts.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "claim-extraction": "You are a **claim extraction agent** designed to identify verifiable factual statements from technical content.\n\n## Task\n\nAnalyze the provided content and extract all factual claims that can be verified against external sources.\n\n## What to Extract\n\nExtract statements that make claims about:\n\n- **Technical facts**: Specific features, capabilities, or behaviors of tools/technologies\n- **Quantitative data**: Statistics, performance metrics, version numbers, dates\n- **Attributions**: Statements about who created, maintains, or endorses something\n- **Historical facts**: When something was released, deprecated, or changed\n- **Comparisons**: Claims about relative performance, popularity, or capabilities\n\n## What to Skip\n\nDo NOT extract:\n\n- Opinions or preferences (\"I think...\", \"in my opinion...\")\n- Generic statements without specifics (\"many developers use...\")\n- Instructions or recommendations (\"you should...\", \"it's best to...\")\n- Questions\n- Examples or hypotheticals clearly marked as such\n\n## Guidelines\n\nEach extracted claim should be:\n\n- **Complete**: Include enough context to be independently verifiable\n- **Specific**: Avoid vague or general statements\n- **Factual**: Make a concrete assertion about reality\n\nExtract between 0 to 10 claims. If there are more than 10 verifiable claims, prioritize the most significant or impactful ones. Extract claims as they appear in the content, maintaining the original phrasing when possible.\n\nFocus on extracting claims that could be false or outdated, not obvious truths." -} diff --git a/src/evaluators/types.ts b/src/evaluators/types.ts deleted file mode 100644 index b17a1686..00000000 --- a/src/evaluators/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Evaluator type constants to avoid magic strings. - */ -export enum Type { - BASE = 'base', - TECHNICAL_ACCURACY = 'technical-accuracy', -} - -export enum Severity { - ERROR = 'error', - WARNING = 'warning', -} diff --git a/src/executors/agent-model-call-executor.ts b/src/executors/agent-model-call-executor.ts new file mode 100644 index 00000000..a9cf46f6 --- /dev/null +++ b/src/executors/agent-model-call-executor.ts @@ -0,0 +1,182 @@ +import type { ToolCallDefinition, ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { ReviewCallContext, RequestBuilder } from '../providers/request-builder'; +import { + buildReviewLLMSchema, + type ReviewLLMResult, +} from '../prompts/schema'; +import { countWords } from '../chunking'; +import { processFindings } from '../findings'; +import { enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, +} from '../review/types'; +import type { ReviewExecutor } from '../review/executor'; +import { TargetReadCapability, buildReadTargetSectionTool } from './target-read-capability-adapter'; +import { + budgetExceededDiagnostic, + buildReviewCallContext, + buildReviewPrompt, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; + +/** + * The agent `modelCall` {@link ReviewExecutor}. + * + * Reviews target content against source-backed rules through a single bounded + * tool-calling run per rule via an injected {@link ToolCallingModelClient}. The + * only executor-owned tool exposed to the model is `read_target_section`, + * which pages through the in-memory `request.target.content`. Rule prompts come verbatim from + * {@link ReviewRule.body} — no model-supplied rule override is introduced — + * candidate findings flow through the shared {@link processFindings} pipeline, + * and the model-call budget is enforced via the review budget module before + * every call. + * + * This is the bounded, target-only review strategy selected by + * `--model-call agent` (and by `auto` for large inputs). + */ +export class AgentModelCallExecutor implements ReviewExecutor { + constructor( + private readonly client: ToolCallingModelClient, + private readonly builder: RequestBuilder, + ) {} + + async run(request: ReviewRequest): Promise { + const schema = buildReviewLLMSchema(); + const capability = new TargetReadCapability(request.target.content); + // Exactly one executor-owned tool is exposed: target-section paging. + const tools = buildReadTargetSectionTool(capability); + const context = { + ...buildReviewCallContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const contentReview = await this.reviewTargetWithRule( + request, + rule, + schema, + tools, + context, + capability.lineCount, + counters, + elapsedMs, + ); + findings.push(...contentReview.findings); + scores.push(...contentReview.scores); + diagnostics.push(...contentReview.diagnostics); + if (contentReview.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { + hadOperationalErrors = true; + diagnostics.push(diagnostic); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: buildReviewUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: makes one bounded tool-calling run that lets the + * model page through the target via `read_target_section`, then projects the + * returned violations through {@link processFindings}. + */ + private async reviewTargetWithRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + tools: Record, + context: ReviewCallContext, + targetLineCount: number, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + // Enforce the model-call budget before committing to the run. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the run that would push the review over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runWithTools({ + // The source-backed rule body, wrapped with the directive/user + // instructions exactly as the single-call path does. No model-supplied + // rule override is introduced. + systemPrompt: this.builder.buildPromptBodyForStructured( + buildReviewPrompt(rule.body, request.context), + context, + ), + prompt: this.buildTargetPrompt(request.target.uri, targetLineCount), + tools, + schema, + options: { + // Each run is bounded by the budget's per-rule section limit: the + // model may page through at most maxChunksPerRule sections before + // emitting its structured finding set. + maxSteps: request.budget.maxChunksPerRule, + maxParallelToolCalls: 1, + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }, + }); + + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + + const { pack, ruleId } = splitRuleId(rule.id); + const wordCount = countWords(request.target.content) || 1; + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: data.violations, + wordCount, + promptMeta: { + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), + }, + targetContent: request.target.content, + }); + } + + private buildTargetPrompt(uri: string, lineCount: number): string { + return [ + `Target URI: ${uri}`, + `Target lines: ${lineCount}`, + '', + 'Page through the target with the read_target_section tool, then report verified violations in the structured output.', + ].join('\n'); + } +} diff --git a/src/executors/errors.ts b/src/executors/errors.ts new file mode 100644 index 00000000..5f2b146a --- /dev/null +++ b/src/executors/errors.ts @@ -0,0 +1,11 @@ +import { VectorlintError } from '../errors'; + +export class TargetSectionRangeError extends VectorlintError { + constructor( + message: string, + public readonly lineCount: number, + ) { + super(message, 'TARGET_SECTION_RANGE'); + this.name = 'TargetSectionRangeError'; + } +} diff --git a/src/executors/index.ts b/src/executors/index.ts new file mode 100644 index 00000000..1e5eb1bd --- /dev/null +++ b/src/executors/index.ts @@ -0,0 +1,38 @@ +import type { StructuredModelClient } from '../providers/structured-model-client'; +import type { ToolCallingModelClient } from '../providers/tool-calling-model-client'; +import type { RequestBuilder } from '../providers/request-builder'; +import type { ReviewExecutor } from '../review/executor'; +import { SingleModelCallExecutor } from './single-model-call-executor'; +import { AgentModelCallExecutor } from './agent-model-call-executor'; + +/** + * The model-client dependencies every executor composes. Both capabilities are + * supplied because the resolved {@link ModelCall} is not known until + * {@link chooseModelCall} runs at review time. + */ +export interface ExecutorDeps { + structuredModelClient: StructuredModelClient; + toolCallingModelClient: ToolCallingModelClient; + builder: RequestBuilder; +} + +/** A resolved reviewer model-call strategy (after `chooseModelCall`). */ +export type ModelCall = 'single' | 'agent'; + +/** + * Selects the {@link ReviewExecutor} for a resolved model call. `single` maps + * to {@link SingleModelCallExecutor}; `agent` maps to + * {@link AgentModelCallExecutor}. Callers resolve `auto` via + * {@link chooseModelCall} before calling this factory. + */ +export function executorFor(modelCall: ModelCall, deps: ExecutorDeps): ReviewExecutor { + if (modelCall === 'agent') { + return new AgentModelCallExecutor(deps.toolCallingModelClient, deps.builder); + } + return new SingleModelCallExecutor(deps.structuredModelClient); +} + +export { SingleModelCallExecutor } from './single-model-call-executor'; +export { AgentModelCallExecutor } from './agent-model-call-executor'; +export { TargetReadCapability } from './target-read-capability-adapter'; +export { REVIEW_BUDGET_EXCEEDED_CODE, splitRuleId } from './shared'; diff --git a/src/executors/shared.ts b/src/executors/shared.ts new file mode 100644 index 00000000..6eef5635 --- /dev/null +++ b/src/executors/shared.ts @@ -0,0 +1,127 @@ +import path from 'path'; + +import { Severity } from '../review/severity'; +import type { ReviewCallContext } from '../providers/request-builder'; +import { BudgetExceededError } from '../review/errors'; +import type { + ReviewDiagnostic, + ReviewContext, + ReviewRequest, + ReviewSeverity, + ReviewUsage, +} from '../review/types'; + +/** + * Stable diagnostic code recorded when a run stops because the model-call + * budget was exhausted before every rule could be reviewed. + */ +export const REVIEW_BUDGET_EXCEEDED_CODE = 'review-budget-exceeded'; + +/** + * Mutable run-wide counters shared across rules (and chunks/sections) by both + * the single and agent model-call executors. + */ +export interface RunCounters { + modelCalls: number; + inputTokens: number; + outputTokens: number; +} + +/** + * Splits a `Pack.RuleId` review rule id into its `pack` and `ruleId` parts. + * The review contract carries the composite id, while {@link processFindings} + * rebuilds the same id from the parts via `buildRuleId`. Splits on the first + * dot; pack names are single path segments and rule ids are PascalCase. + */ +export function splitRuleId(id: string): { pack: string; ruleId: string } { + const dot = id.indexOf('.'); + if (dot === -1) { + return { pack: id, ruleId: id }; + } + return { pack: id.slice(0, dot), ruleId: id.slice(dot + 1) }; +} + +/** + * Builds the provider {@link ReviewCallContext} (file-type hint) for a target URI. + * Shared by both executors so structured and tool-calling calls receive the + * same file-type context for directive substitution. + */ +export function buildReviewCallContext(uri: string): ReviewCallContext { + const ext = path.extname(uri); + return ext ? { fileType: ext } : {}; +} + +/** Adds caller-supplied reference context to a source-backed rule prompt. */ +export function buildReviewPrompt( + ruleBody: string, + context: readonly ReviewContext[] | undefined, +): string { + if (!context || context.length === 0) { + return ruleBody; + } + + const contextSections = context.map((item) => { + const metadata = [ + item.relation ? `Relation: ${item.relation}` : undefined, + item.uri ? `Source: ${item.uri}` : undefined, + ].filter((value): value is string => value !== undefined); + + return [ + `### ${item.label}`, + ...metadata, + item.content, + ].join('\n'); + }); + + return [ + ruleBody, + '', + '## Caller-supplied context', + ...contextSections, + ].join('\n'); +} + +/** + * Maps the review contract's plain {@link ReviewSeverity} union onto the + * finding processor's {@link Severity} enum at the executor boundary (no + * unsafe cast). + */ +export function toFindingSeverity(severity: ReviewSeverity): Severity { + return severity === 'error' ? Severity.ERROR : Severity.WARNING; +} + +/** + * Aggregates run-wide counters into a {@link ReviewUsage}, including token + * counts only when the output policy opts in to usage reporting. + */ +export function buildReviewUsage( + request: ReviewRequest, + counters: RunCounters, + wallClockMs: number, +): ReviewUsage { + const usage: ReviewUsage = { modelCalls: counters.modelCalls, wallClockMs }; + if (request.outputPolicy.includeUsage) { + usage.inputTokens = counters.inputTokens; + usage.outputTokens = counters.outputTokens; + } + return usage; +} + +/** + * Returns a `review-budget-exceeded` error diagnostic when `error` is a + * {@link BudgetExceededError}; otherwise returns `undefined` so the caller can + * rethrow non-budget errors unchanged. Shared by both executors' run loops so + * budget exhaustion surfaces as a partial-result operational failure instead + * of throwing past the {@link ReviewExecutor} contract. + */ +export function budgetExceededDiagnostic(error: unknown): ReviewDiagnostic | undefined { + if (!(error instanceof BudgetExceededError)) { + return undefined; + } + return { + level: 'error', + code: REVIEW_BUDGET_EXCEEDED_CODE, + message: error.message, + context: { limit: error.limit, actual: error.actual }, + }; +} diff --git a/src/executors/single-model-call-executor.ts b/src/executors/single-model-call-executor.ts new file mode 100644 index 00000000..baffc8d3 --- /dev/null +++ b/src/executors/single-model-call-executor.ts @@ -0,0 +1,183 @@ +import type { ReviewExecutor } from '../review/executor'; +import { enforceBudget } from '../review/budget'; +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewRequest, + ReviewResult, + ReviewRule, + ReviewScore, +} from '../review/types'; +import type { ReviewCallContext } from '../providers/request-builder'; +import type { StructuredModelClient } from '../providers/structured-model-client'; +import { + buildReviewLLMSchema, + type ReviewLLMResult, +} from '../prompts/schema'; +import { countWords, mergeViolations, RecursiveChunker, type Chunk } from '../chunking'; +import { prependLineNumbers } from '../output/line-numbering'; +import { processFindings } from '../findings'; +import { + budgetExceededDiagnostic, + buildReviewCallContext, + buildReviewPrompt, + buildReviewUsage, + splitRuleId, + toFindingSeverity, + type RunCounters, +} from './shared'; + +/** + * Word-count threshold above which the single-call executor chunks the target + * before reviewing it. The threshold preserves the existing chunk/merge + * behavior for large documents. + */ +const CHUNKING_WORD_THRESHOLD = 600; +const MAX_CHUNK_WORDS = 500; + +/** + * The single modelCall {@link ReviewExecutor}. + * + * Reviews target content against source-backed rules with one structured model + * call per (rule, chunk) through an injected {@link StructuredModelClient}. It + * owns no tool surface and no autonomous loop: rule prompts come verbatim from + * {@link ReviewRule.body}, candidate findings flow through the shared + * {@link processFindings} pipeline, and the model-call budget is enforced via + * the review budget module before every call. + * + * This is the bounded, transport-only review strategy selected by + * `--model-call single` (and by `auto` for normal-sized inputs). + */ +export class SingleModelCallExecutor implements ReviewExecutor { + constructor(private readonly client: StructuredModelClient) {} + + async run(request: ReviewRequest): Promise { + const schema = buildReviewLLMSchema(); + const context = { + ...buildReviewCallContext(request.target.uri), + recordPayloadTelemetry: request.outputPolicy.recordPayloadTelemetry, + }; + + const findings: ReviewFinding[] = []; + const scores: ReviewScore[] = []; + const diagnostics: ReviewDiagnostic[] = []; + let hadOperationalErrors = false; + + const counters: RunCounters = { modelCalls: 0, inputTokens: 0, outputTokens: 0 }; + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + + try { + for (const rule of request.rules) { + const contentReview = await this.reviewTargetWithRule( + request, + rule, + schema, + context, + counters, + elapsedMs, + ); + findings.push(...contentReview.findings); + scores.push(...contentReview.scores); + diagnostics.push(...contentReview.diagnostics); + if (contentReview.hadOperationalErrors) { + hadOperationalErrors = true; + } + } + } catch (error: unknown) { + // Surface budget exhaustion as an operational failure: the run returns + // partial results plus an error diagnostic rather than throwing past the + // ReviewExecutor contract. Non-budget errors propagate unchanged. + const diagnostic = budgetExceededDiagnostic(error); + if (diagnostic) { + hadOperationalErrors = true; + diagnostics.push(diagnostic); + } else { + throw error; + } + } + + return { + findings, + scores, + diagnostics, + hadOperationalErrors, + usage: buildReviewUsage(request, counters, elapsedMs()), + }; + } + + /** + * Reviews a single rule: chunks the line-numbered target, makes one + * structured model call per chunk, merges violations across chunks, and + * projects the merged candidates through {@link processFindings}. + */ + private async reviewTargetWithRule( + request: ReviewRequest, + rule: ReviewRule, + schema: ReturnType, + context: ReviewCallContext, + counters: RunCounters, + elapsedMs: () => number, + ): Promise { + const numberedContent = prependLineNumbers(request.target.content); + const wordCount = countWords(request.target.content) || 1; + const chunks = this.chunkTarget(numberedContent, wordCount, request.budget.maxChunksPerRule); + const reviewPrompt = buildReviewPrompt(rule.body, request.context); + + const chunkViolations: ReviewLLMResult['violations'][] = []; + for (const chunk of chunks) { + // Enforce the model-call budget before committing to another call. The + // prospective count (calls made so far plus this one) lets enforceBudget + // reject the call that would push the run over maxModelCallsPerReview. + enforceBudget(request.budget, { + modelCalls: counters.modelCalls + 1, + elapsedMs: elapsedMs(), + }); + + const { data, usage } = await this.client.runPromptStructured( + chunk.content, + reviewPrompt, + schema, + context, + ); + counters.modelCalls += 1; + if (usage) { + counters.inputTokens += usage.inputTokens; + counters.outputTokens += usage.outputTokens; + } + chunkViolations.push(data.violations); + } + + const { pack, ruleId } = splitRuleId(rule.id); + const mergedViolations = mergeViolations(chunkViolations); + + return processFindings({ + pack, + ruleId, + ruleSource: rule.source, + candidateFindings: mergedViolations, + wordCount, + promptMeta: { + ...(rule.severity !== undefined ? { severity: toFindingSeverity(rule.severity) } : {}), + }, + targetContent: request.target.content, + }); + } + + /** + * Chunks line-numbered target content for context management. Small targets + * review as a single chunk; larger targets are split recursively and capped + * at the review budget's per-rule chunk limit. + */ + private chunkTarget( + numberedContent: string, + wordCount: number, + maxChunks: number, + ): Chunk[] { + if (wordCount <= CHUNKING_WORD_THRESHOLD) { + return [{ content: numberedContent, index: 0 }]; + } + const chunker = new RecursiveChunker(); + return chunker.chunk(numberedContent, { maxChunkSize: MAX_CHUNK_WORDS }).slice(0, maxChunks); + } +} diff --git a/src/executors/target-read-capability-adapter.ts b/src/executors/target-read-capability-adapter.ts new file mode 100644 index 00000000..95d8b028 --- /dev/null +++ b/src/executors/target-read-capability-adapter.ts @@ -0,0 +1,144 @@ +import { z } from 'zod'; + +import type { ToolCallDefinition } from '../providers/tool-calling-model-client'; +import type { ReviewTargetReadCapability } from '../review/executor'; +import { TargetSectionRangeError } from './errors'; + +/** + * The executor-owned tool for paging through the target under review by + * 1-based line range. + */ +export const READ_TARGET_SECTION_TOOL_NAME = 'read_target_section'; + +/** + * Zod schema for {@link TargetReadCapability.readTargetSection} arguments: + * 1-based positive integers. The tool adapter parses model input against this + * before slicing, so malformed arguments become model-visible errors. + */ +export const READ_TARGET_SECTION_PARAMETERS = z + .object({ + startLine: z.number().int().positive(), + endLine: z.number().int().positive(), + }) + .strict(); + +const READ_TARGET_SECTION_DESCRIPTION = [ + 'Read a 1-based inclusive [startLine, endLine] window of the target content under review.', + 'Returns the window with original line numbers prepended so findings can cite exact lines.', + 'A window outside [1, targetLineCount], or with startLine > endLine, returns an error result', + 'describing the valid range instead of aborting the review.', +].join(' '); + +/** Success result of a {@link TargetReadCapability.readTargetSection} call. */ +export interface TargetSectionResult { + startLine: number; + endLine: number; + /** The requested window, each line prefixed with its 1-based line number. */ + content: string; +} + +/** Model-visible error result returned in place of a window for invalid calls. */ +export interface TargetSectionErrorResult { + error: string; + /** Total lines in the target, so the model can retry with a valid range. */ + lineCount: number; +} + +/** + * A target-only {@link ReviewTargetReadCapability} bound to the in-memory + * `ReviewTarget.content`. It performs NO filesystem access and reads no URI + * other than the target it was constructed from. The agent executor exposes + * this single capability to the model via {@link buildReadTargetSectionTool}. + */ +export class TargetReadCapability implements ReviewTargetReadCapability { + private readonly lines: readonly string[]; + + constructor(content: string) { + // Mirrors prependLineNumbers: a trailing newline yields a final empty line. + this.lines = content.split('\n'); + } + + /** Number of addressable lines in the target. */ + get lineCount(): number { + return this.lines.length; + } + + readTargetSection(startLine: number, endLine: number): Promise { + this.assertValidRange(startLine, endLine); + const slice = this.lines.slice(startLine - 1, endLine); + const content = slice.map((line, index) => `${startLine + index}\t${line}`).join('\n'); + return Promise.resolve({ startLine, endLine, content }); + } + + private assertValidRange(startLine: number, endLine: number): void { + const lineCount = this.lineCount; + if (!Number.isInteger(startLine) || !Number.isInteger(endLine)) { + throw new TargetSectionRangeError( + `read_target_section requires integer line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine < 1 || endLine < 1) { + throw new TargetSectionRangeError( + `read_target_section requires 1-based positive line numbers; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > endLine) { + throw new TargetSectionRangeError( + `read_target_section requires startLine <= endLine; got startLine=${startLine}, endLine=${endLine}. Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (startLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section startLine=${startLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + if (endLine > lineCount) { + throw new TargetSectionRangeError( + `read_target_section endLine=${endLine} is beyond the target's ${lineCount} line(s). Valid range is [1, ${lineCount}].`, + lineCount, + ); + } + } +} + +/** + * Builds the single executor-owned tool map exposed to the + * {@link ToolCallingModelClient}: exactly one entry, `read_target_section`, + * bound to the target-only {@link TargetReadCapability}. Argument-parse and + * range failures are returned as model-visible error results (never thrown + * past the tool boundary) so the bounded review run continues. + */ +export function buildReadTargetSectionTool( + capability: TargetReadCapability, +): Record { + return { + [READ_TARGET_SECTION_TOOL_NAME]: { + description: READ_TARGET_SECTION_DESCRIPTION, + parameters: READ_TARGET_SECTION_PARAMETERS, + execute: async ( + input: unknown, + ): Promise => { + const parsed = READ_TARGET_SECTION_PARAMETERS.safeParse(input); + if (!parsed.success) { + return { + error: `Invalid read_target_section arguments: ${parsed.error.message}.`, + lineCount: capability.lineCount, + }; + } + const { startLine, endLine } = parsed.data; + try { + return await capability.readTargetSection(startLine, endLine); + } catch (error: unknown) { + if (error instanceof TargetSectionRangeError) { + return { error: error.message, lineCount: error.lineCount }; + } + throw error; + } + }, + }, + }; +} diff --git a/src/evaluators/violation-filter.ts b/src/findings/filter-decision.ts similarity index 100% rename from src/evaluators/violation-filter.ts rename to src/findings/filter-decision.ts diff --git a/src/findings/processor.ts b/src/findings/processor.ts index af9a0641..9973e24d 100644 --- a/src/findings/processor.ts +++ b/src/findings/processor.ts @@ -4,7 +4,7 @@ import type { ReviewResult, ReviewScore, } from '../review/types'; -import { computeFilterDecision } from '../evaluators/violation-filter'; +import { computeFilterDecision } from '../findings/filter-decision'; import { verifyFindingEvidence, FINDING_EVIDENCE_NOT_LOCATABLE, diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts index 349a3b49..882cdf93 100644 --- a/src/findings/scorer.ts +++ b/src/findings/scorer.ts @@ -1,5 +1,5 @@ import { calculateScore, type ScoringOptions } from '../scoring'; -import type { ScoredEvaluation } from '../prompts/schema'; +import type { ScoredReview } from '../prompts/schema'; import { resolveSeverity } from './severity'; import type { RawViolation, RuleSeverity } from './types'; @@ -13,7 +13,7 @@ export interface ScoredFindings { scoreText: string; severity: RuleSeverity; findingCount: number; - scored: ScoredEvaluation; + scored: ScoredReview; } /** Scores verified findings using violation density. */ diff --git a/src/findings/severity.ts b/src/findings/severity.ts index 6bb448bc..74e38590 100644 --- a/src/findings/severity.ts +++ b/src/findings/severity.ts @@ -1,12 +1,12 @@ -import type { ScoredEvaluation } from '../prompts/schema'; +import type { ScoredReview } from '../prompts/schema'; import type { FindingsCriterion, RuleSeverity } from './types'; /** Input to {@link resolveSeverity}. */ export interface SeverityInput { - scored: ScoredEvaluation; + scored: ScoredReview; } -/** Resolves severity from a scored evaluation. */ +/** Resolves severity from a scored review. */ export function resolveSeverity(input: SeverityInput): RuleSeverity { return input.scored.severity; } diff --git a/src/findings/types.ts b/src/findings/types.ts index 9c8c0b85..2310e6a7 100644 --- a/src/findings/types.ts +++ b/src/findings/types.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; /** Severity assigned to a rule and its findings. */ export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; diff --git a/src/index.ts b/src/index.ts index 2f2be86d..c6e48d9c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,9 +10,6 @@ import { loadGlobalConfig } from "./config/global-config"; import { CLI_DESCRIPTION, CLI_VERSION } from "./config/constants"; -// Import evaluators module to trigger self-registration of all evaluators -import "./evaluators/index"; - /* * Loads environment variables from Global Config and .env files. * Hierarchy: CLI/Shell > Local.env > Global Config diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts index 5369c639..90cf82e8 100644 --- a/src/observability/ai-observability.ts +++ b/src/observability/ai-observability.ts @@ -1,9 +1,10 @@ export interface AIExecutionContext { - operation: 'structured-eval' | 'agent-tool-loop'; + operation: 'structured-review' | 'tool-calling'; provider: string; model: string; - evaluator?: string; + reviewer?: string; rule?: string; + recordPayloadTelemetry?: boolean; } export interface AIObservability { diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index 25f4d32c..708ebb51 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -60,11 +60,11 @@ export class LangfuseObservability implements AIObservability { metadata: { provider: context.provider, model: context.model, - ...(context.evaluator ? { evaluator: context.evaluator } : {}), + ...(context.reviewer ? { reviewer: context.reviewer } : {}), ...(context.rule ? { rule: context.rule } : {}), }, - recordInputs: true, - recordOutputs: true, + recordInputs: context.recordPayloadTelemetry === true, + recordOutputs: context.recordPayloadTelemetry === true, }, }; } diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index 0d589d14..e4909f66 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -1,4 +1,4 @@ -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import { CLI_VERSION } from '../config/constants'; export interface ScoreComponent { criterion?: string; @@ -8,7 +8,7 @@ export interface ScoreComponent { normalizedMaxScore: number; } -export interface EvaluationScore { +export interface ReviewScoreOutput { id: string; scores: ScoreComponent[]; } @@ -28,7 +28,7 @@ export interface Issue { export interface FileResult { issues: Issue[]; - evaluationScores: EvaluationScore[]; + reviewScores: ReviewScoreOutput[]; } export interface Result { @@ -51,7 +51,7 @@ export class JsonFormatter { addIssue(file: string, issue: Issue): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } this.files[file].issues.push(issue); @@ -62,11 +62,11 @@ export class JsonFormatter { } } - addEvaluationScore(file: string, score: EvaluationScore): void { + addReviewScore(file: string, score: ReviewScoreOutput): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } - this.files[file].evaluationScores.push(score); + this.files[file].reviewScores.push(score); } toJson(): string { diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts index d1daed7e..946008ed 100644 --- a/src/output/rdjson-formatter.ts +++ b/src/output/rdjson-formatter.ts @@ -1,5 +1,5 @@ -import type { Issue, EvaluationScore } from './json-formatter'; -import { Severity } from '../evaluators/types'; +import type { Issue, ReviewScoreOutput } from './json-formatter'; +import { Severity } from '../review/severity'; export interface RdJsonResult { source: { @@ -48,7 +48,7 @@ export interface RdJsonSuggestion { interface FileResult { issues: Issue[]; - evaluationScores: EvaluationScore[]; + reviewScores: ReviewScoreOutput[]; } export class RdJsonFormatter { @@ -56,16 +56,16 @@ export class RdJsonFormatter { addIssue(file: string, issue: Issue): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } this.files[file].issues.push(issue); } - addEvaluationScore(file: string, score: EvaluationScore): void { + addReviewScore(file: string, score: ReviewScoreOutput): void { if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; + this.files[file] = { issues: [], reviewScores: [] }; } - this.files[file].evaluationScores.push(score); + this.files[file].reviewScores.push(score); } toRdJsonFormat(): RdJsonResult { diff --git a/src/output/reporter.ts b/src/output/reporter.ts index 9e69124a..ccd32992 100644 --- a/src/output/reporter.ts +++ b/src/output/reporter.ts @@ -1,10 +1,10 @@ import chalk from 'chalk'; import stripAnsi from 'strip-ansi'; import path from 'path'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import { TokenUsageStats } from '../providers/token-usage'; -export interface EvaluationSummary { +export interface ReviewSummary { id: string; scoreText: string; score?: number; @@ -135,16 +135,16 @@ export function printGlobalSummary(files: number, errors: number, warnings: numb } } -export function printEvaluationSummaries( - summaries: Map +export function printReviewSummaries( + summaries: Map ) { if (summaries.size === 0) return; console.log(''); console.log(chalk.bold('\nQuality Scores:')); - for (const [evalName, items] of summaries) { - console.log(` ${chalk.cyan(evalName)}:`); + for (const [reviewName, items] of summaries) { + console.log(` ${chalk.cyan(reviewName)}:`); // Find max ID length for alignment const maxIdLen = Math.max(...items.map(i => i.id.length)); diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index 13e28ca4..a70a4fab 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -2,7 +2,7 @@ import { existsSync, readFileSync } from "fs"; import path from "path"; /** - * Load a directive to append to evaluation prompts. + * Load a directive to append to review prompts. * Precedence: * 1) Project override: .vectorlint/directive.md in current working directory * 2) Built-in: prompts/directive.md shipped with the CLI @@ -10,7 +10,7 @@ import path from "path"; */ const DEFAULT_DIRECTIVE = ` ## Role -You are VectorLint. You evaluate technical content and flag issues based on a user's style guide and rules, putting into context that the content is a technical documentation. +You are VectorLint. You review technical content and flag issues based on a user's style guide and rules, accounting for the conventions of technical documentation. Your goal is to surface issues that would @@ -32,7 +32,7 @@ flowing text, not across structural boundaries. In plain markdown or text files, unless separated by headings or horizontal rules. - + The user-defined Rule is your only criteria for flagging violations. Do not flag issues if they aren't mentioned in the user's rule or style guide. The goal and context sections exist solely to help you determine whether a pattern match is worth surfacing. @@ -47,7 +47,7 @@ In practice this means: lower the confidence — do not omit the finding entirely Treat the goal and context as your system guidance. Do not leak your internal knowledge to the user. - + A finding is worth surfacing when: @@ -60,7 +60,7 @@ Assign lower confidence (≤ 0.5) when the content's structure or domain makes t ## Task -Evaluate the provided Input against the Rule, identifying +Review the provided Input against the Rule, identifying every instance where the content violates the specified standards. diff --git a/src/prompts/prompt-loader.ts b/src/prompts/prompt-loader.ts index f0fc5e3b..987d0a66 100644 --- a/src/prompts/prompt-loader.ts +++ b/src/prompts/prompt-loader.ts @@ -2,7 +2,7 @@ import { readdirSync, readFileSync, statSync } from 'fs'; import path from 'path'; import YAML from 'yaml'; import { PROMPT_META_SCHEMA, type PromptFile, type PromptMeta } from '../schemas/prompt-schemas'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; // Re-export types for backward compatibility export type { PromptFile, PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; diff --git a/src/prompts/prompt-validator.ts b/src/prompts/prompt-validator.ts index 6275f731..617ee8c8 100644 --- a/src/prompts/prompt-validator.ts +++ b/src/prompts/prompt-validator.ts @@ -1,5 +1,5 @@ import { PromptFile, PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; -import { Severity } from '../evaluators/types'; +import { Severity } from '../review/severity'; import { DEFAULT_TARGET_FLAGS } from './target'; export type ValidationLevel = Severity; diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index a370f52c..37027609 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -1,4 +1,4 @@ -import { Severity } from "../evaluators/types"; +import { Severity } from "../review/severity"; import type { TokenUsage } from "../providers/token-usage"; export type GateChecks = { @@ -19,9 +19,9 @@ export type GateCheckNotes = { fix_preserves_meaning: string; }; -export function buildEvaluationLLMSchema() { +export function buildReviewLLMSchema() { return { - name: "vectorlint_evaluation_result", + name: "vectorlint_review_result", strict: true, schema: { type: "object", @@ -132,7 +132,7 @@ export function buildEvaluationLLMSchema() { } as const; } -export type EvaluationLLMResult = { +export type ReviewLLMResult = { reasoning: string; violations: Array<{ line: number; @@ -151,7 +151,7 @@ export type EvaluationLLMResult = { }>; }; -export type EvaluationItem = { +export type ReviewItem = { line?: number; description?: string; analysis: string; @@ -167,23 +167,15 @@ export type EvaluationItem = { confidence?: number; }; -export type ScoredEvaluation = { +export type ScoredReview = { final_score: number; // 1-10 percentage: number; violation_count: number; - items: Array; + items: Array; severity: typeof Severity.WARNING | typeof Severity.ERROR; message: string; reasoning?: string; - violations: Array; - usage?: TokenUsage; - raw_model_output?: unknown; -}; - -export type PromptEvaluationResult = { - violations: ScoredEvaluation["violations"]; - word_count: number; - reasoning?: string; + violations: Array; usage?: TokenUsage; raw_model_output?: unknown; }; diff --git a/src/providers/index.ts b/src/providers/index.ts index 4bdb538c..ae8bb341 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,7 +1,11 @@ -export { LLMProvider, type LLMResult } from './llm-provider'; +export { LLMProvider } from './llm-provider'; +export { type StructuredModelClient, type LLMResult } from './structured-model-client'; +export { + type ToolCallingModelClient, + type ToolCallDefinition, + type ToolCallRunOptions, +} from './tool-calling-model-client'; export { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; -export { SearchProvider } from './search-provider'; -export { PerplexitySearchProvider, type PerplexitySearchConfig } from './perplexity-provider'; export { createProvider, type ProviderOptions, ProviderType } from './provider-factory'; export { RequestBuilder, DefaultRequestBuilder } from './request-builder'; export { TokenUsage, TokenUsageStats, PricingConfig, calculateCost } from './token-usage'; diff --git a/src/providers/llm-provider.ts b/src/providers/llm-provider.ts index f87304dd..bd4902aa 100644 --- a/src/providers/llm-provider.ts +++ b/src/providers/llm-provider.ts @@ -1,31 +1,11 @@ -import type { TokenUsage } from './token-usage'; -import type { EvalContext } from './request-builder'; +import type { StructuredModelClient } from './structured-model-client'; -export interface LLMResult { - data: T; - usage?: TokenUsage; -} +/** + * `LLMResult` lives on the permanent structured-output capability. It is + * re-exported here so existing deep imports (`from './llm-provider'`) keep + * compiling. + */ +export type { LLMResult } from './structured-model-client'; -export interface AgentToolDefinition { - description: string; - inputSchema: unknown; - execute: (input: unknown) => Promise; -} - -export interface AgentToolLoopParams { - systemPrompt: string; - prompt: string; - tools: Record; - maxSteps?: number; - maxRetries?: number; - maxParallelToolCalls?: number; -} - -export interface AgentToolLoopResult { - usage?: TokenUsage; -} - -export interface LLMProvider { - runPromptStructured(content: string, promptText: string, schema: { name: string; schema: Record }, context?: EvalContext): Promise>; - runAgentToolLoop(params: AgentToolLoopParams): Promise; -} +/** Compatibility alias for the structured-output provider capability. */ +export type LLMProvider = StructuredModelClient; diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts deleted file mode 100644 index 432f8c09..00000000 --- a/src/providers/perplexity-provider.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { generateText } from 'ai'; -import type { LanguageModel } from 'ai'; -import { z } from 'zod'; -import { createPerplexity } from '@ai-sdk/perplexity'; -import type { SearchProvider } from './search-provider'; -import type { PerplexityResult } from '../schemas/perplexity-responses'; -import { createNoopLogger, type Logger } from '../logging/logger'; -import { handleUnknownError } from '../errors'; - -// Boundary validation schema for Perplexity source data. -// The AI SDK's typed Source may not include provider-specific fields (text, publishedDate), -// so we validate the raw data at the boundary to safely extract them. -const PERPLEXITY_SOURCE_SCHEMA = z.object({ - title: z.string().optional(), - text: z.string().optional(), - url: z.string().optional(), - publishedDate: z.string().optional(), -}).passthrough(); -const PERPLEXITY_SOURCES_SCHEMA = z.array(PERPLEXITY_SOURCE_SCHEMA); - -export interface PerplexitySearchConfig { - apiKey?: string; - maxResults?: number; - logger?: Logger; -} - -export class PerplexitySearchProvider implements SearchProvider { - private client: ReturnType; - private maxResults: number; - private logger: Logger; - - constructor(config: PerplexitySearchConfig = {}) { - // Use provided API key or fall back to environment variable - const apiKey = config.apiKey || process.env.PERPLEXITY_API_KEY; - if (!apiKey) { - throw new Error('Perplexity API key is required. Set PERPLEXITY_API_KEY environment variable or pass apiKey in config.'); - } - this.client = createPerplexity({ apiKey }); - this.maxResults = config.maxResults ?? 5; - this.logger = config.logger ?? createNoopLogger(); - } - - async search(query: string): Promise { - if (!query?.trim()) throw new Error('Search query cannot be empty.'); - - this.logger.debug('Perplexity search started', { query }); - - try { - const result = await generateText({ - // @ai-sdk/perplexity@1 exposes LanguageModelV1 models, while ai@6's - // generateText types require a LanguageModel (V2/V3). This is a - // third-party SDK version skew, not a repo type hole; resolve by - // upgrading @ai-sdk/perplexity to a V2 release in a follow-up. - model: this.client('sonar-pro') as unknown as LanguageModel, - prompt: query, - }); - - // Validate sources at the boundary — the SDK may include provider-specific - // fields not present in the typed Source interface - const rawSources: unknown[] = Array.isArray(result.sources) ? result.sources.slice(0, this.maxResults) : []; - const parseResult = PERPLEXITY_SOURCES_SCHEMA.safeParse(rawSources); - if (!parseResult.success) { - this.logger.warn('Perplexity source validation failed', { - error: parseResult.error.message, - }); - } - const sources = parseResult.success ? parseResult.data : []; - - const results: PerplexityResult[] = sources.map(source => ({ - title: source.title || 'Untitled', - snippet: source.text || '', - url: source.url || '', - date: source.publishedDate || '', - })); - - this.logger.debug('Perplexity search completed', { resultCount: results.length }); - this.logger.debug('Perplexity result preview', { - results: results.slice(0, 2), - }); - - return results; - } catch (e: unknown) { - const err = handleUnknownError(e, 'Perplexity API call'); - throw new Error(`Perplexity API call failed: ${err.message}`); - } - } -} diff --git a/src/providers/provider-factory.ts b/src/providers/provider-factory.ts index 3234087d..cbef40ef 100644 --- a/src/providers/provider-factory.ts +++ b/src/providers/provider-factory.ts @@ -4,7 +4,8 @@ import { createAnthropic } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; import type { LanguageModel } from 'ai'; -import { LLMProvider } from './llm-provider'; +import type { StructuredModelClient } from './structured-model-client'; +import type { ToolCallingModelClient } from './tool-calling-model-client'; import { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; import { RequestBuilder } from './request-builder'; import type { EnvConfig } from '../schemas/env-schemas'; @@ -38,7 +39,7 @@ export function createProvider( envConfig: EnvConfig, options: ProviderOptions = {}, builder?: RequestBuilder -): LLMProvider { +): StructuredModelClient & ToolCallingModelClient { let model: LanguageModel; let modelName: string; let temperature = 0.2; diff --git a/src/providers/request-builder.ts b/src/providers/request-builder.ts index 806c53e8..f79c6de6 100644 --- a/src/providers/request-builder.ts +++ b/src/providers/request-builder.ts @@ -1,11 +1,12 @@ // Centralized request construction for provider-agnostic use -export interface EvalContext { +export interface ReviewCallContext { fileType?: string; + recordPayloadTelemetry?: boolean; } export interface RequestBuilder { - buildPromptBodyForStructured(originalBody: string, context?: EvalContext): string; + buildPromptBodyForStructured(originalBody: string, context?: ReviewCallContext): string; } export class DefaultRequestBuilder implements RequestBuilder { @@ -17,7 +18,7 @@ export class DefaultRequestBuilder implements RequestBuilder { this.userInstructions = (userInstructions || '').trim(); } - buildPromptBodyForStructured(originalBody: string, context?: EvalContext): string { + buildPromptBodyForStructured(originalBody: string, context?: ReviewCallContext): string { let directive = this.directive; if (directive) { directive = directive.replaceAll('{{file_type}}', context?.fileType ?? ''); diff --git a/src/providers/search-provider.ts b/src/providers/search-provider.ts deleted file mode 100644 index dfac8818..00000000 --- a/src/providers/search-provider.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Search provider interface for fact verification and research. - * Implementations query external search APIs and return results. - */ -export interface SearchProvider { - search(query: string): Promise; -} diff --git a/src/providers/structured-model-client.ts b/src/providers/structured-model-client.ts new file mode 100644 index 00000000..beb123f6 --- /dev/null +++ b/src/providers/structured-model-client.ts @@ -0,0 +1,20 @@ +import type { TokenUsage } from './token-usage'; +import type { ReviewCallContext } from './request-builder'; + +/** + * Result of a structured model call: validated output plus optional usage. + */ +export interface LLMResult { + data: T; + usage?: TokenUsage; +} + +/** Makes one structured model call and returns validated output. */ +export interface StructuredModelClient { + runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record }, + context?: ReviewCallContext, + ): Promise>; +} diff --git a/src/providers/tool-calling-model-client.ts b/src/providers/tool-calling-model-client.ts new file mode 100644 index 00000000..0e17d46c --- /dev/null +++ b/src/providers/tool-calling-model-client.ts @@ -0,0 +1,46 @@ +import type { ZodType } from 'zod'; +import type { LLMResult } from './structured-model-client'; + +/** + * An executor-owned tool definition supplied to the bounded tool-calling + * transport. The provider is transport only: it does not define or name + * product tools. Descriptions and parameter schemas come from the caller (the + * agent executor), keyed by the caller-supplied tool name, never from the + * provider. + */ +export interface ToolCallDefinition { + description: string; + parameters: ZodType; + execute: (input: unknown) => Promise; +} + +/** + * Bounded execution metadata for a single tool-calling run. The executor sets + * the bounds from the review budget and the transport honors them. + */ +export interface ToolCallRunOptions { + /** Opt in to recording prompt and generated payloads in telemetry. */ + recordPayloadTelemetry?: boolean; + /** Maximum number of model steps (tool-call rounds) before the run stops. */ + maxSteps?: number; + /** Provider retry budget for transient failures. */ + maxRetries?: number; + /** Maximum concurrent tool executions per model step. Defaults to 1. */ + maxParallelToolCalls?: number; + /** Optional abort signal for cooperative cancellation. */ + signal?: AbortSignal; +} + +/** + * Performs one bounded generation with caller-supplied tools and structured + * output. The executor owns the tool map and orchestration limits. + */ +export interface ToolCallingModelClient { + runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise>; +} diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 5cd833fd..022985b1 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -2,12 +2,21 @@ import { generateText, Output, NoObjectGeneratedError, stepCountIs, tool } from import type { LanguageModel } from 'ai'; import { z } from 'zod'; import pLimit from 'p-limit'; -import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; +import type { LLMProvider } from './llm-provider'; +import type { LLMResult } from './structured-model-client'; +import type { ToolCallDefinition, ToolCallRunOptions, ToolCallingModelClient } from './tool-calling-model-client'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from '../observability/ai-observability'; import { handleUnknownError } from '../errors'; +/** + * Conservative default step cap for a single bounded tool-calling run when the + * caller does not supply one. Executors should pass an explicit `maxSteps` + * derived from the review budget. + */ +const DEFAULT_TOOL_CALLING_MAX_STEPS = 10; + export interface VercelAIConfig { model: LanguageModel; providerName?: string; @@ -21,7 +30,7 @@ export interface VercelAIConfig { observability?: AIObservability; } -export class VercelAIProvider implements LLMProvider { +export class VercelAIProvider implements LLMProvider, ToolCallingModelClient { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; @@ -42,7 +51,7 @@ export class VercelAIProvider implements LLMProvider { content: string, promptText: string, schema: { name: string; schema: Record }, - context?: import('./request-builder').EvalContext + context?: import('./request-builder').ReviewCallContext ): Promise> { const systemPrompt = this.builder.buildPromptBodyForStructured(promptText, context); @@ -73,14 +82,17 @@ export class VercelAIProvider implements LLMProvider { } try { - const evaluator = this.extractContextValue(context, 'evaluatorName', 'evaluator'); + const reviewer = this.extractContextValue(context, 'reviewerName', 'reviewer'); const rule = this.extractContextValue(context, 'ruleName', 'rule'); const observabilityOptions = this.getObservabilityOptions({ - operation: 'structured-eval', + operation: 'structured-review', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', - ...(evaluator ? { evaluator } : {}), + ...(reviewer ? { reviewer } : {}), ...(rule ? { rule } : {}), + ...(context?.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: context.recordPayloadTelemetry } + : {}), }); const result = await generateText({ @@ -135,68 +147,109 @@ export class VercelAIProvider implements LLMProvider { } } - runAgentToolLoop = async (params: AgentToolLoopParams): Promise => { - const maxParallel = params.maxParallelToolCalls ?? 1; + /** Runs bounded tool calling with executor-owned tools and output schema. */ + async runWithTools(params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }): Promise> { + const options = params.options ?? {}; + const maxParallel = options.maxParallelToolCalls ?? 1; const limit = pLimit(maxParallel); + const zodSchema = this.jsonSchemaToZod(params.schema); + const mappedTools = Object.fromEntries( Object.entries(params.tools).map(([name, definition]) => [ name, tool({ description: definition.description, - inputSchema: definition.inputSchema as z.ZodType, + inputSchema: definition.parameters, execute: (input: unknown) => limit(() => definition.execute(input)), }), - ]) + ]), ); - const result = await generateText({ - model: this.config.model, - system: params.systemPrompt, - prompt: params.prompt, - ...(params.maxRetries !== undefined ? { maxRetries: params.maxRetries } : {}), - ...this.getObservabilityOptions({ - operation: 'agent-tool-loop', - provider: this.config.providerName ?? 'unknown', - model: this.config.modelName ?? 'unknown', - }), - stopWhen: stepCountIs(params.maxSteps ?? 1000), - providerOptions: { - openai: { - parallelToolCalls: maxParallel > 1, + if (this.config.debug) { + this.logger.debug('[vectorlint] Sending tool-calling request via Vercel AI SDK', { + model: this.config.model, + toolNames: Object.keys(params.tools), + maxSteps: options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS, + }); + } + + try { + const result = await generateText({ + model: this.config.model, + system: params.systemPrompt, + prompt: params.prompt, + ...(this.config.temperature !== undefined && { temperature: this.config.temperature }), + ...(this.config.maxTokens !== undefined && { maxTokens: this.config.maxTokens }), + ...(options.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + ...(options.signal !== undefined ? { abortSignal: options.signal } : {}), + ...this.getObservabilityOptions({ + operation: 'tool-calling', + provider: this.config.providerName ?? 'unknown', + model: this.config.modelName ?? 'unknown', + ...(options.recordPayloadTelemetry !== undefined + ? { recordPayloadTelemetry: options.recordPayloadTelemetry } + : {}), + }), + stopWhen: stepCountIs(options.maxSteps ?? DEFAULT_TOOL_CALLING_MAX_STEPS), + providerOptions: { + openai: { + parallelToolCalls: maxParallel > 1, + }, }, - }, - tools: mappedTools, - }); + tools: mappedTools, + output: Output.object({ + schema: zodSchema, + }), + }); - if (this.config.debug) { - for (const [i, step] of result.steps.entries()) { - const toolNames = step.toolCalls.map((c) => c.toolName).join(', ') || '(none)'; - this.logger.debug( - `[agent] step ${i + 1}: finishReason=${step.finishReason} tools=[${toolNames}]` + if (this.config.debug && result.usage) { + this.logger.debug('[vectorlint] tool-calling LLM response meta', { + usage: { + input_tokens: result.usage.inputTokens, + output_tokens: result.usage.outputTokens, + total_tokens: result.usage.totalTokens, + }, + finish_reason: result.finishReason, + steps: result.steps.length, + }); + } + + const usage = result.usage + ? { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + } + : undefined; + + const output: unknown = result.output; + if (output === undefined || output === null) { + throw new Error( + `LLM returned no structured output. Raw text: ${result.text?.slice(0, 500) ?? '(empty)'}` ); - if (step.text) { - this.logger.debug( - `[agent] step ${i + 1} text: ${step.text.slice(0, 500)}${step.text.length > 500 ? '...' : ''}` - ); - } } - this.logger.debug(`[agent] final finishReason=${result.finishReason} steps=${result.steps.length}`); - if (result.text) { - this.logger.debug( - `[agent] final text: ${result.text.slice(0, 500)}${result.text.length > 500 ? '...' : ''}` + + const llmResult: LLMResult = { data: output as T }; + if (usage) { + llmResult.usage = usage; + } + return llmResult; + } catch (e: unknown) { + if (NoObjectGeneratedError.isInstance(e)) { + const rawText = e instanceof Error && 'text' in e ? String(e.text) : 'unknown'; + throw new Error( + `LLM failed to generate valid structured output. Raw text: ${rawText}` ); } + const err = e instanceof Error ? e : new Error(String(e)); + throw new Error(`Vercel AI SDK tool-calling call failed: ${err.message}`); } - - return result.usage - ? { - usage: { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, - }, - } - : {}; } private getObservabilityOptions(context: AIExecutionContext): Record { @@ -217,7 +270,7 @@ export class VercelAIProvider implements LLMProvider { } private extractContextValue( - context: import('./request-builder').EvalContext | undefined, + context: import('./request-builder').ReviewCallContext | undefined, ...keys: string[] ): string | undefined { if (!context) { diff --git a/src/review/severity.ts b/src/review/severity.ts new file mode 100644 index 00000000..0dad5967 --- /dev/null +++ b/src/review/severity.ts @@ -0,0 +1,5 @@ +/** Severity assigned to rules and surfaced findings. */ +export enum Severity { + ERROR = 'error', + WARNING = 'warning', +} diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 654f41df..468a0aca 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODE, OutputFormat, REVIEW_MODES } from '../cli/types'; +import { DEFAULT_OUTPUT_FORMAT, DEFAULT_REVIEW_MODEL_CALL, OutputFormat, REVIEW_MODEL_CALLS } from '../cli/types'; // CLI options schema for command line argument validation export const CLI_OPTIONS_SCHEMA = z.object({ @@ -8,8 +8,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.nativeEnum(OutputFormat).default(DEFAULT_OUTPUT_FORMAT), - mode: z.enum(REVIEW_MODES).default(DEFAULT_REVIEW_MODE), - print: z.boolean().default(false), + modelCall: z.enum(REVIEW_MODEL_CALLS).default(DEFAULT_REVIEW_MODEL_CALL), prompts: z.string().optional(), config: z.string().optional(), }); diff --git a/src/schemas/index.ts b/src/schemas/index.ts index d17675be..052c1e80 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -3,4 +3,3 @@ export * from './prompt-schemas'; export * from './cli-schemas'; export * from './config-schemas'; export * from './env-schemas'; -export * from './perplexity-responses'; diff --git a/src/schemas/perplexity-responses.ts b/src/schemas/perplexity-responses.ts deleted file mode 100644 index c7b5d0b7..00000000 --- a/src/schemas/perplexity-responses.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { z } from 'zod'; -export const PERPLEXITY_RESULT_SCHEMA = z.object({ - title: z.string().default('Untitled'), - snippet: z.string().default(''), - url: z.string().default(''), - date: z.string().default(''), -}); - -export const PERPLEXITY_RESPONSE_SCHEMA = z.object({ - results: z.array(PERPLEXITY_RESULT_SCHEMA).default([]), -}); - -export type PerplexityResult = z.infer; -export type PerplexityResponse = z.infer; diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index cb246335..b0133947 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { Severity } from "../evaluators/types"; +import { Severity } from "../review/severity"; // Target specification schema for regex matching export const TARGET_SPEC_SCHEMA = z @@ -21,7 +21,6 @@ export const PROMPT_CRITERION_SCHEMA = z.object({ // Prompt metadata schema for YAML frontmatter. export const PROMPT_META_SCHEMA = z.object({ specVersion: z.union([z.string(), z.number()]).optional(), - evaluator: z.enum(["base", "technical-accuracy"]).optional(), id: z.string(), name: z.string(), severity: z.nativeEnum(Severity).optional(), @@ -30,7 +29,6 @@ export const PROMPT_META_SCHEMA = z.object({ .optional(), target: TARGET_SPEC_SCHEMA.optional(), criteria: z.array(PROMPT_CRITERION_SCHEMA).optional(), - evaluateAs: z.enum(["document", "chunk"]).optional(), }); diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index b0560b52..1b5383cb 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -1,5 +1,5 @@ -import type { EvaluationItem, ScoredEvaluation } from "../prompts/schema"; -import { Severity } from "../evaluators/types"; +import type { ReviewItem, ScoredReview } from "../prompts/schema"; +import { Severity } from "../review/severity"; export interface ScoringOptions { strictness?: number | "lenient" | "strict" | "standard" | undefined; @@ -30,10 +30,10 @@ function resolveStrictness( * Formula: Score = (100 - (violations/wordCount * 100 * strictness)) / 10 */ export function calculateScore( - violations: EvaluationItem[], + violations: ReviewItem[], wordCount: number, options: ScoringOptions = {} -): ScoredEvaluation { +): ScoredReview { const strictness = resolveStrictness(options.strictness); const mappedViolations = violations.map((item) => ({ ...item, diff --git a/src/types/external.d.ts b/src/types/external.d.ts deleted file mode 100644 index b7935cc5..00000000 --- a/src/types/external.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -declare module '@perplexity-ai/perplexity_ai' { - export default class Perplexity { - constructor(); - search: { - create(params: { - query: string; - max_results?: number; - max_tokens_per_page?: number; - }): Promise; - }; - } -} diff --git a/tests/agent/agent-executor.test.ts b/tests/agent/agent-executor.test.ts deleted file mode 100644 index e3a47ef2..00000000 --- a/tests/agent/agent-executor.test.ts +++ /dev/null @@ -1,743 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, describe, expect, it } from 'vitest'; -import type { PromptFile } from '../../src/prompts/prompt-loader'; -import type { LLMProvider } from '../../src/providers/llm-provider'; -import { Severity } from '../../src/evaluators/types'; -import { OutputFormat } from '../../src/cli/types'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; -import { AgentToolError } from '../../src/errors'; - -function makePrompt(): PromptFile { - return { - id: 'consistency', - filename: 'consistency.md', - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording.', - meta: { - id: 'Consistency', - name: 'Consistency', - severity: Severity.WARNING, - }, - }; -} - -function makeProvider( - script: (params: Record) => Promise<{ usage?: { inputTokens: number; outputTokens: number } }> -): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: script as unknown as never, - } as unknown as LLMProvider; -} - -describe('agent executor', () => { - const tempDirs: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-agent-')); - tempDirs.push(repo); - return repo; - } - - afterEach(() => { - for (const dir of tempDirs.splice(0, tempDirs.length)) { - rmSync(dir, { recursive: true, force: true }); - } - }); - - it('exposes only non-mutating analysis tools plus finalize_review', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = (params.tools ?? {}) as Record; - const names = Object.keys(tools).sort(); - expect(names).toEqual([ - 'finalize_review', - 'lint', - 'list_directory', - 'read_file', - 'report_finding', - 'search_content', - 'search_files', - ]); - - const finalize = tools.finalize_review as { execute: (input: unknown) => Promise }; - await finalize.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('returns explicit tool error for unknown ruleSource with valid-source hints', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - let errorMessage = ''; - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - expect(error).toBeInstanceOf(AgentToolError); - errorMessage = error instanceof Error ? error.message : String(error); - } - expect(errorMessage).toContain('Unknown ruleSource'); - expect(errorMessage).toContain('Valid sources'); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('returns explicit tool error for unknown ruleSource in report_finding', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/does-not-exist.md', - message: 'Unknown source', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('Unknown ruleSource'); - expect(result.errorMessage).toContain('Valid sources'); - }); - - it('returns findings reconstructed from persisted inline and top-level session events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({ summary: 'done' }); - return { usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - const inlineEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedInline - ).length; - const topLevelEventCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.FindingRecordedTopLevel - ).length; - const replayableFindingEvents = inlineEventCount + topLevelEventCount; - - expect(replayableFindingEvents).toBeGreaterThanOrEqual(2); - expect(result.findings.length).toBe(replayableFindingEvents); - expect(result.findings.some((finding: { line: number }) => finding.line > 1)).toBe(false); - expect( - result.findings.some( - (finding: { ruleId: string; ruleSource: string }) => - finding.ruleId === 'Default.Consistency' && - finding.ruleSource === 'packs/default/consistency.md' - ) - ).toBe(true); - expect(result.events.some((event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized)).toBe(true); - }); - - it('aggregates nested lint usage with agent loop usage', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [], - }, - usage: { - inputTokens: 7, - outputTokens: 3, - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - }; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.usage).toEqual({ - inputTokens: 18, - outputTokens: 8, - }); - }); - - it('falls back to matching all prompts when scanPaths is empty', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.fileRuleMatches).toEqual([ - { file: 'doc.md', ruleSource: 'packs/default/consistency.md' }, - ]); - }); - - it('records the required session event stream and preserves lifecycle ordering', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const eventTypes = result.events.map((event: { eventType: string }) => event.eventType); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallStarted); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.ToolCallFinished); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - expect(eventTypes).toContain(SESSION_EVENT_TYPE.SessionFinalized); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.SessionStarted)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - expect(eventTypes.indexOf(SESSION_EVENT_TYPE.FindingRecordedInline)).toBeLessThan( - eventTypes.indexOf(SESSION_EVENT_TYPE.SessionFinalized) - ); - }); - - it('returns an operational error when finalize_review is called more than once', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - await tools.finalize_review.execute({ summary: 'first' }); - await tools.finalize_review.execute({ summary: 'second' }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - const finalizedCount = result.events.filter( - (event: { eventType: string }) => event.eventType === SESSION_EVENT_TYPE.SessionFinalized - ).length; - expect(finalizedCount).toBe(1); - }); - - it.each([ - { - toolName: 'read_file', - input: { path: '../outside.md' }, - }, - { - toolName: 'search_files', - input: { pattern: '../*.md' }, - }, - { - toolName: 'list_directory', - input: { path: '../' }, - }, - { - toolName: 'search_content', - input: { pattern: 'bad phrase', path: '../', glob: '**/*.md' }, - }, - ])( - 'returns explicit tool errors when $toolName is asked to access outside repository bounds', - async ({ toolName, input }) => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - let toolErrorValue: unknown; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools[toolName]!.execute(input); - } catch (error) { - toolErrorValue = error; - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(toolErrorValue).toBeInstanceOf(AgentToolError); - expect(toolError).toBeTruthy(); - expect(toolError).toMatch(/outside|workspace|root|bounds/i); - } - ); - - it('continues producing findings after a recoverable tool error and reports the tool diagnostic', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - let toolError = ''; - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - - try { - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/does-not-exist.md', - }); - } catch (error) { - toolError = error instanceof Error ? error.message : String(error); - } - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({ summary: 'done' }); - - return { usage: { inputTokens: 2, outputTokens: 2 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(toolError).toContain('Unknown ruleSource'); - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings.length).toBeGreaterThan(0); - expect( - result.events.some( - (event: { eventType: string; payload?: { ok?: boolean } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.ok === false - ) - ).toBe(true); - }); - - it('allows read-only search_content tool usage without requiring mutation capabilities', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\nanother line\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - const output = await tools.search_content.execute({ - pattern: 'bad phrase', - path: '.', - glob: '**/*.md', - }); - - expect(output).toBeTruthy(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - }); - - it('marks the run as operationally failed but preserves findings when finalize_review is missing', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.errorMessage).toContain('finalize_review'); - expect(result.findings.length).toBeGreaterThan(0); - }); - - it('uses reviewInstruction to override the prompt body for that lint invocation', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for wording consistency using the evidence you gathered.', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe( - 'Review this file for wording consistency using the evidence you gathered.' - ); - }); - - it('keeps lint prompt body unchanged when reviewInstruction is not provided', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const promptBodies: string[] = []; - const provider: LLMProvider = { - runPromptStructured(_content, promptText: string) { - promptBodies.push(promptText); - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(promptBodies.length).toBeGreaterThan(0); - expect(promptBodies[0]).toBe('Find inconsistent wording.'); - }); - - it('redacts raw read_file content from persisted tool_call_finished events', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'secret text\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const readFileEvent = result.events.find( - (event: { - eventType: string; - payload?: { toolName?: string; output?: { path?: string; contentLength?: number } }; - }) => event.eventType === SESSION_EVENT_TYPE.ToolCallFinished && event.payload?.toolName === 'read_file' - ); - - expect(readFileEvent?.payload?.output).toEqual({ - path: 'doc.md', - contentLength: 'secret text\n'.length, - }); - }); - - it('records started and failed events for visible-tool path errors', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const provider = makeProvider(async (params) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }); - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [makePrompt()], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - const started = result.events.find( - (event: { eventType: string; payload?: { toolName?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallStarted && event.payload?.toolName === 'read_file' - ); - const failed = result.events.find( - (event: { eventType: string; payload?: { toolName?: string; ok?: boolean; error?: string } }) => - event.eventType === SESSION_EVENT_TYPE.ToolCallFinished - && event.payload?.toolName === 'read_file' - && event.payload?.ok === false - ); - - expect(started).toBeDefined(); - expect(failed?.payload?.error).toContain('outside workspace root'); - }); -}); diff --git a/tests/agent/progress.test.ts b/tests/agent/progress.test.ts deleted file mode 100644 index df213ab4..00000000 --- a/tests/agent/progress.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { AgentProgressReporter } from '../../src/agent/progress'; - -describe('agent progress reporter', () => { - afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - it('animates spinner frames independently of content changes while active', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - reporter.showVisibleToolStart({ - toolName: 'lint', - path: 'README.md', - ruleName: 'Repetition', - ruleText: '# Repetition Flag any instance where the same wording repeats', - }); - - const initialWrites = writeSpy.mock.calls.length; - vi.advanceTimersByTime(250); - - expect(writeSpy.mock.calls.length).toBeGreaterThan(initialWrites); - - reporter.finishRun(); - }); - - it('is a no-op when disabled', () => { - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(false); - reporter.startFile('README.md', 'Repetition'); - reporter.finishRun(); - - expect(writeSpy).not.toHaveBeenCalled(); - }); - - it.each([ - { elapsedMs: 0, expected: 'Completed review in 0s.' }, - { elapsedMs: 60_000, expected: 'Completed review in 1m 0s.' }, - { elapsedMs: 3_600_000, expected: 'Completed review in 1h 0m 0s.' }, - ])('formats elapsed time boundaries for $expected', ({ elapsedMs, expected }) => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(elapsedMs); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain(expected); - }); - - it('formats the completion footer with elapsed time', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun(); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Completed review in 1m 25s.'); - }); - - it('formats a failed footer when the run ends with errors', () => { - vi.useFakeTimers(); - - const writeSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const reporter = new AgentProgressReporter(true); - reporter.startFile('README.md', 'Repetition'); - - vi.advanceTimersByTime(85_000); - reporter.finishRun('failed'); - - const output = writeSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(output).toContain('Review failed after 1m 25s.'); - }); -}); diff --git a/tests/agent/prompt-builder.test.ts b/tests/agent/prompt-builder.test.ts deleted file mode 100644 index fe31074c..00000000 --- a/tests/agent/prompt-builder.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('agent prompt builder', () => { - it('builds a non-empty system prompt for valid inputs', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [ - { file: 'README.md', ruleSource: 'packs/default/ai-pattern.md' }, - { file: 'README.md', ruleSource: 'packs/default/consistency.md' }, - ], - availableTools: [ - { name: 'read_file', description: 'Read a file inside the workspace root.' }, - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - { name: 'finalize_review', description: 'Finalize review output and close the session.' }, - ], - }); - - expect(typeof prompt).toBe('string'); - expect(prompt.length).toBeGreaterThan(0); - expect(prompt).toContain('Workspace root: /workspace'); - expect(prompt).toContain('matched rules'); - }); - - it('supports optional user instructions without breaking prompt generation', async () => { - const { buildAgentSystemPrompt } = await import('../../src/agent/prompt-builder'); - - const withoutUserInstructions = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - }); - - const prompt = buildAgentSystemPrompt({ - workspaceRoot: '/workspace', - fileRuleMatches: [{ file: 'README.md', ruleSource: 'packs/default/consistency.md' }], - availableTools: [ - { name: 'lint', description: 'Review a file against a source-backed rule, optionally using an override review instruction for that call.' }, - ], - userInstructions: 'Always enforce concise phrasing.', - }); - - expect(typeof withoutUserInstructions).toBe('string'); - expect(typeof prompt).toBe('string'); - expect(withoutUserInstructions.length).toBeGreaterThan(0); - expect(prompt.length).toBeGreaterThan(withoutUserInstructions.length); - }); -}); diff --git a/tests/agent/review-session-store.test.ts b/tests/agent/review-session-store.test.ts deleted file mode 100644 index 0258b561..00000000 --- a/tests/agent/review-session-store.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { appendFileSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -beforeEach(() => { - vi.resetModules(); -}); - -afterEach(() => { - vi.doUnmock('crypto'); - vi.resetModules(); -}); - -describe('review session store', () => { - it('creates session files in the VectorLint reviews directory', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(store.sessionFilePath).toContain( - `${path.sep}.vectorlint${path.sep}reviews${path.sep}` - ); - expect(path.basename(store.sessionFilePath)).toMatch(/\.jsonl$/); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); - - it('persists each appended event as a valid JSONL record', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const raw = readFileSync(store.sessionFilePath, 'utf8'); - const lines = raw.trim().split('\n'); - expect(lines).toHaveLength(2); - - const parsed = lines.map((line) => JSON.parse(line) as { eventType?: string }); - expect(parsed.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('reads persisted session events in order and reports completion state', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - expect(await store.hasFinalizedEvent()).toBe(false); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0, summary: 'done' }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - expect(await store.hasFinalizedEvent()).toBe(true); - }); - - it('recovers valid events when an existing session file contains malformed JSONL lines', async () => { - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const store = await createReviewSessionStore({ homeDir: home }); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionStarted, - payload: { cwd: '/repo', targets: ['doc.md'] }, - }); - - appendFileSync(store.sessionFilePath, 'not-json\n', 'utf8'); - appendFileSync(store.sessionFilePath, '{"eventType":"unknown"}\n', 'utf8'); - - await store.append({ - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 0 }, - }); - - const events = await store.replay(); - expect(events.map((event) => event.eventType)).toEqual([ - SESSION_EVENT_TYPE.SessionStarted, - SESSION_EVENT_TYPE.SessionFinalized, - ]); - }); - - it('creates a unique session file when an initial generated session id collides', async () => { - const home = mkdtempSync(path.join(os.tmpdir(), 'vectorlint-home-')); - const reviewsDir = path.join(home, '.vectorlint', 'reviews'); - mkdirSync(reviewsDir, { recursive: true }); - writeFileSync(path.join(reviewsDir, 'collision-id.jsonl'), '', 'utf8'); - - const randomUUIDMock = vi - .fn() - .mockReturnValueOnce('collision-id') - .mockReturnValueOnce('fresh-id'); - - vi.doMock('crypto', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - randomUUID: randomUUIDMock, - }; - }); - - const { createReviewSessionStore } = await import( - '../../src/agent/review-session-store' - ); - - const store = await createReviewSessionStore({ homeDir: home }); - expect(path.basename(store.sessionFilePath)).toBe('fresh-id.jsonl'); - expect(readFileSync(store.sessionFilePath, 'utf8')).toBe(''); - }); -}); diff --git a/tests/agent/types-contract.test.ts b/tests/agent/types-contract.test.ts deleted file mode 100644 index 6363587a..00000000 --- a/tests/agent/types-contract.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { SESSION_EVENT_TYPE } from '../../src/agent/types'; - -describe('agent contracts', () => { - it('accepts ruleSource-based inputs for lint and top-level findings', async () => { - const contracts = await import('../../src/agent/types'); - - const lintInput = contracts.LINT_TOOL_INPUT_SCHEMA.parse({ - file: 'docs/guide.md', - ruleSource: 'packs/default/consistency.md', - reviewInstruction: 'Review this file for consistency.', - }); - expect(lintInput.ruleSource).toBe('packs/default/consistency.md'); - - const topLevel = contracts.TOP_LEVEL_REPORT_INPUT_SCHEMA.parse({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file inconsistency', - references: [{ file: 'docs/guide.md', startLine: 2, endLine: 4 }], - }); - expect(topLevel.kind).toBe('top-level'); - expect(topLevel.ruleSource).toBe('packs/default/consistency.md'); - }); - - it('accepts finalized session events with completion metadata', async () => { - const contracts = await import('../../src/agent/types'); - - const event = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.SessionFinalized, - payload: { totalFindings: 1, summary: 'done' }, - }); - - expect(event.eventType).toBe(SESSION_EVENT_TYPE.SessionFinalized); - expect(event.payload.totalFindings).toBe(1); - }); - - it('accepts required tool and finding session event variants for deterministic replay', async () => { - const contracts = await import('../../src/agent/types'); - - const started = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:00.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallStarted, - payload: { toolName: 'lint', input: { file: 'docs/guide.md' } }, - }); - - const finished = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:01.000Z', - eventType: SESSION_EVENT_TYPE.ToolCallFinished, - payload: { toolName: 'lint', ok: true }, - }); - - const inlineFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:02.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedInline, - payload: { - file: 'docs/guide.md', - line: 2, - message: 'Inconsistent term', - ruleSource: 'packs/default/consistency.md', - }, - }); - - const topLevelFinding = contracts.SESSION_EVENT_SCHEMA.parse({ - sessionId: 'session-1', - timestamp: '2026-03-31T00:00:03.000Z', - eventType: SESSION_EVENT_TYPE.FindingRecordedTopLevel, - payload: { - message: 'Cross-file mismatch', - ruleSource: 'packs/default/consistency.md', - }, - }); - - expect(started.eventType).toBe(SESSION_EVENT_TYPE.ToolCallStarted); - expect(finished.eventType).toBe(SESSION_EVENT_TYPE.ToolCallFinished); - expect(inlineFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedInline); - expect(topLevelFinding.eventType).toBe(SESSION_EVENT_TYPE.FindingRecordedTopLevel); - }); -}); diff --git a/tests/cli-mode-rejection.test.ts b/tests/cli-mode-rejection.test.ts new file mode 100644 index 00000000..38c0f3db --- /dev/null +++ b/tests/cli-mode-rejection.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; + +describe('CLI --mode rejection', () => { + let exitSpy: ReturnType; + + beforeEach(() => { + vi.spyOn(console, 'error').mockImplementation(() => undefined); + exitSpy = vi.spyOn(process, 'exit').mockImplementation( + ((code?: string | number | null) => { + throw new Error(`exit:${code ?? ''}`); + }) as never, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('rejects --mode and points users at --model-call instead of mapping it', async () => { + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect( + program.parseAsync(['node', 'test', 'README.md', '--mode', 'agent']), + ).rejects.toThrow('exit:1'); + + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--mode is no longer supported'), + ); + expect(vi.mocked(console.error)).toHaveBeenCalledWith( + expect.stringContaining('--model-call'), + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); +}); diff --git a/tests/config-loader-integration.test.ts b/tests/config-loader-integration.test.ts index f5efedae..0028101c 100644 --- a/tests/config-loader-integration.test.ts +++ b/tests/config-loader-integration.test.ts @@ -24,7 +24,7 @@ RulesPath = ./prompts [docs/**/*.md] RunRules = VectorLint -technical-accuracy.strictness = 9 +clarity.strictness = 9 [blog/**/*.md] RunRules = BlogPack, SEOPack @@ -41,7 +41,7 @@ readability.severity = error expect(config.scanPaths[0]!.pattern).toBe('docs/**/*.md'); expect(config.scanPaths[0]!.runRules).toEqual(['VectorLint']); expect(config.scanPaths[0]!.overrides).toEqual({ - 'technical-accuracy.strictness': '9' + 'clarity.strictness': '9' }); // Second section @@ -63,7 +63,7 @@ strictness = 7 [content/api/**/*.md] RunRules = APIPack strictness = 9 -technical-accuracy.depth = high +clarity.depth = high [content/archived/**/*.md] RunRules = diff --git a/tests/evaluations/README.md b/tests/evaluations/README.md deleted file mode 100644 index bc6fb185..00000000 --- a/tests/evaluations/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Manual Evaluations - -This directory contains test fixtures for running VectorLint manually against real LLM providers. Use it to evaluate accuracy, compare models, and inspect gate check behavior. - -## Contents - -``` -tests/evaluations/ -├── .vectorlint.ini # Config pointing at test-rules/ -├── TEST_FILE.md # Sample document to evaluate -└── test-rules/ - └── Test/ # Rule pack with general-purpose test rules - ├── clarity.md - ├── consistency.md - ├── passive-voice.md - ├── readability.md - └── wordiness.md -``` - -## Running an evaluation - -From the repo root: - -```bash -# Basic run -npm run dev -- tests/evaluations/TEST_FILE.md \ - --config tests/evaluations/.vectorlint.ini - -# With debug artifacts (writes raw model output + gate check decisions) -npm run dev -- tests/evaluations/TEST_FILE.md \ - --config tests/evaluations/.vectorlint.ini \ - --debug-json - -# With verbose output -npm run dev -- tests/evaluations/TEST_FILE.md \ - --config tests/evaluations/.vectorlint.ini \ - --debug-json --verbose -``` - -Debug artifacts are written to `.vectorlint/runs//.json` and are gitignored. - -## Switching models - -Set your provider and model via environment variables before running: - -```bash -# OpenAI -LLM_PROVIDER=openai OPENAI_MODEL=gpt-4o npm run dev -- ... - -# Anthropic -LLM_PROVIDER=anthropic ANTHROPIC_MODEL=claude-sonnet-4-6 npm run dev -- ... - -# Gemini -LLM_PROVIDER=gemini GEMINI_MODEL=gemini-1.5-pro npm run dev -- ... -``` - -Or configure them in `~/.vectorlint/config.toml`. - -## Inspecting debug artifacts - -Each artifact under `.vectorlint/runs/` contains: - -- `raw_model_output` — exact JSON returned by the model, including all gate check fields -- `filter_decisions` — deterministic surface/hide decision per violation candidate with reasons -- `surfaced_violations` — candidates that passed all gates - -Use these to compare how different models respond to the same rules and content, and to tune gate check thresholds. diff --git a/tests/evaluator.test.ts b/tests/evaluator.test.ts deleted file mode 100644 index 79f0bb91..00000000 --- a/tests/evaluator.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, mkdirSync, writeFileSync } from 'fs'; -import path from 'path'; -import { tmpdir } from 'os'; -import { loadRules } from '../src/prompts/prompt-loader.js'; - -// Fake provider implementing LLMProvider -class FakeProvider { - calls: Array<{ content: string; prompt: string }> = []; - runPromptStructured(content: string, promptText: string): Promise { - this.calls.push({ content, prompt: promptText }); - const injected = content && content.length > 0 ? 'OK' : 'MISSING'; - return Promise.resolve({ result: `RESULT:${injected}` } as T); - } -} - -function setupEnv() { - const root = mkdtempSync(path.join(tmpdir(), 'vlint-')); - const promptsDir = path.join(root, 'prompts'); - mkdirSync(promptsDir, { recursive: true }); - // Two prompts with minimal frontmatter criteria - writeFileSync( - path.join(promptsDir, 'p1.md'), - `---\nid: P1\nname: Prompt 1\ncriteria:\n - name: A\n id: A\n---\nBody 1\n` - ); - writeFileSync( - path.join(promptsDir, 'p2.md'), - `---\nid: P2\nname: Prompt 2\ncriteria:\n - name: B\n id: B\n---\nBody 2\n` - ); - const files = [ - path.join(root, 'a.md'), - path.join(root, 'b.txt'), - ]; - writeFileSync(files[0], '# title'); - writeFileSync(files[1], 'plain'); - return { root, promptsDir, files }; -} - -describe('Evaluation aggregation', () => { - it('runs all prompts for all files and injects content if placeholder exists', async () => { - const { promptsDir, files } = setupEnv(); - const provider = new FakeProvider(); - const { prompts } = loadRules(promptsDir); - // Simulate aggregation: 2 prompts x 2 files = 4 calls - for (let i = 0; i < files.length; i++) { - const content = '# test'; - for (const p of prompts) { - await provider.runPromptStructured(content, p.body); - } - } - expect(provider.calls.length).toBe(4); - }); -}); diff --git a/tests/executors/agent-model-call-executor.test.ts b/tests/executors/agent-model-call-executor.test.ts new file mode 100644 index 00000000..170b0970 --- /dev/null +++ b/tests/executors/agent-model-call-executor.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it } from 'vitest'; + +import { AgentModelCallExecutor } from '../../src/executors/agent-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { ReviewRequest, ReviewRule, ReviewTarget } from '../../src/review'; +import { DefaultRequestBuilder } from '../../src/providers/request-builder'; +import type { ToolCallDefinition, ToolCallingModelClient, ToolCallRunOptions } from '../../src/providers/tool-calling-model-client'; +import type { LLMResult } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { ReviewLLMResult } from '../../src/prompts/schema'; +import type { TargetSectionErrorResult, TargetSectionResult } from '../../src/executors/target-read-capability-adapter'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = ReviewLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +interface CapturedCall { + systemPrompt: string; + prompt: string; + tools: Record; + options?: ToolCallRunOptions; +} + +interface FakeToolClient extends ToolCallingModelClient { + calls: number; + captured: CapturedCall[]; +} + +function makeFakeClient( + respond: () => { data: ReviewLLMResult; usage?: TokenUsage }, +): FakeToolClient { + const fake = { + calls: 0, + captured: [] as CapturedCall[], + runWithTools: ( + params: { + systemPrompt: string; + prompt: string; + tools: Record; + schema: { name: string; schema: Record }; + options?: ToolCallRunOptions; + }, + ): Promise> => { + fake.calls += 1; + fake.captured.push({ + systemPrompt: params.systemPrompt, + prompt: params.prompt, + tools: params.tools, + ...(params.options !== undefined ? { options: params.options } : {}), + }); + const { data, usage } = respond(); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + }; + return fake; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\nthird line\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'agent', + ...overrides, + }; +} + +describe('AgentModelCallExecutor', () => { + it('exposes exactly one tool (read_target_section) and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + // Exactly one bounded model run, exposing exactly one tool. + expect(client.calls).toBe(1); + expect(Object.keys(client.captured[0]!.tools)).toEqual(['read_target_section']); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('passes the source-backed rule body verbatim as the system prompt (no model-supplied rule override)', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule({ body: 'Flag vague advice.' })])); + + // With an empty directive/user-instructions builder, the system prompt is + // the rule body verbatim — no model-supplied rule override is introduced. + expect(client.captured[0]!.systemPrompt).toBe('Flag vague advice.'); + // The agent instruction names the only tool and the target. + expect(client.captured[0]!.prompt).toContain('read_target_section'); + expect(client.captured[0]!.prompt).toContain('file:///repo/docs/guide.md'); + }); + + it('includes caller-supplied context in the system prompt', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule()], { + context: [{ + label: 'Current API contract', + relation: 'reference', + content: 'The endpoint returns HTTP 202.', + }], + })); + + expect(client.captured[0]!.systemPrompt).toContain('## Caller-supplied context'); + expect(client.captured[0]!.systemPrompt).toContain('### Current API contract'); + expect(client.captured[0]!.systemPrompt).toContain('The endpoint returns HTTP 202.'); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's run fits within maxModelCallsPerReview = 1. + expect(client.calls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('reviews each rule independently and aggregates usage across runs', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.calls).toBe(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('exposes a read_target_section that slices only request.target.content', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run(makeRequest([makeRule()])); + + const tool = client.captured[0]!.tools['read_target_section']!; + const result = (await tool.execute({ startLine: 2, endLine: 3 })) as TargetSectionResult; + + // The window matches the in-memory target lines with their real line numbers. + expect(result).toEqual({ + startLine: 2, + endLine: 3, + content: '2\tsecond line is fine\n3\tthird line', + }); + }); + + it('returns a model-visible error for out-of-range windows without aborting the run', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + // The run completes and returns a normal ReviewResult despite the model + // (simulated below) requesting an out-of-range section. + const result = await executor.run(makeRequest([makeRule()])); + expect(result.hadOperationalErrors).toBe(false); + + const tool = client.captured[0]!.tools['read_target_section']!; + const errorResult = (await tool.execute({ startLine: 1, endLine: 999 })) as TargetSectionErrorResult; + + expect(errorResult.error).toContain('999'); + expect(errorResult.lineCount).toBe(4); + }); + + it('bounds each run by the budget maxChunksPerRule step limit', async () => { + const client = makeFakeClient(() => ({ data: { reasoning: 'r', violations: [] } })); + const executor = new AgentModelCallExecutor(client, new DefaultRequestBuilder()); + + await executor.run( + makeRequest([makeRule()], { budget: { ...DEFAULT_REVIEW_BUDGET, maxChunksPerRule: 7 } }), + ); + + expect(client.captured[0]!.options?.maxSteps).toBe(7); + expect(client.captured[0]!.options?.maxParallelToolCalls).toBe(1); + }); +}); diff --git a/tests/executors/single-model-call-executor.test.ts b/tests/executors/single-model-call-executor.test.ts new file mode 100644 index 00000000..0b5ee7f2 --- /dev/null +++ b/tests/executors/single-model-call-executor.test.ts @@ -0,0 +1,268 @@ +import { describe, expect, it } from 'vitest'; + +import { SingleModelCallExecutor } from '../../src/executors/single-model-call-executor'; +import { DEFAULT_REVIEW_BUDGET } from '../../src/review'; +import type { + ReviewRequest, + ReviewRule, + ReviewTarget, +} from '../../src/review'; +import type { LLMResult, StructuredModelClient } from '../../src/providers/structured-model-client'; +import type { TokenUsage } from '../../src/providers/token-usage'; +import type { ReviewLLMResult } from '../../src/prompts/schema'; + +const SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const SUPPORTED_NOTES = { + rule_supports_claim: 'yes', + evidence_exact: 'yes', + context_supports_violation: 'yes', + plausible_non_violation: 'no', + fix_is_drop_in: 'yes', + fix_preserves_meaning: 'yes', +}; + +type ModelViolation = ReviewLLMResult['violations'][number]; + +function modelViolation(overrides: Partial = {}): ModelViolation { + return { + line: 1, + quoted_text: 'vague text', + context_before: '', + context_after: '', + description: 'desc', + analysis: 'analysis', + message: 'message', + suggestion: 'suggestion', + fix: 'fix', + rule_quote: 'rule quote', + checks: SUPPORTED_CHECKS, + check_notes: SUPPORTED_NOTES, + confidence: 0.9, + ...overrides, + }; +} + +type FakeStructuredClient = StructuredModelClient & { + runWithTools: () => Promise; + structuredCalls: number; + toolCalls: number; + lastPromptText?: string; +}; + +function makeFakeClient( + respond: (content: string, promptText: string) => { data: ReviewLLMResult; usage?: TokenUsage }, +): FakeStructuredClient { + const client = { + structuredCalls: 0, + toolCalls: 0, + lastPromptText: undefined as string | undefined, + runPromptStructured: ( + content: string, + promptText: string, + ): Promise> => { + client.structuredCalls += 1; + client.lastPromptText = promptText; + const { data, usage } = respond(content, promptText); + const result: LLMResult = { data: data as unknown as T }; + if (usage) { + result.usage = usage; + } + return Promise.resolve(result); + }, + runWithTools: (): Promise => { + client.toolCalls += 1; + return Promise.reject( + new Error('SingleModelCallExecutor must not call runWithTools'), + ); + }, + }; + return client; +} + +const TARGET: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: 'vague text here\nsecond line is fine\n', + contentType: 'text/markdown', +}; + +function makeRule(overrides: Partial = {}): ReviewRule { + return { + id: 'VectorLint.PseudoAdvice', + source: '/repo/presets/VectorLint/pseudo-advice.md', + body: 'Flag vague advice.', + ...overrides, + }; +} + +function makeRequest(rules: ReviewRule[], overrides: Partial = {}): ReviewRequest { + return { + target: TARGET, + rules, + budget: { ...DEFAULT_REVIEW_BUDGET }, + outputPolicy: { includeUsage: true, recordPayloadTelemetry: false }, + modelCall: 'single', + ...overrides, + }; +} + +describe('SingleModelCallExecutor', () => { + it('makes one structured call per rule and projects findings/scores through processFindings', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text', message: 'Too vague.' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + // One structured call, with the rule body as the source-backed prompt. + expect(client.structuredCalls).toBe(1); + expect(client.lastPromptText).toBe('Flag vague advice.'); + + // The finding is verified + anchored through the shared processor. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.findings[0]?.ruleSource).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(result.findings[0]?.line).toBeGreaterThanOrEqual(1); + expect(result.findings[0]?.match).toBe('vague text'); + + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + expect(result.scores[0]?.findingCount).toBe(1); + + expect(result.usage?.modelCalls).toBe(1); + }); + + it('includes caller-supplied context in the review prompt', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + await executor.run(makeRequest([makeRule()], { + context: [{ + label: 'Current API contract', + relation: 'reference', + uri: 'file:///repo/openapi.md', + content: 'The endpoint returns HTTP 202.', + }], + })); + + expect(client.lastPromptText).toContain('## Caller-supplied context'); + expect(client.lastPromptText).toContain('### Current API contract'); + expect(client.lastPromptText).toContain('The endpoint returns HTTP 202.'); + }); + + it('routes unanchored evidence to a warn diagnostic without emitting a finding', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'quantum entanglement device' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run(makeRequest([makeRule()])); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe('finding-evidence-not-locatable'); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('reviews each rule independently and aggregates usage across calls', async () => { + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text' })], + }, + usage: { inputTokens: 10, outputTokens: 5 }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([ + makeRule({ id: 'VectorLint.RuleA', source: '/repo/a.md' }), + makeRule({ id: 'VectorLint.RuleB', source: '/repo/b.md' }), + ]), + ); + + expect(client.structuredCalls).toBe(2); + expect(result.scores).toHaveLength(2); + expect(result.scores.map((s) => s.ruleId).sort()).toEqual(['VectorLint.RuleA', 'VectorLint.RuleB']); + expect(result.usage?.modelCalls).toBe(2); + expect(result.usage?.inputTokens).toBe(20); + expect(result.usage?.outputTokens).toBe(10); + }); + + it('stops and records an operational error when the model-call budget is exhausted', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest( + [makeRule({ id: 'VectorLint.RuleA' }), makeRule({ id: 'VectorLint.RuleB' })], + { budget: { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 1 } }, + ), + ); + + // Only the first rule's single call fits within maxModelCallsPerReview = 1. + expect(client.structuredCalls).toBe(1); + expect(result.usage?.modelCalls).toBe(1); + expect(result.hadOperationalErrors).toBe(true); + expect(result.diagnostics.some((d) => d.code === 'review-budget-exceeded')).toBe(true); + // The completed rule is still projected; the budget-exceeded rule is not. + expect(result.scores).toHaveLength(1); + expect(result.scores[0]?.ruleId).toBe('VectorLint.RuleA'); + }); + + it('never invokes the tool-calling surface', async () => { + const client = makeFakeClient(() => ({ + data: { reasoning: 'r', violations: [] }, + })); + const executor = new SingleModelCallExecutor(client); + + await executor.run(makeRequest([makeRule()])); + + expect(client.toolCalls).toBe(0); + expect(client.structuredCalls).toBe(1); + }); + + it('chunks large targets into multiple model calls and merges per-chunk violations', async () => { + // > 600 words triggers recursive chunking. + const targetContent = Array.from({ length: 200 }, (_, i) => `vague text line ${i}`).join('\n'); + const client = makeFakeClient(() => ({ + data: { + reasoning: 'r', + violations: [modelViolation({ quoted_text: 'vague text line 0' })], + }, + })); + const executor = new SingleModelCallExecutor(client); + + const result = await executor.run( + makeRequest([makeRule()], { + target: { uri: 'file:///repo/docs/big.md', content: targetContent, contentType: 'text/markdown' }, + }), + ); + + expect(client.structuredCalls).toBeGreaterThan(1); + expect(result.usage?.modelCalls).toBe(client.structuredCalls); + expect(result.scores).toHaveLength(1); + // Identical violations across chunks dedupe to a single anchored finding. + expect(result.findings).toHaveLength(1); + expect(result.findings[0]?.ruleId).toBe('VectorLint.PseudoAdvice'); + }); +}); diff --git a/tests/executors/target-read-capability-adapter.test.ts b/tests/executors/target-read-capability-adapter.test.ts new file mode 100644 index 00000000..c46016af --- /dev/null +++ b/tests/executors/target-read-capability-adapter.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildReadTargetSectionTool, + READ_TARGET_SECTION_TOOL_NAME, + TargetReadCapability, + type TargetSectionErrorResult, + type TargetSectionResult, +} from '../../src/executors/target-read-capability-adapter'; + +const CONTENT = 'alpha line\nbeta line\ngamma line\n'; + +describe('TargetReadCapability', () => { + it('counts lines including the trailing empty line (mirrors prependLineNumbers)', () => { + expect(new TargetReadCapability(CONTENT).lineCount).toBe(4); + expect(new TargetReadCapability('only one').lineCount).toBe(1); + }); + + it('slices only the in-memory target content with original line numbers', async () => { + const capability = new TargetReadCapability(CONTENT); + + const first = await capability.readTargetSection(1, 2); + expect(first).toEqual({ + startLine: 1, + endLine: 2, + content: '1\talpha line\n2\tbeta line', + }); + + const tail = await capability.readTargetSection(3, 4); + expect(tail).toEqual({ + startLine: 3, + endLine: 4, + content: '3\tgamma line\n4\t', + }); + }); +}); + +describe('buildReadTargetSectionTool', () => { + function toolFor(content: string) { + return buildReadTargetSectionTool(new TargetReadCapability(content)); + } + + it('exposes exactly one tool named read_target_section', () => { + const tools = toolFor(CONTENT); + expect(Object.keys(tools)).toEqual([READ_TARGET_SECTION_TOOL_NAME]); + }); + + it('returns numbered target windows for valid ranges (target-only slicing)', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 3, + })) as TargetSectionResult; + + expect(result).toEqual({ + startLine: 1, + endLine: 3, + content: '1\talpha line\n2\tbeta line\n3\tgamma line', + }); + }); + + it('returns a model-visible error (never throws) when endLine exceeds the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 1, + endLine: 99, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('99'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error when startLine is past the target', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 10, + endLine: 12, + })) as TargetSectionErrorResult; + + expect(result.error).toContain('10'); + expect(result.lineCount).toBe(4); + }); + + it('returns a model-visible error for an inverted range', async () => { + const tools = toolFor(CONTENT); + const result = (await tools[READ_TARGET_SECTION_TOOL_NAME]!.execute({ + startLine: 3, + endLine: 1, + })) as TargetSectionErrorResult; + + expect(result.error).toMatch(/startLine <= endLine/); + }); + + it('returns a model-visible error for malformed arguments', async () => { + const tools = toolFor(CONTENT); + const execute = tools[READ_TARGET_SECTION_TOOL_NAME]!.execute; + + const missing = (await execute({ startLine: 1 })) as TargetSectionErrorResult; + expect(missing.error).toMatch(/Invalid read_target_section arguments/); + expect(missing.lineCount).toBe(4); + + const negative = (await execute({ startLine: 0, endLine: 2 })) as TargetSectionErrorResult; + expect(negative.error).toMatch(/Invalid read_target_section arguments/); + + const floats = (await execute({ startLine: 1.5, endLine: 2 })) as TargetSectionErrorResult; + expect(floats.error).toMatch(/Invalid read_target_section arguments/); + }); +}); diff --git a/tests/file-sections.test.ts b/tests/file-sections.test.ts index e29998a9..61b0ec2d 100644 --- a/tests/file-sections.test.ts +++ b/tests/file-sections.test.ts @@ -51,7 +51,7 @@ describe('File-centric configuration (File Sections)', () => { const config = { 'critical/**/*.md': { RunRules: 'VectorLint', - 'technical-accuracy.strictness': '9', + 'clarity.strictness': '9', 'readability.severity': 'error' } }; @@ -59,7 +59,7 @@ describe('File-centric configuration (File Sections)', () => { const sections = parser.parseSections(config); expect(sections[0].overrides).toEqual({ - 'technical-accuracy.strictness': '9', + 'clarity.strictness': '9', 'readability.severity': 'error' }); }); @@ -142,15 +142,15 @@ describe('File-centric configuration (File Sections)', () => { describe('Integration: Pattern priority and override merging', () => { it('applies prompts with overrides based on file path patterns', () => { const sections: FilePatternConfig[] = [ - createFilePatternConfig('**/*.md', ['VectorLint'], { 'technical-accuracy.strictness': 7 }), - createFilePatternConfig('docs/api/**/*.md', ['APIPack'], { 'technical-accuracy.strictness': 9 }) + createFilePatternConfig('**/*.md', ['VectorLint'], { 'clarity.strictness': 7 }), + createFilePatternConfig('docs/api/**/*.md', ['APIPack'], { 'clarity.strictness': 9 }) ]; const result = resolver.resolveConfiguration('docs/api/users.md', sections); // Cascading: APIPack applies on top of VectorLint expect(result.packs).toEqual(['VectorLint', 'APIPack']); - expect(result.overrides['technical-accuracy.strictness']).toBe(9); // Later wins + expect(result.overrides['clarity.strictness']).toBe(9); // Later wins }); }); }); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts index 8e660bf2..7edffc11 100644 --- a/tests/findings/scorer.test.ts +++ b/tests/findings/scorer.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { calculateScore } from '../../src/scoring'; -import { Severity } from '../../src/evaluators/types'; +import { Severity } from '../../src/review/severity'; import { scoreFindings } from '../../src/findings/scorer'; import type { RawViolation } from '../../src/findings/types'; diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts index 031115f1..dde92f43 100644 --- a/tests/findings/severity.test.ts +++ b/tests/findings/severity.test.ts @@ -4,10 +4,10 @@ import { resolveCriterionId, resolveSeverity, } from '../../src/findings/severity'; -import { Severity } from '../../src/evaluators/types'; -import type { ScoredEvaluation } from '../../src/prompts/schema'; +import { Severity } from '../../src/review/severity'; +import type { ScoredReview } from '../../src/prompts/schema'; -function makeScoredEvaluation(severity: Severity): ScoredEvaluation { +function makeScoredReview(severity: Severity): ScoredReview { return { final_score: 5, percentage: 50, @@ -20,11 +20,11 @@ function makeScoredEvaluation(severity: Severity): ScoredEvaluation { } describe('resolveSeverity', () => { - it('returns the density-derived severity from the scored evaluation', () => { - expect(resolveSeverity({ scored: makeScoredEvaluation(Severity.WARNING) })).toBe( + it('returns the density-derived severity from the scored review', () => { + expect(resolveSeverity({ scored: makeScoredReview(Severity.WARNING) })).toBe( Severity.WARNING, ); - expect(resolveSeverity({ scored: makeScoredEvaluation(Severity.ERROR) })).toBe( + expect(resolveSeverity({ scored: makeScoredReview(Severity.ERROR) })).toBe( Severity.ERROR, ); }); diff --git a/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md b/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md index bdb8cae5..b590be1f 100644 --- a/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md +++ b/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Capitalization name: Capitalization severity: warning -evaluateAs: document --- # Capitalization diff --git a/tests/fixtures/consistency/rules/consistency-rules/consistency.md b/tests/fixtures/consistency/rules/consistency-rules/consistency.md index 219d1022..04c9e717 100644 --- a/tests/fixtures/consistency/rules/consistency-rules/consistency.md +++ b/tests/fixtures/consistency/rules/consistency-rules/consistency.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Consistency name: Consistency severity: warning -evaluateAs: document --- # Consistency diff --git a/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md b/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md index ab5bda8f..a0bf644f 100644 --- a/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md +++ b/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Inclusivity name: Inclusivity severity: warning -evaluateAs: document --- # Inclusivity diff --git a/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md b/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md index bfa21666..9bcda8dd 100644 --- a/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md +++ b/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md @@ -1,9 +1,7 @@ --- -evaluator: base id: PassiveVoice name: Passive Voice severity: warning -evaluateAs: document --- # Passive Voice diff --git a/tests/fixtures/repetition/rules/repetition-rules/repetition.md b/tests/fixtures/repetition/rules/repetition-rules/repetition.md index d102e343..66e87219 100644 --- a/tests/fixtures/repetition/rules/repetition-rules/repetition.md +++ b/tests/fixtures/repetition/rules/repetition-rules/repetition.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Repetition name: Repetition severity: warning -evaluateAs: document --- # Repetition diff --git a/tests/fixtures/technical-accuracy/test.md b/tests/fixtures/technical-accuracy/test.md deleted file mode 100644 index 391efe75..00000000 --- a/tests/fixtures/technical-accuracy/test.md +++ /dev/null @@ -1,31 +0,0 @@ -# Evaluating Modern AI-Assisted Developer Tools - -AI-driven developer productivity has surged in 2025, with new tools claiming to automate everything from documentation to deployment. - -**1. GitHub Copilot** has become one of the most widely adopted AI pair-programming tools, integrated directly into VS Code and JetBrains IDEs. - -**2. Codeium** offers similar capabilities to Copilot and provides enterprise-grade privacy controls for on-prem deployments. - -**3. JetBrains AI Assistant** integrates with IntelliJ and PyCharm, suggesting code completions and explaining errors inline. - -**4. DeepDeploy claims to create and manage cloud deployment pipelines automatically, requiring no configuration.** - -**5. TypeScript ensures that JavaScript applications will never encounter runtime errors once properly typed.** - -**6. CloudLint** reportedly scans and fixes cloud infrastructure misconfigurations instantly using natural language prompts. - -**7. OpenAI’s GPT-4 Turbo API allows developers to build contextual assistants that integrate directly into CI/CD systems.** - -**8. StackSynth is described as a next-generation AI framework that composes entire full-stack apps based on Figma designs.** - -**9. Always trust AI-generated code suggestions—they are trained on billions of high-quality code examples.** - -**10. Vercel and Netlify have become leading platforms for frontend deployments, offering serverless hosting and instant previews.** - -**11. Rust is now the most popular systems language, surpassing C++ in developer usage according to Stack Overflow’s 2025 survey.** - -**12. Some claim that “Python eliminates all memory safety issues,” but Python only abstracts memory management—it doesn’t guarantee safety.** - -**13. CloudLint has no public GitHub repository or documentation at this time.** - -**14. Always verify AI-generated claims before deploying to production.** diff --git a/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md b/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md index 07c74cff..4c943158 100644 --- a/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md +++ b/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md @@ -1,9 +1,7 @@ --- -evaluator: base id: UnsupportedClaims name: Unsupported Claims severity: warning -evaluateAs: document --- # Unsupported Claims diff --git a/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md b/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md index cc8529fc..2da17f1b 100644 --- a/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md +++ b/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Wordiness name: Wordiness severity: warning -evaluateAs: document --- # Wordiness diff --git a/tests/fixtures/wordiness/test.md b/tests/fixtures/wordiness/test.md index 940fca40..3e7396fc 100644 --- a/tests/fixtures/wordiness/test.md +++ b/tests/fixtures/wordiness/test.md @@ -20,4 +20,4 @@ Avoid phrases that add length without adding meaning. "Gather together your depe ## Review process -Prior to publishing, ask a colleague to read the draft. In order to get useful feedback, be specific about what you want them to evaluate. At this point in time, most teams review docs informally, which often leads to inconsistencies that are hard to catch after the fact. +Prior to publishing, ask a colleague to read the draft. In order to get useful feedback, be specific about what you want them to assess. At this point in time, most teams review docs informally, which often leads to inconsistencies that are hard to catch after the fact. diff --git a/tests/global-config.test.ts b/tests/global-config.test.ts index 2a8d8c7b..acee276c 100644 --- a/tests/global-config.test.ts +++ b/tests/global-config.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import { loadGlobalConfig } from '../src/config/global-config'; +import { ensureGlobalConfig, loadGlobalConfig } from '../src/config/global-config'; import { GLOBAL_CONFIG_DIR, GLOBAL_CONFIG_FILE } from '../src/config/constants'; // Mock fs and os @@ -49,6 +49,21 @@ BOOL_KEY = true expect(fs.readFileSync).toHaveBeenCalledWith(mockConfigPath, 'utf-8'); }); + it('creates the global config directory and file with restrictive modes', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + + expect(ensureGlobalConfig()).toBe(mockConfigPath); + expect(fs.mkdirSync).toHaveBeenCalledWith( + path.dirname(mockConfigPath), + { recursive: true, mode: 0o700 }, + ); + expect(fs.writeFileSync).toHaveBeenCalledWith( + mockConfigPath, + expect.any(String), + { encoding: 'utf-8', mode: 0o600 }, + ); + }); + it('should NOT overwrite existing env vars', () => { const mockToml = ` [env] diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 359da161..0837162f 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -12,7 +12,7 @@ const MOCK_LOAD_CONFIG = vi.hoisted(() => vi.fn()); const MOCK_RESOLVE_TARGETS = vi.hoisted(() => vi.fn()); const MOCK_CREATE_OBSERVABILITY = vi.hoisted(() => vi.fn()); const MOCK_CREATE_PROVIDER = vi.hoisted(() => vi.fn()); -const MOCK_EVALUATE_FILES = vi.hoisted(() => vi.fn()); +const MOCK_REVIEW_FILES = vi.hoisted(() => vi.fn()); const MOCK_LOAD_RULE_FILE = vi.hoisted(() => vi.fn()); const MOCK_LIST_ALL_PACKS = vi.hoisted(() => vi.fn()); const MOCK_FIND_RULE_FILES = vi.hoisted(() => vi.fn()); @@ -55,7 +55,7 @@ vi.mock('../src/providers/provider-factory', async (importOriginal) => { }); vi.mock('../src/cli/orchestrator', () => ({ - evaluateFiles: MOCK_EVALUATE_FILES, + reviewFiles: MOCK_REVIEW_FILES, })); vi.mock('../src/prompts/prompt-loader', () => ({ @@ -97,8 +97,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'single', config: undefined, }); MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); @@ -116,7 +115,7 @@ describe('Main command observability lifecycle', () => { MOCK_LOAD_RULE_FILE.mockReturnValue({ prompt: undefined, warning: undefined }); MOCK_RESOLVE_TARGETS.mockReturnValue(['README.md']); MOCK_CREATE_PROVIDER.mockReturnValue({ mocked: true }); - MOCK_EVALUATE_FILES.mockResolvedValue({ + MOCK_REVIEW_FILES.mockResolvedValue({ totalFiles: 1, totalErrors: 0, totalWarnings: 0, @@ -142,8 +141,7 @@ describe('Main command observability lifecycle', () => { showPromptTrunc: false, debugJson: false, output: 'line', - mode: 'standard', - print: false, + modelCall: 'single', config: undefined, }); const observability = { diff --git a/tests/observability/langfuse-observability.test.ts b/tests/observability/langfuse-observability.test.ts index 56448489..a125447a 100644 --- a/tests/observability/langfuse-observability.test.ts +++ b/tests/observability/langfuse-observability.test.ts @@ -26,7 +26,7 @@ describe('LangfuseObservability', () => { SHUTDOWN_MOCK.mockResolvedValue(undefined); }); - it('returns AI SDK telemetry options with full payload recording enabled', () => { + it('does not record payloads by default', () => { const subject = new LangfuseObservability({ publicKey: 'pk-lf-test', secretKey: 'sk-lf-test', @@ -34,21 +34,40 @@ describe('LangfuseObservability', () => { }); expect(subject.decorateCall({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', - evaluator: 'clarity', + reviewer: 'clarity', rule: 'no-fluff', })).toEqual({ experimental_telemetry: { isEnabled: true, - functionId: 'vectorlint.structured-eval', + functionId: 'vectorlint.structured-review', metadata: { provider: 'openai', model: 'gpt-4o', - evaluator: 'clarity', + reviewer: 'clarity', rule: 'no-fluff', }, + recordInputs: false, + recordOutputs: false, + }, + }); + }); + + it('records payloads only when the review output policy opts in', () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + }); + + expect(subject.decorateCall({ + operation: 'structured-review', + provider: 'openai', + model: 'gpt-4o', + recordPayloadTelemetry: true, + })).toMatchObject({ + experimental_telemetry: { recordInputs: true, recordOutputs: true, }, diff --git a/tests/observability/noop-observability.test.ts b/tests/observability/noop-observability.test.ts index 877b00f7..ce29e1d6 100644 --- a/tests/observability/noop-observability.test.ts +++ b/tests/observability/noop-observability.test.ts @@ -6,7 +6,7 @@ describe('NoopObservability', () => { it('returns an empty option object for any AI execution context', () => { expect(subject.decorateCall({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', })).toEqual({}); diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts deleted file mode 100644 index baf5ae98..00000000 --- a/tests/orchestrator-agent-output.test.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; -import * as path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; -import type { PromptFile } from '../src/prompts/prompt-loader'; -import type { LLMProvider } from '../src/providers/llm-provider'; -import type { Logger } from '../src/logging/logger'; -import { Severity } from '../src/evaluators/types'; - -function makePrompt(): PromptFile { - const id = 'consistency'; - const name = 'Consistency'; - return { - id, - filename: `${id}.md`, - fullPath: 'packs/default/consistency.md', - pack: 'Default', - body: 'Find inconsistent wording', - meta: { - id: name, - name, - severity: Severity.WARNING, - }, - }; -} - -type LoggerSpy = Logger & { warn: ReturnType }; - -function makeLogger(): LoggerSpy { - return { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - } as unknown as LoggerSpy; -} - -interface StandardProviderSpies { - provider: LLMProvider; - runPromptStructured: ReturnType; - runAgentToolLoop: ReturnType; -} - -function makeStandardProvider(): StandardProviderSpies { - const runPromptStructured = vi.fn().mockResolvedValue({ - data: { reasoning: 'ok', violations: [] }, - }); - const runAgentToolLoop = vi.fn().mockResolvedValue({ - usage: { inputTokens: 0, outputTokens: 0 }, - }); - const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; - return { provider, runPromptStructured, runAgentToolLoop }; -} - -describe('agent mode fallback', () => { - const tempRepos: string[] = []; - - function createTempRepo(): string { - const repo = mkdtempSync(path.join(process.cwd(), 'tmp-agent-orch-')); - tempRepos.push(repo); - return repo; - } - - beforeEach(() => { - vi.spyOn(console, 'log').mockImplementation(() => undefined); - vi.spyOn(console, 'warn').mockImplementation(() => undefined); - vi.spyOn(console, 'error').mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - for (const repo of tempRepos.splice(0, tempRepos.length)) { - rmSync(repo, { recursive: true, force: true }); - } - }); - - it('warns through the injected logger and falls back to standard evaluation', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('falls back to standard evaluation'), - ); - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); - - it('does not invoke the agent executor in line output mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); - const logger = makeLogger(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - logger, - }); - - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('falls back to standard evaluation')); - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(runPromptStructured).toHaveBeenCalled(); - }); - - it('stays silent in standard mode without a logger', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const { provider, runAgentToolLoop } = makeStandardProvider(); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: DEFAULT_REVIEW_MODE, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - }); - - expect(runAgentToolLoop).not.toHaveBeenCalled(); - expect(result.totalFiles).toBe(1); - }); -}); diff --git a/tests/orchestrator-executor-dispatch.test.ts b/tests/orchestrator-executor-dispatch.test.ts new file mode 100644 index 00000000..c89e3840 --- /dev/null +++ b/tests/orchestrator-executor-dispatch.test.ts @@ -0,0 +1,229 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import * as path from 'path'; +import { reviewFiles } from '../src/cli/orchestrator'; +import { OutputFormat, type ReviewOptions } from '../src/cli/types'; +import { Severity } from '../src/review/severity'; +import type { PromptFile } from '../src/prompts/prompt-loader'; +import type { ReviewRequest, ReviewResult } from '../src/review/types'; + +const { EXECUTOR_FOR_MOCK, FAKE_EXECUTOR_RUN } = vi.hoisted(() => ({ + EXECUTOR_FOR_MOCK: vi.fn(), + FAKE_EXECUTOR_RUN: vi.fn(), +})); + +// Replace executorFor with a spy that returns a controllable executor so the +// orchestrator's wiring (ReviewRequest build -> chooseModelCall -> dispatch -> +// ReviewResult routing) is exercised without a real model call. +vi.mock('../src/executors', () => ({ + executorFor: EXECUTOR_FOR_MOCK, +})); + +function makePrompt(id: string): PromptFile { + return { + id, + filename: `${id}.md`, + fullPath: path.join(process.cwd(), 'prompts', `${id}.md`), + pack: 'TestPack', + body: `Rule body for ${id}`, + meta: { id, name: id, type: 'check', severity: Severity.WARNING }, + }; +} + +function makeOptions(prompts: PromptFile[], overrides: Partial = {}): ReviewOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + requestBuilder: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Json, + modelCall: 'auto', + ...overrides, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), 'vectorlint-dispatch-')); + const filePath = path.join(dir, 'input.md'); + writeFileSync(filePath, content); + return filePath; +} + +function makeReviewResult(overrides: Partial = {}): ReviewResult { + return { + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: path.join(process.cwd(), 'prompts', 'CheckPrompt.md'), + severity: 'warning', + message: 'Vague advice found', + line: 1, + column: 1, + match: 'vague text', + suggestion: 'Be specific.', + }, + ], + scores: [ + { + ruleId: 'TestPack.CheckPrompt', + score: 8, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 1, + }, + ], + diagnostics: [], + hadOperationalErrors: false, + usage: { modelCalls: 1, inputTokens: 10, outputTokens: 5 }, + ...overrides, + }; +} + +describe('orchestrator executor dispatch', () => { + beforeEach(() => { + EXECUTOR_FOR_MOCK.mockReset(); + FAKE_EXECUTOR_RUN.mockReset(); + EXECUTOR_FOR_MOCK.mockReturnValue({ run: FAKE_EXECUTOR_RUN }); + FAKE_EXECUTOR_RUN.mockResolvedValue(makeReviewResult()); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + vi.spyOn(console, 'warn').mockImplementation(() => undefined); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('resolves auto to single for a normal-sized target and routes findings to JSON output', async () => { + const file = createTempFile('vague text here\n'); + const run = await reviewFiles([file], makeOptions([makePrompt('CheckPrompt')])); + + // auto + small target + one rule resolves to the single executor. + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + + // The verified finding reaches the JSON sink. + const parsed = JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])) as { + files: Record> }>; + }; + const issues = Object.values(parsed.files).flatMap((f) => f.issues); + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + line: 1, + severity: Severity.WARNING, + message: 'Vague advice found', + rule: 'TestPack.CheckPrompt', + match: 'vague text', + }); + + expect(run.totalWarnings).toBe(1); + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it('resolves auto to agent for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await reviewFiles([file], makeOptions([makePrompt('BigPrompt')])); + + expect(EXECUTOR_FOR_MOCK).toHaveBeenCalledTimes(1); + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('honors an explicit single modelCall even for a large target', async () => { + const file = createTempFile(`${'x'.repeat(650_000)}\n`); + await reviewFiles([file], makeOptions([makePrompt('ForcedSingle')], { modelCall: 'single' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('single'); + }); + + it('honors an explicit agent modelCall even for a small target', async () => { + const file = createTempFile('small\n'); + await reviewFiles([file], makeOptions([makePrompt('ForcedAgent')], { modelCall: 'agent' })); + + expect(EXECUTOR_FOR_MOCK.mock.calls[0]![0]).toBe('agent'); + }); + + it('forwards the built ReviewRequest (target + rules + modelCall) to the executor', async () => { + const file = createTempFile('target content line one\n'); + await reviewFiles([file], makeOptions([makePrompt('FwdPrompt')], { modelCall: 'agent' })); + + expect(FAKE_EXECUTOR_RUN).toHaveBeenCalledTimes(1); + const request = FAKE_EXECUTOR_RUN.mock.calls[0]![0] as ReviewRequest; + expect(request.target.content).toBe('target content line one\n'); + expect(request.rules).toHaveLength(1); + expect(request.rules[0]?.body).toBe('Rule body for FwdPrompt'); + expect(request.modelCall).toBe('agent'); + }); + + it('aggregates error-severity findings and flags hadSeverityErrors', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [ + { + ruleId: 'TestPack.CheckPrompt', + ruleSource: 'src', + severity: 'error', + message: 'Severe issue', + line: 2, + column: 1, + match: 'vague text', + }, + ], + scores: [ + { ruleId: 'TestPack.CheckPrompt', score: 0, scoreText: '0.0/10', severity: 'error', findingCount: 1 }, + ], + }), + ); + const file = createTempFile('vague text\n'); + + const run = await reviewFiles([file], makeOptions([makePrompt('ErrorPrompt')])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it('routes ReviewResult diagnostics to verbose console output', async () => { + FAKE_EXECUTOR_RUN.mockResolvedValue( + makeReviewResult({ + findings: [], + diagnostics: [{ level: 'warn', code: 'finding-evidence-not-locatable', message: 'could not anchor quote' }], + }), + ); + const file = createTempFile('vague text\n'); + + await reviewFiles([file], makeOptions([makePrompt('DiagPrompt')], { verbose: true })); + + expect(vi.mocked(console.warn)).toHaveBeenCalledWith( + expect.stringContaining('could not anchor quote'), + ); + }); + + it('aggregates token usage from the ReviewResult', async () => { + const file = createTempFile('vague text\n'); + const run = await reviewFiles([file], makeOptions([makePrompt('UsagePrompt')])); + + expect(run.tokenUsage?.totalInputTokens).toBe(10); + expect(run.tokenUsage?.totalOutputTokens).toBe(5); + }); + + it('skips the executor when no prompts apply', async () => { + const file = createTempFile('content\n'); + // A scan-path config that runs a pack none of the prompts belong to. + const run = await reviewFiles( + [file], + makeOptions([makePrompt('OrphanPrompt')], { + scanPaths: [{ pattern: '**/*.md', runRules: ['OtherPack'], overrides: {} }], + }), + ); + + expect(EXECUTOR_FOR_MOCK).not.toHaveBeenCalled(); + expect(run.totalFiles).toBe(1); + expect(run.totalWarnings).toBe(0); + }); +}); diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts deleted file mode 100644 index c2ce4926..00000000 --- a/tests/orchestrator-filtering.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { PromptEvaluationResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type Violation = PromptEvaluationResult["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-filtering-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -function makeViolation( - overrides: Partial = {} -): Violation { - return { - line: 1, - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeResult(params: { - violations: Violation[]; - wordCount?: number; -}): PromptEvaluationResult { - return { - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -describe("CLI violation filtering", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("filters low-confidence violations from CLI counts by default", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "RulePrompt", - name: "Rule Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation(), - makeViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation(), - makeViolation({ - line: 2, - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - quoted_text: "Beta text", - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); - }); - - it("does not mark severity error when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckErrorPrompt", - name: "Check Error Prompt", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalErrors).toBe(0); - expect(defaultRun.hadSeverityErrors).toBe(false); - - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalErrors).toBe(1); - expect(zeroThresholdRun.hadSeverityErrors).toBe(true); - }); - - it("score reflects only surfaced violations, not filtered-out ones", async () => { - const content = new Array(100).fill("word").join(" ") + "\n"; - const targetFile = createTempFile(content); - - const prompt = createPrompt({ - id: "ScorePrompt", - name: "Score Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ quoted_text: content.split(" ")[0] ?? "word" }), - makeViolation({ - quoted_text: content.split(" ")[1] ?? "word", - confidence: 0.2, - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - const scoreLine = logCalls.find(l => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - expect(scoreLine).not.toContain("8.0/10"); - }); - - it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); - - it("does not emit dummy issues in Vale JSON output when no violations are surfaced", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValeJsonPrompt", - name: "Check Vale JSON Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ - confidence: 0.2, - }), - ], - }) - ); - - const run = await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - expect(run.totalWarnings).toBe(0); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const allIssues = Object.values(parsed).flat(); - - expect(allIssues).toHaveLength(0); - expect(JSON.stringify(parsed)).not.toContain("No issues found"); - }); -}); diff --git a/tests/orchestrator-finding-processor.test.ts b/tests/orchestrator-finding-processor.test.ts deleted file mode 100644 index e15ef2ef..00000000 --- a/tests/orchestrator-finding-processor.test.ts +++ /dev/null @@ -1,334 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { Severity } from "../src/evaluators/types"; -import type { Result } from "../src/output/json-formatter"; -import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { PromptEvaluationResult } from "../src/prompts/schema"; - -const { EVALUATE_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), -})); - -type Violation = PromptEvaluationResult["violations"][number]; - -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -} as const; - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-finding-proc-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -function makeViolation( - overrides: Partial = {} -): Violation { - return { - line: 1, - analysis: "Issue 1", - message: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - quoted_text: "Alpha text", - context_before: "", - context_after: "", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - confidence: 0.9, - ...overrides, - }; -} - -function makeResult(params: { - violations: Violation[]; - wordCount?: number; -}): PromptEvaluationResult { - return { - violations: params.violations, - word_count: params.wordCount ?? 100, - }; -} - -describe("evaluation via the shared finding processor", () => { - const originalThreshold = process.env.CONFIDENCE_THRESHOLD; - - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - delete process.env.CONFIDENCE_THRESHOLD; - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - if (originalThreshold === undefined) { - delete process.env.CONFIDENCE_THRESHOLD; - } else { - process.env.CONFIDENCE_THRESHOLD = originalThreshold; - } - vi.restoreAllMocks(); - }); - - it("reports fully locatable findings, score, and counts unchanged", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "RulePrompt", - name: "Rule Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ message: "First issue", analysis: "First issue" }), - makeViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second issue", - analysis: "Second issue", - suggestion: "Suggestion 2", - fix: "Fix 2", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalWarnings).toBe(2); - expect(run.totalErrors).toBe(0); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("8.0/10"); - expect(scoreLine).toContain("RulePrompt"); - }); - - it("counts only verified findings and excludes unanchored quotes from the score", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CountingFixPrompt", - name: "Counting Fix Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ quoted_text: "Alpha text" }), - makeViolation({ - line: 9, - quoted_text: "this quote is not anywhere in the content", - message: "Ghost issue", - analysis: "Ghost issue", - }), - ], - wordCount: 100, - }) - ); - - const logCalls: string[] = []; - vi.spyOn(console, "log").mockImplementation((...args) => { - logCalls.push(args.map(String).join(" ")); - }); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalWarnings).toBe(1); - expect(run.hadOperationalErrors).toBe(false); - - const scoreLine = logCalls.find((l) => l.includes("/10")); - expect(scoreLine).toBeDefined(); - expect(scoreLine).toContain("9.0/10"); - }); - - it("does not flag severity errors when no finding can be verified", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorNoVerifiedPrompt", - name: "Error No Verified Prompt", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ - line: 9, - quoted_text: "nowhere to be found", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.hadSeverityErrors).toBe(false); - expect(run.hadOperationalErrors).toBe(false); - }); - - it("flags severity errors when verified error-severity findings exist", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "ErrorVerifiedPrompt", - name: "Error Verified Prompt", - severity: Severity.ERROR, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [makeViolation({ quoted_text: "Alpha text" })], - wordCount: 10, - }) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(1); - expect(run.hadSeverityErrors).toBe(true); - }); - - it("emits verified findings with anchored location through the JSON sink", async () => { - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "CheckJsonPrompt", - name: "Check JSON Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ message: "First", analysis: "First" }), - makeViolation({ - line: 2, - quoted_text: "Beta text", - message: "Second", - analysis: "Second", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.Json, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Result; - const issues = Object.values(parsed.files).flatMap((file) => file.issues); - - expect(issues).toHaveLength(2); - expect(issues[0]).toMatchObject({ - line: 1, - column: 1, - severity: Severity.WARNING, - message: "First", - rule: "TestPack.CheckJsonPrompt", - match: "Alpha text", - }); - expect(issues[1]).toMatchObject({ - line: 2, - message: "Second", - match: "Beta text", - }); - }); - - it("omits unanchored quotes from the Vale JSON output", async () => { - const targetFile = createTempFile("Alpha text\n"); - const prompt = createPrompt({ - id: "CheckValePrompt", - name: "Check Vale Prompt", - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeResult({ - violations: [ - makeViolation({ quoted_text: "Alpha text" }), - makeViolation({ - line: 9, - quoted_text: "missing quote", - message: "Ghost", - analysis: "Ghost", - }), - ], - wordCount: 100, - }) - ); - - await evaluateFiles([targetFile], { - ...createBaseOptions([prompt]), - outputFormat: OutputFormat.ValeJson, - }); - - const parsed = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as ValeOutput; - const issues = Object.values(parsed).flat(); - - expect(issues).toHaveLength(1); - expect(issues[0]).toMatchObject({ - Check: "TestPack.CheckValePrompt", - Line: 1, - Match: "Alpha text", - Severity: "warning", - }); - }); -}); diff --git a/tests/package-entrypoint.test.ts b/tests/package-entrypoint.test.ts new file mode 100644 index 00000000..ff99ac77 --- /dev/null +++ b/tests/package-entrypoint.test.ts @@ -0,0 +1,21 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const PACKAGE_PATH = resolve(dirname(fileURLToPath(import.meta.url)), '../package.json'); + +describe('package entrypoints', () => { + it('keeps both published bin aliases pointed at the built CLI', () => { + const packageJson = JSON.parse(readFileSync(PACKAGE_PATH, 'utf8')) as { + main?: string; + bin?: Record; + }; + + expect(packageJson.main).toBe('dist/index.js'); + expect(packageJson.bin).toEqual({ + vectorlint: './dist/index.js', + veclint: './dist/index.js', + }); + }); +}); diff --git a/tests/perplexity-provider.test.ts b/tests/perplexity-provider.test.ts deleted file mode 100644 index 0c68154e..00000000 --- a/tests/perplexity-provider.test.ts +++ /dev/null @@ -1,286 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { PerplexitySearchProvider } from '../src/providers/perplexity-provider'; -import { createMockLogger } from './utils'; - - -// Mock the Vercel AI SDK — use vi.hoisted so the mock is available in the vi.mock factory -const MOCK_GENERATE_TEXT = vi.hoisted(() => vi.fn()); - -vi.mock('ai', () => ({ - generateText: MOCK_GENERATE_TEXT, -})); - -vi.mock('@ai-sdk/perplexity', () => ({ - createPerplexity: vi.fn(() => vi.fn((model: string) => ({ _type: 'perplexity', model }))), -})); - -// Mock data matching the raw Perplexity API source shape (uses `text`, not `snippet`) -const MOCK_SOURCES = [ - { - title: 'AI Overview', - text: 'AI tools in 2025 are evolving fast.', - url: 'https://example.com/ai-overview', - }, - { - title: 'Developer Productivity', - text: 'AI improves developer efficiency by 40%.', - url: 'https://example.com/dev-productivity', - }, -]; - -// Expected mapped output (provider maps `text` → `snippet`, adds `date` default) -const EXPECTED_RESULTS = [ - { - title: 'AI Overview', - snippet: 'AI tools in 2025 are evolving fast.', - url: 'https://example.com/ai-overview', - date: '', - }, - { - title: 'Developer Productivity', - snippet: 'AI improves developer efficiency by 40%.', - url: 'https://example.com/dev-productivity', - date: '', - }, -]; - -describe('PerplexitySearchProvider', () => { - const ORIGINAL_ENV = { ...process.env }; - - beforeEach(() => { - vi.clearAllMocks(); - // Mock process.env.PERPLEXITY_API_KEY for tests - process.env.PERPLEXITY_API_KEY = 'test-api-key'; - }); - - afterEach(() => { - process.env = { ...ORIGINAL_ENV }; - }); - - describe('Constructor', () => { - it('initializes with defaults using environment variable', () => { - const provider = new PerplexitySearchProvider(); - expect(provider).toBeInstanceOf(PerplexitySearchProvider); - }); - - it('accepts override config with apiKey', () => { - const provider = new PerplexitySearchProvider({ apiKey: 'custom-key', maxResults: 10 }); - expect(provider).toBeInstanceOf(PerplexitySearchProvider); - }); - - it('throws error when no API key is provided', () => { - delete process.env.PERPLEXITY_API_KEY; - expect(() => new PerplexitySearchProvider()).toThrow('Perplexity API key is required'); - }); - - it('accepts partial config with maxResults', () => { - const provider = new PerplexitySearchProvider({ maxResults: 2 }); - expect(provider).toBeInstanceOf(PerplexitySearchProvider); - }); - }); - - describe('search', () => { - it('executes search query successfully', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ maxResults: 2 }); - const results = await provider.search('AI tools for developers'); - - expect(results).toHaveLength(2); - expect(results[0]).toEqual(EXPECTED_RESULTS[0]); - }); - - it('throws error for empty query', async () => { - const provider = new PerplexitySearchProvider(); - - await expect(provider.search('')).rejects.toThrow('Search query cannot be empty'); - await expect(provider.search(' ')).rejects.toThrow('Search query cannot be empty'); - }); - - it('handles empty sources array', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: [], - }); - - const provider = new PerplexitySearchProvider(); - const results = await provider.search('unknown topic'); - - expect(results).toHaveLength(0); - }); - - it('limits results to maxResults', async () => { - const manyResults = Array.from({ length: 10 }, (_, i) => ({ - title: `Result ${i}`, - text: `Snippet ${i}`, - url: `https://example.com/${i}`, - publishedDate: '', - })); - - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: manyResults, - }); - - const provider = new PerplexitySearchProvider({ maxResults: 5 }); - const results = await provider.search('test query'); - - expect(results).toHaveLength(5); - }); - - // This test relies on PERPLEXITY_SOURCE_SCHEMA marking all fields as optional - // with .passthrough(), so objects with missing fields still pass validation. - it('handles missing fields gracefully', async () => { - const incompleteResults = [ - { - // Missing all fields - }, - { - title: 'Has Title', - // Missing other fields - }, - { - text: 'Has snippet', - url: 'https://example.com', - publishedDate: '2025-01-01', - }, - ]; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: incompleteResults, - }); - - const provider = new PerplexitySearchProvider(); - const results = await provider.search('test'); - - expect(results).toHaveLength(3); - expect(results[0]!.title).toBe('Untitled'); - expect(results[0]!.snippet).toBe(''); - expect(results[1]!.title).toBe('Has Title'); - expect(results[1]!.snippet).toBe(''); - }); - }); - - describe('Logging', () => { - const logger = createMockLogger(); - - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('logs the search query through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ logger }); - await provider.search('test query'); - - expect(logger.debug).toHaveBeenCalledWith('Perplexity search started', { - query: 'test query', - }); - }); - - it('logs the result count through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ logger }); - await provider.search('test query'); - - expect(logger.debug).toHaveBeenCalledWith('Perplexity search completed', { - resultCount: 2, - }); - }); - - it('logs a structured result preview', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: MOCK_SOURCES, - }); - - const provider = new PerplexitySearchProvider({ logger }); - await provider.search('test query'); - - expect(logger.debug).toHaveBeenCalledWith('Perplexity result preview', { - results: EXPECTED_RESULTS, - }); - }); - - it('warns when source validation fails', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - sources: [null], - }); - - const provider = new PerplexitySearchProvider({ logger }); - const results = await provider.search('test query'); - - expect(results).toEqual([]); - expect(logger.warn).toHaveBeenCalled(); - const warnCall = logger.warn.mock.calls.at(-1); - expect(warnCall?.[0]).toBe('Perplexity source validation failed'); - const warnMeta: unknown = warnCall?.[1]; - expect(warnMeta).toBeDefined(); - expect(warnMeta).not.toBeNull(); - expect(typeof warnMeta).toBe('object'); - if (!warnMeta || typeof warnMeta !== 'object' || !('error' in warnMeta)) { - throw new Error('Expected warning metadata with an error field'); - } - expect(warnMeta.error).toContain('Expected object, received null'); - }); - }); - - describe('Error Handling', () => { - it('throws descriptive error for API failures', async () => { - MOCK_GENERATE_TEXT.mockRejectedValue(new Error('Network error')); - - const provider = new PerplexitySearchProvider(); - - await expect(provider.search('test')).rejects.toThrow('Perplexity API call failed: Network error'); - }); - - it('handles unknown error types', async () => { - MOCK_GENERATE_TEXT.mockRejectedValue('String error'); - - const provider = new PerplexitySearchProvider(); - - await expect(provider.search('test')).rejects.toThrow('Perplexity API call failed: Perplexity API call: String error'); - }); - }); - - describe('Configuration', () => { - it('returns all sources when maxResults exceeds available count', async () => { - const results = Array.from({ length: 3 }, (_, i) => ({ - title: `Result ${i}`, - text: `Snippet ${i}`, - url: `https://example.com/${i}`, - publishedDate: '', - })); - - MOCK_GENERATE_TEXT.mockResolvedValue({ sources: results }); - - const provider = new PerplexitySearchProvider({ maxResults: 10 }); - const searchResults = await provider.search('test'); - - // maxResults (10) > available sources (3), so all 3 should be returned - expect(searchResults).toHaveLength(3); - }); - - it('uses default maxResults when not specified', async () => { - const results = Array.from({ length: 10 }, (_, i) => ({ - title: `Result ${i}`, - text: `Snippet ${i}`, - url: `https://example.com/${i}`, - publishedDate: '', - })); - - MOCK_GENERATE_TEXT.mockResolvedValue({ sources: results }); - - const provider = new PerplexitySearchProvider(); - const searchResults = await provider.search('test'); - - // Default maxResults is 5 - expect(searchResults).toHaveLength(5); - }); - }); -}); diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index 80caac1a..8957db5b 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -22,12 +22,11 @@ describe('Prompt Loader Validation', () => { } }); - describe('Base Evaluator', () => { - it('should load base prompt with criteria', () => { + describe('rule frontmatter', () => { + it('should load a rule with criteria', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule criteria: - name: Quality id: QualityCheck @@ -37,15 +36,13 @@ Check content.`); const { prompts, warnings } = loadRules(tmpDir); expect(warnings).toHaveLength(0); expect(prompts).toHaveLength(1); - expect(prompts[0].meta.evaluator).toBe('base'); expect(prompts[0].meta.criteria).toHaveLength(1); }); - it('should load base prompt without criteria', () => { + it('should load a rule without criteria', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule --- Check content.`); @@ -54,10 +51,9 @@ Check content.`); expect(prompts).toHaveLength(1); }); - it('should reject base prompt missing id', () => { + it('should reject a rule missing id', () => { createPrompt('test.md', `--- -evaluator: base -name: Test Evaluator +name: Test Rule --- Check content.`); @@ -67,9 +63,8 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); - it('should reject base prompt missing name', () => { + it('should reject a rule missing name', () => { createPrompt('test.md', `--- -evaluator: base id: Test --- Check content.`); @@ -80,11 +75,10 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); - it('should reject base prompt with criterion missing id', () => { + it('should reject a rule with criterion missing id', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule criteria: - name: Quality --- @@ -96,11 +90,10 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); - it('should reject base prompt with criterion missing name', () => { + it('should reject a rule with criterion missing name', () => { createPrompt('test.md', `--- -evaluator: base id: Test -name: Test Evaluator +name: Test Rule criteria: - id: QualityCheck --- @@ -116,7 +109,6 @@ Check content.`); describe('ignored frontmatter', () => { it('ignores a supplied type field', () => { createPrompt('test.md', `--- -evaluator: base id: Test name: Test Prompt type: unused diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts index 383bad08..f033f46c 100644 --- a/tests/prompt-schema.test.ts +++ b/tests/prompt-schema.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from "vitest"; -import { buildEvaluationLLMSchema } from "../src/prompts/schema"; +import { buildReviewLLMSchema } from "../src/prompts/schema"; describe("prompt schema verbosity constraints", () => { it("includes concise analysis and suggestion descriptions", () => { - const schema = buildEvaluationLLMSchema(); + const schema = buildReviewLLMSchema(); const violationProperties = schema.schema.properties.violations.items.properties; expect(violationProperties.analysis).toEqual({ @@ -17,7 +17,7 @@ describe("prompt schema verbosity constraints", () => { }); it("includes the concise user-facing message field", () => { - const schema = buildEvaluationLLMSchema(); + const schema = buildReviewLLMSchema(); const violationProperties = schema.schema.properties.violations.items.properties; expect(violationProperties.message).toEqual({ diff --git a/tests/providers/llm-provider-contract.test.ts b/tests/providers/llm-provider-contract.test.ts deleted file mode 100644 index 249854ca..00000000 --- a/tests/providers/llm-provider-contract.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { LanguageModel } from 'ai'; -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('LLMProvider agent contract', () => { - it('supports agent-mode execution via a provider loop interface', () => { - const provider = new VercelAIProvider({ - model: {} as unknown as LanguageModel, - }); - - const runAgentToolLoop = (provider as { runAgentToolLoop?: unknown }).runAgentToolLoop; - expect(typeof runAgentToolLoop).toBe('function'); - }); -}); diff --git a/tests/providers/structured-model-client.test.ts b/tests/providers/structured-model-client.test.ts new file mode 100644 index 00000000..45db05cc --- /dev/null +++ b/tests/providers/structured-model-client.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import type { + LLMResult, + StructuredModelClient, +} from '../../src/providers/structured-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Schema', schema: { type: 'object' } }; + +describe('StructuredModelClient contract', () => { + it('exposes runPromptStructured and returns an LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: { ok: true } }), + }; + + expect(typeof client.runPromptStructured).toBe('function'); + + const result = await client.runPromptStructured('content', 'prompt', SCHEMA); + expect(result.data).toEqual({ ok: true }); + expect(result.usage).toBeUndefined(); + }); + + it('carries optional token usage on LLMResult', async () => { + const client: StructuredModelClient = { + runPromptStructured: () => + Promise.resolve({ data: 1, usage: { inputTokens: 3, outputTokens: 4 } }), + }; + + const result: LLMResult = await client.runPromptStructured( + 'content', + 'prompt', + SCHEMA, + ); + + expect(result.data).toBe(1); + expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 4 }); + }); + + it('does not declare an autonomous agent-loop method', () => { + // Compile-time guard: the capability surface is exactly runPromptStructured. + // If a second member were added, this conditional type would resolve to + // `false` and the assignment below would fail tsc. + type StructuredKeys = keyof StructuredModelClient; + const onlyRunPromptStructured: StructuredKeys extends 'runPromptStructured' ? true : false = true; + expect(onlyRunPromptStructured).toBe(true); + + // Runtime guard: a minimal implementation exposes exactly one capability. + const client: StructuredModelClient = { + runPromptStructured: () => Promise.resolve({ data: null }), + }; + expect(Object.keys(client)).toEqual(['runPromptStructured']); + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy StructuredModelClient). + const client: StructuredModelClient = provider; + expect(typeof client.runPromptStructured).toBe('function'); + }); +}); diff --git a/tests/providers/tool-calling-model-client.test.ts b/tests/providers/tool-calling-model-client.test.ts new file mode 100644 index 00000000..a2239552 --- /dev/null +++ b/tests/providers/tool-calling-model-client.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import type { LanguageModel } from 'ai'; +import { z } from 'zod'; +import type { + ToolCallDefinition, + ToolCallRunOptions, + ToolCallingModelClient, +} from '../../src/providers/tool-calling-model-client'; +import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; + +const SCHEMA = { name: 'Findings', schema: { type: 'object' } }; + +describe('ToolCallingModelClient contract', () => { + it('exposes a single bounded runWithTools method', () => { + // Compile-time guard: the capability surface is exactly runWithTools. + type ToolCallingKeys = keyof ToolCallingModelClient; + const onlyRunWithTools: ToolCallingKeys extends 'runWithTools' ? true : false = true; + expect(onlyRunWithTools).toBe(true); + + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: { findings: [] } }), + }; + expect(typeof client.runWithTools).toBe('function'); + }); + + it('returns structured output plus optional usage', async () => { + const client: ToolCallingModelClient = { + runWithTools: () => + Promise.resolve({ + data: { findings: [{ line: 1 }] }, + usage: { inputTokens: 5, outputTokens: 6 }, + }), + }; + + const result = await client.runWithTools({ + systemPrompt: 'system', + prompt: 'prompt', + tools: { read: makeTool() }, + schema: SCHEMA, + options: { maxSteps: 3, maxParallelToolCalls: 1 } satisfies ToolCallRunOptions, + }); + + expect(result.data).toEqual({ findings: [{ line: 1 }] }); + expect(result.usage).toEqual({ inputTokens: 5, outputTokens: 6 }); + }); + + it('accepts executor-owned tool definitions with typed parameters', () => { + const parameters = z.object({ startLine: z.number().int().positive() }); + const definition: ToolCallDefinition = { + description: 'paged read', + parameters, + execute: (input) => Promise.resolve(input), + }; + + expect(definition.parameters).toBe(parameters); + expect(typeof definition.execute).toBe('function'); + }); + + it('does not name workspace-agent or product-finding concepts on the capability surface', () => { + const client: ToolCallingModelClient = { + runWithTools: () => Promise.resolve({ data: {} }), + }; + const keys = Object.keys(client as Record); + + const forbidden = [ + 'read_file', + 'search_content', + 'report_finding', + ]; + for (const concept of forbidden) { + expect(keys).not.toContain(concept); + } + }); + + it('is satisfied by VercelAIProvider', () => { + const provider = new VercelAIProvider({ + model: {} as unknown as LanguageModel, + }); + + // Type-level assignability is enforced by this assignment (tsc fails if + // VercelAIProvider does not satisfy ToolCallingModelClient). + const client: ToolCallingModelClient = provider; + expect(typeof client.runWithTools).toBe('function'); + }); +}); + +function makeTool(): ToolCallDefinition { + return { + description: 'a bounded executor-owned tool', + parameters: z.object({ value: z.number().int().positive() }), + execute: () => Promise.resolve({ ok: true }), + }; +} diff --git a/tests/providers/vercel-ai-provider-agent-loop.test.ts b/tests/providers/vercel-ai-provider-agent-loop.test.ts deleted file mode 100644 index b1bca660..00000000 --- a/tests/providers/vercel-ai-provider-agent-loop.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { z } from 'zod'; -import type { LanguageModel } from 'ai'; -import { createMockLogger } from '../utils'; - -const MOCK_GENERATE_TEXT = vi.hoisted(() => vi.fn()); -const MOCK_STEP_COUNT_IS = vi.hoisted(() => vi.fn(() => ({ type: 'stepCount' }))); -const MOCK_TOOL = vi.hoisted(() => - vi.fn((definition: Record) => definition) -); - -const ERROR_CLASSES = vi.hoisted(() => { - class NoObjectGeneratedError extends Error { - text: string; - - constructor(message: string, text: string) { - super(message); - this.name = 'NoObjectGeneratedError'; - this.text = text; - } - - static isInstance(error: unknown): error is NoObjectGeneratedError { - return error instanceof NoObjectGeneratedError; - } - } - - return { NoObjectGeneratedError }; -}); - -vi.mock('ai', () => { - const { NoObjectGeneratedError } = ERROR_CLASSES; - return { - generateText: MOCK_GENERATE_TEXT, - stepCountIs: MOCK_STEP_COUNT_IS, - tool: MOCK_TOOL, - Output: { - object: vi.fn((schema: unknown) => ({ - _outputType: 'object', - schema, - })), - }, - NoObjectGeneratedError, - }; -}); - -import { VercelAIProvider } from '../../src/providers/vercel-ai-provider'; - -describe('VercelAIProvider agent loop', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('applies configured retry and tool-concurrency limits during agent execution', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - const runAgentToolLoop = ( - provider as unknown as { - runAgentToolLoop?: (params: Record) => Promise; - } - ).runAgentToolLoop; - - expect(typeof runAgentToolLoop).toBe('function'); - if (typeof runAgentToolLoop !== 'function') { - throw new Error('runAgentToolLoop is not implemented'); - } - - await runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxRetries: 4, - maxSteps: 42, - maxParallelToolCalls: 1, - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - maxRetries?: number; - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - - expect(call.maxRetries).toBe(4); - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: false }, - }); - }); - - it('adds observability options to agent-loop generateText calls', async () => { - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => ({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.agent-tool-loop' }, - })), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(observability.decorateCall).toHaveBeenCalledWith({ - operation: 'agent-tool-loop', - provider: 'openai', - model: 'gpt-4o-mini', - }); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).toHaveProperty('experimental_telemetry'); - }); - - it('continues agent-loop AI calls when observability decoration fails', async () => { - const logger = createMockLogger(); - const observability = { - init: vi.fn(), - decorateCall: vi.fn(() => { - throw new Error('telemetry failed'); - }), - shutdown: vi.fn(), - }; - - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'done', - usage: { inputTokens: 10, outputTokens: 5 }, - steps: [], - finishReason: 'stop', - }); - - const provider = new VercelAIProvider({ - model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, - providerName: 'openai', - modelName: 'gpt-4o-mini', - logger, - observability, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; - expect(call).not.toHaveProperty('experimental_telemetry'); - expect(logger.warn).toHaveBeenCalledWith( - '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', - expect.objectContaining({ error: 'telemetry failed', operation: 'agent-tool-loop' }) - ); - }); - - it('limits concurrent tool executes to maxParallelToolCalls', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - maxParallelToolCalls: 2, - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - tool_d: { description: 'd', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBeLessThanOrEqual(2); - expect(maxConcurrent).toBeGreaterThan(1); - const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as { - providerOptions?: { openai?: { parallelToolCalls?: boolean } }; - }; - expect(call.providerOptions).toEqual({ - openai: { parallelToolCalls: true }, - }); - }); - - it('defaults tool concurrency to 1 when maxParallelToolCalls is not supplied', async () => { - let maxConcurrent = 0; - let currentConcurrent = 0; - - MOCK_GENERATE_TEXT.mockImplementation(async (args: Record) => { - const tools = args.tools as Record Promise }>; - await Promise.all(Object.values(tools).map((t) => t.execute({}))); - return { text: 'done', usage: { inputTokens: 10, outputTokens: 5 } }; - }); - - const trackingExecute = async () => { - currentConcurrent++; - maxConcurrent = Math.max(maxConcurrent, currentConcurrent); - await new Promise((resolve) => setTimeout(resolve, 10)); - currentConcurrent--; - return {}; - }; - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - tool_a: { description: 'a', inputSchema: z.object({}), execute: trackingExecute }, - tool_b: { description: 'b', inputSchema: z.object({}), execute: trackingExecute }, - tool_c: { description: 'c', inputSchema: z.object({}), execute: trackingExecute }, - }, - }); - - expect(maxConcurrent).toBe(1); - }); - - it('emits agent-loop debug output through the injected logger', async () => { - MOCK_GENERATE_TEXT.mockResolvedValue({ - text: 'final summary', - finishReason: 'stop', - steps: [ - { - finishReason: 'tool-calls', - text: 'step text', - toolCalls: [{ toolName: 'lint' }], - }, - ], - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const logger = createMockLogger(); - - const provider = new VercelAIProvider({ - model: { provider: 'openai' } as unknown as LanguageModel, - debug: true, - logger, - }); - - await provider.runAgentToolLoop({ - systemPrompt: 'system', - prompt: 'prompt', - tools: { - finalize_review: { - description: 'Finalize review session', - inputSchema: z.object({ summary: z.string().optional() }), - execute: () => Promise.resolve({ ok: true }), - }, - }, - }); - - expect(logger.debug).toHaveBeenCalled(); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] step 1: finishReason=tool-calls tools=[lint]') - ) - ).toBe(true); - expect( - logger.debug.mock.calls.some(([message]) => - String(message).includes('[agent] final finishReason=stop steps=1') - ) - ).toBe(true); - }); -}); diff --git a/tests/rdjson-formatter.test.ts b/tests/rdjson-formatter.test.ts index 29a5b830..043b8669 100644 --- a/tests/rdjson-formatter.test.ts +++ b/tests/rdjson-formatter.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { RdJsonFormatter, type RdJsonResult } from '../src/output/rdjson-formatter'; -import { Severity } from '../src/evaluators/types'; +import { Severity } from '../src/review/severity'; describe('RdJsonFormatter', () => { it('should produce valid RDJSON output', () => { diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts index 11381ac1..100ec9a7 100644 --- a/tests/review/request-builder.test.ts +++ b/tests/review/request-builder.test.ts @@ -5,7 +5,7 @@ import { buildReviewRequest, } from '../../src/review'; import { ValidationError } from '../../src/errors'; -import { Severity } from '../../src/evaluators/types'; +import { Severity } from '../../src/review/severity'; import type { PromptFile } from '../../src/schemas/prompt-schemas'; import type { ReviewContext, ReviewTarget } from '../../src/review'; diff --git a/tests/evaluations/.vectorlint.ini b/tests/reviews/.vectorlint.ini similarity index 100% rename from tests/evaluations/.vectorlint.ini rename to tests/reviews/.vectorlint.ini diff --git a/tests/evaluations/TEST_FILE.md b/tests/reviews/TEST_FILE.md similarity index 100% rename from tests/evaluations/TEST_FILE.md rename to tests/reviews/TEST_FILE.md diff --git a/tests/evaluations/test-rules/Test/clarity.md b/tests/reviews/test-rules/Test/clarity.md similarity index 89% rename from tests/evaluations/test-rules/Test/clarity.md rename to tests/reviews/test-rules/Test/clarity.md index 372b0688..2c88ec19 100644 --- a/tests/evaluations/test-rules/Test/clarity.md +++ b/tests/reviews/test-rules/Test/clarity.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Clarity name: Clarity severity: warning -evaluateAs: document --- Identify unclear or ambiguous statements in the content: diff --git a/tests/evaluations/test-rules/Test/consistency.md b/tests/reviews/test-rules/Test/consistency.md similarity index 91% rename from tests/evaluations/test-rules/Test/consistency.md rename to tests/reviews/test-rules/Test/consistency.md index 248d903b..f8f238f1 100644 --- a/tests/evaluations/test-rules/Test/consistency.md +++ b/tests/reviews/test-rules/Test/consistency.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Consistency name: Consistency severity: warning -evaluateAs: document --- Check for terminology consistency across the document: diff --git a/tests/evaluations/test-rules/Test/passive-voice.md b/tests/reviews/test-rules/Test/passive-voice.md similarity index 90% rename from tests/evaluations/test-rules/Test/passive-voice.md rename to tests/reviews/test-rules/Test/passive-voice.md index 0df1eda6..0a72f774 100644 --- a/tests/evaluations/test-rules/Test/passive-voice.md +++ b/tests/reviews/test-rules/Test/passive-voice.md @@ -1,9 +1,7 @@ --- -evaluator: base id: PassiveVoice name: Passive Voice severity: warning -evaluateAs: document --- Identify passive voice constructions in the content: diff --git a/tests/evaluations/test-rules/Test/readability.md b/tests/reviews/test-rules/Test/readability.md similarity index 75% rename from tests/evaluations/test-rules/Test/readability.md rename to tests/reviews/test-rules/Test/readability.md index e943f04b..a49ad16f 100644 --- a/tests/evaluations/test-rules/Test/readability.md +++ b/tests/reviews/test-rules/Test/readability.md @@ -1,12 +1,10 @@ --- -evaluator: base id: Readability name: Readability severity: warning -evaluateAs: document --- -Evaluate the readability of the content: +Review the readability of the content: - Flag sentences over 25 words - Flag paragraphs over 150 words without breaks diff --git a/tests/evaluations/test-rules/Test/wordiness.md b/tests/reviews/test-rules/Test/wordiness.md similarity index 90% rename from tests/evaluations/test-rules/Test/wordiness.md rename to tests/reviews/test-rules/Test/wordiness.md index 77b42f41..a6509cfd 100644 --- a/tests/evaluations/test-rules/Test/wordiness.md +++ b/tests/reviews/test-rules/Test/wordiness.md @@ -1,9 +1,7 @@ --- -evaluator: base id: Wordiness name: Wordiness severity: warning -evaluateAs: document --- Flag wordy or redundant phrases in the content: diff --git a/tests/rule-pack-e2e.test.ts b/tests/rule-pack-e2e.test.ts index 41477cab..5226e722 100644 --- a/tests/rule-pack-e2e.test.ts +++ b/tests/rule-pack-e2e.test.ts @@ -7,7 +7,7 @@ import { loadConfig } from '../src/boundaries/config-loader.js'; import { ScanPathResolver } from '../src/boundaries/scan-path-resolver.js'; import { DEFAULT_CONFIG_FILENAME } from '../src/config/constants.js'; -describe('Eval Pack System End-to-End', () => { +describe('Rule Pack System End-to-End', () => { let tempDir: string; let promptsDir: string; @@ -25,7 +25,7 @@ describe('Eval Pack System End-to-End', () => { }); it('complete workflow: config → packs → files → resolution', async () => { - // 1. Setup eval pack structure + // 1. Setup rule pack structure const vectorLintDir = path.join(promptsDir, 'VectorLint'); const customPackDir = path.join(promptsDir, 'CustomPack'); @@ -33,10 +33,10 @@ describe('Eval Pack System End-to-End', () => { mkdirSync(path.join(vectorLintDir, 'Technical')); mkdirSync(customPackDir); - // Create eval files + // Create rule files writeFileSync( - path.join(vectorLintDir, 'technical-accuracy.md'), - '---\nid: technical-accuracy\n---\n# Technical Accuracy' + path.join(vectorLintDir, 'clarity.md'), + '---\nid: clarity\n---\n# Clarity' ); writeFileSync( path.join(vectorLintDir, 'readability.md'), @@ -47,8 +47,8 @@ describe('Eval Pack System End-to-End', () => { '---\nid: deep-check\n---\n# Deep Check' ); writeFileSync( - path.join(customPackDir, 'custom-eval.md'), - '---\nid: custom-eval\n---\n# Custom' + path.join(customPackDir, 'custom-rule.md'), + '---\nid: custom-rule\n---\n# Custom' ); // 2. Create config file @@ -57,7 +57,7 @@ RulesPath = ${promptsDir} [docs/**/*.md] RunRules = VectorLint -technical-accuracy.strictness = 9 +clarity.strictness = 9 [docs/blog/**/*.md] RunRules = VectorLint, CustomPack @@ -71,7 +71,7 @@ readability.severity = error expect(config.scanPaths).toHaveLength(2); expect(config.rulesPath).toBe(promptsDir); - // 4. Discover eval packs + // 4. Discover rule packs const loader = new RulePackLoader(); const packs = await loader.listAllPacks(promptsDir); const packNames = packs.map(p => p.name); @@ -80,7 +80,7 @@ readability.severity = error expect(packNames).toContain('VectorLint'); expect(packNames).toContain('CustomPack'); - // 5. Load eval files from packs + // 5. Load rule files from packs const vectorLintPack = packs.find(p => p.name === 'VectorLint')!; expect(packs.find(p => p.name === 'VectorLint')?.isPreset).toBe(false); const customPack = packs.find(p => p.name === 'CustomPack')!; @@ -103,7 +103,7 @@ readability.severity = error expect(docsFileResolution.packs).toEqual(['VectorLint']); expect(docsFileResolution.overrides).toEqual({ - 'technical-accuracy.strictness': '9' + 'clarity.strictness': '9' }); // Test file in docs/blog/ @@ -116,7 +116,7 @@ readability.severity = error expect(blogFileResolution.packs).toContain('VectorLint'); expect(blogFileResolution.packs).toContain('CustomPack'); expect(blogFileResolution.overrides).toEqual({ - 'technical-accuracy.strictness': '9', + 'clarity.strictness': '9', 'readability.severity': 'error' }); }); @@ -126,7 +126,7 @@ readability.severity = error const vectorLintDir = path.join(promptsDir, 'VectorLint'); mkdirSync(vectorLintDir); writeFileSync( - path.join(vectorLintDir, 'eval.md'), + path.join(vectorLintDir, 'rule.md'), '---\nid: test\n---\n# Test' ); diff --git a/tests/rule-pack-loader.test.ts b/tests/rule-pack-loader.test.ts index d8bcfcb4..dd3560df 100644 --- a/tests/rule-pack-loader.test.ts +++ b/tests/rule-pack-loader.test.ts @@ -110,15 +110,15 @@ describe('RulePackLoader', () => { mkdirSync(path.join(packDir, 'Advanced', 'Deep')); // Create .md files - writeFileSync(path.join(packDir, 'technical-accuracy.md'), '# Eval'); - writeFileSync(path.join(packDir, 'readability.md'), '# Eval'); - writeFileSync(path.join(packDir, 'Advanced', 'deep-check.md'), '# Eval'); - writeFileSync(path.join(packDir, 'Advanced', 'Deep', 'nested.md'), '# Eval'); + writeFileSync(path.join(packDir, 'clarity.md'), '# Rule'); + writeFileSync(path.join(packDir, 'readability.md'), '# Rule'); + writeFileSync(path.join(packDir, 'Advanced', 'deep-check.md'), '# Rule'); + writeFileSync(path.join(packDir, 'Advanced', 'Deep', 'nested.md'), '# Rule'); const files = await loader.findRuleFiles(packDir); expect(files).toHaveLength(4); - expect(files).toContain(path.join(packDir, 'technical-accuracy.md')); + expect(files).toContain(path.join(packDir, 'clarity.md')); expect(files).toContain(path.join(packDir, 'readability.md')); expect(files).toContain(path.join(packDir, 'Advanced', 'deep-check.md')); expect(files).toContain(path.join(packDir, 'Advanced', 'Deep', 'nested.md')); @@ -129,7 +129,7 @@ describe('RulePackLoader', () => { mkdirSync(packDir); // Create various file types - writeFileSync(path.join(packDir, 'eval.md'), '# Eval'); + writeFileSync(path.join(packDir, 'rule.md'), '# Rule'); writeFileSync(path.join(packDir, 'README.txt'), 'text file'); writeFileSync(path.join(packDir, 'script.js'), 'console.log()'); writeFileSync(path.join(packDir, 'config.json'), '{}'); @@ -137,7 +137,7 @@ describe('RulePackLoader', () => { const files = await loader.findRuleFiles(packDir); expect(files).toHaveLength(1); - expect(files[0]).toBe(path.join(packDir, 'eval.md')); + expect(files[0]).toBe(path.join(packDir, 'rule.md')); }); it('returns empty array when pack directory is empty', async () => { diff --git a/tests/schemas/mock-schemas.ts b/tests/schemas/mock-schemas.ts index 19a3de28..c0b2a1fc 100644 --- a/tests/schemas/mock-schemas.ts +++ b/tests/schemas/mock-schemas.ts @@ -117,38 +117,3 @@ export function createMockAnthropicClient(createFn: (params: unknown) => Promise }, }; } - - -/** - * Perplexity mock schemas + factory for tests - */ - -export const MOCK_PERPLEXITY_SEARCH_PARAMS_SCHEMA = z.object({ - query: z.string(), - max_results: z.number().optional(), - max_tokens_per_page: z.number().optional(), -}); - -export type MockPerplexitySearchParams = z.infer; - -export interface MockPerplexityClient { - search: { - create: (params: MockPerplexitySearchParams) => Promise; - }; -} - -/** - * Factory that validates params at runtime and forwards to provided createFn. - */ -export function createMockPerplexityClient( - createFn: (params: MockPerplexitySearchParams) => Promise -): MockPerplexityClient { - return { - search: { - create: async (params: MockPerplexitySearchParams) => { - MOCK_PERPLEXITY_SEARCH_PARAMS_SCHEMA.parse(params); - return createFn(params); - }, - }, - }; -} diff --git a/tests/scoring-types.test.ts b/tests/scoring-types.test.ts deleted file mode 100644 index ba799007..00000000 --- a/tests/scoring-types.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { BaseEvaluator } from "../src/evaluators/base-evaluator"; -import type { LLMProvider, LLMResult } from "../src/providers/llm-provider"; -import type { PromptFile } from "../src/schemas/prompt-schemas"; -import type { EvaluationLLMResult } from "../src/prompts/schema"; -import type { SearchProvider } from "../src/providers/search-provider"; - -const FULLY_SUPPORTED_CHECKS = { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, -}; - -const CHECK_NOTES = { - rule_supports_claim: "Supported", - evidence_exact: "Exact", - context_supports_violation: "Supported", - plausible_non_violation: "None", - fix_is_drop_in: "Drop-in", - fix_preserves_meaning: "Preserved", -}; - -describe("Scoring Types", () => { - const mockLlmProvider = { - runPromptStructured: vi.fn(), - } as unknown as LLMProvider; - - describe("Evaluation", () => { - const prompt: PromptFile = { - id: "test-rule", - filename: "test.md", - fullPath: "/test.md", - body: "Find violations.", - pack: "test", - meta: { - id: "test-rule", - name: "Test Rule", - }, - }; - - it("should calculate score correctly based on violation count", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, prompt); - - // Mock LLM returning violations only wrapped in LLMResult - const mockLlmResponse: LLMResult = { - data: { - reasoning: "Two issues found", - violations: [ - { - line: 1, - description: "Issue 1", - analysis: "First issue found", - message: "First issue", - suggestion: "", - fix: "", - quoted_text: "", - context_before: "", - context_after: "", - rule_quote: "", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: CHECK_NOTES, - confidence: 0.9, - }, - { - line: 2, - description: "Issue 2", - analysis: "Second issue found", - message: "Second issue", - suggestion: "", - fix: "", - quoted_text: "", - context_before: "", - context_after: "", - rule_quote: "", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: CHECK_NOTES, - confidence: 0.9, - }, - ], - }, - }; - - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce(mockLlmResponse); - - const content = new Array(100).fill("word").join(" "); - const result = await evaluator.evaluate("file.md", content); - - expect(result.violations).toHaveLength(2); - expect(result.word_count).toBe(100); - }); - - it("should handle empty violations list (perfect score)", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, prompt); - - const mockLlmResponse: LLMResult = { - data: { - reasoning: "No issues found", - violations: [], - }, - }; - - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce(mockLlmResponse); - - const result = await evaluator.evaluate("file.md", "content"); - - expect(result.violations).toHaveLength(0); - expect(result.word_count).toBeGreaterThan(0); - }); - }); - - describe("Technical Accuracy Evaluator", () => { - it("should return perfect score when no claims are found", async () => { - // Reset modules to ensure clean state for test-scoped mocking - vi.resetModules(); - - // Mock prompt-loader for this test only - vi.doMock("../src/evaluators/prompt-loader", () => ({ - getPrompt: vi.fn().mockReturnValue({ body: "Extract claims" }), - })); - - const { TechnicalAccuracyEvaluator } = await import( - "../src/evaluators/accuracy-evaluator" - ); - - const mockSearchProvider: SearchProvider = { - search: vi.fn().mockResolvedValue({ results: [] }), - }; - - const prompt: PromptFile = { - id: "tech-acc", - filename: "tech.md", - fullPath: "/tech.md", - body: "Check accuracy", - pack: "test", - meta: { id: "tech-acc", name: "Tech Acc" }, - }; - - const evaluator = new TechnicalAccuracyEvaluator( - mockLlmProvider, - prompt, - mockSearchProvider - ); - - // Mock claim extraction to return empty list wrapped in LLMResult - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce({ data: { claims: [] } }); - - const result = await evaluator.evaluate("file.md", "content"); - - expect(result.violations).toHaveLength(0); - expect(result.word_count).toBeGreaterThan(0); - }); - }); -}); diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index ab258dd4..77b7aced 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -132,7 +132,7 @@ describe('VercelAIProvider', () => { const provider = new VercelAIProvider(config); const schema = { - name: 'submit_evaluation', + name: 'submit_review', schema: { properties: { score: { type: 'number' }, @@ -226,7 +226,7 @@ describe('VercelAIProvider', () => { const observability = { init: vi.fn(), decorateCall: vi.fn(() => ({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-eval' }, + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-review' }, })), shutdown: vi.fn(), }; @@ -250,15 +250,15 @@ describe('VercelAIProvider', () => { ); expect(observability.decorateCall).toHaveBeenCalledWith({ - operation: 'structured-eval', + operation: 'structured-review', provider: 'openai', model: 'gpt-4o', - evaluator: undefined, + reviewer: undefined, rule: undefined, }); expect(MOCK_GENERATE_TEXT).toHaveBeenCalledWith( expect.objectContaining({ - experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-eval' }, + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-review' }, }) ); }); @@ -296,7 +296,7 @@ describe('VercelAIProvider', () => { expect(call).not.toHaveProperty('experimental_telemetry'); expect(logger.warn).toHaveBeenCalledWith( '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', - expect.objectContaining({ error: 'telemetry failed', operation: 'structured-eval' }) + expect.objectContaining({ error: 'telemetry failed', operation: 'structured-review' }) ); }); }); @@ -391,6 +391,7 @@ describe('VercelAIProvider', () => { }) as Record, }) ); + expect(logger.debug).not.toHaveBeenCalledWith('Test content'); }); it('does not log when debug is disabled', async () => { diff --git a/tests/violation-filter.test.ts b/tests/violation-filter.test.ts index 8c933d64..9d2110cd 100644 --- a/tests/violation-filter.test.ts +++ b/tests/violation-filter.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; -import { computeFilterDecision } from "../src/evaluators/violation-filter"; +import { computeFilterDecision } from "../src/findings/filter-decision"; const ORIGINAL_THRESHOLD = process.env.CONFIDENCE_THRESHOLD;