From b0cb61f7b0a4552585505d22fe5c955784a313ca Mon Sep 17 00:00:00 2001 From: JeremyDev87 Date: Fri, 10 Apr 2026 02:37:46 +0900 Subject: [PATCH] feat(mcp/plugin): add self-evolving rules pipeline and suggest_rules MCP tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire PatternDetector → RuleSuggester pipeline in Python (SuggestPipeline) - Add suggest_rules MCP handler that spawns Python pipeline via child_process - Add effectiveness scoring to RuleInsightsService for auto-generated rules - Render effectiveness scores in rule impact report - Register SuggestRulesHandler in index.ts and mcp.module.ts - Add comprehensive tests for all new code (Python + TypeScript) Closes #1437 --- apps/mcp-server/src/mcp/handlers/index.ts | 6 + .../mcp/handlers/rule-impact.handler.spec.ts | 2 + .../src/mcp/handlers/rule-impact.handler.ts | 25 +++ .../handlers/rule-insights.handler.spec.ts | 1 + .../handlers/suggest-rules.handler.spec.ts | 201 ++++++++++++++++++ .../src/mcp/handlers/suggest-rules.handler.ts | 140 ++++++++++++ apps/mcp-server/src/mcp/mcp.module.ts | 2 + .../src/rules/rule-insights.service.spec.ts | 67 ++++++ .../src/rules/rule-insights.service.ts | 57 +++++ .../hooks/lib/suggest_pipeline.py | 87 ++++++++ .../hooks/lib/test_suggest_pipeline.py | 116 ++++++++++ 11 files changed, 704 insertions(+) create mode 100644 apps/mcp-server/src/mcp/handlers/suggest-rules.handler.spec.ts create mode 100644 apps/mcp-server/src/mcp/handlers/suggest-rules.handler.ts create mode 100644 packages/claude-code-plugin/hooks/lib/suggest_pipeline.py create mode 100644 packages/claude-code-plugin/hooks/lib/test_suggest_pipeline.py diff --git a/apps/mcp-server/src/mcp/handlers/index.ts b/apps/mcp-server/src/mcp/handlers/index.ts index acef0cb0..c373f433 100644 --- a/apps/mcp-server/src/mcp/handlers/index.ts +++ b/apps/mcp-server/src/mcp/handlers/index.ts @@ -174,6 +174,12 @@ export { ReviewPrHandler } from './review-pr.handler'; */ export { ActivateHandler } from './activate.handler'; +/** + * Handler for suggest_rules tool (self-evolving rules pipeline) + * @see {@link SuggestRulesHandler} + */ +export { SuggestRulesHandler } from './suggest-rules.handler'; + /** * Injection token for the array of all tool handlers. * diff --git a/apps/mcp-server/src/mcp/handlers/rule-impact.handler.spec.ts b/apps/mcp-server/src/mcp/handlers/rule-impact.handler.spec.ts index 8f6c94f0..e287763a 100644 --- a/apps/mcp-server/src/mcp/handlers/rule-impact.handler.spec.ts +++ b/apps/mcp-server/src/mcp/handlers/rule-impact.handler.spec.ts @@ -35,6 +35,7 @@ describe('RuleImpactHandler', () => { emerging: ['security'], }, suggestions: ['Some suggestion about high-frequency rules'], + effectivenessScores: [], }; beforeEach(() => { @@ -140,6 +141,7 @@ describe('RuleImpactHandler', () => { suggestions: [ 'No tracking data available yet — use parse_mode to start collecting rule usage data', ], + effectivenessScores: [], }; (mockInsightsService.generateInsights as ReturnType).mockReturnValue( emptyInsight, diff --git a/apps/mcp-server/src/mcp/handlers/rule-impact.handler.ts b/apps/mcp-server/src/mcp/handlers/rule-impact.handler.ts index 4ec08da4..aae605bc 100644 --- a/apps/mcp-server/src/mcp/handlers/rule-impact.handler.ts +++ b/apps/mcp-server/src/mcp/handlers/rule-impact.handler.ts @@ -131,6 +131,9 @@ export class RuleImpactHandler extends AbstractHandler { // Domain Coverage this.appendDomainCoverage(lines, insights); + // Effectiveness Scores + this.appendEffectivenessScores(lines, insights); + // Unused Rules this.appendUnusedRules(lines, insights); @@ -216,6 +219,28 @@ export class RuleImpactHandler extends AbstractHandler { lines.push(''); } + private appendEffectivenessScores(lines: string[], insights: RuleInsight): void { + const scores = insights.effectivenessScores; + if (!scores || scores.length === 0) return; + + lines.push('## Auto-Generated Rule Effectiveness'); + lines.push(''); + lines.push('| Rule | Baseline | Current | Reduction | Verdict |'); + lines.push('| --- | ---: | ---: | ---: | --- |'); + for (const score of scores) { + const verdictBadge = + score.verdict === 'effective' + ? '**EFFECTIVE**' + : score.verdict === 'needs-review' + ? 'needs-review' + : '_ineffective_'; + lines.push( + `| ${score.ruleName} | ${(score.baselineFailureRate * 100).toFixed(1)}% | ${(score.currentFailureRate * 100).toFixed(1)}% | ${score.reductionPercent}% | ${verdictBadge} |`, + ); + } + lines.push(''); + } + private appendUnusedRules(lines: string[], insights: RuleInsight): void { lines.push('## Unused Rules'); lines.push(''); diff --git a/apps/mcp-server/src/mcp/handlers/rule-insights.handler.spec.ts b/apps/mcp-server/src/mcp/handlers/rule-insights.handler.spec.ts index 147829eb..0ecbfb3c 100644 --- a/apps/mcp-server/src/mcp/handlers/rule-insights.handler.spec.ts +++ b/apps/mcp-server/src/mcp/handlers/rule-insights.handler.spec.ts @@ -27,6 +27,7 @@ describe('RuleInsightsHandler', () => { emerging: ['new-rule'], }, suggestions: ['Some suggestion'], + effectivenessScores: [], }; beforeEach(() => { diff --git a/apps/mcp-server/src/mcp/handlers/suggest-rules.handler.spec.ts b/apps/mcp-server/src/mcp/handlers/suggest-rules.handler.spec.ts new file mode 100644 index 00000000..86fd9ad2 --- /dev/null +++ b/apps/mcp-server/src/mcp/handlers/suggest-rules.handler.spec.ts @@ -0,0 +1,201 @@ +import { SuggestRulesHandler } from './suggest-rules.handler'; +import * as child_process from 'child_process'; + +vi.mock('child_process'); + +describe('SuggestRulesHandler', () => { + let handler: SuggestRulesHandler; + + const mockSuggestions = [ + { + title: 'Repeated Bash failure: rm -rf /bad/path', + description: + 'The `Bash` tool failed 5 times across 3 sessions with input `rm -rf /bad/path`.', + rule_content: '# Repeated Bash failure\n\n> Auto-detected rule\n', + pattern: { + tool_name: 'Bash', + input_summary: 'rm -rf /bad/path', + failure_count: 5, + session_count: 3, + first_seen: 1700000000, + last_seen: 1700100000, + }, + }, + { + title: 'Repeated Read failure: /nonexistent/file.ts', + description: + 'The `Read` tool failed 3 times across 3 sessions with input `/nonexistent/file.ts`.', + rule_content: '# Repeated Read failure\n\n> Auto-detected rule\n', + pattern: { + tool_name: 'Read', + input_summary: '/nonexistent/file.ts', + failure_count: 3, + session_count: 3, + first_seen: 1700000000, + last_seen: 1700100000, + }, + }, + ]; + + beforeEach(() => { + vi.mocked(child_process.execFile).mockImplementation( + (_cmd: string, _args: readonly string[] | undefined | null, _opts: unknown, cb: unknown) => { + const callback = cb as (err: Error | null, stdout: string, stderr: string) => void; + callback(null, JSON.stringify(mockSuggestions), ''); + return {} as child_process.ChildProcess; + }, + ); + + handler = new SuggestRulesHandler(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should return null for unhandled tools', async () => { + const result = await handler.handle('unknown_tool', {}); + expect(result).toBeNull(); + }); + + describe('suggest_rules', () => { + it('should return suggestions from pipeline', async () => { + const result = await handler.handle('suggest_rules', {}); + + expect(result).not.toBeNull(); + expect(result?.isError).toBeFalsy(); + + const parsed = JSON.parse(result!.content[0].text); + expect(parsed.suggestions).toHaveLength(2); + expect(parsed.suggestions[0].title).toContain('Bash'); + }); + + it('should pass minOccurrences parameter to pipeline', async () => { + await handler.handle('suggest_rules', { minOccurrences: 5 }); + + expect(child_process.execFile).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['--min-occurrences', '5']), + expect.any(Object), + expect.any(Function), + ); + }); + + it('should pass days parameter to pipeline', async () => { + await handler.handle('suggest_rules', { days: 14 }); + + expect(child_process.execFile).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['--days', '14']), + expect.any(Object), + expect.any(Function), + ); + }); + + it('should pass dbPath parameter to pipeline', async () => { + await handler.handle('suggest_rules', { dbPath: '/custom/history.db' }); + + expect(child_process.execFile).toHaveBeenCalledWith( + expect.any(String), + expect.arrayContaining(['--db-path', '/custom/history.db']), + expect.any(Object), + expect.any(Function), + ); + }); + + it('should handle pipeline returning empty suggestions', async () => { + vi.mocked(child_process.execFile).mockImplementation( + ( + _cmd: string, + _args: readonly string[] | undefined | null, + _opts: unknown, + cb: unknown, + ) => { + const callback = cb as (err: Error | null, stdout: string, stderr: string) => void; + callback(null, '[]', ''); + return {} as child_process.ChildProcess; + }, + ); + + const result = await handler.handle('suggest_rules', {}); + + expect(result).not.toBeNull(); + const parsed = JSON.parse(result!.content[0].text); + expect(parsed.suggestions).toHaveLength(0); + }); + + it('should return error when pipeline fails', async () => { + vi.mocked(child_process.execFile).mockImplementation( + ( + _cmd: string, + _args: readonly string[] | undefined | null, + _opts: unknown, + cb: unknown, + ) => { + const callback = cb as (err: Error | null, stdout: string, stderr: string) => void; + callback(new Error('Python not found'), '', 'error'); + return {} as child_process.ChildProcess; + }, + ); + + const result = await handler.handle('suggest_rules', {}); + + expect(result).not.toBeNull(); + expect(result?.isError).toBe(true); + expect(result!.content[0].text).toContain('Pipeline execution failed'); + }); + + it('should return error when pipeline outputs invalid JSON', async () => { + vi.mocked(child_process.execFile).mockImplementation( + ( + _cmd: string, + _args: readonly string[] | undefined | null, + _opts: unknown, + cb: unknown, + ) => { + const callback = cb as (err: Error | null, stdout: string, stderr: string) => void; + callback(null, 'not valid json', ''); + return {} as child_process.ChildProcess; + }, + ); + + const result = await handler.handle('suggest_rules', {}); + + expect(result).not.toBeNull(); + expect(result?.isError).toBe(true); + expect(result!.content[0].text).toContain('Failed to parse'); + }); + + it('should include metadata in response', async () => { + const result = await handler.handle('suggest_rules', {}); + + const parsed = JSON.parse(result!.content[0].text); + expect(parsed).toHaveProperty('generatedAt'); + expect(parsed).toHaveProperty('count'); + expect(parsed.count).toBe(2); + }); + }); + + describe('getToolDefinitions', () => { + it('should return suggest_rules definition', () => { + const definitions = handler.getToolDefinitions(); + + expect(definitions).toHaveLength(1); + expect(definitions[0].name).toBe('suggest_rules'); + }); + + it('should have correct input schema properties', () => { + const definitions = handler.getToolDefinitions(); + const schema = definitions[0].inputSchema; + + expect(schema.properties).toHaveProperty('minOccurrences'); + expect(schema.properties).toHaveProperty('days'); + expect(schema.properties).toHaveProperty('dbPath'); + }); + + it('should have no required parameters', () => { + const definitions = handler.getToolDefinitions(); + expect(definitions[0].inputSchema.required).toEqual([]); + }); + }); +}); diff --git a/apps/mcp-server/src/mcp/handlers/suggest-rules.handler.ts b/apps/mcp-server/src/mcp/handlers/suggest-rules.handler.ts new file mode 100644 index 00000000..9e5795a5 --- /dev/null +++ b/apps/mcp-server/src/mcp/handlers/suggest-rules.handler.ts @@ -0,0 +1,140 @@ +import { Injectable } from '@nestjs/common'; +import { execFile } from 'child_process'; +import { join } from 'path'; +import { AbstractHandler } from './abstract-handler'; +import type { ToolDefinition } from './base.handler'; +import type { ToolResponse } from '../response.utils'; +import { createJsonResponse, createErrorResponse } from '../response.utils'; +import { extractOptionalString } from '../../shared/validation.constants'; + +const PIPELINE_SCRIPT = join( + __dirname, + '..', + '..', + '..', + '..', + 'packages', + 'claude-code-plugin', + 'hooks', + 'lib', + 'suggest_pipeline.py', +); + +const PIPELINE_TIMEOUT_MS = 30_000; + +interface PipelineSuggestion { + title: string; + description: string; + rule_content: string; + pattern: { + tool_name: string; + input_summary: string; + failure_count: number; + session_count: number; + first_seen: number; + last_seen: number; + }; +} + +@Injectable() +export class SuggestRulesHandler extends AbstractHandler { + protected getHandledTools(): string[] { + return ['suggest_rules']; + } + + protected async handleTool( + _toolName: string, + args: Record | undefined, + ): Promise { + const pipelineArgs = this.buildPipelineArgs(args); + + let stdout: string; + try { + stdout = await this.runPipeline(pipelineArgs); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return createErrorResponse(`Pipeline execution failed: ${message}`); + } + + let suggestions: PipelineSuggestion[]; + try { + suggestions = JSON.parse(stdout); + } catch { + return createErrorResponse(`Failed to parse pipeline output: ${stdout.slice(0, 200)}`); + } + + return createJsonResponse({ + generatedAt: Date.now(), + count: suggestions.length, + suggestions, + }); + } + + getToolDefinitions(): ToolDefinition[] { + return [ + { + name: 'suggest_rules', + description: + 'Analyze execution history for repeated failure patterns and generate draft rule suggestions. ' + + 'Rules are proposed for human review — never auto-applied. ' + + 'Powered by PatternDetector → RuleSuggester pipeline.', + inputSchema: { + type: 'object' as const, + properties: { + minOccurrences: { + type: 'number', + description: 'Minimum number of failures to count as a pattern (default: 3)', + }, + days: { + type: 'number', + description: 'How many days back to search (default: 30)', + }, + dbPath: { + type: 'string', + description: 'Path to history.db file. Defaults to ~/.codingbuddy/history.db', + }, + }, + required: [], + }, + }, + ]; + } + + private buildPipelineArgs(args: Record | undefined): string[] { + const pipelineArgs: string[] = []; + + const dbPath = extractOptionalString(args, 'dbPath'); + if (dbPath) { + pipelineArgs.push('--db-path', dbPath); + } + + const minOccurrences = args?.minOccurrences; + if (typeof minOccurrences === 'number' && Number.isInteger(minOccurrences)) { + pipelineArgs.push('--min-occurrences', String(minOccurrences)); + } + + const days = args?.days; + if (typeof days === 'number' && Number.isInteger(days)) { + pipelineArgs.push('--days', String(days)); + } + + return pipelineArgs; + } + + private runPipeline(pipelineArgs: string[]): Promise { + return new Promise((resolve, reject) => { + execFile( + 'python3', + [PIPELINE_SCRIPT, ...pipelineArgs], + { timeout: PIPELINE_TIMEOUT_MS }, + (err, stdout, stderr) => { + if (err) { + reject(new Error(`${err.message}${stderr ? `: ${stderr}` : ''}`)); + return; + } + resolve(stdout.trim()); + }, + ); + }); + } +} diff --git a/apps/mcp-server/src/mcp/mcp.module.ts b/apps/mcp-server/src/mcp/mcp.module.ts index 553e5c92..ee95f54c 100644 --- a/apps/mcp-server/src/mcp/mcp.module.ts +++ b/apps/mcp-server/src/mcp/mcp.module.ts @@ -44,6 +44,7 @@ import { ResumeHandler, RuleImpactHandler, ActivateHandler, + SuggestRulesHandler, } from './handlers'; const handlers = [ @@ -70,6 +71,7 @@ const handlers = [ ResumeHandler, RuleImpactHandler, ActivateHandler, + SuggestRulesHandler, ]; @Module({ diff --git a/apps/mcp-server/src/rules/rule-insights.service.spec.ts b/apps/mcp-server/src/rules/rule-insights.service.spec.ts index 9bd0d736..0c7be2dd 100644 --- a/apps/mcp-server/src/rules/rule-insights.service.spec.ts +++ b/apps/mcp-server/src/rules/rule-insights.service.spec.ts @@ -205,5 +205,72 @@ describe('RuleInsightsService', () => { expect(result.suggestions.some(s => s.includes('No tracking data'))).toBe(true); }); }); + + describe('effectiveness scoring', () => { + it('should compute effectiveness score for rules with declining failure rates', () => { + const stats: Record = { + 'auto-bash-guard': { + count: 8, + lastUsed: NOW - DAY_MS, + generatedRule: true, + baselineFailureRate: 0.25, + currentFailureRate: 0.05, + } as unknown as RuleStats, + }; + + const result = service.generateInsights(stats, [], NOW); + + expect(result.effectivenessScores).toBeDefined(); + expect(result.effectivenessScores).toHaveLength(1); + expect(result.effectivenessScores[0].ruleName).toBe('auto-bash-guard'); + expect(result.effectivenessScores[0].reductionPercent).toBe(80); + expect(result.effectivenessScores[0].verdict).toBe('effective'); + }); + + it('should classify rules with no improvement as ineffective', () => { + const stats: Record = { + 'auto-read-guard': { + count: 3, + lastUsed: NOW - DAY_MS, + generatedRule: true, + baselineFailureRate: 0.2, + currentFailureRate: 0.22, + } as unknown as RuleStats, + }; + + const result = service.generateInsights(stats, [], NOW); + + expect(result.effectivenessScores).toHaveLength(1); + expect(result.effectivenessScores[0].verdict).toBe('ineffective'); + }); + + it('should classify rules with moderate improvement as needs-review', () => { + const stats: Record = { + 'auto-write-guard': { + count: 5, + lastUsed: NOW - DAY_MS, + generatedRule: true, + baselineFailureRate: 0.3, + currentFailureRate: 0.2, + } as unknown as RuleStats, + }; + + const result = service.generateInsights(stats, [], NOW); + + expect(result.effectivenessScores).toHaveLength(1); + expect(result.effectivenessScores[0].reductionPercent).toBeCloseTo(33.3, 0); + expect(result.effectivenessScores[0].verdict).toBe('needs-review'); + }); + + it('should return empty effectivenessScores when no generated rules exist', () => { + const stats: Record = { + core: { count: 10, lastUsed: NOW - DAY_MS }, + }; + + const result = service.generateInsights(stats, [], NOW); + + expect(result.effectivenessScores).toEqual([]); + }); + }); }); }); diff --git a/apps/mcp-server/src/rules/rule-insights.service.ts b/apps/mcp-server/src/rules/rule-insights.service.ts index 53f64f82..aac4cf91 100644 --- a/apps/mcp-server/src/rules/rule-insights.service.ts +++ b/apps/mcp-server/src/rules/rule-insights.service.ts @@ -1,6 +1,14 @@ import { Injectable } from '@nestjs/common'; import type { RuleStats } from './rule-tracker'; +export interface EffectivenessScore { + ruleName: string; + baselineFailureRate: number; + currentFailureRate: number; + reductionPercent: number; + verdict: 'effective' | 'needs-review' | 'ineffective'; +} + export interface RuleInsight { generatedAt: number; summary: { @@ -22,6 +30,7 @@ export interface RuleInsight { emerging: string[]; }; suggestions: string[]; + effectivenessScores: EffectivenessScore[]; } const WEEK_MS = 7 * 24 * 60 * 60 * 1000; @@ -30,6 +39,10 @@ const TOP_RULES_LIMIT = 10; const EMERGING_THRESHOLD = 3; const SUGGESTION_UNUSED_PREVIEW = 5; +/** Thresholds for effectiveness verdict */ +const EFFECTIVE_THRESHOLD = 50; +const NEEDS_REVIEW_THRESHOLD = 10; + @Injectable() export class RuleInsightsService { generateInsights( @@ -52,6 +65,7 @@ export class RuleInsightsService { trends.declining, entries.length, ); + const effectivenessScores = this.computeEffectivenessScores(entries); return { generatedAt: now, @@ -65,6 +79,7 @@ export class RuleInsightsService { unusedRules, trends, suggestions, + effectivenessScores, }; } @@ -120,6 +135,48 @@ export class RuleInsightsService { return { recentlyActive, declining, emerging }; } + private computeEffectivenessScores(entries: Array<[string, RuleStats]>): EffectivenessScore[] { + const scores: EffectivenessScore[] = []; + + for (const [name, stat] of entries) { + const extended = stat as RuleStats & { + generatedRule?: boolean; + baselineFailureRate?: number; + currentFailureRate?: number; + }; + + if (!extended.generatedRule) continue; + if ( + typeof extended.baselineFailureRate !== 'number' || + typeof extended.currentFailureRate !== 'number' + ) + continue; + + const baseline = extended.baselineFailureRate; + const current = extended.currentFailureRate; + const reductionPercent = baseline > 0 ? ((baseline - current) / baseline) * 100 : 0; + + let verdict: EffectivenessScore['verdict']; + if (reductionPercent >= EFFECTIVE_THRESHOLD) { + verdict = 'effective'; + } else if (reductionPercent >= NEEDS_REVIEW_THRESHOLD) { + verdict = 'needs-review'; + } else { + verdict = 'ineffective'; + } + + scores.push({ + ruleName: name, + baselineFailureRate: baseline, + currentFailureRate: current, + reductionPercent: Math.round(reductionPercent * 10) / 10, + verdict, + }); + } + + return scores; + } + private generateSuggestions( topRules: RuleInsight['topRules'], unusedRules: string[], diff --git a/packages/claude-code-plugin/hooks/lib/suggest_pipeline.py b/packages/claude-code-plugin/hooks/lib/suggest_pipeline.py new file mode 100644 index 00000000..8d02c49a --- /dev/null +++ b/packages/claude-code-plugin/hooks/lib/suggest_pipeline.py @@ -0,0 +1,87 @@ +"""Pipeline wiring PatternDetector → RuleSuggester for self-evolving rules.""" +import json +import logging +import os +import sys + +# Ensure both hooks/lib (local) and plugin root (for hooks.lib.* imports) are on path +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_PLUGIN_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR)) +for _p in (_THIS_DIR, _PLUGIN_ROOT): + if _p not in sys.path: + sys.path.insert(0, _p) + +from history_db import HistoryDB +from pattern_detector import PatternDetector +from rule_suggester import RuleSuggester + +logger = logging.getLogger(__name__) + + +class SuggestPipeline: + """Connects pattern detection to rule suggestion generation. + + Usage: + pipeline = SuggestPipeline(db) + suggestions = pipeline.run(min_occurrences=3, days=30) + """ + + def __init__(self, db: HistoryDB): + self._detector = PatternDetector(db) + self._suggester = RuleSuggester() + + def run(self, min_occurrences: int = 3, days: int = 30) -> list: + """Run the full pipeline: detect patterns → generate rule suggestions. + + Args: + min_occurrences: Minimum failures to count as a pattern. + days: How many days back to search. + + Returns: + List of suggestion dicts with keys: title, description, + rule_content, pattern. + """ + try: + patterns = self._detector.detect_patterns( + min_occurrences=min_occurrences, days=days + ) + except Exception as e: + logger.error("Pattern detection failed: %s", e) + return [] + + return self._suggester.suggest_rules(patterns) + + @staticmethod + def to_json(suggestions: list) -> str: + """Serialize suggestions to JSON string.""" + return json.dumps(suggestions, default=str) + + +def main(): + """CLI entry point: outputs suggestions as JSON to stdout.""" + import argparse + + parser = argparse.ArgumentParser(description="Run suggest-rules pipeline") + parser.add_argument("--db-path", help="Path to history.db") + parser.add_argument( + "--min-occurrences", type=int, default=3, help="Minimum failures" + ) + parser.add_argument( + "--days", type=int, default=30, help="Days to look back" + ) + args = parser.parse_args() + + db = HistoryDB(db_path=args.db_path) if args.db_path else HistoryDB.get_instance() + try: + pipeline = SuggestPipeline(db) + suggestions = pipeline.run( + min_occurrences=args.min_occurrences, days=args.days + ) + print(pipeline.to_json(suggestions)) + finally: + if args.db_path: + db.close() + + +if __name__ == "__main__": + main() diff --git a/packages/claude-code-plugin/hooks/lib/test_suggest_pipeline.py b/packages/claude-code-plugin/hooks/lib/test_suggest_pipeline.py new file mode 100644 index 00000000..be1a07d8 --- /dev/null +++ b/packages/claude-code-plugin/hooks/lib/test_suggest_pipeline.py @@ -0,0 +1,116 @@ +"""Tests for SuggestPipeline — wires PatternDetector → RuleSuggester.""" +import json +import sqlite3 +import tempfile +import time + +import pytest + +from history_db import HistoryDB +from suggest_pipeline import SuggestPipeline + + +@pytest.fixture +def tmp_db(tmp_path): + """Create a temporary HistoryDB with test data.""" + db_path = str(tmp_path / "test_history.db") + db = HistoryDB(db_path=db_path) + return db + + +@pytest.fixture +def db_with_patterns(tmp_db): + """HistoryDB populated with repeated failure patterns.""" + session_ids = ["sess-1", "sess-2", "sess-3"] + for sid in session_ids: + tmp_db.start_session(sid, project="/test/project") + + # Create a repeated failure pattern: Bash tool fails 5 times across 3 sessions + for sid in session_ids: + tmp_db.record_tool_call(sid, "Bash", "rm -rf /bad/path", success=False) + tmp_db.record_tool_call("sess-1", "Bash", "rm -rf /bad/path", success=False) + tmp_db.record_tool_call("sess-2", "Bash", "rm -rf /bad/path", success=False) + + # Create another pattern: Read tool fails 3 times + for sid in session_ids: + tmp_db.record_tool_call(sid, "Read", "/nonexistent/file.ts", success=False) + + # Successful calls should not appear as patterns + for sid in session_ids: + tmp_db.record_tool_call(sid, "Write", "output.txt", success=True) + + return tmp_db + + +@pytest.fixture +def db_no_patterns(tmp_db): + """HistoryDB with no failure patterns (below threshold).""" + tmp_db.start_session("sess-1", project="/test/project") + tmp_db.record_tool_call("sess-1", "Bash", "echo hello", success=False) + return tmp_db + + +class TestSuggestPipeline: + def test_run_returns_suggestions_for_detected_patterns(self, db_with_patterns): + pipeline = SuggestPipeline(db_with_patterns) + result = pipeline.run() + + assert len(result) >= 2 + titles = [s["title"] for s in result] + assert any("Bash" in t for t in titles) + assert any("Read" in t for t in titles) + + def test_run_returns_empty_when_no_patterns(self, db_no_patterns): + pipeline = SuggestPipeline(db_no_patterns) + result = pipeline.run() + + assert result == [] + + def test_run_passes_parameters_to_detector(self, db_with_patterns): + pipeline = SuggestPipeline(db_with_patterns) + + # With high min_occurrences, only the Bash pattern qualifies (5 failures) + result = pipeline.run(min_occurrences=4) + assert len(result) == 1 + assert "Bash" in result[0]["title"] + + def test_suggestions_contain_required_fields(self, db_with_patterns): + pipeline = SuggestPipeline(db_with_patterns) + result = pipeline.run() + + for suggestion in result: + assert "title" in suggestion + assert "description" in suggestion + assert "rule_content" in suggestion + assert "pattern" in suggestion + + def test_suggestions_contain_pattern_metadata(self, db_with_patterns): + pipeline = SuggestPipeline(db_with_patterns) + result = pipeline.run() + + bash_suggestion = next(s for s in result if "Bash" in s["title"]) + assert bash_suggestion["pattern"]["failure_count"] >= 5 + assert bash_suggestion["pattern"]["session_count"] >= 2 + + def test_to_json_returns_valid_json(self, db_with_patterns): + pipeline = SuggestPipeline(db_with_patterns) + result = pipeline.run() + json_str = pipeline.to_json(result) + + parsed = json.loads(json_str) + assert isinstance(parsed, list) + assert len(parsed) >= 2 + + def test_run_with_empty_db(self, tmp_db): + pipeline = SuggestPipeline(tmp_db) + result = pipeline.run() + + assert result == [] + + def test_run_handles_db_error_gracefully(self, tmp_db): + # Close the DB to simulate an error + tmp_db.close() + pipeline = SuggestPipeline(tmp_db) + result = pipeline.run() + + assert result == []