Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions apps/mcp-server/src/mcp/handlers/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,12 @@ export { ReviewPrHandler } from './review-pr.handler';
*/
export { ActivateHandler } from './activate.handler';

/**
* Handler for suggest_rules tool (self-evolving rules pipeline)
* @see {@link SuggestRulesHandler}
*/
export { SuggestRulesHandler } from './suggest-rules.handler';

/**
* Injection token for the array of all tool handlers.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ describe('RuleImpactHandler', () => {
emerging: ['security'],
},
suggestions: ['Some suggestion about high-frequency rules'],
effectivenessScores: [],
};

beforeEach(() => {
Expand Down Expand Up @@ -140,6 +141,7 @@ describe('RuleImpactHandler', () => {
suggestions: [
'No tracking data available yet — use parse_mode to start collecting rule usage data',
],
effectivenessScores: [],
};
(mockInsightsService.generateInsights as ReturnType<typeof vi.fn>).mockReturnValue(
emptyInsight,
Expand Down
25 changes: 25 additions & 0 deletions apps/mcp-server/src/mcp/handlers/rule-impact.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ export class RuleImpactHandler extends AbstractHandler {
// Domain Coverage
this.appendDomainCoverage(lines, insights);

// Effectiveness Scores
this.appendEffectivenessScores(lines, insights);

// Unused Rules
this.appendUnusedRules(lines, insights);

Expand Down Expand Up @@ -216,6 +219,28 @@ export class RuleImpactHandler extends AbstractHandler {
lines.push('');
}

private appendEffectivenessScores(lines: string[], insights: RuleInsight): void {
const scores = insights.effectivenessScores;
if (!scores || scores.length === 0) return;

lines.push('## Auto-Generated Rule Effectiveness');
lines.push('');
lines.push('| Rule | Baseline | Current | Reduction | Verdict |');
lines.push('| --- | ---: | ---: | ---: | --- |');
for (const score of scores) {
const verdictBadge =
score.verdict === 'effective'
? '**EFFECTIVE**'
: score.verdict === 'needs-review'
? 'needs-review'
: '_ineffective_';
lines.push(
`| ${score.ruleName} | ${(score.baselineFailureRate * 100).toFixed(1)}% | ${(score.currentFailureRate * 100).toFixed(1)}% | ${score.reductionPercent}% | ${verdictBadge} |`,
);
}
lines.push('');
}

private appendUnusedRules(lines: string[], insights: RuleInsight): void {
lines.push('## Unused Rules');
lines.push('');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ describe('RuleInsightsHandler', () => {
emerging: ['new-rule'],
},
suggestions: ['Some suggestion'],
effectivenessScores: [],
};

beforeEach(() => {
Expand Down
201 changes: 201 additions & 0 deletions apps/mcp-server/src/mcp/handlers/suggest-rules.handler.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
import { SuggestRulesHandler } from './suggest-rules.handler';
import * as child_process from 'child_process';

vi.mock('child_process');

describe('SuggestRulesHandler', () => {
let handler: SuggestRulesHandler;

const mockSuggestions = [
{
title: 'Repeated Bash failure: rm -rf /bad/path',
description:
'The `Bash` tool failed 5 times across 3 sessions with input `rm -rf /bad/path`.',
rule_content: '# Repeated Bash failure\n\n> Auto-detected rule\n',
pattern: {
tool_name: 'Bash',
input_summary: 'rm -rf /bad/path',
failure_count: 5,
session_count: 3,
first_seen: 1700000000,
last_seen: 1700100000,
},
},
{
title: 'Repeated Read failure: /nonexistent/file.ts',
description:
'The `Read` tool failed 3 times across 3 sessions with input `/nonexistent/file.ts`.',
rule_content: '# Repeated Read failure\n\n> Auto-detected rule\n',
pattern: {
tool_name: 'Read',
input_summary: '/nonexistent/file.ts',
failure_count: 3,
session_count: 3,
first_seen: 1700000000,
last_seen: 1700100000,
},
},
];

beforeEach(() => {
vi.mocked(child_process.execFile).mockImplementation(
(_cmd: string, _args: readonly string[] | undefined | null, _opts: unknown, cb: unknown) => {
const callback = cb as (err: Error | null, stdout: string, stderr: string) => void;
callback(null, JSON.stringify(mockSuggestions), '');
return {} as child_process.ChildProcess;
},
);

handler = new SuggestRulesHandler();
});

afterEach(() => {
vi.restoreAllMocks();
});

it('should return null for unhandled tools', async () => {
const result = await handler.handle('unknown_tool', {});
expect(result).toBeNull();
});

describe('suggest_rules', () => {
it('should return suggestions from pipeline', async () => {
const result = await handler.handle('suggest_rules', {});

expect(result).not.toBeNull();
expect(result?.isError).toBeFalsy();

const parsed = JSON.parse(result!.content[0].text);
expect(parsed.suggestions).toHaveLength(2);
expect(parsed.suggestions[0].title).toContain('Bash');
});

it('should pass minOccurrences parameter to pipeline', async () => {
await handler.handle('suggest_rules', { minOccurrences: 5 });

expect(child_process.execFile).toHaveBeenCalledWith(
expect.any(String),
expect.arrayContaining(['--min-occurrences', '5']),
expect.any(Object),
expect.any(Function),
);
});

it('should pass days parameter to pipeline', async () => {
await handler.handle('suggest_rules', { days: 14 });

expect(child_process.execFile).toHaveBeenCalledWith(
expect.any(String),
expect.arrayContaining(['--days', '14']),
expect.any(Object),
expect.any(Function),
);
});

it('should pass dbPath parameter to pipeline', async () => {
await handler.handle('suggest_rules', { dbPath: '/custom/history.db' });

expect(child_process.execFile).toHaveBeenCalledWith(
expect.any(String),
expect.arrayContaining(['--db-path', '/custom/history.db']),
expect.any(Object),
expect.any(Function),
);
});

it('should handle pipeline returning empty suggestions', async () => {
vi.mocked(child_process.execFile).mockImplementation(
(
_cmd: string,
_args: readonly string[] | undefined | null,
_opts: unknown,
cb: unknown,
) => {
const callback = cb as (err: Error | null, stdout: string, stderr: string) => void;
callback(null, '[]', '');
return {} as child_process.ChildProcess;
},
);

const result = await handler.handle('suggest_rules', {});

expect(result).not.toBeNull();
const parsed = JSON.parse(result!.content[0].text);
expect(parsed.suggestions).toHaveLength(0);
});

it('should return error when pipeline fails', async () => {
vi.mocked(child_process.execFile).mockImplementation(
(
_cmd: string,
_args: readonly string[] | undefined | null,
_opts: unknown,
cb: unknown,
) => {
const callback = cb as (err: Error | null, stdout: string, stderr: string) => void;
callback(new Error('Python not found'), '', 'error');
return {} as child_process.ChildProcess;
},
);

const result = await handler.handle('suggest_rules', {});

expect(result).not.toBeNull();
expect(result?.isError).toBe(true);
expect(result!.content[0].text).toContain('Pipeline execution failed');
});

it('should return error when pipeline outputs invalid JSON', async () => {
vi.mocked(child_process.execFile).mockImplementation(
(
_cmd: string,
_args: readonly string[] | undefined | null,
_opts: unknown,
cb: unknown,
) => {
const callback = cb as (err: Error | null, stdout: string, stderr: string) => void;
callback(null, 'not valid json', '');
return {} as child_process.ChildProcess;
},
);

const result = await handler.handle('suggest_rules', {});

expect(result).not.toBeNull();
expect(result?.isError).toBe(true);
expect(result!.content[0].text).toContain('Failed to parse');
});

it('should include metadata in response', async () => {
const result = await handler.handle('suggest_rules', {});

const parsed = JSON.parse(result!.content[0].text);
expect(parsed).toHaveProperty('generatedAt');
expect(parsed).toHaveProperty('count');
expect(parsed.count).toBe(2);
});
});

describe('getToolDefinitions', () => {
it('should return suggest_rules definition', () => {
const definitions = handler.getToolDefinitions();

expect(definitions).toHaveLength(1);
expect(definitions[0].name).toBe('suggest_rules');
});

it('should have correct input schema properties', () => {
const definitions = handler.getToolDefinitions();
const schema = definitions[0].inputSchema;

expect(schema.properties).toHaveProperty('minOccurrences');
expect(schema.properties).toHaveProperty('days');
expect(schema.properties).toHaveProperty('dbPath');
});

it('should have no required parameters', () => {
const definitions = handler.getToolDefinitions();
expect(definitions[0].inputSchema.required).toEqual([]);
});
});
});
Loading
Loading