Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions apps/mcp-server/src/rules/rule-insights.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ describe('RuleInsightsService', () => {
generatedRule: true,
baselineFailureRate: 0.25,
currentFailureRate: 0.05,
} as unknown as RuleStats,
},
};

const result = service.generateInsights(stats, [], NOW);
Expand All @@ -235,7 +235,7 @@ describe('RuleInsightsService', () => {
generatedRule: true,
baselineFailureRate: 0.2,
currentFailureRate: 0.22,
} as unknown as RuleStats,
},
};

const result = service.generateInsights(stats, [], NOW);
Expand All @@ -252,7 +252,7 @@ describe('RuleInsightsService', () => {
generatedRule: true,
baselineFailureRate: 0.3,
currentFailureRate: 0.2,
} as unknown as RuleStats,
},
};

const result = service.generateInsights(stats, [], NOW);
Expand Down
16 changes: 5 additions & 11 deletions apps/mcp-server/src/rules/rule-insights.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,21 +139,15 @@ export class RuleInsightsService {
const scores: EffectivenessScore[] = [];

for (const [name, stat] of entries) {
const extended = stat as RuleStats & {
generatedRule?: boolean;
baselineFailureRate?: number;
currentFailureRate?: number;
};

if (!extended.generatedRule) continue;
if (!stat.generatedRule) continue;
if (
typeof extended.baselineFailureRate !== 'number' ||
typeof extended.currentFailureRate !== 'number'
typeof stat.baselineFailureRate !== 'number' ||
typeof stat.currentFailureRate !== 'number'
)
continue;

const baseline = extended.baselineFailureRate;
const current = extended.currentFailureRate;
const baseline = stat.baselineFailureRate;
const current = stat.currentFailureRate;
const reductionPercent = baseline > 0 ? ((baseline - current) / baseline) * 100 : 0;

let verdict: EffectivenessScore['verdict'];
Expand Down
90 changes: 90 additions & 0 deletions apps/mcp-server/src/rules/rule-tracker.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,4 +189,94 @@ describe('RuleTracker', () => {
expect(loaded.getStats()).toEqual({});
});
});

describe('markRuleAsGenerated (#1444)', () => {
it('should tag an existing rule as auto-generated', () => {
tracker.trackRuleUsage(['auto-bash-guard']);
tracker.markRuleAsGenerated('auto-bash-guard');

const stats = tracker.getStats();
expect(stats['auto-bash-guard'].generatedRule).toBe(true);
});

it('should create a fresh stat entry when marking an unseen rule', () => {
tracker.markRuleAsGenerated('fresh-rule');

const stats = tracker.getStats();
expect(stats['fresh-rule']).toBeDefined();
expect(stats['fresh-rule'].generatedRule).toBe(true);
expect(stats['fresh-rule'].count).toBe(0);
});

it('should ignore empty rule name', () => {
tracker.markRuleAsGenerated('');

expect(tracker.getStats()).toEqual({});
});

it('should not wipe generatedRule flag on subsequent trackRuleUsage', () => {
tracker.markRuleAsGenerated('auto-rule');
tracker.trackRuleUsage(['auto-rule']);
tracker.trackRuleUsage(['auto-rule']);

const stats = tracker.getStats();
expect(stats['auto-rule'].generatedRule).toBe(true);
expect(stats['auto-rule'].count).toBe(2);
});
});

describe('recordFailureRate (#1444)', () => {
it('should record baseline failure rate for a generated rule', () => {
tracker.markRuleAsGenerated('auto-rule');
tracker.recordFailureRate('auto-rule', 0.25, 'baseline');

const stats = tracker.getStats();
expect(stats['auto-rule'].baselineFailureRate).toBe(0.25);
});

it('should record current failure rate for a generated rule', () => {
tracker.markRuleAsGenerated('auto-rule');
tracker.recordFailureRate('auto-rule', 0.05, 'current');

const stats = tracker.getStats();
expect(stats['auto-rule'].currentFailureRate).toBe(0.05);
});

it('should overwrite current failure rate on repeat calls', () => {
tracker.markRuleAsGenerated('auto-rule');
tracker.recordFailureRate('auto-rule', 0.2, 'current');
tracker.recordFailureRate('auto-rule', 0.1, 'current');

const stats = tracker.getStats();
expect(stats['auto-rule'].currentFailureRate).toBe(0.1);
});

it('should be a no-op for rules not marked as generated', () => {
tracker.trackRuleUsage(['manual-rule']);
tracker.recordFailureRate('manual-rule', 0.5, 'baseline');

const stats = tracker.getStats();
expect(stats['manual-rule'].baselineFailureRate).toBeUndefined();
});

it('should be a no-op for unknown rules', () => {
tracker.recordFailureRate('unknown-rule', 0.5, 'baseline');

expect(tracker.getStats()).toEqual({});
});

it('should persist failure rates through save/load cycle', async () => {
vi.mocked(existsSync).mockReturnValue(true);
tracker.markRuleAsGenerated('auto-rule');
tracker.recordFailureRate('auto-rule', 0.3, 'baseline');
tracker.recordFailureRate('auto-rule', 0.05, 'current');

await tracker.save();

const written = JSON.parse(vi.mocked(fs.writeFile).mock.calls[0][1] as string);
expect(written['auto-rule'].generatedRule).toBe(true);
expect(written['auto-rule'].baselineFailureRate).toBe(0.3);
expect(written['auto-rule'].currentFailureRate).toBe(0.05);
});
});
});
47 changes: 47 additions & 0 deletions apps/mcp-server/src/rules/rule-tracker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,22 @@ import { dirname } from 'path';
export interface RuleStats {
count: number;
lastUsed: number;
/**
* True when the rule was generated by the self-evolving pipeline
* (#1444). Used to separate auto-generated rules from manual rules in
* effectiveness reports.
*/
generatedRule?: boolean;
/**
* Baseline failure rate captured when a generated rule was first
* introduced. Combined with {@link currentFailureRate} to compute
* effectiveness reduction percent.
*/
baselineFailureRate?: number;
/**
* Latest observed failure rate for a generated rule.
*/
currentFailureRate?: number;
}

export interface RuleEffectivenessReport {
Expand Down Expand Up @@ -46,6 +62,37 @@ export class RuleTracker {
}
}

/**
* Mark a rule as auto-generated by the self-evolving pipeline (#1444).
* Creates the stat entry if the rule has never been tracked.
*/
markRuleAsGenerated(ruleName: string): void {
if (!ruleName) return;
if (!this.stats[ruleName]) {
this.stats[ruleName] = { count: 0, lastUsed: Date.now() };
}
this.stats[ruleName].generatedRule = true;
}

/**
* Record a baseline or current failure rate for a generated rule
* (#1444). Baseline is captured once when the rule is introduced;
* current is updated as new execution data arrives.
*
* A rule must be marked as generated (see {@link markRuleAsGenerated})
* before failure rates are recorded — otherwise the call is a no-op.
*/
recordFailureRate(ruleName: string, rate: number, kind: 'baseline' | 'current'): void {
if (!ruleName) return;
const stat = this.stats[ruleName];
if (!stat || !stat.generatedRule) return;
if (kind === 'baseline') {
stat.baselineFailureRate = rate;
} else {
stat.currentFailureRate = rate;
}
}

getStats(): Record<string, RuleStats> {
return JSON.parse(JSON.stringify(this.stats));
}
Expand Down
2 changes: 1 addition & 1 deletion packages/claude-code-plugin/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# CodingBuddy Claude Code Plugin

> Version 5.4.0
> Version 5.4.1

Multi-AI Rules for consistent coding practices - PLAN/ACT/EVAL workflow, specialist agents, and reusable skills for systematic development.

Expand Down
2 changes: 1 addition & 1 deletion packages/claude-code-plugin/namespace-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@
"namespaced": "codingbuddy:plan"
}
],
"generatedAt": "2026-04-06T09:54:05.647Z"
"generatedAt": "2026-04-10T09:03:53.981Z"
}
6 changes: 5 additions & 1 deletion packages/claude-code-plugin/scripts/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,18 @@ The plugin works standalone with core features. MCP integration unlocks the full
| Feature | Standalone | With MCP |
| ------------------------------------ | :--------: | :------: |
| PLAN/ACT/EVAL/AUTO keyword triggers | ✅ | ✅ |
| Staged planning (Discover→Design→Plan) | ✅ | ✅ |
| Clarification gate (ambiguous prompts) | ✅ | ✅ |
| Permission forecast (prompt-aware) | ✅ | ✅ |
| Council scene (agent eye glyphs) | ✅ ¹ | ✅ |
| Specialist agent prompts | ⚠️ ¹ | ✅ |
| Slash commands (codingbuddy:\\*) | ✅ | ✅ |
| Dynamic checklists | — | ✅ |
| Context persistence across modes | — | ✅ |
| Rule search & impact reports | — | ✅ |
| Session briefings & recovery | — | ✅ |

> ¹ Standalone specialist prompts require a local \`.ai-rules/\` directory. Run \`npx codingbuddy-rules init\` to scaffold it. See [#1216](https://github.com/JeremyDev87/codingbuddy/issues/1216) for the standalone enhancement roadmap.
> ¹ Standalone features that read agent definitions require a local \`.ai-rules/\` directory. Run \`npx codingbuddy-rules init\` to scaffold it. See [#1216](https://github.com/JeremyDev87/codingbuddy/issues/1216) for the standalone enhancement roadmap.

## MCP Integration (Recommended)

Expand Down
Loading