Skip to content

perf(campaign): eliminate O(N*M) bottleneck in edge case detection analysis #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
theRebelliousNerd wants to merge 1 commit into
base: main
Choose a base branch
from
Open

perf(campaign): eliminate O(N*M) bottleneck in edge case detection analysis #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 0 additions & 32 deletions fix_test.py
View file
Open in desktop

This file was deleted.

96 changes: 61 additions & 35 deletions internal/campaign/edge_case_detector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"time"

"codenerd/internal/core"
Expand Down Expand Up @@ -92,6 +93,12 @@ type SplitSuggestion struct {
Reason string `json:"reason"`
}

// kernelFacts caches kernel facts to avoid O(N*M) query costs
type kernelFacts struct {
dependencies []core.Fact
complexities []core.Fact
}

// EdgeCaseDetector analyzes files to determine appropriate actions.
type EdgeCaseDetector struct {
kernel *core.RealKernel
Expand Down Expand Up @@ -160,21 +167,57 @@ func (d *EdgeCaseDetector) AnalyzeFiles(ctx context.Context, paths []string, int
timer := logging.StartTimer(logging.CategoryCampaign, "AnalyzeFiles")
defer timer.Stop()

ctx, cancel := context.WithTimeout(ctx, d.config.AnalysisTimeout*time.Duration(len(paths)))
// Prevent immediate timeout if paths is empty
timeoutDuration := d.config.AnalysisTimeout * time.Duration(len(paths))
if len(paths) == 0 {
timeoutDuration = d.config.AnalysisTimeout // at least one unit of time
}
ctx, cancel := context.WithTimeout(ctx, timeoutDuration)
defer cancel()

// Pre-fetch facts to avoid O(N*M) performance penalty
var kFacts kernelFacts
if d.kernel != nil {
kFacts.dependencies, _ = d.kernel.Query("dependency_link")
kFacts.complexities, _ = d.kernel.Query("cyclomatic_complexity")
}

decisions := make([]FileDecision, 0, len(paths))
decisionCh := make(chan FileDecision, len(paths))

// Run in parallel with a bounded number of workers
var wg sync.WaitGroup
sem := make(chan struct{}, 8)

for _, path := range paths {
select {
case <-ctx.Done():
logging.Campaign("Edge case analysis interrupted: %d/%d files analyzed before timeout", len(decisions), len(paths))
return decisions, ctx.Err()
default:
}
wg.Add(1)
go func(p string) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()

select {
case <-ctx.Done():
return
default:
decisionCh <- d.analyzeFile(ctx, p, intelligence, &kFacts)
}
}(path)
}

// Wait for all workers in a separate goroutine
go func() {
wg.Wait()
close(decisionCh)
}()

decision := d.analyzeFile(ctx, path, intelligence)
decisions = append(decisions, decision)
for dec := range decisionCh {
decisions = append(decisions, dec)
}

if ctx.Err() != nil {
logging.Campaign("Edge case analysis interrupted: %d/%d files analyzed before timeout", len(decisions), len(paths))
return decisions, ctx.Err()
}

// Sort by priority: refactor_first > modularize > create > extend > skip
Expand Down Expand Up @@ -252,7 +295,7 @@ func (d *EdgeCaseDetector) logDecisionSummary(decisions []FileDecision) {
}

// analyzeFile performs analysis on a single file.
func (d *EdgeCaseDetector) analyzeFile(ctx context.Context, path string, intel *IntelligenceReport) FileDecision {
func (d *EdgeCaseDetector) analyzeFile(ctx context.Context, path string, intel *IntelligenceReport, kFacts *kernelFacts) FileDecision {
// Check for context cancellation
if err := ctx.Err(); err != nil {
return FileDecision{Path: path, RecommendedAction: ActionSkip, Reasoning: "Analysis cancelled"}
Expand Down Expand Up @@ -282,7 +325,7 @@ func (d *EdgeCaseDetector) analyzeFile(ctx context.Context, path string, intel *
}

// Gather metrics from intelligence report
d.gatherMetrics(&decision, path, intel)
d.gatherMetrics(&decision, path, intel, kFacts)

// Apply decision logic
decision.RecommendedAction, decision.Reasoning = d.determineAction(decision)
Expand All @@ -294,7 +337,7 @@ func (d *EdgeCaseDetector) analyzeFile(ctx context.Context, path string, intel *
}

// gatherMetrics populates decision metrics from intelligence data.
func (d *EdgeCaseDetector) gatherMetrics(decision *FileDecision, path string, intel *IntelligenceReport) {
func (d *EdgeCaseDetector) gatherMetrics(decision *FileDecision, path string, intel *IntelligenceReport, kFacts *kernelFacts) {
if intel == nil {
return
}
Expand Down Expand Up @@ -327,20 +370,14 @@ func (d *EdgeCaseDetector) gatherMetrics(decision *FileDecision, path string, in

// Query kernel for dependencies
if d.kernel != nil {
d.queryDependencies(decision, path)
d.queryComplexity(decision, path)
d.queryDependencies(decision, path, kFacts)
d.queryComplexity(decision, path, kFacts)
}
}

// queryDependencies gets file dependencies from the kernel.
func (d *EdgeCaseDetector) queryDependencies(decision *FileDecision, path string) {
// Query dependency_link for dependencies
facts, err := d.kernel.Query("dependency_link")
if err != nil {
return
}

for _, fact := range facts {
func (d *EdgeCaseDetector) queryDependencies(decision *FileDecision, path string, kFacts *kernelFacts) {
for _, fact := range kFacts.dependencies {
if len(fact.Args) >= 3 {
file := d.parseArg(fact.Args[0])
imported := d.parseArg(fact.Args[2])
Expand All @@ -359,18 +396,12 @@ func (d *EdgeCaseDetector) queryDependencies(decision *FileDecision, path string
}

// queryComplexity estimates complexity from kernel facts.
func (d *EdgeCaseDetector) queryComplexity(decision *FileDecision, path string) {
// Query for complexity-related facts
facts, err := d.kernel.Query("cyclomatic_complexity")
if err != nil {
return
}

func (d *EdgeCaseDetector) queryComplexity(decision *FileDecision, path string, kFacts *kernelFacts) {
var (
maxComplexity float64
hasComplexity bool
)
for _, fact := range facts {
for _, fact := range kFacts.complexities {
if len(fact.Args) >= 3 {
file := d.parseArg(fact.Args[0])
if d.matchesPath(file, path) {
Expand Down Expand Up @@ -820,10 +851,5 @@ func (a *EdgeCaseAnalysis) GetPreworkTasks() []string {
// Files marked `Exists: true` in intelligence might have been deleted. Should verify
// against `os.Stat(path)` to prevent invalid `ActionExtend` or `ActionModularize` commands.

// TODO: Missing Edge Case - Performance Vector: Massive volume of facts in kernel.
// `queryDependencies` and `queryComplexity` executes an O(N) fetch of all facts for *each file*.
// For campaigns on large repos, this becomes O(N * M) and hangs the orchestrator.
// Need to parallelize AnalyzeFiles or parameterize kernel queries.

// TODO: Missing Edge Case - Extreme Values: Max file size boundaries.
// `LineCount` bounds checking should prevent `float64` precision or overflow issues
12 changes: 0 additions & 12 deletions plan.txt
View file
Open in desktop

This file was deleted.