diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index aaf5ac6..c026b3d 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -593,12 +593,15 @@ func (i *Indexer) run(ctx context.Context, opts Options) (store.ScanSummary, err names = append(names, name) } crossStart := time.Now() - n, err := i.store.ResolveEdgesForNames(ctx, repo.ID, names) + stats, err := i.store.ResolveEdgesForNamesWithStats(ctx, repo.ID, names) if err != nil { _ = i.store.CompleteScan(ctx, scanID, summary, started, "failed", err.Error()) return summary, err } - summary.ResolveCrossFileTargets = n + if stats != (store.ResolveEdgesForNamesStats{}) { + summary.ResolveCrossFile = &stats + } + summary.ResolveCrossFileTargets = stats.TargetsSelected summary.ResolveCrossFileMS = time.Since(crossStart).Milliseconds() summary.ResolveMode = "paths+names" } diff --git a/internal/store/cross_file_resolution_test.go b/internal/store/cross_file_resolution_test.go new file mode 100644 index 0000000..c1d2a0c --- /dev/null +++ b/internal/store/cross_file_resolution_test.go @@ -0,0 +1,60 @@ +package store_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/isink17/codegraph/internal/indexer" + "github.com/isink17/codegraph/internal/parser" + goparser "github.com/isink17/codegraph/internal/parser/golang" + "github.com/isink17/codegraph/internal/store" +) + +func TestCrossFileResolution_PartialRunIntroducedSymbolResolvesOtherFile(t *testing.T) { + ctx := context.Background() + repoRoot := t.TempDir() + + if err := os.WriteFile(filepath.Join(repoRoot, "b.go"), []byte("package main\nfunc Bar() { Foo() }\n"), 0o644); err != nil { + t.Fatalf("WriteFile(b.go) error = %v", err) + } + + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + defer s.Close() + + idx := indexer.New(s, parser.NewRegistry(goparser.New()), nil) + if _, err := idx.Index(ctx, indexer.Options{RepoRoot: repoRoot}); err != nil { + t.Fatalf("Index() error = %v", err) + } + + if err := os.WriteFile(filepath.Join(repoRoot, "a.go"), []byte("package main\nfunc Foo() {}\n"), 0o644); err != nil { + t.Fatalf("WriteFile(a.go) error = %v", err) + } + summary, err := idx.Update(ctx, indexer.Options{RepoRoot: repoRoot, Paths: []string{"a.go"}}) + if err != nil { + t.Fatalf("Update() error = %v", err) + } + if summary.ResolveMode != "paths+names" { + t.Fatalf("ResolveMode = %q, want %q", summary.ResolveMode, "paths+names") + } + if summary.ResolveCrossFile == nil || summary.ResolveCrossFile.TargetsSelected == 0 { + t.Fatalf("ResolveCrossFile stats missing or empty: %#v", summary.ResolveCrossFile) + } + + repo, err := s.UpsertRepo(ctx, repoRoot) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + unresolved, err := s.CountUnresolvedEdgesByDstName(ctx, repo.ID, "Foo") + if err != nil { + t.Fatalf("CountUnresolvedEdgesByDstName() error = %v", err) + } + if unresolved != 0 { + t.Fatalf("unresolved edges for Foo = %d, want 0", unresolved) + } +} diff --git a/internal/store/resolve_edges_for_names_test.go b/internal/store/resolve_edges_for_names_test.go new file mode 100644 index 0000000..46f356a --- /dev/null +++ b/internal/store/resolve_edges_for_names_test.go @@ -0,0 +1,152 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" +) + +func TestResolveEdgesForNames_ExactName(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + + fileA, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + t.Fatalf("insertTestFile(a.go) error = %v", err) + } + fileB, err := insertTestFile(ctx, s, repo.ID, "b.go") + if err != nil { + t.Fatalf("insertTestFile(b.go) error = %v", err) + } + + dstID, err := insertTestSymbol(ctx, s, repo.ID, fileA, "Foo", "Foo") + if err != nil { + t.Fatalf("insertTestSymbol(dst) error = %v", err) + } + srcID, err := insertTestSymbol(ctx, s, repo.ID, fileB, "Bar", "Bar") + if err != nil { + t.Fatalf("insertTestSymbol(src) error = %v", err) + } + edgeID, err := insertTestEdge(ctx, s, repo.ID, fileB, srcID, "Foo") + if err != nil { + t.Fatalf("insertTestEdge() error = %v", err) + } + + stats, err := s.ResolveEdgesForNamesWithStats(ctx, repo.ID, []string{"Foo"}) + if err != nil { + t.Fatalf("ResolveEdgesForNamesWithStats() error = %v", err) + } + if stats.TargetsSelected != 1 { + t.Fatalf("TargetsSelected = %d, want 1", stats.TargetsSelected) + } + if stats.ExactHits != 1 { + t.Fatalf("ExactHits = %d, want 1", stats.ExactHits) + } + + var gotDst sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT dst_symbol_id FROM edges WHERE id = ?`, edgeID).Scan(&gotDst); err != nil { + t.Fatalf("QueryRow(dst_symbol_id) error = %v", err) + } + if !gotDst.Valid || gotDst.Int64 != dstID { + t.Fatalf("dst_symbol_id = (%v,%d), want (%v,%d)", gotDst.Valid, gotDst.Int64, true, dstID) + } +} + +func TestResolveEdgesForNames_QualifiedSuffix(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := Open(dbPath) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + repo, err := s.UpsertRepo(ctx, t.TempDir()) + if err != nil { + t.Fatalf("UpsertRepo() error = %v", err) + } + fileA, err := insertTestFile(ctx, s, repo.ID, "a.go") + if err != nil { + t.Fatalf("insertTestFile(a.go) error = %v", err) + } + fileB, err := insertTestFile(ctx, s, repo.ID, "b.go") + if err != nil { + t.Fatalf("insertTestFile(b.go) error = %v", err) + } + + dstID, err := insertTestSymbol(ctx, s, repo.ID, fileA, "Foo", "pkg.Foo") + if err != nil { + t.Fatalf("insertTestSymbol(dst) error = %v", err) + } + srcID, err := insertTestSymbol(ctx, s, repo.ID, fileB, "Bar", "Bar") + if err != nil { + t.Fatalf("insertTestSymbol(src) error = %v", err) + } + edgeID, err := insertTestEdge(ctx, s, repo.ID, fileB, srcID, "pkg.Foo") + if err != nil { + t.Fatalf("insertTestEdge() error = %v", err) + } + + stats, err := s.ResolveEdgesForNamesWithStats(ctx, repo.ID, []string{"Foo"}) + if err != nil { + t.Fatalf("ResolveEdgesForNamesWithStats() error = %v", err) + } + if stats.TargetsSelected != 1 { + t.Fatalf("TargetsSelected = %d, want 1", stats.TargetsSelected) + } + if stats.SuffixHits != 1 { + t.Fatalf("SuffixHits = %d, want 1", stats.SuffixHits) + } + + var gotDst sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT dst_symbol_id FROM edges WHERE id = ?`, edgeID).Scan(&gotDst); err != nil { + t.Fatalf("QueryRow(dst_symbol_id) error = %v", err) + } + if !gotDst.Valid || gotDst.Int64 != dstID { + t.Fatalf("dst_symbol_id = (%v,%d), want (%v,%d)", gotDst.Valid, gotDst.Int64, true, dstID) + } +} + +func insertTestFile(ctx context.Context, s *Store, repoID int64, path string) (int64, error) { + res, err := s.db.ExecContext(ctx, `INSERT INTO files(repo_id, path, language, indexed_at) VALUES(?, ?, ?, '')`, repoID, path, "go") + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func insertTestSymbol(ctx context.Context, s *Store, repoID, fileID int64, name, qualified string) (int64, error) { + res, err := s.db.ExecContext(ctx, ` + INSERT INTO symbols( + repo_id, file_id, language, kind, name, qualified_name, + start_line, start_col, end_line, end_col, stable_key + ) + VALUES(?, ?, ?, ?, ?, ?, 1, 1, 1, 1, ?) + `, repoID, fileID, "go", "function", name, qualified, qualified) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func insertTestEdge(ctx context.Context, s *Store, repoID, fileID, srcSymbolID int64, dstName string) (int64, error) { + res, err := s.db.ExecContext(ctx, ` + INSERT INTO edges(repo_id, src_symbol_id, dst_symbol_id, dst_name, edge_kind, evidence, file_id, line) + VALUES(?, ?, NULL, ?, 'call', '', ?, 1) + `, repoID, srcSymbolID, dstName, fileID) + if err != nil { + return 0, err + } + return res.LastInsertId() +} diff --git a/internal/store/store.go b/internal/store/store.go index 7cd6458..dabdde1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -92,44 +92,58 @@ type FileMetadataUpdate struct { } type ScanSummary struct { - RepoID int64 `json:"repo_id"` - ScanID int64 `json:"scan_id"` - FilesSeen int `json:"files_seen"` - FilesIndexed int `json:"files_indexed"` - FilesSkipped int `json:"files_skipped"` - FilesChanged int `json:"files_changed"` - FilesDeleted int `json:"files_deleted"` - FilesTotal int `json:"files_total,omitempty"` - FilesDeletedPct float64 `json:"files_deleted_pct,omitempty"` - ParseErrors int `json:"parse_errors,omitempty"` - ParseSamples []string `json:"parse_samples,omitempty"` - LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` - PhaseTimings []ScanPhaseTiming `json:"phase_timings,omitempty"` - ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` - WalkMS int64 `json:"walk_ms,omitempty"` - ProcessWallMS int64 `json:"process_wall_ms,omitempty"` - TaskMS int64 `json:"task_ms,omitempty"` - TaskOtherMS int64 `json:"task_other_ms,omitempty"` - ParseMS int64 `json:"parse_ms,omitempty"` - ReadMS int64 `json:"read_ms,omitempty"` - HashMS int64 `json:"hash_ms,omitempty"` - AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` - WriteMS int64 `json:"write_ms,omitempty"` - WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` - WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` - WriteMarkSeenFlushes int `json:"write_mark_seen_flushes,omitempty"` - WriteMarkSeenSkipped int `json:"write_mark_seen_skipped,omitempty"` - WriteTouchFlushes int `json:"write_touch_flushes,omitempty"` - WriteParseFailedFlushes int `json:"write_parse_failed_flushes,omitempty"` - WriteReplaceFlushes int `json:"write_replace_flushes,omitempty"` - WriteStats *WriteStats `json:"write_stats,omitempty"` - EmbedMS int64 `json:"embed_ms,omitempty"` - MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` - ResolveMS int64 `json:"resolve_ms,omitempty"` - ResolveMode string `json:"resolve_mode,omitempty"` - ResolveCrossFileMS int64 `json:"resolve_cross_file_ms,omitempty"` - ResolveCrossFileTargets int `json:"resolve_cross_file_targets,omitempty"` - DurationMS int64 `json:"duration_ms"` + RepoID int64 `json:"repo_id"` + ScanID int64 `json:"scan_id"` + FilesSeen int `json:"files_seen"` + FilesIndexed int `json:"files_indexed"` + FilesSkipped int `json:"files_skipped"` + FilesChanged int `json:"files_changed"` + FilesDeleted int `json:"files_deleted"` + FilesTotal int `json:"files_total,omitempty"` + FilesDeletedPct float64 `json:"files_deleted_pct,omitempty"` + ParseErrors int `json:"parse_errors,omitempty"` + ParseSamples []string `json:"parse_samples,omitempty"` + LanguageCoverage map[string]LanguageCounts `json:"language_coverage,omitempty"` + PhaseTimings []ScanPhaseTiming `json:"phase_timings,omitempty"` + ExistingLoadMS int64 `json:"existing_load_ms,omitempty"` + WalkMS int64 `json:"walk_ms,omitempty"` + ProcessWallMS int64 `json:"process_wall_ms,omitempty"` + TaskMS int64 `json:"task_ms,omitempty"` + TaskOtherMS int64 `json:"task_other_ms,omitempty"` + ParseMS int64 `json:"parse_ms,omitempty"` + ReadMS int64 `json:"read_ms,omitempty"` + HashMS int64 `json:"hash_ms,omitempty"` + AdapterParseMS int64 `json:"adapter_parse_ms,omitempty"` + WriteMS int64 `json:"write_ms,omitempty"` + WriteMetadataMS int64 `json:"write_metadata_ms,omitempty"` + WriteReplaceMS int64 `json:"write_replace_ms,omitempty"` + WriteMarkSeenFlushes int `json:"write_mark_seen_flushes,omitempty"` + WriteMarkSeenSkipped int `json:"write_mark_seen_skipped,omitempty"` + WriteTouchFlushes int `json:"write_touch_flushes,omitempty"` + WriteParseFailedFlushes int `json:"write_parse_failed_flushes,omitempty"` + WriteReplaceFlushes int `json:"write_replace_flushes,omitempty"` + WriteStats *WriteStats `json:"write_stats,omitempty"` + EmbedMS int64 `json:"embed_ms,omitempty"` + MarkMissingMS int64 `json:"mark_missing_ms,omitempty"` + ResolveMS int64 `json:"resolve_ms,omitempty"` + ResolveMode string `json:"resolve_mode,omitempty"` + ResolveCrossFileMS int64 `json:"resolve_cross_file_ms,omitempty"` + ResolveCrossFileTargets int `json:"resolve_cross_file_targets,omitempty"` + ResolveCrossFile *ResolveEdgesForNamesStats `json:"resolve_cross_file,omitempty"` + DurationMS int64 `json:"duration_ms"` +} + +type ResolveEdgesForNamesStats struct { + NamesInput int `json:"names_input,omitempty"` + NamesUnique int `json:"names_unique,omitempty"` + ExactQueryBatches int `json:"exact_query_batches,omitempty"` + ExactHits int `json:"exact_hits,omitempty"` + QualifiedScanned int `json:"qualified_scanned,omitempty"` + SuffixHits int `json:"suffix_hits,omitempty"` + TargetsSelected int `json:"targets_selected,omitempty"` + ExactSelectMS int64 `json:"exact_select_ms,omitempty"` + SuffixSelectMS int64 `json:"suffix_select_ms,omitempty"` + ResolveTargetsMS int64 `json:"resolve_targets_ms,omitempty"` } type ScanPhaseTiming struct { @@ -1831,9 +1845,19 @@ func (s *Store) ResolveEdgesForPaths(ctx context.Context, repoID int64, paths [] // // It returns the number of candidate edges selected for resolution. func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names []string) (int, error) { + stats, err := s.ResolveEdgesForNamesWithStats(ctx, repoID, names) + if err != nil { + return 0, err + } + return stats.TargetsSelected, nil +} + +func (s *Store) ResolveEdgesForNamesWithStats(ctx context.Context, repoID int64, names []string) (ResolveEdgesForNamesStats, error) { + var stats ResolveEdgesForNamesStats if len(names) == 0 { - return 0, nil + return stats, nil } + stats.NamesInput = len(names) seen := make(map[string]struct{}, len(names)) unique := make([]string, 0, len(names)) for _, name := range names { @@ -1848,8 +1872,9 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] unique = append(unique, name) } if len(unique) == 0 { - return 0, nil + return stats, nil } + stats.NamesUnique = len(unique) nameSet := make(map[string]struct{}, len(unique)) for _, name := range unique { @@ -1867,6 +1892,7 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] // unresolved edges have simple (non-qualified) dst_name values. targetByID := make(map[int64]edgeTarget, 64) + exactStarted := time.Now() // Keep under sqliteDefaultMaxVariables (repoID + N names). for start := 0; start < len(unique); start += sqliteInClauseBatchSize { end := min(start+sqliteInClauseBatchSize, len(unique)) @@ -1882,69 +1908,79 @@ func (s *Store) ResolveEdgesForNames(ctx context.Context, repoID int64, names [] rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { - return 0, err + return stats, err } for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { _ = rows.Close() - return 0, err + return stats, err } targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + stats.ExactHits++ } if err := rows.Err(); err != nil { _ = rows.Close() - return 0, err + return stats, err } if err := rows.Close(); err != nil { - return 0, err + return stats, err } + stats.ExactQueryBatches++ } + stats.ExactSelectMS = time.Since(exactStarted).Milliseconds() // Suffix matching requires looking at qualified dst_name values. Keep this // as a single pass over the qualified unresolved set (no repeated LIKE // queries), but avoid scanning simple dst_name values entirely. + suffixStarted := time.Now() rows, err := s.db.QueryContext(ctx, ` SELECT id, dst_name FROM edges WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name != '' AND instr(dst_name, '.') > 0 `, repoID) if err != nil { - return 0, err + return stats, err } for rows.Next() { var id int64 var dstName string if err := rows.Scan(&id, &dstName); err != nil { _ = rows.Close() - return 0, err + return stats, err } + stats.QualifiedScanned++ if _, ok := targetByID[id]; ok { continue } if dot := strings.LastIndexByte(dstName, '.'); dot >= 0 && dot+1 < len(dstName) { if _, ok := nameSet[dstName[dot+1:]]; ok { targetByID[id] = edgeTarget{edgeID: id, dstName: dstName} + stats.SuffixHits++ } } } if err := rows.Err(); err != nil { _ = rows.Close() - return 0, err + return stats, err } if err := rows.Close(); err != nil { - return 0, err + return stats, err } + stats.SuffixSelectMS = time.Since(suffixStarted).Milliseconds() targets := make([]edgeTarget, 0, len(targetByID)) for _, target := range targetByID { targets = append(targets, target) } + stats.TargetsSelected = len(targets) + resolveStarted := time.Now() if err := s.resolveEdgeTargets(ctx, repoID, targets); err != nil { - return 0, err + return stats, err } - return len(targets), nil + stats.ResolveTargetsMS = time.Since(resolveStarted).Milliseconds() + return stats, nil } func scanEdgeTargets(rows *sql.Rows) ([]edgeTarget, error) { @@ -1963,6 +1999,16 @@ func scanEdgeTargets(rows *sql.Rows) ([]edgeTarget, error) { return targets, nil } +func (s *Store) CountUnresolvedEdgesByDstName(ctx context.Context, repoID int64, dstName string) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(*) + FROM edges + WHERE repo_id = ? AND dst_symbol_id IS NULL AND dst_name = ? + `, repoID, dstName).Scan(&n) + return n, err +} + func (s *Store) resolveEdgeTargets(ctx context.Context, repoID int64, targets []edgeTarget) error { if len(targets) == 0 { return nil