diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a1161cca37..4a1206a799 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -221,6 +221,7 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc | `procedure` | Procedure declarations in languages such as Fortran | Callable definition | | `program` | Program block declarations in languages such as Fortran | Definition target and container | | `protocol` | Protocol declarations in languages that distinguish protocols from interfaces | Definition target and container | +| `protocol_impl` | Elixir `defimpl` protocol implementation declarations | Definition target and container for implementation blocks | | `reference` | Secondary extracted symbolic references, such as HTML classes or metadata keys | Search/filter symbol | | `rule` | CSS/SCSS rule container context used by nested references | Container context | | `route` | Razor route directives | Context/search symbol | diff --git a/changelog.d/unreleased/1664.fixed.md b/changelog.d/unreleased/1664.fixed.md new file mode 100644 index 0000000000..a6dbf9e3bc --- /dev/null +++ b/changelog.d/unreleased/1664.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1664 +affected: + - src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **Elixir pipe chains now preserve call targets (#1664)** — `|>` calls to both `Module.function(...)` and local `function(...)` targets are indexed as call references. + +## 日本語 + +- **Elixir pipe chain の呼び出し先を保持するようになりました (#1664)** — `|>` による `Module.function(...)` とローカル `function(...)` の両方を call reference として index します。 diff --git a/changelog.d/unreleased/1824.fixed.md b/changelog.d/unreleased/1824.fixed.md new file mode 100644 index 0000000000..0300ba7041 --- /dev/null +++ b/changelog.d/unreleased/1824.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 1824 +affected: + - DEVELOPER_GUIDE.md + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs + - src/CodeIndex/Models/SymbolKindCatalog.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **Elixir protocol implementations are now indexed (#1824)** — `defimpl Protocol, for: Type` blocks produce `protocol_impl` symbols and type references for both the protocol and implemented type names. + +## 日本語 + +- **Elixir protocol implementation を index するようになりました (#1824)** — `defimpl Protocol, for: Type` block は `protocol_impl` symbol と、protocol/type 双方への type reference を生成します。 diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 8d9b18e2cb..3b0ce57d78 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1558,6 +1558,7 @@ value TEXT EnforceRequiredFileIdConstraints(); EnforceReferenceLineSetNullConstraint(); EnsureReferenceLinesContextKey(); + EnsureKindCheckConstraintsCurrent(); // Indexes / インデックス Execute("CREATE INDEX IF NOT EXISTS idx_files_lang ON files(lang)"); @@ -1942,6 +1943,104 @@ private bool ReferenceLinesHasContextUniqueKey() return false; } + private void EnsureKindCheckConstraintsCurrent() + { + var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); + var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolsCreateSql = + $""" + CREATE TABLE symbols ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + kind TEXT CHECK (kind IN ({symbolKindCheck})), + sub_kind TEXT, + name TEXT, + line INTEGER, + start_line INTEGER, + start_column INTEGER, + end_line INTEGER, + body_start_line INTEGER, + body_end_line INTEGER, + signature TEXT, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})), + container_name TEXT, + container_qualified_name TEXT, + family_key TEXT, + visibility TEXT, + return_type TEXT, + is_metadata_target INTEGER, + name_folded TEXT + ) + """; + const string symbolsColumns = "id, file_id, kind, sub_kind, name, line, start_line, start_column, end_line, body_start_line, body_end_line, signature, container_kind, container_name, container_qualified_name, family_key, visibility, return_type, is_metadata_target, name_folded"; + var symbolReferencesCreateSql = + $""" + CREATE TABLE symbol_references ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + symbol_name TEXT, + reference_kind TEXT CHECK (reference_kind IN ({referenceKindCheck})), + line INTEGER, + column_number INTEGER, + context TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})), + container_name TEXT, + symbol_name_folded TEXT, + container_name_folded TEXT, + is_self_reference INTEGER NOT NULL DEFAULT 0, + is_mutual_recursion INTEGER NOT NULL DEFAULT 0 + ) + """; + const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion"; + + var foreignKeys = ReadPragmaLong("foreign_keys"); + Execute("PRAGMA foreign_keys=OFF"); + try + { + if (!TableCheckContainsAll("symbols", SymbolKindCatalog.SymbolKinds)) + RebuildTableWithCurrentKindChecks("symbols", "_symbols_kind_check", symbolsCreateSql, symbolsColumns); + + if (!TableCheckContainsAll("symbol_references", SymbolKindCatalog.SymbolKinds.Concat(SymbolKindCatalog.ReferenceKinds))) + RebuildTableWithCurrentKindChecks("symbol_references", "_symbol_references_kind_check", symbolReferencesCreateSql, symbolReferencesColumns); + } + finally + { + Execute($"PRAGMA foreign_keys={foreignKeys}"); + } + } + + private bool TableCheckContainsAll(string tableName, IEnumerable allowedValues) + { + var createSql = GetTableCreateSql(tableName); + if (createSql == null) + return true; + + if (!createSql.Contains("CHECK", StringComparison.OrdinalIgnoreCase)) + return true; + + return allowedValues.All(value => createSql.Contains($"'{value.Replace("'", "''")}'", StringComparison.Ordinal)); + } + + private string? GetTableCreateSql(string tableName) + { + using var cmd = _connection.CreateCommand(); + if (_activeMigrationTransaction != null) + cmd.Transaction = _activeMigrationTransaction; + cmd.CommandText = "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = @table"; + cmd.Parameters.AddWithValue("@table", tableName); + return cmd.ExecuteScalar() as string; + } + + private void RebuildTableWithCurrentKindChecks(string tableName, string oldTableName, string createSql, string columns) + { + Execute($"DROP TABLE IF EXISTS {oldTableName}"); + Execute($"ALTER TABLE {tableName} RENAME TO {oldTableName}"); + Execute(createSql); + Execute($"INSERT INTO {tableName} ({columns}) SELECT {columns} FROM {oldTableName}"); + Execute($"DROP TABLE {oldTableName}"); + } + private void RebuildTableWithRequiredFileId(string tableName, string createSql, string columns) { if (ColumnIsNotNull(tableName, "file_id")) diff --git a/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs index d5f1bec25a..2c8f06e2de 100644 --- a/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs @@ -1,9 +1,37 @@ using CodeIndex.Models; +using System.Text.RegularExpressions; namespace CodeIndex.Indexer; internal static class ElixirReferenceExtractor { + private static readonly Regex PipeCallRegex = new( + @"\|>\s*(?:(?:[A-Z_]\w*(?:\.[A-Z_]\w*)*)\.)?(?[a-z_]\w*[!?]?)\s*(?=\(|$)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex DefimplRegex = new( + @"^\s*defimpl\s+(?[\w.]+)\s*,\s*for:\s*(?\[[^\]]+\]|[\w.{}]+)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex DefimplTypeRegex = new( + @"[\w.{}]+", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly HashSet IgnoredPipeCallNames = new(StringComparer.Ordinal) + { + "and", + "catch", + "do", + "else", + "end", + "fn", + "in", + "not", + "or", + "rescue", + "when", + }; + public static void EmitTypePositionReferences( string preparedLine, List references, @@ -13,6 +41,8 @@ public static void EmitTypePositionReferences( int lineNumber, SymbolRecord? container) { + EmitDefimplReferences(preparedLine, references, seen, fileId, context, lineNumber, container); + LanguageReferenceExtractionSupport.EmitTypePositionReferences( "elixir", preparedLine, @@ -26,11 +56,64 @@ public static void EmitTypePositionReferences( container); } + private static void EmitDefimplReferences( + string preparedLine, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + SymbolRecord? container) + { + var match = DefimplRegex.Match(preparedLine); + if (!match.Success) + return; + + AddDefimplGroupReference(match.Groups["protocol"]); + + var typesGroup = match.Groups["types"]; + foreach (Match typeMatch in DefimplTypeRegex.Matches(typesGroup.Value)) + AddDefimplTypeReference(typeMatch.Groups[0], typesGroup.Index + typeMatch.Index); + + void AddDefimplGroupReference(Group group) + => AddDefimplTypeReference(group, group.Index); + + void AddDefimplTypeReference(Group group, int column) + { + var name = group.Value.Trim(); + if (name.Length == 0) + return; + + var key = $"type_reference:{name}:{lineNumber}:{column}"; + if (!seen.Add(key)) + return; + + references.Add(new ReferenceRecord + { + FileId = fileId, + SymbolName = name, + ReferenceKind = "type_reference", + Line = lineNumber, + Column = column, + Context = context.Trim(), + ContainerKind = container?.Kind, + ContainerName = container?.Name, + }); + } + } + public static void EmitAdditionalCallReferences( string preparedLine, Action addCallLikeReference, IReadOnlySet? definitionNames) { + foreach (Match match in PipeCallRegex.Matches(preparedLine)) + { + var name = match.Groups["name"].Value; + if (!IgnoredPipeCallNames.Contains(name)) + addCallLikeReference(name, match.Groups["name"].Index); + } + LanguageReferenceExtractionSupport.EmitAdditionalCallReferences( "elixir", preparedLine, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index f7947ecaa1..8e9ecfb3d0 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1743,6 +1743,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("function", new Regex(@"^\s*(?:def|defp|defmacro|defguardp?)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.ElixirEnd), new("class", new Regex(@"^\s*defmodule\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd), new("interface", new Regex(@"^\s*defprotocol\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd), + new("protocol_impl", new Regex(@"^\s*defimpl\s+(?[\w.]+(?:\s*,\s*for:\s*(?:\[[^\]]+\]|[\w.{}]+))?)", RegexOptions.Compiled), BodyStyle.ElixirEnd), new("import", new Regex(@"^\s*(?:import|alias|use|require)\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.None), ], ["dart"] = @@ -2115,7 +2116,7 @@ public static IReadOnlyCollection GetSupportedLanguages() private static readonly HashSet ContainerKinds = [ - "class", "struct", "interface", "protocol", "namespace", "enum", "object", "heading", "specialization", "class_hook" + "class", "struct", "interface", "protocol", "protocol_impl", "namespace", "enum", "object", "heading", "specialization", "class_hook" ]; private static bool IsRustDirectTraitBodyMember(List symbols, int candidateLine) diff --git a/src/CodeIndex/Models/SymbolKindCatalog.cs b/src/CodeIndex/Models/SymbolKindCatalog.cs index c2afb6f78e..a1bde5ea47 100644 --- a/src/CodeIndex/Models/SymbolKindCatalog.cs +++ b/src/CodeIndex/Models/SymbolKindCatalog.cs @@ -42,6 +42,7 @@ public static class SymbolKindCatalog "procedure", "program", "protocol", + "protocol_impl", "reference", "rule", "route", diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 2a0df4d06f..db42de811d 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -419,6 +419,140 @@ INSERT INTO symbols (file_id, kind, name, line) Assert.Equal(19, ex.SqliteErrorCode); } + [Fact] + public void InitializeSchema_RefreshesLegacyKindCheckConstraints() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_kind_check_{Guid.NewGuid():N}.db"); + try + { + var builder = new SqliteConnectionStringBuilder { DataSource = dbPath }; + using (var conn = new SqliteConnection(builder.ConnectionString)) + { + conn.Open(); + ExecuteNonQuery(conn, """ + CREATE TABLE files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL UNIQUE, + lang TEXT, + size INTEGER, + lines INTEGER, + checksum TEXT, + modified DATETIME, + generated INTEGER NOT NULL DEFAULT 0, + indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """); + ExecuteNonQuery(conn, """ + CREATE TABLE chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + start_line INTEGER, + end_line INTEGER, + content TEXT, + UNIQUE(file_id, chunk_index) + ) + """); + ExecuteNonQuery(conn, """ + CREATE TABLE reference_lines ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + line INTEGER NOT NULL, + context TEXT NOT NULL, + UNIQUE(file_id, line, context) + ) + """); + ExecuteNonQuery(conn, """ + CREATE TABLE symbols ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + kind TEXT CHECK (kind IN ('class','function','module')), + sub_kind TEXT, + name TEXT, + line INTEGER, + start_line INTEGER, + start_column INTEGER, + end_line INTEGER, + body_start_line INTEGER, + body_end_line INTEGER, + signature TEXT, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ('class','function','module')), + container_name TEXT, + container_qualified_name TEXT, + family_key TEXT, + visibility TEXT, + return_type TEXT, + is_metadata_target INTEGER, + name_folded TEXT + ) + """); + ExecuteNonQuery(conn, """ + CREATE TABLE symbol_references ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + symbol_name TEXT, + reference_kind TEXT CHECK (reference_kind IN ('call','type_reference')), + line INTEGER, + column_number INTEGER, + context TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ('class','function','module')), + container_name TEXT, + symbol_name_folded TEXT, + container_name_folded TEXT, + is_self_reference INTEGER NOT NULL DEFAULT 0, + is_mutual_recursion INTEGER NOT NULL DEFAULT 0 + ) + """); + } + + using var db = new DbContext(dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + var fileId = writer.UpsertFile(new FileRecord + { + Path = "lib/inspect_impl.ex", + Lang = "elixir", + Size = 64, + Lines = 4, + Modified = new DateTime(2026, 5, 31, 0, 0, 0, DateTimeKind.Utc), + Checksum = Guid.NewGuid().ToString("N"), + }); + writer.InsertSymbols( + [ + new SymbolRecord + { + FileId = fileId, + Kind = "protocol_impl", + Name = "String.Chars, for: User", + Line = 1, + ContainerKind = "protocol_impl", + ContainerName = "String.Chars, for: User", + }, + ]); + writer.InsertReferences( + [ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "User", + ReferenceKind = "type_reference", + Line = 1, + Column = 23, + ContainerKind = "protocol_impl", + ContainerName = "String.Chars, for: User", + }, + ]); + + Assert.Equal(1, ExecuteScalarLong(db.Connection, "SELECT COUNT(*) FROM symbols WHERE kind = 'protocol_impl'")); + Assert.Equal(1, ExecuteScalarLong(db.Connection, "SELECT COUNT(*) FROM symbol_references WHERE container_kind = 'protocol_impl'")); + } + finally + { + DeleteDbFiles(dbPath); + } + } + [Fact] public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime() { @@ -2289,4 +2423,11 @@ private static long ExecuteScalarLong(SqliteConnection connection, string sql) cmd.CommandText = sql; return Convert.ToInt64(cmd.ExecuteScalar()); } + + private static void ExecuteNonQuery(SqliteConnection connection, string sql) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } } diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 73ba753b46..a34bdd6a0c 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -14790,6 +14790,52 @@ public void Extract_ElixirCall_DetectsReferences() Assert.Contains(references, r => r.SymbolName == "start_link"); } + [Fact] + public void Extract_ElixirPipeCalls_DetectsQualifiedAndUnqualifiedTargets() + { + const string content = """ + defmodule MyApp do + def run(items) do + items + |> Enum.map(&process/1) + |> Enum.filter(&valid?/1) + |> normalize(:strict) + |> persist! + end + end + """; + + var symbols = SymbolExtractor.Extract(1, "elixir", content); + var references = ReferenceExtractor.Extract(1, "elixir", content, symbols); + + Assert.Contains(references, r => r.SymbolName == "map" && r.ReferenceKind == "call"); + Assert.Contains(references, r => r.SymbolName == "filter" && r.ReferenceKind == "call"); + Assert.Contains(references, r => r.SymbolName == "normalize" && r.ReferenceKind == "call"); + Assert.Contains(references, r => r.SymbolName == "persist!" && r.ReferenceKind == "call"); + } + + [Fact] + public void Extract_ElixirDefimpl_CapturesProtocolAndImplementedTypes() + { + const string content = """ + defimpl Enumerable, for: MyApp.Stream do + def count(stream), do: {:ok, length(stream.items)} + end + + defimpl Inspect, for: [MyApp.Stream, Other.Stream] do + def inspect(stream, _opts), do: "#Stream<#{length(stream.items)}>" + end + """; + + var symbols = SymbolExtractor.Extract(1, "elixir", content); + var references = ReferenceExtractor.Extract(1, "elixir", content, symbols); + + Assert.Contains(references, r => r.SymbolName == "Enumerable" && r.ReferenceKind == "type_reference"); + Assert.Contains(references, r => r.SymbolName == "Inspect" && r.ReferenceKind == "type_reference"); + Assert.Contains(references, r => r.SymbolName == "MyApp.Stream" && r.ReferenceKind == "type_reference"); + Assert.Contains(references, r => r.SymbolName == "Other.Stream" && r.ReferenceKind == "type_reference"); + } + [Fact] public void Extract_ElixirNestedBlocks_AssignsCorrectCallerContainers() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index bfe3c009bf..17ce14fcf8 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -20789,6 +20789,31 @@ public void Extract_Elixir_DetectsModuleAndFunctions() Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "parse"); } + [Fact] + public void Extract_Elixir_DetectsProtocolImplementations() + { + const string content = """ + defmodule MyApp.Stream do + defstruct [:items] + end + + defimpl Enumerable, for: MyApp.Stream do + def count(stream), do: {:ok, length(stream.items)} + end + + defimpl Inspect, for: [MyApp.Stream, Other.Stream] do + def inspect(stream, _opts), do: "#Stream<#{length(stream.items)}>" + end + """; + + var symbols = SymbolExtractor.Extract(1, "elixir", content); + + Assert.Contains(symbols, s => s.Kind == "protocol_impl" && s.Name.StartsWith("Enumerable", StringComparison.Ordinal)); + Assert.Contains(symbols, s => s.Kind == "protocol_impl" && s.Name.StartsWith("Inspect", StringComparison.Ordinal)); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "count" && s.ContainerKind == "protocol_impl"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "inspect" && s.ContainerKind == "protocol_impl"); + } + [Fact] public void Extract_Elixir_NestedBlocks_AndDoShorthand_HaveMatchingBodyRanges() {