diff --git a/changelog.d/unreleased/2102.fixed.md b/changelog.d/unreleased/2102.fixed.md new file mode 100644 index 0000000000..45d868bc6e --- /dev/null +++ b/changelog.d/unreleased/2102.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 2102 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **SQL generated/computed columns are now indexed (#2102)** — `CREATE TABLE` and `ALTER TABLE ... ADD` generated/computed columns emit column symbols and dependency references for generation and `DEFAULT NEXT VALUE FOR` expressions. + +## 日本語 + +- **SQL の generated/computed column を index するようにしました (#2102)** — `CREATE TABLE` と `ALTER TABLE ... ADD` の生成/計算列で列シンボルを出し、生成式と `DEFAULT NEXT VALUE FOR` 式の依存参照も抽出します。 diff --git a/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs index 3a895868f3..36f09072ff 100644 --- a/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs @@ -285,6 +285,18 @@ internal sealed class State private static readonly Regex CreateTempRoutineRegex = new( $@"(?{TempIdentifierPattern})", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex GeneratedColumnMarkerRegex = new( + @"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|NEXT\s+VALUE\s+FOR)\b|(?{QualifiedIdentifierNoCapturePattern})", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SqlExpressionIdentifierRegex = new( + $@"(?{QualifiedIdentifierNoCapturePattern})", + RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex TrailingTempIdentifierRegex = new( $@"^(?:(?:ONLY)\b\s+)?(?(?:{TempIdentifierPattern}|{QualifiedIdentifierNoCapturePattern}))(?:\s+(?:AS\s+)?(?:{QuotedIdentifierPattern}|{BareIdentifierPattern}))?\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -509,6 +521,19 @@ private static void EmitStatementReferences( fileId, resolveContainerForCall); + EmitGeneratedColumnDependencyReferences( + statement, + statementStart, + statementLineOffset, + lineOffset, + context, + lineNumber, + references, + seen, + fileId, + resolveContainerForCall, + shouldIgnoreName); + EmitSourceCaptureReferences( FromSourceListRegex.Matches(statement), statement, @@ -1547,6 +1572,133 @@ private static void EmitMergeUsingReferences( } } + private static void EmitGeneratedColumnDependencyReferences( + string statement, + int statementStart, + int statementLineOffset, + int lineOffset, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + Func resolveContainerForCall, + Func shouldIgnoreName) + { + if (!GeneratedColumnMarkerRegex.IsMatch(statement)) + return; + + foreach (Match match in GeneratedColumnExpressionStartRegex.Matches(statement)) + { + if (match.Index < statementLineOffset || IsInsideDoubleQuotedRegion(statement, match.Index)) + continue; + if (match.Value.TrimStart().StartsWith("AS", StringComparison.OrdinalIgnoreCase) + && !IsLikelyComputedColumnAsExpression(statement, match.Index)) + { + continue; + } + + var openParenIndex = statement.IndexOf('(', match.Index + match.Length - 1); + if (openParenIndex < 0) + continue; + + var closeParenIndex = FindMatchingParen(statement, openParenIndex); + if (closeParenIndex <= openParenIndex) + continue; + + EmitSqlExpressionIdentifierDependencies( + statement, + openParenIndex + 1, + closeParenIndex, + statementStart, + statementLineOffset, + lineOffset, + context, + lineNumber, + references, + seen, + fileId, + resolveContainerForCall, + shouldIgnoreName); + } + + foreach (Match match in DefaultNextValueForExpressionRegex.Matches(statement)) + { + if (match.Index < statementLineOffset || IsInsideDoubleQuotedRegion(statement, match.Index)) + continue; + + var sequence = match.Groups["name"]; + EmitSqlExpressionIdentifierDependencies( + statement, + sequence.Index, + sequence.Index + sequence.Length, + statementStart, + statementLineOffset, + lineOffset, + context, + lineNumber, + references, + seen, + fileId, + resolveContainerForCall, + shouldIgnoreName); + } + } + + private static void EmitSqlExpressionIdentifierDependencies( + string statement, + int startIndex, + int endIndexExclusive, + int statementStart, + int statementLineOffset, + int lineOffset, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + Func resolveContainerForCall, + Func shouldIgnoreName) + { + var expression = statement[startIndex..endIndexExclusive]; + foreach (Match match in SqlExpressionIdentifierRegex.Matches(expression)) + { + var rawIndex = startIndex + match.Index; + if (rawIndex < statementLineOffset || IsInsideDoubleQuotedRegion(statement, rawIndex)) + continue; + + var rawName = match.Value; + NormalizeIdentifier(rawName, rawIndex, out var resolvedName, out var nameIndex, out var wasQuoted); + if (!wasQuoted && (shouldIgnoreName(resolvedName) || IsGeneratedColumnDependencyKeyword(resolvedName))) + continue; + + var nameColumn = nameIndex + statementStart - lineOffset; + var container = resolveContainerForCall(rawIndex); + ReferenceExtractor.AddReference(references, seen, fileId, resolvedName, nameColumn, "generated_column_dependency", context, lineNumber, container); + } + } + + private static bool IsGeneratedColumnDependencyKeyword(string name) + => name.Equals("GENERATED", StringComparison.OrdinalIgnoreCase) + || name.Equals("ALWAYS", StringComparison.OrdinalIgnoreCase) + || name.Equals("AS", StringComparison.OrdinalIgnoreCase) + || name.Equals("DEFAULT", StringComparison.OrdinalIgnoreCase) + || name.Equals("NEXT", StringComparison.OrdinalIgnoreCase) + || name.Equals("VALUE", StringComparison.OrdinalIgnoreCase) + || name.Equals("FOR", StringComparison.OrdinalIgnoreCase) + || name.Equals("STORED", StringComparison.OrdinalIgnoreCase) + || name.Equals("VIRTUAL", StringComparison.OrdinalIgnoreCase) + || name.Equals("PERSISTED", StringComparison.OrdinalIgnoreCase) + || name.Equals("NULL", StringComparison.OrdinalIgnoreCase) + || name.Equals("NOT", StringComparison.OrdinalIgnoreCase); + + private static bool IsLikelyComputedColumnAsExpression(string statement, int asIndex) + { + var prefix = statement[..asIndex]; + return Regex.IsMatch(prefix, @"(?{SqlQualifiedIdentifierSegmentPattern})(?:\s*\([^)]*\))?\s+AS\s*\(", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex SqlAlterTableAddGeneratedColumnRegex = new( + $@"(?{SqlQualifiedIdentifierPattern})\s+ADD(?:\s+COLUMN)?\s+(?!CONSTRAINT\b)(?{SqlQualifiedIdentifierSegmentPattern})\b(?=[^;]*?\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS\s*\(|DEFAULT\s+NEXT\s+VALUE\s+FOR)\b)", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex SqlCreateTableBodyRegex = new( + $@"(?{SqlQualifiedIdentifierPattern})\s*\((?[\s\S]*?)\)\s*;", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex SqlGeneratedColumnDefinitionMarkerRegex = new( + @"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS\s*\(|DEFAULT\s+NEXT\s+VALUE\s+FOR)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex SqlColumnDefinitionNameRegex = new( + $@"^\s*(?{SqlQualifiedIdentifierSegmentPattern})\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex SqlReturnsTableRegex = new( @"\bRETURNS\s+TABLE\s*\((?(?:[^()]|\([^()]*\))*)\)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Singleline); @@ -1391,6 +1403,8 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // file module declarations and inline modules / ファイルモジュール宣言とインラインモジュール new("file_module", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?mod\s+(?(?:r#)?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility"), new("namespace", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?mod\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + // Trait associated type defaults / trait 関連型のデフォルト + new("property", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?type\s+(?(?:r#)?\w+)(?:\s*<[^=>]+>)?(?:\s*:\s*[^=;]+)?\s*=\s*(?[^;]+)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), // type alias / 型エイリアス new("import", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?type\s+(?(?:r#)?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility"), new("import", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?use\s+(?.+);", RegexOptions.Compiled), BodyStyle.None, "visibility"), @@ -2099,6 +2113,22 @@ public static IReadOnlyCollection GetSupportedLanguages() "class", "struct", "interface", "protocol", "namespace", "enum", "object", "heading", "specialization", "class_hook" ]; + private static bool IsRustDirectTraitBodyMember(List symbols, int candidateLine) + { + SymbolRecord? innermostContainer = null; + foreach (var symbol in symbols) + { + if (!symbol.BodyStartLine.HasValue || !symbol.BodyEndLine.HasValue) + continue; + if (candidateLine < symbol.BodyStartLine.Value || candidateLine > symbol.BodyEndLine.Value) + continue; + if (innermostContainer == null || symbol.StartLine >= innermostContainer.StartLine) + innermostContainer = symbol; + } + + return innermostContainer?.Kind == "protocol"; + } + /// /// Extract symbols from the given source content. /// 指定されたソース内容からシンボルを抽出する。 @@ -2687,6 +2717,14 @@ public static List Extract(long fileId, string? lang, string conte lineOffset = FindNextSameLineBraceStatementStart(matchLine, absoluteStartColumn + Math.Max(1, match.Length), lang); continue; } + if (lang == "rust" + && pattern.Kind == "property" + && pattern.BodyStyle == BodyStyle.None + && pattern.ReturnTypeGroup != null + && !IsRustDirectTraitBodyMember(symbols, i + 1)) + { + break; + } var rawReturnType = NormalizeCSharpImplicitPartialMethodReturnType( lang, pattern, @@ -3939,6 +3977,7 @@ public static List Extract(long fileId, string? lang, string conte ExtractSqlCteSymbols(fileId, lines, symbols); ExtractSqlDefinerSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols); ExtractSqlRoutineResultColumnSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols); + ExtractSqlGeneratedColumnSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols); } if (IsRazorLanguage(originalLang) || IsRazorFilePath(filePath)) ExtractRazorDirectiveSymbols(fileId, lines, symbols); @@ -4046,6 +4085,119 @@ private static int GetLineNumberFromOffset(List lineStarts, int offset) return ~index; } + private static void ExtractSqlGeneratedColumnSymbols(long fileId, string[] lines, string[] structuralLines, List symbols) + { + var structuralContent = string.Join('\n', structuralLines); + if (structuralContent.IndexOf("GENERATED", StringComparison.OrdinalIgnoreCase) < 0 + && structuralContent.IndexOf("NEXT VALUE FOR", StringComparison.OrdinalIgnoreCase) < 0 + && structuralContent.IndexOf(" AS ", StringComparison.OrdinalIgnoreCase) < 0) + { + return; + } + + var lineStarts = BuildLineStarts(structuralContent); + foreach (Match match in SqlAlterTableAddGeneratedColumnRegex.Matches(structuralContent)) + { + var nameGroup = match.Groups["name"]; + AddSqlGeneratedColumnSymbol( + fileId, + lines, + lineStarts, + new GroupProxy(nameGroup.Value, nameGroup.Index), + match.Groups["table"].Value, + symbols); + } + + foreach (Match tableMatch in SqlCreateTableBodyRegex.Matches(structuralContent)) + { + var tableName = tableMatch.Groups["table"].Value; + var bodyGroup = tableMatch.Groups["body"]; + foreach (var column in EnumerateSqlColumnDefinitions(bodyGroup.Value, bodyGroup.Index)) + { + if (!SqlGeneratedColumnDefinitionMarkerRegex.IsMatch(column.Text)) + continue; + + var nameMatch = SqlColumnDefinitionNameRegex.Match(column.Text); + if (!nameMatch.Success) + continue; + + AddSqlGeneratedColumnSymbol( + fileId, + lines, + lineStarts, + new GroupProxy(nameMatch.Groups["name"].Value, column.StartIndex + nameMatch.Groups["name"].Index), + tableName, + symbols); + } + } + } + + private static void AddSqlGeneratedColumnSymbol( + long fileId, + string[] lines, + List lineStarts, + IGroupLike nameGroup, + string rawTableName, + List symbols) + { + var name = NormalizeSqlIdentifierSegment(nameGroup.Value); + if (string.IsNullOrWhiteSpace(name)) + return; + + var lineNumber = GetLineNumberFromOffset(lineStarts, nameGroup.Index); + AddSymbolRecord( + symbols, + null, + lineNumber, + new SymbolRecord + { + FileId = fileId, + Kind = "property", + SubKind = "generated_column", + Name = name, + Line = lineNumber, + StartLine = lineNumber, + StartColumn = nameGroup.Index - lineStarts[lineNumber - 1], + EndLine = lineNumber, + Signature = lines[lineNumber - 1].Trim(), + ContainerKind = "class", + ContainerName = NormalizeSqlIdentifierSegment(SqlNameResolver.GetLeafName(rawTableName)), + }, + lines[lineNumber - 1]); + } + + private interface IGroupLike + { + string Value { get; } + int Index { get; } + } + + private readonly record struct GroupProxy(string Value, int Index) : IGroupLike; + + private readonly record struct SqlColumnDefinitionSlice(string Text, int StartIndex); + + private static IEnumerable EnumerateSqlColumnDefinitions(string body, int bodyStartIndex) + { + var start = 0; + var depth = 0; + for (var i = 0; i <= body.Length; i++) + { + if (i == body.Length || (body[i] == ',' && depth == 0)) + { + var text = body[start..i].Trim(); + if (text.Length > 0) + yield return new SqlColumnDefinitionSlice(text, bodyStartIndex + start + body[start..i].Length - body[start..i].TrimStart().Length); + start = i + 1; + continue; + } + + if (body[i] == '(') + depth++; + else if (body[i] == ')' && depth > 0) + depth--; + } + } + private static string NormalizeSqlIdentifierSegment(string value) { if (value.Length >= 2 && value[0] == '[' && value[^1] == ']') diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index bd96c02ca1..c9c28eed05 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -14718,6 +14718,32 @@ public void Extract_SQL_AlterTableCapturesTargetReference() Assert.DoesNotContain(references, r => r.SymbolName == "sales" && r.ReferenceKind == "reference"); } + [Fact] + public void Extract_SQL_GeneratedColumnsCaptureExpressionDependencies() + { + const string content = """ + CREATE TABLE dbo.Orders ( + subtotal int, + tax int, + total int GENERATED ALWAYS AS (round(subtotal + tax, 2)) STORED, + invoice_no int DEFAULT NEXT VALUE FOR billing.invoice_seq, + created_at timestamp DEFAULT CURRENT_TIMESTAMP + ); + ALTER TABLE dbo.Orders ADD computed_total AS (subtotal + tax) PERSISTED; + """; + + var symbols = SymbolExtractor.Extract(1, "sql", content); + var references = ReferenceExtractor.Extract(1, "sql", content, symbols); + + Assert.Contains(references, r => r.SymbolName == "round" && r.ReferenceKind == "generated_column_dependency"); + Assert.Contains(references, r => r.SymbolName == "subtotal" && r.ReferenceKind == "generated_column_dependency"); + Assert.Contains(references, r => r.SymbolName == "tax" && r.ReferenceKind == "generated_column_dependency"); + Assert.Contains(references, r => r.SymbolName.EndsWith("invoice_seq", StringComparison.Ordinal) && r.ReferenceKind == "generated_column_dependency"); + Assert.DoesNotContain(references, r => r.SymbolName == "GENERATED" && r.ReferenceKind == "generated_column_dependency"); + Assert.DoesNotContain(references, r => r.SymbolName == "DEFAULT" && r.ReferenceKind == "generated_column_dependency"); + Assert.DoesNotContain(references, r => r.SymbolName == "CURRENT_TIMESTAMP" && r.ReferenceKind == "generated_column_dependency"); + } + [Fact] public void Extract_SQL_DropTableCapturesAllTargetReferences() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index b410cbca67..8ad5d20da5 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -1156,6 +1156,54 @@ CREATE PROCEDURE [dbo].[proc]]name] && s.Name.Contains("name", StringComparison.OrdinalIgnoreCase)); } + [Fact] + public void Extract_SqlGeneratedColumns_DetectsColumnSymbols() + { + var content = """ + CREATE TABLE dbo.Orders ( + subtotal int, + tax int, + total int GENERATED ALWAYS AS (subtotal + tax) STORED, + invoice_no int DEFAULT NEXT VALUE FOR billing.invoice_seq, + created_at timestamp DEFAULT CURRENT_TIMESTAMP + ); + ALTER TABLE dbo.Orders ADD COLUMN net_total int GENERATED ALWAYS AS (total - tax) STORED; + ALTER TABLE dbo.Orders ADD computed_total AS (subtotal + tax) PERSISTED; + ALTER TABLE dbo.Orders ADD CONSTRAINT df_orders_created DEFAULT 0 FOR created_at; + """; + + var symbols = SymbolExtractor.Extract(1, "sql", content); + + Assert.Contains(symbols, s => + s.Kind == "property" + && s.SubKind == "generated_column" + && s.Name == "total" + && s.ContainerName == "Orders"); + Assert.Contains(symbols, s => + s.Kind == "property" + && s.SubKind == "generated_column" + && s.Name == "invoice_no" + && s.ContainerName == "Orders"); + Assert.Contains(symbols, s => + s.Kind == "property" + && s.SubKind == "generated_column" + && s.Name == "net_total" + && s.ContainerName == "Orders"); + Assert.Contains(symbols, s => + s.Kind == "property" + && s.SubKind == "generated_column" + && s.Name == "computed_total" + && s.ContainerName == "Orders"); + Assert.DoesNotContain(symbols, s => + s.Kind == "property" + && s.SubKind == "generated_column" + && s.Name == "created_at"); + Assert.DoesNotContain(symbols, s => + s.Kind == "property" + && s.SubKind == "generated_column" + && s.Name == "CONSTRAINT"); + } + [Fact] public void Extract_CobolProgramId_DetectsProgramSymbol() { @@ -14382,13 +14430,13 @@ fn build(&self) { Assert.Contains(symbols, s => s.Kind == "property" && s.Name == "Output" - && s.ContainerKind == "interface" + && s.ContainerKind == "protocol" && s.ContainerName == "Builder" && s.ReturnType == "()"); Assert.Contains(symbols, s => s.Kind == "property" && s.Name == "Error" - && s.ContainerKind == "interface" + && s.ContainerKind == "protocol" && s.ContainerName == "Builder" && s.ReturnType == "String"); Assert.DoesNotContain(symbols, s => s.Kind == "property" && s.Name == "Pending");