Skip to content

Commit f8eb26a

Browse files
WidthdomWidthdom
authored
Merge pull request #2600 from Widthdom/fix-issue2102
Fix SQL generated column indexing
2 parents 482464a + 50245cc commit f8eb26a

5 files changed

Lines changed: 396 additions & 0 deletions

File tree

‎changelog.d/unreleased/2102.fixed.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
category: fixed
3+
issues:
4+
- 2102
5+
affected:
6+
- src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
7+
- src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs
8+
- tests/CodeIndex.Tests/SymbolExtractorTests.cs
9+
- tests/CodeIndex.Tests/ReferenceExtractorTests.cs
10+
---
11+
12+
## English
13+
14+
- **SQL generated/computed columns are now indexed (#2102)** — `CREATE TABLE` and `ALTER TABLE ... ADD` generated/computed columns emit column symbols and dependency references for generation and `DEFAULT NEXT VALUE FOR` expressions.
15+
16+
## 日本語
17+
18+
- **SQL の generated/computed column を index するようにしました (#2102)** — `CREATE TABLE` と `ALTER TABLE ... ADD` の生成/計算列で列シンボルを出し、生成式と `DEFAULT NEXT VALUE FOR` 式の依存参照も抽出します。

‎src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,18 @@ internal sealed class State
285285
private static readonly Regex CreateTempRoutineRegex = new(
286286
$@"(?<![\w$])CREATE(?:\s+OR\s+(?:REPLACE|ALTER))?(?:\s+(?:TEMP|TEMPORARY))?\s+(?:PROC(?:EDURE)?|FUNCTION)\b(?:\s+IF\s+NOT\s+EXISTS)?\s+(?<name>{TempIdentifierPattern})",
287287
RegexOptions.Compiled | RegexOptions.IgnoreCase);
288+
private static readonly Regex GeneratedColumnMarkerRegex = new(
289+
@"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|NEXT\s+VALUE\s+FOR)\b|(?<![\w$])AS\s*\(",
290+
RegexOptions.Compiled | RegexOptions.IgnoreCase);
291+
private static readonly Regex GeneratedColumnExpressionStartRegex = new(
292+
@"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS)\s*\(",
293+
RegexOptions.Compiled | RegexOptions.IgnoreCase);
294+
private static readonly Regex DefaultNextValueForExpressionRegex = new(
295+
$@"\bDEFAULT\s+NEXT\s+VALUE\s+FOR\s+(?<name>{QualifiedIdentifierNoCapturePattern})",
296+
RegexOptions.Compiled | RegexOptions.IgnoreCase);
297+
private static readonly Regex SqlExpressionIdentifierRegex = new(
298+
$@"(?<name>{QualifiedIdentifierNoCapturePattern})",
299+
RegexOptions.Compiled | RegexOptions.IgnoreCase);
288300
private static readonly Regex TrailingTempIdentifierRegex = new(
289301
$@"^(?:(?:ONLY)\b\s+)?(?<item>(?:{TempIdentifierPattern}|{QualifiedIdentifierNoCapturePattern}))(?:\s+(?:AS\s+)?(?:{QuotedIdentifierPattern}|{BareIdentifierPattern}))?\s*$",
290302
RegexOptions.Compiled | RegexOptions.IgnoreCase);
@@ -509,6 +521,19 @@ private static void EmitStatementReferences(
509521
fileId,
510522
resolveContainerForCall);
511523

524+
EmitGeneratedColumnDependencyReferences(
525+
statement,
526+
statementStart,
527+
statementLineOffset,
528+
lineOffset,
529+
context,
530+
lineNumber,
531+
references,
532+
seen,
533+
fileId,
534+
resolveContainerForCall,
535+
shouldIgnoreName);
536+
512537
EmitSourceCaptureReferences(
513538
FromSourceListRegex.Matches(statement),
514539
statement,
@@ -1547,6 +1572,133 @@ private static void EmitMergeUsingReferences(
15471572
}
15481573
}
15491574

1575+
private static void EmitGeneratedColumnDependencyReferences(
1576+
string statement,
1577+
int statementStart,
1578+
int statementLineOffset,
1579+
int lineOffset,
1580+
string context,
1581+
int lineNumber,
1582+
List<ReferenceRecord> references,
1583+
HashSet<string> seen,
1584+
long fileId,
1585+
Func<int, SymbolRecord?> resolveContainerForCall,
1586+
Func<string, bool> shouldIgnoreName)
1587+
{
1588+
if (!GeneratedColumnMarkerRegex.IsMatch(statement))
1589+
return;
1590+
1591+
foreach (Match match in GeneratedColumnExpressionStartRegex.Matches(statement))
1592+
{
1593+
if (match.Index < statementLineOffset || IsInsideDoubleQuotedRegion(statement, match.Index))
1594+
continue;
1595+
if (match.Value.TrimStart().StartsWith("AS", StringComparison.OrdinalIgnoreCase)
1596+
&& !IsLikelyComputedColumnAsExpression(statement, match.Index))
1597+
{
1598+
continue;
1599+
}
1600+
1601+
var openParenIndex = statement.IndexOf('(', match.Index + match.Length - 1);
1602+
if (openParenIndex < 0)
1603+
continue;
1604+
1605+
var closeParenIndex = FindMatchingParen(statement, openParenIndex);
1606+
if (closeParenIndex <= openParenIndex)
1607+
continue;
1608+
1609+
EmitSqlExpressionIdentifierDependencies(
1610+
statement,
1611+
openParenIndex + 1,
1612+
closeParenIndex,
1613+
statementStart,
1614+
statementLineOffset,
1615+
lineOffset,
1616+
context,
1617+
lineNumber,
1618+
references,
1619+
seen,
1620+
fileId,
1621+
resolveContainerForCall,
1622+
shouldIgnoreName);
1623+
}
1624+
1625+
foreach (Match match in DefaultNextValueForExpressionRegex.Matches(statement))
1626+
{
1627+
if (match.Index < statementLineOffset || IsInsideDoubleQuotedRegion(statement, match.Index))
1628+
continue;
1629+
1630+
var sequence = match.Groups["name"];
1631+
EmitSqlExpressionIdentifierDependencies(
1632+
statement,
1633+
sequence.Index,
1634+
sequence.Index + sequence.Length,
1635+
statementStart,
1636+
statementLineOffset,
1637+
lineOffset,
1638+
context,
1639+
lineNumber,
1640+
references,
1641+
seen,
1642+
fileId,
1643+
resolveContainerForCall,
1644+
shouldIgnoreName);
1645+
}
1646+
}
1647+
1648+
private static void EmitSqlExpressionIdentifierDependencies(
1649+
string statement,
1650+
int startIndex,
1651+
int endIndexExclusive,
1652+
int statementStart,
1653+
int statementLineOffset,
1654+
int lineOffset,
1655+
string context,
1656+
int lineNumber,
1657+
List<ReferenceRecord> references,
1658+
HashSet<string> seen,
1659+
long fileId,
1660+
Func<int, SymbolRecord?> resolveContainerForCall,
1661+
Func<string, bool> shouldIgnoreName)
1662+
{
1663+
var expression = statement[startIndex..endIndexExclusive];
1664+
foreach (Match match in SqlExpressionIdentifierRegex.Matches(expression))
1665+
{
1666+
var rawIndex = startIndex + match.Index;
1667+
if (rawIndex < statementLineOffset || IsInsideDoubleQuotedRegion(statement, rawIndex))
1668+
continue;
1669+
1670+
var rawName = match.Value;
1671+
NormalizeIdentifier(rawName, rawIndex, out var resolvedName, out var nameIndex, out var wasQuoted);
1672+
if (!wasQuoted && (shouldIgnoreName(resolvedName) || IsGeneratedColumnDependencyKeyword(resolvedName)))
1673+
continue;
1674+
1675+
var nameColumn = nameIndex + statementStart - lineOffset;
1676+
var container = resolveContainerForCall(rawIndex);
1677+
ReferenceExtractor.AddReference(references, seen, fileId, resolvedName, nameColumn, "generated_column_dependency", context, lineNumber, container);
1678+
}
1679+
}
1680+
1681+
private static bool IsGeneratedColumnDependencyKeyword(string name)
1682+
=> name.Equals("GENERATED", StringComparison.OrdinalIgnoreCase)
1683+
|| name.Equals("ALWAYS", StringComparison.OrdinalIgnoreCase)
1684+
|| name.Equals("AS", StringComparison.OrdinalIgnoreCase)
1685+
|| name.Equals("DEFAULT", StringComparison.OrdinalIgnoreCase)
1686+
|| name.Equals("NEXT", StringComparison.OrdinalIgnoreCase)
1687+
|| name.Equals("VALUE", StringComparison.OrdinalIgnoreCase)
1688+
|| name.Equals("FOR", StringComparison.OrdinalIgnoreCase)
1689+
|| name.Equals("STORED", StringComparison.OrdinalIgnoreCase)
1690+
|| name.Equals("VIRTUAL", StringComparison.OrdinalIgnoreCase)
1691+
|| name.Equals("PERSISTED", StringComparison.OrdinalIgnoreCase)
1692+
|| name.Equals("NULL", StringComparison.OrdinalIgnoreCase)
1693+
|| name.Equals("NOT", StringComparison.OrdinalIgnoreCase);
1694+
1695+
private static bool IsLikelyComputedColumnAsExpression(string statement, int asIndex)
1696+
{
1697+
var prefix = statement[..asIndex];
1698+
return Regex.IsMatch(prefix, @"(?<![\w$])ALTER\s+TABLE\b[\s\S]*\bADD\b", RegexOptions.IgnoreCase)
1699+
|| Regex.IsMatch(prefix, @"(?<![\w$])CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?(?:(?:(?:GLOBAL|LOCAL)\s+)?(?:TEMP|TEMPORARY)\s+|UNLOGGED\s+)?TABLE\b", RegexOptions.IgnoreCase);
1700+
}
1701+
15501702
private static void EmitSourceReference(
15511703
string rawName,
15521704
int rawIndex,

‎src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,18 @@ public static int GetContractVersion(string? lang)
361361
private static readonly Regex SqlCteDefinitionRegex = new(
362362
$@"(?<![\w$])(?:WITH\s+(?:RECURSIVE\s+)?|,\s*)(?<name>{SqlQualifiedIdentifierSegmentPattern})(?:\s*\([^)]*\))?\s+AS\s*\(",
363363
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
364+
private static readonly Regex SqlAlterTableAddGeneratedColumnRegex = new(
365+
$@"(?<![\w$])ALTER\s+TABLE\s+(?<table>{SqlQualifiedIdentifierPattern})\s+ADD(?:\s+COLUMN)?\s+(?!CONSTRAINT\b)(?<name>{SqlQualifiedIdentifierSegmentPattern})\b(?=[^;]*?\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS\s*\(|DEFAULT\s+NEXT\s+VALUE\s+FOR)\b)",
366+
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
367+
private static readonly Regex SqlCreateTableBodyRegex = new(
368+
$@"(?<![\w$])CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?(?:(?:(?:GLOBAL|LOCAL)\s+)?(?:TEMP|TEMPORARY)\s+|UNLOGGED\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?<table>{SqlQualifiedIdentifierPattern})\s*\((?<body>[\s\S]*?)\)\s*;",
369+
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
370+
private static readonly Regex SqlGeneratedColumnDefinitionMarkerRegex = new(
371+
@"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS\s*\(|DEFAULT\s+NEXT\s+VALUE\s+FOR)\b",
372+
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
373+
private static readonly Regex SqlColumnDefinitionNameRegex = new(
374+
$@"^\s*(?<name>{SqlQualifiedIdentifierSegmentPattern})\b",
375+
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
364376
private static readonly Regex SqlReturnsTableRegex = new(
365377
@"\bRETURNS\s+TABLE\s*\((?<columns>(?:[^()]|\([^()]*\))*)\)",
366378
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Singleline);
@@ -1391,6 +1403,8 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult
13911403
// file module declarations and inline modules / ファイルモジュール宣言とインラインモジュール
13921404
new("file_module", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?mod\s+(?<name>(?:r#)?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility"),
13931405
new("namespace", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?mod\s+(?<name>(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"),
1406+
// Trait associated type defaults / trait 関連型のデフォルト
1407+
new("property", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?type\s+(?<name>(?:r#)?\w+)(?:\s*<[^=>]+>)?(?:\s*:\s*[^=;]+)?\s*=\s*(?<returnType>[^;]+)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"),
13941408
// type alias / 型エイリアス
13951409
new("import", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?type\s+(?<name>(?:r#)?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility"),
13961410
new("import", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?use\s+(?<name>.+);", RegexOptions.Compiled), BodyStyle.None, "visibility"),
@@ -2099,6 +2113,22 @@ public static IReadOnlyCollection<string> GetSupportedLanguages()
20992113
"class", "struct", "interface", "protocol", "namespace", "enum", "object", "heading", "specialization", "class_hook"
21002114
];
21012115

2116+
private static bool IsRustDirectTraitBodyMember(List<SymbolRecord> symbols, int candidateLine)
2117+
{
2118+
SymbolRecord? innermostContainer = null;
2119+
foreach (var symbol in symbols)
2120+
{
2121+
if (!symbol.BodyStartLine.HasValue || !symbol.BodyEndLine.HasValue)
2122+
continue;
2123+
if (candidateLine < symbol.BodyStartLine.Value || candidateLine > symbol.BodyEndLine.Value)
2124+
continue;
2125+
if (innermostContainer == null || symbol.StartLine >= innermostContainer.StartLine)
2126+
innermostContainer = symbol;
2127+
}
2128+
2129+
return innermostContainer?.Kind == "protocol";
2130+
}
2131+
21022132
/// <summary>
21032133
/// Extract symbols from the given source content.
21042134
/// 指定されたソース内容からシンボルを抽出する。
@@ -2687,6 +2717,14 @@ public static List<SymbolRecord> Extract(long fileId, string? lang, string conte
26872717
lineOffset = FindNextSameLineBraceStatementStart(matchLine, absoluteStartColumn + Math.Max(1, match.Length), lang);
26882718
continue;
26892719
}
2720+
if (lang == "rust"
2721+
&& pattern.Kind == "property"
2722+
&& pattern.BodyStyle == BodyStyle.None
2723+
&& pattern.ReturnTypeGroup != null
2724+
&& !IsRustDirectTraitBodyMember(symbols, i + 1))
2725+
{
2726+
break;
2727+
}
26902728
var rawReturnType = NormalizeCSharpImplicitPartialMethodReturnType(
26912729
lang,
26922730
pattern,
@@ -3955,6 +3993,7 @@ public static List<SymbolRecord> Extract(long fileId, string? lang, string conte
39553993
ExtractSqlCteSymbols(fileId, lines, symbols);
39563994
ExtractSqlDefinerSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols);
39573995
ExtractSqlRoutineResultColumnSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols);
3996+
ExtractSqlGeneratedColumnSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols);
39583997
}
39593998
if (IsRazorLanguage(originalLang) || IsRazorFilePath(filePath))
39603999
ExtractRazorDirectiveSymbols(fileId, lines, symbols);
@@ -4062,6 +4101,119 @@ private static int GetLineNumberFromOffset(List<int> lineStarts, int offset)
40624101
return ~index;
40634102
}
40644103

4104+
private static void ExtractSqlGeneratedColumnSymbols(long fileId, string[] lines, string[] structuralLines, List<SymbolRecord> symbols)
4105+
{
4106+
var structuralContent = string.Join('\n', structuralLines);
4107+
if (structuralContent.IndexOf("GENERATED", StringComparison.OrdinalIgnoreCase) < 0
4108+
&& structuralContent.IndexOf("NEXT VALUE FOR", StringComparison.OrdinalIgnoreCase) < 0
4109+
&& structuralContent.IndexOf(" AS ", StringComparison.OrdinalIgnoreCase) < 0)
4110+
{
4111+
return;
4112+
}
4113+
4114+
var lineStarts = BuildLineStarts(structuralContent);
4115+
foreach (Match match in SqlAlterTableAddGeneratedColumnRegex.Matches(structuralContent))
4116+
{
4117+
var nameGroup = match.Groups["name"];
4118+
AddSqlGeneratedColumnSymbol(
4119+
fileId,
4120+
lines,
4121+
lineStarts,
4122+
new GroupProxy(nameGroup.Value, nameGroup.Index),
4123+
match.Groups["table"].Value,
4124+
symbols);
4125+
}
4126+
4127+
foreach (Match tableMatch in SqlCreateTableBodyRegex.Matches(structuralContent))
4128+
{
4129+
var tableName = tableMatch.Groups["table"].Value;
4130+
var bodyGroup = tableMatch.Groups["body"];
4131+
foreach (var column in EnumerateSqlColumnDefinitions(bodyGroup.Value, bodyGroup.Index))
4132+
{
4133+
if (!SqlGeneratedColumnDefinitionMarkerRegex.IsMatch(column.Text))
4134+
continue;
4135+
4136+
var nameMatch = SqlColumnDefinitionNameRegex.Match(column.Text);
4137+
if (!nameMatch.Success)
4138+
continue;
4139+
4140+
AddSqlGeneratedColumnSymbol(
4141+
fileId,
4142+
lines,
4143+
lineStarts,
4144+
new GroupProxy(nameMatch.Groups["name"].Value, column.StartIndex + nameMatch.Groups["name"].Index),
4145+
tableName,
4146+
symbols);
4147+
}
4148+
}
4149+
}
4150+
4151+
private static void AddSqlGeneratedColumnSymbol(
4152+
long fileId,
4153+
string[] lines,
4154+
List<int> lineStarts,
4155+
IGroupLike nameGroup,
4156+
string rawTableName,
4157+
List<SymbolRecord> symbols)
4158+
{
4159+
var name = NormalizeSqlIdentifierSegment(nameGroup.Value);
4160+
if (string.IsNullOrWhiteSpace(name))
4161+
return;
4162+
4163+
var lineNumber = GetLineNumberFromOffset(lineStarts, nameGroup.Index);
4164+
AddSymbolRecord(
4165+
symbols,
4166+
null,
4167+
lineNumber,
4168+
new SymbolRecord
4169+
{
4170+
FileId = fileId,
4171+
Kind = "property",
4172+
SubKind = "generated_column",
4173+
Name = name,
4174+
Line = lineNumber,
4175+
StartLine = lineNumber,
4176+
StartColumn = nameGroup.Index - lineStarts[lineNumber - 1],
4177+
EndLine = lineNumber,
4178+
Signature = lines[lineNumber - 1].Trim(),
4179+
ContainerKind = "class",
4180+
ContainerName = NormalizeSqlIdentifierSegment(SqlNameResolver.GetLeafName(rawTableName)),
4181+
},
4182+
lines[lineNumber - 1]);
4183+
}
4184+
4185+
private interface IGroupLike
4186+
{
4187+
string Value { get; }
4188+
int Index { get; }
4189+
}
4190+
4191+
private readonly record struct GroupProxy(string Value, int Index) : IGroupLike;
4192+
4193+
private readonly record struct SqlColumnDefinitionSlice(string Text, int StartIndex);
4194+
4195+
private static IEnumerable<SqlColumnDefinitionSlice> EnumerateSqlColumnDefinitions(string body, int bodyStartIndex)
4196+
{
4197+
var start = 0;
4198+
var depth = 0;
4199+
for (var i = 0; i <= body.Length; i++)
4200+
{
4201+
if (i == body.Length || (body[i] == ',' && depth == 0))
4202+
{
4203+
var text = body[start..i].Trim();
4204+
if (text.Length > 0)
4205+
yield return new SqlColumnDefinitionSlice(text, bodyStartIndex + start + body[start..i].Length - body[start..i].TrimStart().Length);
4206+
start = i + 1;
4207+
continue;
4208+
}
4209+
4210+
if (body[i] == '(')
4211+
depth++;
4212+
else if (body[i] == ')' && depth > 0)
4213+
depth--;
4214+
}
4215+
}
4216+
40654217
private static string NormalizeSqlIdentifierSegment(string value)
40664218
{
40674219
if (value.Length >= 2 && value[0] == '[' && value[^1] == ']')

0 commit comments

Comments
 (0)