diff --git a/changelog.d/unreleased/2099.fixed.md b/changelog.d/unreleased/2099.fixed.md new file mode 100644 index 0000000000..650117ab29 --- /dev/null +++ b/changelog.d/unreleased/2099.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 2099 +affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **SQL window `OVER` clauses now emit column references (#2099)** — `PARTITION BY` and `ORDER BY` identifiers inside window functions are recorded as `column_reference` edges instead of being lost among window-frame syntax. + +## 日本語 + +- **SQL window `OVER` 句が column reference を出力するようになりました (#2099)** — window 関数内の `PARTITION BY` / `ORDER BY` 識別子を window frame 構文に埋もれさせず、`column_reference` edge として記録します。 diff --git a/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs index 3a895868f3..b2c752a900 100644 --- a/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs @@ -291,6 +291,12 @@ internal sealed class State private static readonly Regex MergeTargetHintContinuationPrefixRegex = new( $@"(?ROWS|RANGE|GROUPS|BETWEEN|UNBOUNDED|PRECEDING|FOLLOWING|CURRENT|ROW|EXCLUDE|TIES|OTHERS|NO)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase); public static State CreateState() => new(); @@ -325,6 +331,43 @@ public static Dictionary> BuildDefinitionLeafSpans return spansByLine; } + public static HashSet<(int LineNumber, int ColumnIndex)> BuildWindowFunctionCallSiteSuppressions(string[] lines) + { + var suppressed = new HashSet<(int LineNumber, int ColumnIndex)>(); + if (lines.Length == 0) + return suppressed; + + var lineStarts = new int[lines.Length]; + var textBuilder = new StringBuilder(); + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + if (lineIndex > 0) + textBuilder.Append('\n'); + lineStarts[lineIndex] = textBuilder.Length; + textBuilder.Append(lines[lineIndex]); + } + + var text = textBuilder.ToString(); + var searchStart = 0; + while (TryFindNextWindowClause( + text, + searchStart, + out var overKeywordIndex, + out _, + out var closeParenIndex)) + { + if (TryFindWindowFunctionNameIndex(text, overKeywordIndex, out var functionNameIndex) + && TryMapJoinedOffsetToLine(lineStarts, text, functionNameIndex, out var lineNumber, out var columnIndex)) + { + suppressed.Add((lineNumber, columnIndex)); + } + + searchStart = closeParenIndex + 1; + } + + return suppressed; + } + public static bool ShouldSuppressDefinitionCall( IReadOnlyList? definitionLeafSpans, string resolvedName, @@ -482,6 +525,20 @@ private static void EmitStatementReferences( suppressedCallIndices.Add(nameGroup.Index + statementStart - lineOffset); } + EmitWindowClauseReferences( + statement, + statementStart, + statementLineOffset, + lineOffset, + context, + lineNumber, + references, + seen, + fileId, + suppressedCallIndices, + resolveContainerForCall, + shouldIgnoreName); + EmitProcedureCalls( statement, statementStart, @@ -1394,6 +1451,292 @@ private static void EmitStatementReferences( shouldIgnoreName); } + private static void EmitWindowClauseReferences( + string statement, + int statementStart, + int statementLineOffset, + int lineOffset, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + HashSet suppressedCallIndices, + Func resolveContainerForCall, + Func shouldIgnoreName) + { + int searchStart = 0; + while (TryFindNextWindowClause( + statement, + searchStart, + out var overKeywordIndex, + out var openParenIndex, + out var closeParenIndex)) + { + if (TryFindWindowFunctionNameIndex(statement, overKeywordIndex, out var functionNameIndex) + && functionNameIndex >= statementLineOffset) + { + suppressedCallIndices.Add(functionNameIndex + statementStart - lineOffset); + } + + var bodyStart = openParenIndex + 1; + if (closeParenIndex > statementLineOffset) + { + EmitWindowClauseColumnReferences( + statement, + bodyStart, + closeParenIndex, + statementStart, + statementLineOffset, + lineOffset, + context, + lineNumber, + references, + seen, + fileId, + suppressedCallIndices, + resolveContainerForCall, + shouldIgnoreName); + } + + searchStart = closeParenIndex + 1; + } + } + + private static bool TryFindNextWindowClause( + string statement, + int searchStart, + out int overKeywordIndex, + out int openParenIndex, + out int closeParenIndex) + { + overKeywordIndex = -1; + openParenIndex = -1; + closeParenIndex = -1; + + for (int i = searchStart; i < statement.Length;) + { + if (!IsKeywordAt(statement, i, "OVER")) + { + i++; + continue; + } + + if (IsInsideDoubleQuotedRegion(statement, i)) + { + i += "OVER".Length; + continue; + } + + int probe = SkipWhitespaceAhead(statement, i + "OVER".Length); + if (probe >= statement.Length || statement[probe] != '(') + { + i += "OVER".Length; + continue; + } + + int close = FindMatchingParen(statement, probe); + if (close < 0) + { + i += "OVER".Length; + continue; + } + + overKeywordIndex = i; + openParenIndex = probe; + closeParenIndex = close; + return true; + } + + return false; + } + + private static bool TryFindWindowFunctionNameIndex(string statement, int overKeywordIndex, out int functionNameIndex) + { + functionNameIndex = -1; + int probe = overKeywordIndex - 1; + while (probe >= 0 && char.IsWhiteSpace(statement[probe])) + probe--; + if (probe < 0 || statement[probe] != ')') + return false; + + int openParen = FindMatchingOpenParen(statement, probe); + if (openParen <= 0) + return false; + + probe = openParen - 1; + while (probe >= 0 && char.IsWhiteSpace(statement[probe])) + probe--; + int nameEnd = probe; + while (probe >= 0 && IsSqlIdentifierPart(statement[probe])) + probe--; + int nameStart = probe + 1; + if (nameStart > nameEnd) + return false; + + functionNameIndex = nameStart; + return true; + } + + private static bool TryMapJoinedOffsetToLine(int[] lineStarts, string text, int offset, out int lineNumber, out int columnIndex) + { + lineNumber = 0; + columnIndex = 0; + if (offset < 0 || offset >= text.Length) + return false; + + var lineIndex = Array.BinarySearch(lineStarts, offset); + if (lineIndex < 0) + lineIndex = ~lineIndex - 1; + if (lineIndex < 0 || lineIndex >= lineStarts.Length) + return false; + + lineNumber = lineIndex + 1; + columnIndex = offset - lineStarts[lineIndex]; + return true; + } + + private static void EmitWindowClauseColumnReferences( + string statement, + int bodyStart, + int bodyEnd, + int statementStart, + int statementLineOffset, + int lineOffset, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + HashSet suppressedCallIndices, + Func resolveContainerForCall, + Func shouldIgnoreName) + { + foreach (Match keywordMatch in WindowFrameKeywordRegex.Matches(statement)) + { + if (keywordMatch.Index >= bodyStart && keywordMatch.Index < bodyEnd && keywordMatch.Index >= statementLineOffset) + suppressedCallIndices.Add(keywordMatch.Index + statementStart - lineOffset); + } + + foreach (var (start, end) in EnumerateWindowColumnListSpans(statement, bodyStart, bodyEnd)) + { + foreach (Match match in WindowClauseColumnRegex.Matches(statement)) + { + var nameGroup = match.Groups["name"]; + if (!nameGroup.Success || nameGroup.Index < start || nameGroup.Index >= end || nameGroup.Index < statementLineOffset) + continue; + if (IsSqlWindowKeyword(nameGroup.Value)) + continue; + if (IsImmediatelyFollowedByOpenParen(statement, nameGroup.Index + nameGroup.Length)) + continue; + + NormalizeIdentifier(nameGroup.Value, nameGroup.Index, out var resolvedName, out var nameIndex, out var wasQuoted); + if (!wasQuoted && shouldIgnoreName(resolvedName)) + continue; + + int nameColumn = nameIndex + statementStart - lineOffset; + var container = resolveContainerForCall(nameGroup.Index); + ReferenceExtractor.AddReference( + references, + seen, + fileId, + resolvedName, + nameColumn, + "column_reference", + context, + lineNumber, + container); + } + } + } + + private static IEnumerable<(int Start, int End)> EnumerateWindowColumnListSpans(string statement, int bodyStart, int bodyEnd) + { + int position = bodyStart; + while (position < bodyEnd) + { + if (IsKeywordAt(statement, position, "PARTITION")) + { + int byIndex = SkipWhitespaceAhead(statement, position + "PARTITION".Length); + if (IsKeywordAt(statement, byIndex, "BY")) + { + int start = SkipWhitespaceAhead(statement, byIndex + "BY".Length); + int end = FindWindowListEnd(statement, start, bodyEnd); + yield return (start, end); + position = end; + continue; + } + } + + if (IsKeywordAt(statement, position, "ORDER")) + { + int byIndex = SkipWhitespaceAhead(statement, position + "ORDER".Length); + if (IsKeywordAt(statement, byIndex, "BY")) + { + int start = SkipWhitespaceAhead(statement, byIndex + "BY".Length); + int end = FindWindowListEnd(statement, start, bodyEnd); + yield return (start, end); + position = end; + continue; + } + } + + position++; + } + } + + private static int FindWindowListEnd(string statement, int start, int bodyEnd) + { + for (int i = start; i < bodyEnd; i++) + { + if (IsKeywordAt(statement, i, "PARTITION") + || IsKeywordAt(statement, i, "ORDER") + || IsKeywordAt(statement, i, "ROWS") + || IsKeywordAt(statement, i, "RANGE") + || IsKeywordAt(statement, i, "GROUPS")) + { + return i; + } + } + + return bodyEnd; + } + + private static bool IsKeywordAt(string text, int index, string keyword) + { + if (index < 0 || index + keyword.Length > text.Length) + return false; + if (string.Compare(text, index, keyword, 0, keyword.Length, StringComparison.OrdinalIgnoreCase) != 0) + return false; + if (index > 0 && IsSqlIdentifierPart(text[index - 1])) + return false; + var after = index + keyword.Length; + return after >= text.Length || !IsSqlIdentifierPart(text[after]); + } + + private static bool IsSqlIdentifierPart(char value) + { + return char.IsLetterOrDigit(value) || value == '_' || value == '$' || value == '#'; + } + + private static bool IsSqlWindowKeyword(string value) + { + return string.Equals(value, "PARTITION", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "ORDER", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "BY", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "ASC", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "DESC", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "NULLS", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "FIRST", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "LAST", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsImmediatelyFollowedByOpenParen(string text, int index) + { + int probe = SkipWhitespaceAhead(text, index); + return probe < text.Length && text[probe] == '('; + } + private static void EmitProcedureCalls( string statement, int statementStart, @@ -1644,6 +1987,28 @@ private static int FindMatchingParen(string text, int openParenIndex) return -1; } + private static int FindMatchingOpenParen(string text, int closeParenIndex) + { + var depth = 0; + for (var i = closeParenIndex; i >= 0; i--) + { + if (text[i] == ')') + { + depth++; + continue; + } + + if (text[i] != '(') + continue; + + depth--; + if (depth == 0) + return i; + } + + return -1; + } + private static void EmitSelectIntoTargetReferences( string statement, int statementStart, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 6e6acdcf73..c815212333 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -1008,6 +1008,9 @@ internal static List ExtractCore(ReferenceExtractionContext req var sqlDefinitionLeafSpansByLine = language == "sql" ? SqlReferenceExtractor.BuildDefinitionLeafSpansByLine(lines, symbols) : null; + var sqlWindowFunctionCallSiteSuppressions = language == "sql" + ? SqlReferenceExtractor.BuildWindowFunctionCallSiteSuppressions(structuralLines) + : null; var cobolCallableSymbols = language == "cobol" ? symbols .Where(symbol => symbol.Kind == "function") @@ -2550,6 +2553,9 @@ bool TryAddCallLikeReference(string name, int callIndex) continue; if (sqlSuppressedCallIndices != null && sqlSuppressedCallIndices.Contains(callIndex)) continue; + if (sqlWindowFunctionCallSiteSuppressions != null + && sqlWindowFunctionCallSiteSuppressions.Contains((lineNumber, callIndex))) + continue; matchedCallIndices.Add(callIndex); if (TryAddCallLikeReference(name, callIndex)) { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index bd96c02ca1..c134572539 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -14925,6 +14925,47 @@ public void Extract_SQL_ForeignKeyReferencesCapturesTargetTableReference() Assert.DoesNotContain(references, r => r.SymbolName == "Customers" && r.ReferenceKind == "call"); } + [Fact] + public void Extract_SQL_WindowOverClausesEmitColumnReferences() + { + const string content = """ + SELECT + ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY created_at DESC) AS rn, + SUM(amount) OVER (PARTITION BY [region] ORDER BY COALESCE(sale_date, fallback_date) ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total + FROM sales.orders; + """; + + var symbols = SymbolExtractor.Extract(1, "sql", content); + var references = ReferenceExtractor.Extract(1, "sql", content, symbols); + + Assert.Contains(references, r => r.SymbolName == "customer_id" && r.ReferenceKind == "column_reference" && r.Line == 2); + Assert.Contains(references, r => r.SymbolName == "created_at" && r.ReferenceKind == "column_reference" && r.Line == 2); + Assert.Contains(references, r => r.SymbolName == "region" && r.ReferenceKind == "column_reference" && r.Line == 3); + Assert.Contains(references, r => r.SymbolName == "sale_date" && r.ReferenceKind == "column_reference" && r.Line == 3); + Assert.Contains(references, r => r.SymbolName == "fallback_date" && r.ReferenceKind == "column_reference" && r.Line == 3); + Assert.DoesNotContain(references, r => r.SymbolName == "COALESCE" && r.ReferenceKind == "column_reference"); + Assert.DoesNotContain(references, r => (r.SymbolName is "ROW_NUMBER" or "SUM") && r.ReferenceKind == "call"); + Assert.DoesNotContain(references, r => r.SymbolName is "ROWS" or "UNBOUNDED" or "PRECEDING" or "CURRENT" or "ROW"); + } + + [Fact] + public void Extract_SQL_MultilineWindowOverClausesSuppressFunctionCalls() + { + const string content = """ + SELECT + SUM(amount) + OVER (PARTITION BY customer_id ORDER BY created_at DESC) AS running_total + FROM sales.orders; + """; + + var symbols = SymbolExtractor.Extract(1, "sql", content); + var references = ReferenceExtractor.Extract(1, "sql", content, symbols); + + Assert.Contains(references, r => r.SymbolName == "customer_id" && r.ReferenceKind == "column_reference" && r.Line == 3); + Assert.Contains(references, r => r.SymbolName == "created_at" && r.ReferenceKind == "column_reference" && r.Line == 3); + Assert.DoesNotContain(references, r => r.SymbolName == "SUM" && r.ReferenceKind == "call"); + } + [Fact] public void Extract_SQL_CreateSynonymCapturesBaseObjectReference() {