From 739989591b8d571a217caf636464f9a4b8f54fab Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Mon, 18 May 2026 23:03:43 +0100 Subject: [PATCH 01/16] fix: parenthesise ternary/coalesce in ExprToString to preserve AST on round-trip (#64) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ExprToString did not wrap TernaryExpression or CoalesceExpression in parentheses when they appeared as operands of higher-precedence binary, unary, BETWEEN, IN, or LIKE operators. When the SDK's transformed query was round-tripped through SimplifySdkQuery → re-parse, the missing parentheses produced a different AST — causing COUNT(expr) to evaluate the wrong condition and miscount documents. Added WrapIfLowPrecedence helper that wraps ternary/coalesce nodes in parens when they appear in operator positions that would otherwise re-parse with different associativity. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 5 + .../CosmosSqlParser.cs | 2766 +++++++++-------- src/Directory.Build.props | 2 +- .../Issue64CountTernaryUndefinedTests.cs | 219 ++ 4 files changed, 1614 insertions(+), 1378 deletions(-) create mode 100644 tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue64CountTernaryUndefinedTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6da297f..4ea8bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.0.18] - 2026-05-15 + +### Fixed +- `COUNT(expr)` with nested ternary expressions no longer miscounts documents (Issue #64). `ExprToString` in `CosmosSqlParser` did not parenthesise ternary/coalesce sub-expressions when they appeared as operands of higher-precedence binary operators. When the SDK's transformed query was round-tripped through `SimplifySdkQuery`, the missing parentheses caused re-parsing to produce a different AST — e.g. `(innerTernary > 0) ? 1 : undefined` became `innerTernary ? val : (otherVal > 0 ? 1 : undefined)` — making `COUNT` evaluate the wrong condition. The fix wraps ternary and coalesce expressions in parentheses whenever they appear inside binary, unary, BETWEEN, IN, or LIKE operators. + ## [4.0.17] - 2026-05-14 ### Fixed diff --git a/src/CosmosDB.InMemoryEmulator/CosmosSqlParser.cs b/src/CosmosDB.InMemoryEmulator/CosmosSqlParser.cs index 5647ac2..9625e67 100644 --- a/src/CosmosDB.InMemoryEmulator/CosmosSqlParser.cs +++ b/src/CosmosDB.InMemoryEmulator/CosmosSqlParser.cs @@ -13,74 +13,74 @@ namespace CosmosDB.InMemoryEmulator; public enum CosmosSqlToken { - // Keywords - Select, - Distinct, - Top, - Value, - As, - From, - Join, - In, - Where, - And, - Or, - Not, - Between, - Like, - Is, - Null, - Defined, - Undefined, - True, - False, - Exists, - Order, - By, - Asc, - Desc, - Group, - Having, - Offset, - Limit, - Array, - Escape, - - // Literals & identifiers - Identifier, - StringLiteral, - DoubleQuotedString, - NumberLiteral, - Parameter, - - // Operators & punctuation - Star, - Comma, - Dot, - OpenParen, - CloseParen, - OpenBracket, - CloseBracket, - Equals, - NotEquals, - LessThanOrEqual, - GreaterThanOrEqual, - LessThan, - GreaterThan, - Plus, - Minus, - Slash, - Percent, - Ampersand, - Pipe, - Caret, - Tilde, - QuestionQuestion, - Question, - Colon, - DoublePipe, - OpenBrace, - CloseBrace, + // Keywords + Select, + Distinct, + Top, + Value, + As, + From, + Join, + In, + Where, + And, + Or, + Not, + Between, + Like, + Is, + Null, + Defined, + Undefined, + True, + False, + Exists, + Order, + By, + Asc, + Desc, + Group, + Having, + Offset, + Limit, + Array, + Escape, + + // Literals & identifiers + Identifier, + StringLiteral, + DoubleQuotedString, + NumberLiteral, + Parameter, + + // Operators & punctuation + Star, + Comma, + Dot, + OpenParen, + CloseParen, + OpenBracket, + CloseBracket, + Equals, + NotEquals, + LessThanOrEqual, + GreaterThanOrEqual, + LessThan, + GreaterThan, + Plus, + Minus, + Slash, + Percent, + Ampersand, + Pipe, + Caret, + Tilde, + QuestionQuestion, + Question, + Colon, + DoublePipe, + OpenBrace, + CloseBrace, } // ────────────────────────────────────────────── @@ -89,13 +89,13 @@ public enum CosmosSqlToken public enum ComparisonOp { - Equal, - NotEqual, - LessThan, - GreaterThan, - LessThanOrEqual, - GreaterThanOrEqual, - Like, + Equal, + NotEqual, + LessThan, + GreaterThan, + LessThanOrEqual, + GreaterThanOrEqual, + Like, } public sealed record SelectField(string Expression, string Alias, SqlExpression SqlExpr = null); @@ -151,17 +151,17 @@ public sealed record ArrayLiteralExpression(SqlExpression[] Elements) : SqlExpre public enum BinaryOp { - Equal, NotEqual, LessThan, GreaterThan, LessThanOrEqual, GreaterThanOrEqual, - And, Or, - Add, Subtract, Multiply, Divide, Modulo, - BitwiseAnd, BitwiseOr, BitwiseXor, - Like, - StringConcat, + Equal, NotEqual, LessThan, GreaterThan, LessThanOrEqual, GreaterThanOrEqual, + And, Or, + Add, Subtract, Multiply, Divide, Modulo, + BitwiseAnd, BitwiseOr, BitwiseXor, + Like, + StringConcat, } public enum UnaryOp { - Not, Negate, BitwiseNot, + Not, Negate, BitwiseNot, } // ── Backward-compatible WhereExpression wrappers ── @@ -185,26 +185,26 @@ public sealed record SqlExpressionCondition(SqlExpression Expression) : WhereExp // ── Query ── public sealed record CosmosSqlQuery( - SelectField[] SelectFields, - bool IsSelectAll, - int? TopCount, - string FromAlias, - JoinClause Join, - WhereExpression Where, - int? Offset, - int? Limit, - OrderByClause OrderBy = null, - bool IsDistinct = false, - bool IsValueSelect = false, - JoinClause[] Joins = null, - OrderByField[] OrderByFields = null, - string[] GroupByFields = null, - SqlExpression[] GroupByExpressions = null, - WhereExpression Having = null, - SqlExpression WhereExpr = null, - SqlExpression HavingExpr = null, - string FromSource = null, - SqlExpression RankExpression = null); + SelectField[] SelectFields, + bool IsSelectAll, + int? TopCount, + string FromAlias, + JoinClause Join, + WhereExpression Where, + int? Offset, + int? Limit, + OrderByClause OrderBy = null, + bool IsDistinct = false, + bool IsValueSelect = false, + JoinClause[] Joins = null, + OrderByField[] OrderByFields = null, + string[] GroupByFields = null, + SqlExpression[] GroupByExpressions = null, + WhereExpression Having = null, + SqlExpression WhereExpr = null, + SqlExpression HavingExpr = null, + string FromSource = null, + SqlExpression RankExpression = null); public sealed record OrderByClause(string Field, bool Ascending); @@ -214,133 +214,133 @@ public sealed record OrderByClause(string Field, bool Ascending); public static class CosmosSqlTokenizer { - private static readonly TextParser StringLiteralToken = - from open in Character.EqualTo('\'') - from content in Character.EqualTo('\'').Then(_ => Character.EqualTo('\'')).Try() - .Or(Character.Except('\'')) - .Many() - from close in Character.EqualTo('\'') - select Unit.Value; - - private static readonly TextParser DoubleQuotedStringToken = - from open in Character.EqualTo('"') - from content in Character.Except('"').Many() - from close in Character.EqualTo('"') - select Unit.Value; - - private static readonly TextParser ParameterToken = - from at in Character.EqualTo('@') - from rest in Character.LetterOrDigit.Or(Character.EqualTo('_')).AtLeastOnce() - select Unit.Value; - - private static readonly TextParser NumberToken = - (from number in Numerics.Decimal - from exp in ( - from e in Character.EqualTo('e').Or(Character.EqualTo('E')) - from sign in Character.EqualTo('+').Or(Character.EqualTo('-')).OptionalOrDefault() - from digits in Character.Digit.AtLeastOnce() - select Unit.Value - ).OptionalOrDefault() - select Unit.Value); - - // Ref: EF Core Cosmos provider uses $type as a discriminator column. - // Allow $ as a valid identifier character to support queries like c.$type. - private static readonly TextParser IdentOrKeyword = - from first in Character.Letter.Or(Character.EqualTo('_')).Or(Character.EqualTo('$')) - from rest in Character.LetterOrDigit.Or(Character.EqualTo('_')).Or(Character.EqualTo('$')).Many() - select Unit.Value; - - // Quoted identifiers like [My Column] are not used by the Cosmos SDK's LINQ provider - // (it uses root["name"] with double-quoted strings instead). Removing this tokenizer rule - // allows [expr, expr] array literals and [0] numeric indexers to tokenize correctly. - // If needed in the future, require the content to contain a space to disambiguate. - - private static readonly Dictionary Keywords = new(StringComparer.OrdinalIgnoreCase) - { - ["SELECT"] = CosmosSqlToken.Select, - ["DISTINCT"] = CosmosSqlToken.Distinct, - ["TOP"] = CosmosSqlToken.Top, - ["VALUE"] = CosmosSqlToken.Value, - ["AS"] = CosmosSqlToken.As, - ["FROM"] = CosmosSqlToken.From, - ["JOIN"] = CosmosSqlToken.Join, - ["IN"] = CosmosSqlToken.In, - ["WHERE"] = CosmosSqlToken.Where, - ["AND"] = CosmosSqlToken.And, - ["OR"] = CosmosSqlToken.Or, - ["NOT"] = CosmosSqlToken.Not, - ["BETWEEN"] = CosmosSqlToken.Between, - ["LIKE"] = CosmosSqlToken.Like, - ["IS"] = CosmosSqlToken.Is, - ["NULL"] = CosmosSqlToken.Null, - ["DEFINED"] = CosmosSqlToken.Defined, - ["UNDEFINED"] = CosmosSqlToken.Undefined, - ["TRUE"] = CosmosSqlToken.True, - ["FALSE"] = CosmosSqlToken.False, - ["EXISTS"] = CosmosSqlToken.Exists, - ["ORDER"] = CosmosSqlToken.Order, - ["BY"] = CosmosSqlToken.By, - ["ASC"] = CosmosSqlToken.Asc, - ["DESC"] = CosmosSqlToken.Desc, - ["GROUP"] = CosmosSqlToken.Group, - ["HAVING"] = CosmosSqlToken.Having, - ["OFFSET"] = CosmosSqlToken.Offset, - ["LIMIT"] = CosmosSqlToken.Limit, - ["ARRAY"] = CosmosSqlToken.Array, - ["ESCAPE"] = CosmosSqlToken.Escape, - }; - - private static readonly Tokenizer Inner = new TokenizerBuilder() - .Ignore(Span.WhiteSpace) - .Match(StringLiteralToken, CosmosSqlToken.StringLiteral) - .Match(DoubleQuotedStringToken, CosmosSqlToken.DoubleQuotedString) - .Match(ParameterToken, CosmosSqlToken.Parameter) - .Match(NumberToken, CosmosSqlToken.NumberLiteral) - .Match(IdentOrKeyword, CosmosSqlToken.Identifier) - .Match(Character.EqualTo('*'), CosmosSqlToken.Star) - .Match(Character.EqualTo(','), CosmosSqlToken.Comma) - .Match(Character.EqualTo('.'), CosmosSqlToken.Dot) - .Match(Character.EqualTo('('), CosmosSqlToken.OpenParen) - .Match(Character.EqualTo(')'), CosmosSqlToken.CloseParen) - .Match(Character.EqualTo('['), CosmosSqlToken.OpenBracket) - .Match(Character.EqualTo(']'), CosmosSqlToken.CloseBracket) - .Match(Span.EqualTo("!="), CosmosSqlToken.NotEquals) - .Match(Span.EqualTo("<>"), CosmosSqlToken.NotEquals) - .Match(Span.EqualTo("<="), CosmosSqlToken.LessThanOrEqual) - .Match(Span.EqualTo(">="), CosmosSqlToken.GreaterThanOrEqual) - .Match(Span.EqualTo("??"), CosmosSqlToken.QuestionQuestion) - .Match(Span.EqualTo("||"), CosmosSqlToken.DoublePipe) - .Match(Character.EqualTo('='), CosmosSqlToken.Equals) - .Match(Character.EqualTo('<'), CosmosSqlToken.LessThan) - .Match(Character.EqualTo('>'), CosmosSqlToken.GreaterThan) - .Match(Character.EqualTo('+'), CosmosSqlToken.Plus) - .Match(Character.EqualTo('-'), CosmosSqlToken.Minus) - .Match(Character.EqualTo('/'), CosmosSqlToken.Slash) - .Match(Character.EqualTo('%'), CosmosSqlToken.Percent) - .Match(Character.EqualTo('&'), CosmosSqlToken.Ampersand) - .Match(Character.EqualTo('|'), CosmosSqlToken.Pipe) - .Match(Character.EqualTo('^'), CosmosSqlToken.Caret) - .Match(Character.EqualTo('~'), CosmosSqlToken.Tilde) - .Match(Character.EqualTo('?'), CosmosSqlToken.Question) - .Match(Character.EqualTo(':'), CosmosSqlToken.Colon) - .Match(Character.EqualTo('{'), CosmosSqlToken.OpenBrace) - .Match(Character.EqualTo('}'), CosmosSqlToken.CloseBrace) - .Build(); - - public static TokenList Tokenize(string input) - { - var tokens = Inner.Tokenize(input); - var remapped = tokens.Select(token => - { - if (token.Kind == CosmosSqlToken.Identifier && - Keywords.TryGetValue(token.Span.ToStringValue(), out var keyword)) - { - return new Token(keyword, token.Span); - } - return token; - }).ToArray(); - return new TokenList(remapped); - } + private static readonly TextParser StringLiteralToken = + from open in Character.EqualTo('\'') + from content in Character.EqualTo('\'').Then(_ => Character.EqualTo('\'')).Try() + .Or(Character.Except('\'')) + .Many() + from close in Character.EqualTo('\'') + select Unit.Value; + + private static readonly TextParser DoubleQuotedStringToken = + from open in Character.EqualTo('"') + from content in Character.Except('"').Many() + from close in Character.EqualTo('"') + select Unit.Value; + + private static readonly TextParser ParameterToken = + from at in Character.EqualTo('@') + from rest in Character.LetterOrDigit.Or(Character.EqualTo('_')).AtLeastOnce() + select Unit.Value; + + private static readonly TextParser NumberToken = + (from number in Numerics.Decimal + from exp in ( + from e in Character.EqualTo('e').Or(Character.EqualTo('E')) + from sign in Character.EqualTo('+').Or(Character.EqualTo('-')).OptionalOrDefault() + from digits in Character.Digit.AtLeastOnce() + select Unit.Value + ).OptionalOrDefault() + select Unit.Value); + + // Ref: EF Core Cosmos provider uses $type as a discriminator column. + // Allow $ as a valid identifier character to support queries like c.$type. + private static readonly TextParser IdentOrKeyword = + from first in Character.Letter.Or(Character.EqualTo('_')).Or(Character.EqualTo('$')) + from rest in Character.LetterOrDigit.Or(Character.EqualTo('_')).Or(Character.EqualTo('$')).Many() + select Unit.Value; + + // Quoted identifiers like [My Column] are not used by the Cosmos SDK's LINQ provider + // (it uses root["name"] with double-quoted strings instead). Removing this tokenizer rule + // allows [expr, expr] array literals and [0] numeric indexers to tokenize correctly. + // If needed in the future, require the content to contain a space to disambiguate. + + private static readonly Dictionary Keywords = new(StringComparer.OrdinalIgnoreCase) + { + ["SELECT"] = CosmosSqlToken.Select, + ["DISTINCT"] = CosmosSqlToken.Distinct, + ["TOP"] = CosmosSqlToken.Top, + ["VALUE"] = CosmosSqlToken.Value, + ["AS"] = CosmosSqlToken.As, + ["FROM"] = CosmosSqlToken.From, + ["JOIN"] = CosmosSqlToken.Join, + ["IN"] = CosmosSqlToken.In, + ["WHERE"] = CosmosSqlToken.Where, + ["AND"] = CosmosSqlToken.And, + ["OR"] = CosmosSqlToken.Or, + ["NOT"] = CosmosSqlToken.Not, + ["BETWEEN"] = CosmosSqlToken.Between, + ["LIKE"] = CosmosSqlToken.Like, + ["IS"] = CosmosSqlToken.Is, + ["NULL"] = CosmosSqlToken.Null, + ["DEFINED"] = CosmosSqlToken.Defined, + ["UNDEFINED"] = CosmosSqlToken.Undefined, + ["TRUE"] = CosmosSqlToken.True, + ["FALSE"] = CosmosSqlToken.False, + ["EXISTS"] = CosmosSqlToken.Exists, + ["ORDER"] = CosmosSqlToken.Order, + ["BY"] = CosmosSqlToken.By, + ["ASC"] = CosmosSqlToken.Asc, + ["DESC"] = CosmosSqlToken.Desc, + ["GROUP"] = CosmosSqlToken.Group, + ["HAVING"] = CosmosSqlToken.Having, + ["OFFSET"] = CosmosSqlToken.Offset, + ["LIMIT"] = CosmosSqlToken.Limit, + ["ARRAY"] = CosmosSqlToken.Array, + ["ESCAPE"] = CosmosSqlToken.Escape, + }; + + private static readonly Tokenizer Inner = new TokenizerBuilder() + .Ignore(Span.WhiteSpace) + .Match(StringLiteralToken, CosmosSqlToken.StringLiteral) + .Match(DoubleQuotedStringToken, CosmosSqlToken.DoubleQuotedString) + .Match(ParameterToken, CosmosSqlToken.Parameter) + .Match(NumberToken, CosmosSqlToken.NumberLiteral) + .Match(IdentOrKeyword, CosmosSqlToken.Identifier) + .Match(Character.EqualTo('*'), CosmosSqlToken.Star) + .Match(Character.EqualTo(','), CosmosSqlToken.Comma) + .Match(Character.EqualTo('.'), CosmosSqlToken.Dot) + .Match(Character.EqualTo('('), CosmosSqlToken.OpenParen) + .Match(Character.EqualTo(')'), CosmosSqlToken.CloseParen) + .Match(Character.EqualTo('['), CosmosSqlToken.OpenBracket) + .Match(Character.EqualTo(']'), CosmosSqlToken.CloseBracket) + .Match(Span.EqualTo("!="), CosmosSqlToken.NotEquals) + .Match(Span.EqualTo("<>"), CosmosSqlToken.NotEquals) + .Match(Span.EqualTo("<="), CosmosSqlToken.LessThanOrEqual) + .Match(Span.EqualTo(">="), CosmosSqlToken.GreaterThanOrEqual) + .Match(Span.EqualTo("??"), CosmosSqlToken.QuestionQuestion) + .Match(Span.EqualTo("||"), CosmosSqlToken.DoublePipe) + .Match(Character.EqualTo('='), CosmosSqlToken.Equals) + .Match(Character.EqualTo('<'), CosmosSqlToken.LessThan) + .Match(Character.EqualTo('>'), CosmosSqlToken.GreaterThan) + .Match(Character.EqualTo('+'), CosmosSqlToken.Plus) + .Match(Character.EqualTo('-'), CosmosSqlToken.Minus) + .Match(Character.EqualTo('/'), CosmosSqlToken.Slash) + .Match(Character.EqualTo('%'), CosmosSqlToken.Percent) + .Match(Character.EqualTo('&'), CosmosSqlToken.Ampersand) + .Match(Character.EqualTo('|'), CosmosSqlToken.Pipe) + .Match(Character.EqualTo('^'), CosmosSqlToken.Caret) + .Match(Character.EqualTo('~'), CosmosSqlToken.Tilde) + .Match(Character.EqualTo('?'), CosmosSqlToken.Question) + .Match(Character.EqualTo(':'), CosmosSqlToken.Colon) + .Match(Character.EqualTo('{'), CosmosSqlToken.OpenBrace) + .Match(Character.EqualTo('}'), CosmosSqlToken.CloseBrace) + .Build(); + + public static TokenList Tokenize(string input) + { + var tokens = Inner.Tokenize(input); + var remapped = tokens.Select(token => + { + if (token.Kind == CosmosSqlToken.Identifier && + Keywords.TryGetValue(token.Span.ToStringValue(), out var keyword)) + { + return new Token(keyword, token.Span); + } + return token; + }).ToArray(); + return new TokenList(remapped); + } } // ────────────────────────────────────────────── @@ -349,1152 +349,1164 @@ public static TokenList Tokenize(string input) public static class CosmosSqlParser { - private static readonly HashSet LegacyFunctionNames = new(StringComparer.OrdinalIgnoreCase) - { - "STARTSWITH", "ENDSWITH", "CONTAINS", "ARRAY_CONTAINS", "IS_DEFINED", "IS_NULL", - }; - - // ── Helpers ── - - private static string TokenSpanToString(Token token) => - token.Span.ToStringValue(); - - private static TokenListParser AnyIdentifierOrKeyword => - Token.EqualTo(CosmosSqlToken.Identifier).Select(TokenSpanToString) - .Or(Token.EqualTo(CosmosSqlToken.Value).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Array).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Defined).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Asc).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Desc).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Top).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Limit).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Offset).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.Null).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.True).Select(TokenSpanToString)) - .Or(Token.EqualTo(CosmosSqlToken.False).Select(TokenSpanToString)); - - // ── Dotted path: ident.ident.ident ── - - private static readonly TokenListParser DottedPath = - from first in AnyIdentifierOrKeyword - from rest in ( - from dot in Token.EqualTo(CosmosSqlToken.Dot) - from next in AnyIdentifierOrKeyword - select "." + next - ).Many() - select first + string.Concat(rest); - - // ── Dotted path with optional array indexing: ident.ident[0].ident ── - - // Helper: a bracketed string inside [ ] can be single- or double-quoted - private static readonly TokenListParser BracketedStringContent = - Token.EqualTo(CosmosSqlToken.StringLiteral).Select(t => t.Span.ToStringValue()[1..^1]) - .Or(Token.EqualTo(CosmosSqlToken.DoubleQuotedString).Select(t => t.Span.ToStringValue()[1..^1])); - - private static readonly TokenListParser DottedPathWithIndex = - from first in AnyIdentifierOrKeyword - from rest in ( - from dot in Token.EqualTo(CosmosSqlToken.Dot) - from next in AnyIdentifierOrKeyword - select "." + next - ).Or( - (from open in Token.EqualTo(CosmosSqlToken.OpenBracket) - from idx in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(TokenSpanToString) - from close in Token.EqualTo(CosmosSqlToken.CloseBracket) - select "[" + idx + "]").Try() - ).Or( - (from open in Token.EqualTo(CosmosSqlToken.OpenBracket) - from str in BracketedStringContent - from close in Token.EqualTo(CosmosSqlToken.CloseBracket) - select "." + str).Try() - ).Many() - select first + string.Concat(rest); - - // ── Primary expression ── - - // String literal expression: accepts both single-quoted (Cosmos SQL standard) and - // double-quoted strings (used by the SDK's LINQ provider in generated queries). - private static readonly TokenListParser StringLiteral = - Token.EqualTo(CosmosSqlToken.StringLiteral) - .Select(t => - { - var raw = t.Span.ToStringValue(); - var unquoted = raw[1..^1].Replace("''", "'"); - return (SqlExpression)new LiteralExpression(unquoted); - }) - .Or(Token.EqualTo(CosmosSqlToken.DoubleQuotedString) - .Select(t => - { - var raw = t.Span.ToStringValue(); - var unquoted = raw[1..^1]; - return (SqlExpression)new LiteralExpression(unquoted); - })); - - private static readonly TokenListParser NumberLiteral = - Token.EqualTo(CosmosSqlToken.NumberLiteral) - .Select(t => - { - var raw = t.Span.ToStringValue(); - if (long.TryParse(raw, out var longVal)) - { - return (SqlExpression)new LiteralExpression(longVal); - } - - return (SqlExpression)new LiteralExpression(double.Parse(raw, System.Globalization.CultureInfo.InvariantCulture)); - }); - - private static readonly TokenListParser TrueLiteral = - Token.EqualTo(CosmosSqlToken.True).Select(_ => (SqlExpression)new LiteralExpression(true)); - - private static readonly TokenListParser FalseLiteral = - Token.EqualTo(CosmosSqlToken.False).Select(_ => (SqlExpression)new LiteralExpression(false)); - - private static readonly TokenListParser NullLiteral = - Token.EqualTo(CosmosSqlToken.Null).Select(_ => (SqlExpression)new LiteralExpression(null)); - - private static readonly TokenListParser UndefinedLiteral = - Token.EqualTo(CosmosSqlToken.Undefined).Select(_ => (SqlExpression)new UndefinedLiteralExpression()); - - private static readonly TokenListParser ParameterExpr = - Token.EqualTo(CosmosSqlToken.Parameter) - .Select(t => (SqlExpression)new ParameterExpression(t.Span.ToStringValue())); - - // Function call: FUNC_NAME(args...) - // Supports * as a function argument (e.g. COUNT(*)) - private static readonly TokenListParser FunctionCall = - from name in AnyIdentifierOrKeyword - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from args in Token.EqualTo(CosmosSqlToken.Star).Select(_ => (SqlExpression)new IdentifierExpression("*")) - .Or(Superpower.Parse.Ref(() => Expr)) - .ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (SqlExpression)new FunctionCallExpression(name.ToUpperInvariant(), args); - - // Dotted function call: namespace.FUNC_NAME(args...) — supports udf.xxx() - // UDF names preserve original casing (Cosmos DB UDFs are case-sensitive); - // built-in dotted functions (e.g. ST_DISTANCE) are uppercased. - private static readonly TokenListParser DottedFunctionCall = - from first in AnyIdentifierOrKeyword - from dot in Token.EqualTo(CosmosSqlToken.Dot) - from second in AnyIdentifierOrKeyword - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from args in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (SqlExpression)new FunctionCallExpression( - first.Equals("udf", StringComparison.OrdinalIgnoreCase) - ? "UDF." + second // preserve UDF name casing - : (first + "." + second).ToUpperInvariant(), args); - - // EXISTS( subquery-text ) - private static readonly TokenListParser ExistsExpr = - from kw in Token.EqualTo(CosmosSqlToken.Exists) - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from inner in CaptureBalancedParens() - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (SqlExpression)new ExistsExpression(inner); - - // Subquery expression: (SELECT ...) — a full SELECT statement inside parentheses - private static readonly TokenListParser SubqueryParens = - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from subquery in Superpower.Parse.Ref(() => QueryParser) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (SqlExpression)new SubqueryExpression(subquery); - - // Parenthesized expression (try subquery first, fall back to regular expression) - private static readonly TokenListParser Parens = - SubqueryParens.Try() - .Or( - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from expr in Superpower.Parse.Ref(() => Expr) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select expr); - - // Object literal: { key: expr, key: expr, ... } - // Keys can be identifiers or double-quoted strings (for SDK-generated queries like {"item": root.value}) - private static readonly TokenListParser ObjectLiteral = - from open in Token.EqualTo(CosmosSqlToken.OpenBrace) - from props in ( - from key in AnyIdentifierOrKeyword - .Or(Token.EqualTo(CosmosSqlToken.DoubleQuotedString).Select(t => t.Span.ToStringValue()[1..^1])) - .Or(Token.EqualTo(CosmosSqlToken.StringLiteral).Select(t => t.Span.ToStringValue()[1..^1])) - from colon in Token.EqualTo(CosmosSqlToken.Colon) - from value in Superpower.Parse.Ref(() => Expr) - select new KeyValuePair(key, value) - ).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from close in Token.EqualTo(CosmosSqlToken.CloseBrace) - select (SqlExpression)new ObjectLiteralExpression(props); - - // Array literal: [ expr, expr, ... ] - private static readonly TokenListParser ArrayLiteral = - from open in Token.EqualTo(CosmosSqlToken.OpenBracket) - from elements in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from close in Token.EqualTo(CosmosSqlToken.CloseBracket) - select (SqlExpression)new ArrayLiteralExpression(elements); - - // Identifier or dotted path - private static readonly TokenListParser IdentExpr = - DottedPathWithIndex.Select(path => (SqlExpression)new IdentifierExpression(path)); - - // ARRAY(subquery): ARRAY keyword followed by a parenthesised SELECT - private static readonly TokenListParser ArraySubqueryCall = - from name in Token.EqualTo(CosmosSqlToken.Array) - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from subquery in Superpower.Parse.Ref(() => QueryParser) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (SqlExpression)new FunctionCallExpression("ARRAY", [new SubqueryExpression(subquery)]); - - // Primary: literal | parameter | function call | exists | parens | object | array | ident - private static readonly TokenListParser Primary = - StringLiteral - .Or(NumberLiteral) - .Or(TrueLiteral) - .Or(FalseLiteral) - .Or(NullLiteral) - .Or(UndefinedLiteral) - .Or(ParameterExpr) - .Or(ExistsExpr.Try()) - .Or(ArraySubqueryCall.Try()) - .Or(DottedFunctionCall.Try()) - .Or(FunctionCall.Try()) - .Or(Parens) - .Or(ObjectLiteral.Try()) - .Or(ArrayLiteral.Try()) - .Or(IdentExpr); - - // ── Unary ── - - private static readonly TokenListParser UnaryExpr = - (from op in Token.EqualTo(CosmosSqlToken.Not) - from operand in Superpower.Parse.Ref(() => UnaryExpr) - select (SqlExpression)new UnaryExpression(UnaryOp.Not, operand)) - .Or( - from op in Token.EqualTo(CosmosSqlToken.Minus) - from operand in Primary - select (SqlExpression)new UnaryExpression(UnaryOp.Negate, operand)) - .Or( - from op in Token.EqualTo(CosmosSqlToken.Tilde) - from operand in Primary - select (SqlExpression)new UnaryExpression(UnaryOp.BitwiseNot, operand)) - .Or(Primary); - - // ── Multiplicative: *, /, % ── - - private static readonly TokenListParser Multiplicative = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.Star).Select(_ => BinaryOp.Multiply) - .Or(Token.EqualTo(CosmosSqlToken.Slash).Select(_ => BinaryOp.Divide)) - .Or(Token.EqualTo(CosmosSqlToken.Percent).Select(_ => BinaryOp.Modulo)), - UnaryExpr, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Additive: +, - ── - - private static readonly TokenListParser Additive = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.Plus).Select(_ => BinaryOp.Add) - .Or(Token.EqualTo(CosmosSqlToken.Minus).Select(_ => BinaryOp.Subtract)), - Multiplicative, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Bitwise AND: & ── - - private static readonly TokenListParser BitwiseAndExpr = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.Ampersand).Select(_ => BinaryOp.BitwiseAnd), - Additive, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Bitwise XOR: ^ ── - - private static readonly TokenListParser BitwiseXorExpr = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.Caret).Select(_ => BinaryOp.BitwiseXor), - BitwiseAndExpr, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Bitwise OR: | ── - - private static readonly TokenListParser BitwiseOrExpr = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.Pipe).Select(_ => BinaryOp.BitwiseOr), - BitwiseXorExpr, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── String concat: || ── - - private static readonly TokenListParser StringConcatExpr = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.DoublePipe).Select(_ => BinaryOp.StringConcat), - BitwiseOrExpr, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Comparison: =, !=, <, >, <=, >=, LIKE ── - - private static readonly TokenListParser CompOp = - Token.EqualTo(CosmosSqlToken.Equals).Select(_ => BinaryOp.Equal) - .Or(Token.EqualTo(CosmosSqlToken.NotEquals).Select(_ => BinaryOp.NotEqual)) - .Or(Token.EqualTo(CosmosSqlToken.LessThanOrEqual).Select(_ => BinaryOp.LessThanOrEqual)) - .Or(Token.EqualTo(CosmosSqlToken.GreaterThanOrEqual).Select(_ => BinaryOp.GreaterThanOrEqual)) - .Or(Token.EqualTo(CosmosSqlToken.LessThan).Select(_ => BinaryOp.LessThan)) - .Or(Token.EqualTo(CosmosSqlToken.GreaterThan).Select(_ => BinaryOp.GreaterThan)) - .Or(Token.EqualTo(CosmosSqlToken.Like).Select(_ => BinaryOp.Like)); - - private static readonly TokenListParser Comparison = - from left in StringConcatExpr - from rest in ( - // BETWEEN low AND high - (from kw in Token.EqualTo(CosmosSqlToken.Between) - from low in StringConcatExpr - from and in Token.EqualTo(CosmosSqlToken.And) - from high in StringConcatExpr - select (Func)(l => new BetweenExpression(l, low, high))) - // NOT BETWEEN low AND high - .Or( - (from not in Token.EqualTo(CosmosSqlToken.Not) - from kw in Token.EqualTo(CosmosSqlToken.Between) - from low in StringConcatExpr - from and in Token.EqualTo(CosmosSqlToken.And) - from high in StringConcatExpr - select (Func)(l => new UnaryExpression(UnaryOp.Not, new BetweenExpression(l, low, high)))) - .Try()) - // IN (val, val, ...) - .Or( - from kw in Token.EqualTo(CosmosSqlToken.In) - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from vals in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (Func)(l => new InExpression(l, vals))) - // NOT IN (val, val, ...) - .Or( - (from not in Token.EqualTo(CosmosSqlToken.Not) - from kw in Token.EqualTo(CosmosSqlToken.In) - from open in Token.EqualTo(CosmosSqlToken.OpenParen) - from vals in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from close in Token.EqualTo(CosmosSqlToken.CloseParen) - select (Func)(l => new UnaryExpression(UnaryOp.Not, new InExpression(l, vals)))) - .Try()) - // LIKE pattern ESCAPE char - .Or( - (from kw in Token.EqualTo(CosmosSqlToken.Like) - from pattern in StringConcatExpr - from esc in Token.EqualTo(CosmosSqlToken.Escape) - from escChar in Token.EqualTo(CosmosSqlToken.StringLiteral) - select (Func)(l => new LikeExpression(l, pattern, escChar.Span.ToStringValue()[1..^1]))) - .Try()) - // NOT LIKE pattern ESCAPE char - .Or( - (from not in Token.EqualTo(CosmosSqlToken.Not) - from kw in Token.EqualTo(CosmosSqlToken.Like) - from pattern in StringConcatExpr - from esc in Token.EqualTo(CosmosSqlToken.Escape) - from escChar in Token.EqualTo(CosmosSqlToken.StringLiteral) - select (Func)(l => new UnaryExpression(UnaryOp.Not, new LikeExpression(l, pattern, escChar.Span.ToStringValue()[1..^1])))) - .Try()) - // NOT LIKE pattern (without ESCAPE) - .Or( - (from not in Token.EqualTo(CosmosSqlToken.Not) - from kw in Token.EqualTo(CosmosSqlToken.Like) - from pattern in StringConcatExpr - select (Func)(l => new UnaryExpression(UnaryOp.Not, new BinaryExpression(l, BinaryOp.Like, pattern)))) - .Try()) - // IS NOT NULL (must come before IS NULL to avoid consuming IS and failing on NOT) - .Or( - (from is_ in Token.EqualTo(CosmosSqlToken.Is) - from not_ in Token.EqualTo(CosmosSqlToken.Not) - from null_ in Token.EqualTo(CosmosSqlToken.Null) - select (Func)(l => new BinaryExpression(l, BinaryOp.NotEqual, new LiteralExpression(null)))) - .Try()) - // IS NULL - .Or( - from is_ in Token.EqualTo(CosmosSqlToken.Is) - from null_ in Token.EqualTo(CosmosSqlToken.Null) - select (Func)(l => new BinaryExpression(l, BinaryOp.Equal, new LiteralExpression(null)))) - // op right - .Or( - from op in CompOp - from right in StringConcatExpr - select (Func)(l => new BinaryExpression(l, op, right))) - ).OptionalOrDefault(null) - select rest != null ? rest(left) : left; - - // ── Logical AND ── - - private static readonly TokenListParser AndExpr = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.And).Select(_ => BinaryOp.And), - Comparison, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Logical OR ── - - private static readonly TokenListParser OrExpr = - Superpower.Parse.Chain( - Token.EqualTo(CosmosSqlToken.Or).Select(_ => BinaryOp.Or), - AndExpr, - (op, left, right) => new BinaryExpression(left, op, right)); - - // ── Null coalesce: ?? ── - - private static readonly TokenListParser CoalesceExpr = - from left in OrExpr - from rest in ( - from op in Token.EqualTo(CosmosSqlToken.QuestionQuestion) - from right in Superpower.Parse.Ref(() => CoalesceExpr) - select right - ).OptionalOrDefault(null) - select rest != null ? new CoalesceExpression(left, rest) : left; - - // ── Ternary: cond ? then : else ── - - private static readonly TokenListParser TernaryExpr = - from cond in CoalesceExpr - from rest in ( - from q in Token.EqualTo(CosmosSqlToken.Question) - from ifTrue in Superpower.Parse.Ref(() => Expr) - from colon in Token.EqualTo(CosmosSqlToken.Colon) - from ifFalse in Superpower.Parse.Ref(() => Expr) - select new { ifTrue, ifFalse } - ).OptionalOrDefault(null) - select rest != null ? (SqlExpression)new TernaryExpression(cond, rest.ifTrue, rest.ifFalse) : cond; - - // ── Top-level expression ── - - private static readonly TokenListParser Expr = TernaryExpr; - - // ────────────────────────────────────────────── - // SELECT field parsing - // ────────────────────────────────────────────── - - private static readonly TokenListParser StarField = - Token.EqualTo(CosmosSqlToken.Star).Select(_ => new SelectField("*", null)); - - // Handle "c.*" (alias DOT STAR) as equivalent to "*" - private static readonly TokenListParser AliasDotStarField = - from ident in AnyIdentifierOrKeyword - from dot in Token.EqualTo(CosmosSqlToken.Dot) - from star in Token.EqualTo(CosmosSqlToken.Star) - select new SelectField("*", null); - - private static readonly TokenListParser ExpressionField = - from expr in Expr - from alias in ( - from as_ in Token.EqualTo(CosmosSqlToken.As) - from name in AnyIdentifierOrKeyword - select name - ).OptionalOrDefault(null) - select new SelectField(ExprToString(expr), alias, expr); - - private static readonly TokenListParser SingleSelectField = - StarField.Try().Or(AliasDotStarField.Try()).Or(ExpressionField); - - // ────────────────────────────────────────────── - // JOIN clause parsing - // ────────────────────────────────────────────── - - private static readonly TokenListParser JoinParser = - from kw in Token.EqualTo(CosmosSqlToken.Join) - from alias in AnyIdentifierOrKeyword - from in_ in Token.EqualTo(CosmosSqlToken.In) - from source in DottedPathWithIndex - select ParseJoinSource(alias, source); - - // ────────────────────────────────────────────── - // ORDER BY parsing - // ────────────────────────────────────────────── - - private static readonly TokenListParser OrderByFieldParser = - (from expr in Superpower.Parse.Ref(() => Expr) - from dir in Token.EqualTo(CosmosSqlToken.Asc).Select(_ => true) - .Or(Token.EqualTo(CosmosSqlToken.Desc).Select(_ => false)) - .OptionalOrDefault(true) - select expr is IdentifierExpression ident - ? new OrderByField(ident.Name, dir) - : new OrderByField(null, dir, expr)).Try() - .Or( - from field in DottedPathWithIndex - from dir in Token.EqualTo(CosmosSqlToken.Asc).Select(_ => true) - .Or(Token.EqualTo(CosmosSqlToken.Desc).Select(_ => false)) - .OptionalOrDefault(true) - select new OrderByField(field, dir)); - - // ────────────────────────────────────────────── - // Full query parsing - // ────────────────────────────────────────────── - - private static readonly TokenListParser QueryParser = - from select_ in Token.EqualTo(CosmosSqlToken.Select) - from distinct in Token.EqualTo(CosmosSqlToken.Distinct).OptionalOrDefault() - from top in ( - from topKw in Token.EqualTo(CosmosSqlToken.Top) - from n in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(t => int.Parse(t.Span.ToStringValue())) - select (int?)n - ).OptionalOrDefault(null) - from value_ in Token.EqualTo(CosmosSqlToken.Value).OptionalOrDefault() - from fields in SingleSelectField.ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - from fromKw in Token.EqualTo(CosmosSqlToken.From) - from fromFirstIdent in AnyIdentifierOrKeyword - from fromClause in ( - // FROM alias IN source.path - from in_ in Token.EqualTo(CosmosSqlToken.In) - from source in DottedPath - select (Alias: fromFirstIdent, Source: (string)source) - ).Try().Or( - // FROM source AS alias - from as_ in Token.EqualTo(CosmosSqlToken.As) - from alias in AnyIdentifierOrKeyword - select (Alias: alias, Source: (string)null) - ).Try().Or( - // FROM source alias (implicit alias without AS keyword — only plain identifiers) - from alias in Token.EqualTo(CosmosSqlToken.Identifier).Select(TokenSpanToString) - select (Alias: alias, Source: (string)null) - ).Try().OptionalOrDefault((Alias: fromFirstIdent, Source: (string)null)) - from joins in JoinParser.Many() - from where_ in ( - from whereKw in Token.EqualTo(CosmosSqlToken.Where) - from expr in Expr - select expr - ).OptionalOrDefault(null) - from groupBy in ( - from groupKw in Token.EqualTo(CosmosSqlToken.Group) - from byKw in Token.EqualTo(CosmosSqlToken.By) - from groupFields in Expr.ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - select groupFields - ).OptionalOrDefault(null) - from having in ( - from havingKw in Token.EqualTo(CosmosSqlToken.Having) - from expr in Expr - select expr - ).OptionalOrDefault(null) - from orderByResult in ( - from orderKw in Token.EqualTo(CosmosSqlToken.Order) - from byKw in Token.EqualTo(CosmosSqlToken.By) - from result in ( - from rankKw in Token.EqualTo(CosmosSqlToken.Identifier) - .Where(t => string.Equals(t.Span.ToStringValue(), "RANK", StringComparison.OrdinalIgnoreCase)) - from expr in Expr - select (Fields: (OrderByField[])null, RankExpr: (SqlExpression)expr) - ).Try().Or( - from orderFields in OrderByFieldParser.ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) - select (Fields: orderFields, RankExpr: (SqlExpression)null) - ) - select result - ).OptionalOrDefault(default) - from offset in ( - from offsetKw in Token.EqualTo(CosmosSqlToken.Offset) - from n in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(t => int.Parse(t.Span.ToStringValue())) - select (int?)n - ).OptionalOrDefault(null) - from limit in ( - from limitKw in Token.EqualTo(CosmosSqlToken.Limit) - from n in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(t => int.Parse(t.Span.ToStringValue())) - select (int?)n - ).OptionalOrDefault(null) - select BuildQuery( - distinct.HasValue, top, value_.HasValue, fields, - fromClause.Alias, fromClause.Source, joins, where_, - groupBy?.Select(ExprToString).ToArray(), - groupBy?.ToArray(), - having, orderByResult.Fields, offset, limit, orderByResult.RankExpr); - - // ────────────────────────────────────────────── - // Public API - // ────────────────────────────────────────────── - - private const int ParseCacheMaxSize = 5000; - private static readonly ConcurrentDictionary ParseCache = new(); - - public static CosmosSqlQuery Parse(string sql) - { - if (ParseCache.TryGetValue(sql, out var cached)) - { - return cached; - } - - CosmosSqlQuery parsed; - try - { - var tokens = CosmosSqlTokenizer.Tokenize(sql); - parsed = QueryParser.Parse(tokens); - } - catch (Exception ex) - { - throw new NotSupportedException($"Failed to parse Cosmos SQL query: {sql}", ex); - } - - if (ParseCache.Count < ParseCacheMaxSize) - { - ParseCache.TryAdd(sql, parsed); - } - - return parsed; - } - - public static bool TryParse(string sql, out CosmosSqlQuery result) - { - try - { - result = Parse(sql); - return true; - } - catch - { - result = null; - return false; - } - } - - // Backward-compatible: convert SqlExpression tree to WhereExpression tree - public static WhereExpression ToWhereExpression(SqlExpression expr) - { - if (expr is null) - { - return null; - } - - switch (expr) - { - case BinaryExpression bin when bin.Operator == BinaryOp.And: - return new AndCondition(ToWhereExpression(bin.Left), ToWhereExpression(bin.Right)); - - case BinaryExpression bin when bin.Operator == BinaryOp.Or: - return new OrCondition(ToWhereExpression(bin.Left), ToWhereExpression(bin.Right)); - - case UnaryExpression { Operator: UnaryOp.Not } unary: - return new NotCondition(ToWhereExpression(unary.Operand)); - - case ExistsExpression exists: - return new ExistsCondition(exists.RawSubquery); - - case FunctionCallExpression func: - if (func.FunctionName.StartsWith("UDF.", StringComparison.OrdinalIgnoreCase) || - func.FunctionName.StartsWith("ST_", StringComparison.OrdinalIgnoreCase)) - { - return new SqlExpressionCondition(func); - } - - var hasComplexArgs = func.Arguments.Any(a => a is ObjectLiteralExpression or ArrayLiteralExpression or FunctionCallExpression); - if (hasComplexArgs) - { - return new SqlExpressionCondition(func); - } - - if (LegacyFunctionNames.Contains(func.FunctionName)) - { - var args = func.Arguments.Select(ExprToString).ToArray(); - return new FunctionCondition(func.FunctionName, args); - } - - return new SqlExpressionCondition(func); - - case BinaryExpression bin when IsComparisonOp(bin.Operator): - if (ContainsFunctionCall(bin.Left) || ContainsFunctionCall(bin.Right) || - ContainsArithmetic(bin.Left) || ContainsArithmetic(bin.Right) || - ContainsComplexExpression(bin.Left) || ContainsComplexExpression(bin.Right)) - { - return new SqlExpressionCondition(bin); - } - - var compOp = bin.Operator switch - { - BinaryOp.Equal => ComparisonOp.Equal, - BinaryOp.NotEqual => ComparisonOp.NotEqual, - BinaryOp.LessThan => ComparisonOp.LessThan, - BinaryOp.GreaterThan => ComparisonOp.GreaterThan, - BinaryOp.LessThanOrEqual => ComparisonOp.LessThanOrEqual, - BinaryOp.GreaterThanOrEqual => ComparisonOp.GreaterThanOrEqual, - BinaryOp.Like => ComparisonOp.Like, - _ => ComparisonOp.Equal, - }; - return new ComparisonCondition(ExprToString(bin.Left), compOp, ExprToString(bin.Right)); - - default: - return new SqlExpressionCondition(expr); - } - } - - /// - /// Removes SDK-injected IS_DEFINED(alias) and literal true nodes from - /// AND chains in a WHERE expression AST, returning the simplified user condition. - /// Only strips IS_DEFINED() calls whose argument is exactly the FROM alias - /// (the SDK injects IS_DEFINED(root) for ORDER BY, never IS_DEFINED(root.field)), - /// preserving any legitimate IS_DEFINED() from user code. - /// Returns null when the entire expression reduces to nothing. - /// - public static SqlExpression SimplifySdkWhereExpression( - SqlExpression expr, string fromAlias = null) - { - return SimplifyCore(expr, fromAlias); - } - - private static SqlExpression SimplifyCore(SqlExpression expr, string fromAlias) - { - if (expr is null) - { - return null; - } - - if (expr is LiteralExpression { Value: true }) - { - return null; - } - - if (expr is FunctionCallExpression func && - string.Equals(func.FunctionName, "IS_DEFINED", StringComparison.OrdinalIgnoreCase) && - func.Arguments.Length == 1) - { - var argStr = ExprToString(func.Arguments[0]); - // Only strip when: no alias specified (backward-compat), or arg IS the FROM alias exactly. - // The SDK injects IS_DEFINED(root) — never IS_DEFINED(root.field) — so this - // preserves any user-written IS_DEFINED on specific field paths. - if (fromAlias is null || string.Equals(argStr, fromAlias, StringComparison.OrdinalIgnoreCase)) - { - return null; - } - } - - if (expr is BinaryExpression { Operator: BinaryOp.And } and) - { - var left = SimplifyCore(and.Left, fromAlias); - var right = SimplifyCore(and.Right, fromAlias); - if (left is null && right is null) - { - return null; - } - - if (left is null) - { - return right; - } - - if (right is null) - { - return left; - } - - return new BinaryExpression(left, BinaryOp.And, right); - } - - return expr; - } - - /// - /// Rebuilds a Superpower-parsed into clean SQL that - /// can execute. Strips SDK-injected WHERE clauses - /// (IS_DEFINED(alias), literal true), normalises bracket notation - /// (root["name"]) to dot notation (root.name) via the AST, and - /// reconstructs SELECT, WHERE, ORDER BY, TOP, OFFSET/LIMIT, DISTINCT, and GROUP BY - /// clauses from the parsed structure. - /// - /// For ORDER BY queries where the SDK rewrites the SELECT to include orderByItems - /// and payload, this emits SELECT VALUE alias to return full documents. - /// For all other queries, the original SELECT expressions are emitted from the AST. - /// - /// - public static string SimplifySdkQuery(CosmosSqlQuery parsed) - { - var fromAlias = parsed.FromAlias; - - var isOrderByQuery = parsed.SelectFields.Any(field => - string.Equals(field.Alias, "orderByItems", StringComparison.OrdinalIgnoreCase)); - - // SELECT clause - var sb = new System.Text.StringBuilder("SELECT "); - if (parsed.IsDistinct) - { - sb.Append("DISTINCT "); - } - - if (parsed.TopCount.HasValue) - { - sb.Append($"TOP {parsed.TopCount.Value} "); - } - - if (isOrderByQuery) - { - sb.Append($"VALUE {fromAlias}"); - } - else if (parsed.IsValueSelect) - { - var selectExprs = parsed.SelectFields - .Select(field => ExprToString(field.SqlExpr ?? new IdentifierExpression(field.Expression))) - .ToArray(); - sb.Append("VALUE "); - sb.Append(string.Join(", ", selectExprs)); - } - else if (parsed.IsSelectAll) - { - sb.Append('*'); - } - else - { - var selectParts = parsed.SelectFields.Select(field => - { - var expr = field.SqlExpr is not null ? ExprToString(field.SqlExpr) : field.Expression; - return field.Alias is not null ? $"{expr} AS {field.Alias}" : expr; - }); - sb.Append(string.Join(", ", selectParts)); - } - - // FROM - if (parsed.FromSource is not null) - { - sb.Append($" FROM {fromAlias} IN {parsed.FromSource}"); - } - else - { - sb.Append($" FROM {fromAlias}"); - } - - // JOINs - if (parsed.Joins is { Length: > 0 }) - { - foreach (var join in parsed.Joins) - { - sb.Append($" JOIN {join.Alias} IN {join.SourceAlias}.{join.ArrayField}"); - } - } - - // WHERE — strip SDK-injected nodes - var simplifiedWhere = SimplifySdkWhereExpression(parsed.WhereExpr, fromAlias); - if (simplifiedWhere is not null) - { - sb.Append($" WHERE {ExprToString(simplifiedWhere)}"); - } - - // GROUP BY - if (parsed.GroupByFields is { Length: > 0 }) - { - sb.Append($" GROUP BY {string.Join(", ", parsed.GroupByFields)}"); - } - - // HAVING - if (parsed.HavingExpr is not null) - { - sb.Append($" HAVING {ExprToString(parsed.HavingExpr)}"); - } - - // ORDER BY - if (parsed.OrderByFields is { Length: > 0 }) - { - var orderByStr = string.Join(", ", parsed.OrderByFields.Select(field => - $"{field.Field ?? ExprToString(field.Expression)} {(field.Ascending ? "ASC" : "DESC")}")); - sb.Append($" ORDER BY {orderByStr}"); - } - else if (parsed.RankExpression is not null) - { - sb.Append($" ORDER BY RANK {ExprToString(parsed.RankExpression)}"); - } - - // OFFSET / LIMIT - if (parsed.Offset.HasValue) - { - sb.Append($" OFFSET {parsed.Offset.Value}"); - } - - if (parsed.Limit.HasValue) - { - sb.Append($" LIMIT {parsed.Limit.Value}"); - } - - return sb.ToString(); - } - - // ────────────────────────────────────────────── - // Internal helpers - // ────────────────────────────────────────────── - - internal static WhereExpression ParseWhereExpression(string expression) - { - var wrappedSql = $"SELECT * FROM c WHERE {expression}"; - var query = Parse(wrappedSql); - return query.Where; - } - - private static bool IsComparisonOp(BinaryOp op) => - op is BinaryOp.Equal or BinaryOp.NotEqual or BinaryOp.LessThan or BinaryOp.GreaterThan - or BinaryOp.LessThanOrEqual or BinaryOp.GreaterThanOrEqual or BinaryOp.Like; - - private static bool ContainsFunctionCall(SqlExpression expr) => - expr switch - { - FunctionCallExpression => true, - BinaryExpression bin => ContainsFunctionCall(bin.Left) || ContainsFunctionCall(bin.Right), - UnaryExpression unary => ContainsFunctionCall(unary.Operand), - _ => false - }; - - private static bool ContainsArithmetic(SqlExpression expr) => - expr switch - { - BinaryExpression bin when !IsComparisonOp(bin.Operator) => true, - BinaryExpression bin => ContainsArithmetic(bin.Left) || ContainsArithmetic(bin.Right), - UnaryExpression unary => ContainsArithmetic(unary.Operand), - _ => false - }; - - /// - /// Returns true when the expression contains a subquery, coalesce, or ternary — - /// any of which require full expression evaluation rather than string-based ResolveValue. - /// - private static bool ContainsComplexExpression(SqlExpression expr) => - expr switch - { - SubqueryExpression => true, - CoalesceExpression => true, - TernaryExpression => true, - BinaryExpression bin => ContainsComplexExpression(bin.Left) || ContainsComplexExpression(bin.Right), - UnaryExpression unary => ContainsComplexExpression(unary.Operand), - _ => false - }; - - public static string ExprToString(SqlExpression expr) - { - return expr switch - { - LiteralExpression { Value: null } => "null", - LiteralExpression { Value: string s } => $"'{s}'", - LiteralExpression { Value: bool b } => b ? "true" : "false", - LiteralExpression lit => lit.Value?.ToString() ?? "null", - UndefinedLiteralExpression => "undefined", - IdentifierExpression ident => ident.Name, - ParameterExpression param => param.Name, - PropertyAccessExpression prop => $"{ExprToString(prop.Object)}.{prop.Property}", - IndexAccessExpression idx => $"{ExprToString(idx.Object)}[{ExprToString(idx.Index)}]", - FunctionCallExpression func => $"{func.FunctionName}({string.Join(", ", func.Arguments.Select(ExprToString))})", - BinaryExpression { Operator: BinaryOp.And or BinaryOp.Or } bin => - $"({ExprToString(bin.Left)} {BinaryOpToString(bin.Operator)} {ExprToString(bin.Right)})", - BinaryExpression bin => $"{ExprToString(bin.Left)} {BinaryOpToString(bin.Operator)} {ExprToString(bin.Right)}", - UnaryExpression unary => $"{UnaryOpToString(unary.Operator)} {ExprToString(unary.Operand)}", - BetweenExpression betw => $"{ExprToString(betw.Value)} BETWEEN {ExprToString(betw.Low)} AND {ExprToString(betw.High)}", - InExpression inExpr => $"{ExprToString(inExpr.Value)} IN ({string.Join(", ", inExpr.List.Select(ExprToString))})", - LikeExpression like => like.EscapeChar is not null - ? $"{ExprToString(like.Value)} LIKE {ExprToString(like.Pattern)} ESCAPE '{like.EscapeChar}'" - : $"{ExprToString(like.Value)} LIKE {ExprToString(like.Pattern)}", - ExistsExpression exists => $"EXISTS({exists.RawSubquery})", - SubqueryExpression sub => $"({SubqueryToString(sub.Subquery)})", - TernaryExpression tern => $"{ExprToString(tern.Condition)} ? {ExprToString(tern.IfTrue)} : {ExprToString(tern.IfFalse)}", - CoalesceExpression coal => $"{ExprToString(coal.Left)} ?? {ExprToString(coal.Right)}", - ObjectLiteralExpression obj => "{" + string.Join(", ", obj.Properties.Select(p => $"{p.Key}: {ExprToString(p.Value)}")) + "}", - ArrayLiteralExpression arr => "[" + string.Join(", ", arr.Elements.Select(ExprToString)) + "]", - _ => expr.ToString(), - }; - } - - private static string SubqueryToString(CosmosSqlQuery sub) - { - var sb = new System.Text.StringBuilder("SELECT "); - if (sub.IsDistinct) - { - sb.Append("DISTINCT "); - } - - if (sub.TopCount.HasValue) - { - sb.Append($"TOP {sub.TopCount.Value} "); - } - - if (sub.IsValueSelect) - { - sb.Append("VALUE "); - } - - if (sub.IsSelectAll) - { - sb.Append('*'); - } - else - { - var selectParts = sub.SelectFields.Select(field => - { - var expr = field.SqlExpr is not null ? ExprToString(field.SqlExpr) : field.Expression; - return field.Alias is not null ? $"{expr} AS {field.Alias}" : expr; - }); - sb.Append(string.Join(", ", selectParts)); - } - - if (sub.FromSource is not null) - { - sb.Append($" FROM {sub.FromAlias} IN {sub.FromSource}"); - } - else - { - sb.Append($" FROM {sub.FromAlias}"); - } - - if (sub.Joins is { Length: > 0 }) - { - foreach (var join in sub.Joins) - { - sb.Append($" JOIN {join.Alias} IN {join.SourceAlias}.{join.ArrayField}"); - } - } - - if (sub.WhereExpr is not null) - { - sb.Append($" WHERE {ExprToString(sub.WhereExpr)}"); - } - - if (sub.GroupByFields is { Length: > 0 }) - { - sb.Append($" GROUP BY {string.Join(", ", sub.GroupByFields)}"); - } - - if (sub.HavingExpr is not null) - { - sb.Append($" HAVING {ExprToString(sub.HavingExpr)}"); - } - - if (sub.OrderByFields is { Length: > 0 }) - { - var orderByStr = string.Join(", ", sub.OrderByFields.Select(field => - $"{field.Field} {(field.Ascending ? "ASC" : "DESC")}")); - sb.Append($" ORDER BY {orderByStr}"); - } - - if (sub.Offset.HasValue) - { - sb.Append($" OFFSET {sub.Offset.Value}"); - } - - if (sub.Limit.HasValue) - { - sb.Append($" LIMIT {sub.Limit.Value}"); - } - - return sb.ToString(); - } - - private static string BinaryOpToString(BinaryOp op) => op switch - { - BinaryOp.Equal => "=", - BinaryOp.NotEqual => "!=", - BinaryOp.LessThan => "<", - BinaryOp.GreaterThan => ">", - BinaryOp.LessThanOrEqual => "<=", - BinaryOp.GreaterThanOrEqual => ">=", - BinaryOp.And => "AND", - BinaryOp.Or => "OR", - BinaryOp.Add => "+", - BinaryOp.Subtract => "-", - BinaryOp.Multiply => "*", - BinaryOp.Divide => "/", - BinaryOp.Modulo => "%", - BinaryOp.Like => "LIKE", - BinaryOp.StringConcat => "||", - BinaryOp.BitwiseAnd => "&", - BinaryOp.BitwiseOr => "|", - BinaryOp.BitwiseXor => "^", - _ => op.ToString(), - }; - - private static string UnaryOpToString(UnaryOp op) => op switch - { - UnaryOp.Not => "NOT", - UnaryOp.Negate => "-", - UnaryOp.BitwiseNot => "~", - _ => op.ToString(), - }; - - private static JoinClause ParseJoinSource(string alias, string source) - { - var dotIdx = source.IndexOf('.'); - if (dotIdx < 0) - { - return new JoinClause(alias, source, source); - } - - return new JoinClause(alias, source[..dotIdx], source[(dotIdx + 1)..]); - } - - private static CosmosSqlQuery BuildQuery( - bool isDistinct, int? top, bool isValue, - SelectField[] fields, string fromAlias, string fromSource, - JoinClause[] joins, SqlExpression whereExpr, - string[] groupBy, SqlExpression[] groupByExpressions, SqlExpression havingExpr, - OrderByField[] orderByFields, - int? offset, int? limit, SqlExpression rankExpr = null) - { - var isSelectAll = fields.Length == 1 && fields[0].Expression == "*"; - var where = whereExpr != null ? ToWhereExpression(whereExpr) : null; - // HAVING always uses SqlExpressionCondition to preserve aggregate function expressions - // (ToWhereExpression would decompose AND/OR into AndCondition/OrCondition with string-based - // comparison operands, losing the ability to evaluate aggregate functions like COUNT/SUM). - var having = havingExpr != null ? new SqlExpressionCondition(havingExpr) : null; - var rawHavingExpr = havingExpr; - - // Backward compat: first join, first order by - var firstJoin = joins.Length > 0 ? joins[0] : null; - OrderByClause legacyOrderBy = null; - if (orderByFields is { Length: > 0 }) - { - legacyOrderBy = new OrderByClause(orderByFields[0].Field, orderByFields[0].Ascending); - } - - return new CosmosSqlQuery( - SelectFields: fields, - IsSelectAll: isSelectAll, - TopCount: top, - FromAlias: fromAlias, - Join: firstJoin, - Where: where, - Offset: offset, - Limit: limit, - OrderBy: legacyOrderBy, - IsDistinct: isDistinct, - IsValueSelect: isValue, - Joins: joins, - OrderByFields: orderByFields, - GroupByFields: groupBy, - GroupByExpressions: groupByExpressions, - Having: having, - WhereExpr: whereExpr, - HavingExpr: rawHavingExpr, - FromSource: fromSource, - RankExpression: rankExpr); - } - - // Captures tokens inside balanced parentheses as a raw string - private static TokenListParser CaptureBalancedParens() - { - return input => - { - var depth = 0; - var startPos = input.Position; - var current = input; - var parts = new List(); - - while (!current.IsAtEnd) - { - var token = current.ConsumeToken(); - if (!token.HasValue) - { - break; - } - - if (token.Value.Kind == CosmosSqlToken.OpenParen) - { - depth++; - parts.Add("("); - current = token.Remainder; - } - else if (token.Value.Kind == CosmosSqlToken.CloseParen) - { - if (depth == 0) - { - return TokenListParserResult.Value(string.Join(" ", parts), input, current); - } - depth--; - parts.Add(")"); - current = token.Remainder; - } - else - { - parts.Add(token.Value.Span.ToStringValue()); - current = token.Remainder; - } - } - - return TokenListParserResult.Value(string.Join(" ", parts), input, current); - }; - } + private static readonly HashSet LegacyFunctionNames = new(StringComparer.OrdinalIgnoreCase) + { + "STARTSWITH", "ENDSWITH", "CONTAINS", "ARRAY_CONTAINS", "IS_DEFINED", "IS_NULL", + }; + + // ── Helpers ── + + private static string TokenSpanToString(Token token) => + token.Span.ToStringValue(); + + private static TokenListParser AnyIdentifierOrKeyword => + Token.EqualTo(CosmosSqlToken.Identifier).Select(TokenSpanToString) + .Or(Token.EqualTo(CosmosSqlToken.Value).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Array).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Defined).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Asc).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Desc).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Top).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Limit).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Offset).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.Null).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.True).Select(TokenSpanToString)) + .Or(Token.EqualTo(CosmosSqlToken.False).Select(TokenSpanToString)); + + // ── Dotted path: ident.ident.ident ── + + private static readonly TokenListParser DottedPath = + from first in AnyIdentifierOrKeyword + from rest in ( + from dot in Token.EqualTo(CosmosSqlToken.Dot) + from next in AnyIdentifierOrKeyword + select "." + next + ).Many() + select first + string.Concat(rest); + + // ── Dotted path with optional array indexing: ident.ident[0].ident ── + + // Helper: a bracketed string inside [ ] can be single- or double-quoted + private static readonly TokenListParser BracketedStringContent = + Token.EqualTo(CosmosSqlToken.StringLiteral).Select(t => t.Span.ToStringValue()[1..^1]) + .Or(Token.EqualTo(CosmosSqlToken.DoubleQuotedString).Select(t => t.Span.ToStringValue()[1..^1])); + + private static readonly TokenListParser DottedPathWithIndex = + from first in AnyIdentifierOrKeyword + from rest in ( + from dot in Token.EqualTo(CosmosSqlToken.Dot) + from next in AnyIdentifierOrKeyword + select "." + next + ).Or( + (from open in Token.EqualTo(CosmosSqlToken.OpenBracket) + from idx in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(TokenSpanToString) + from close in Token.EqualTo(CosmosSqlToken.CloseBracket) + select "[" + idx + "]").Try() + ).Or( + (from open in Token.EqualTo(CosmosSqlToken.OpenBracket) + from str in BracketedStringContent + from close in Token.EqualTo(CosmosSqlToken.CloseBracket) + select "." + str).Try() + ).Many() + select first + string.Concat(rest); + + // ── Primary expression ── + + // String literal expression: accepts both single-quoted (Cosmos SQL standard) and + // double-quoted strings (used by the SDK's LINQ provider in generated queries). + private static readonly TokenListParser StringLiteral = + Token.EqualTo(CosmosSqlToken.StringLiteral) + .Select(t => + { + var raw = t.Span.ToStringValue(); + var unquoted = raw[1..^1].Replace("''", "'"); + return (SqlExpression)new LiteralExpression(unquoted); + }) + .Or(Token.EqualTo(CosmosSqlToken.DoubleQuotedString) + .Select(t => + { + var raw = t.Span.ToStringValue(); + var unquoted = raw[1..^1]; + return (SqlExpression)new LiteralExpression(unquoted); + })); + + private static readonly TokenListParser NumberLiteral = + Token.EqualTo(CosmosSqlToken.NumberLiteral) + .Select(t => + { + var raw = t.Span.ToStringValue(); + if (long.TryParse(raw, out var longVal)) + { + return (SqlExpression)new LiteralExpression(longVal); + } + + return (SqlExpression)new LiteralExpression(double.Parse(raw, System.Globalization.CultureInfo.InvariantCulture)); + }); + + private static readonly TokenListParser TrueLiteral = + Token.EqualTo(CosmosSqlToken.True).Select(_ => (SqlExpression)new LiteralExpression(true)); + + private static readonly TokenListParser FalseLiteral = + Token.EqualTo(CosmosSqlToken.False).Select(_ => (SqlExpression)new LiteralExpression(false)); + + private static readonly TokenListParser NullLiteral = + Token.EqualTo(CosmosSqlToken.Null).Select(_ => (SqlExpression)new LiteralExpression(null)); + + private static readonly TokenListParser UndefinedLiteral = + Token.EqualTo(CosmosSqlToken.Undefined).Select(_ => (SqlExpression)new UndefinedLiteralExpression()); + + private static readonly TokenListParser ParameterExpr = + Token.EqualTo(CosmosSqlToken.Parameter) + .Select(t => (SqlExpression)new ParameterExpression(t.Span.ToStringValue())); + + // Function call: FUNC_NAME(args...) + // Supports * as a function argument (e.g. COUNT(*)) + private static readonly TokenListParser FunctionCall = + from name in AnyIdentifierOrKeyword + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from args in Token.EqualTo(CosmosSqlToken.Star).Select(_ => (SqlExpression)new IdentifierExpression("*")) + .Or(Superpower.Parse.Ref(() => Expr)) + .ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (SqlExpression)new FunctionCallExpression(name.ToUpperInvariant(), args); + + // Dotted function call: namespace.FUNC_NAME(args...) — supports udf.xxx() + // UDF names preserve original casing (Cosmos DB UDFs are case-sensitive); + // built-in dotted functions (e.g. ST_DISTANCE) are uppercased. + private static readonly TokenListParser DottedFunctionCall = + from first in AnyIdentifierOrKeyword + from dot in Token.EqualTo(CosmosSqlToken.Dot) + from second in AnyIdentifierOrKeyword + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from args in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (SqlExpression)new FunctionCallExpression( + first.Equals("udf", StringComparison.OrdinalIgnoreCase) + ? "UDF." + second // preserve UDF name casing + : (first + "." + second).ToUpperInvariant(), args); + + // EXISTS( subquery-text ) + private static readonly TokenListParser ExistsExpr = + from kw in Token.EqualTo(CosmosSqlToken.Exists) + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from inner in CaptureBalancedParens() + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (SqlExpression)new ExistsExpression(inner); + + // Subquery expression: (SELECT ...) — a full SELECT statement inside parentheses + private static readonly TokenListParser SubqueryParens = + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from subquery in Superpower.Parse.Ref(() => QueryParser) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (SqlExpression)new SubqueryExpression(subquery); + + // Parenthesized expression (try subquery first, fall back to regular expression) + private static readonly TokenListParser Parens = + SubqueryParens.Try() + .Or( + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from expr in Superpower.Parse.Ref(() => Expr) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select expr); + + // Object literal: { key: expr, key: expr, ... } + // Keys can be identifiers or double-quoted strings (for SDK-generated queries like {"item": root.value}) + private static readonly TokenListParser ObjectLiteral = + from open in Token.EqualTo(CosmosSqlToken.OpenBrace) + from props in ( + from key in AnyIdentifierOrKeyword + .Or(Token.EqualTo(CosmosSqlToken.DoubleQuotedString).Select(t => t.Span.ToStringValue()[1..^1])) + .Or(Token.EqualTo(CosmosSqlToken.StringLiteral).Select(t => t.Span.ToStringValue()[1..^1])) + from colon in Token.EqualTo(CosmosSqlToken.Colon) + from value in Superpower.Parse.Ref(() => Expr) + select new KeyValuePair(key, value) + ).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from close in Token.EqualTo(CosmosSqlToken.CloseBrace) + select (SqlExpression)new ObjectLiteralExpression(props); + + // Array literal: [ expr, expr, ... ] + private static readonly TokenListParser ArrayLiteral = + from open in Token.EqualTo(CosmosSqlToken.OpenBracket) + from elements in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from close in Token.EqualTo(CosmosSqlToken.CloseBracket) + select (SqlExpression)new ArrayLiteralExpression(elements); + + // Identifier or dotted path + private static readonly TokenListParser IdentExpr = + DottedPathWithIndex.Select(path => (SqlExpression)new IdentifierExpression(path)); + + // ARRAY(subquery): ARRAY keyword followed by a parenthesised SELECT + private static readonly TokenListParser ArraySubqueryCall = + from name in Token.EqualTo(CosmosSqlToken.Array) + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from subquery in Superpower.Parse.Ref(() => QueryParser) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (SqlExpression)new FunctionCallExpression("ARRAY", [new SubqueryExpression(subquery)]); + + // Primary: literal | parameter | function call | exists | parens | object | array | ident + private static readonly TokenListParser Primary = + StringLiteral + .Or(NumberLiteral) + .Or(TrueLiteral) + .Or(FalseLiteral) + .Or(NullLiteral) + .Or(UndefinedLiteral) + .Or(ParameterExpr) + .Or(ExistsExpr.Try()) + .Or(ArraySubqueryCall.Try()) + .Or(DottedFunctionCall.Try()) + .Or(FunctionCall.Try()) + .Or(Parens) + .Or(ObjectLiteral.Try()) + .Or(ArrayLiteral.Try()) + .Or(IdentExpr); + + // ── Unary ── + + private static readonly TokenListParser UnaryExpr = + (from op in Token.EqualTo(CosmosSqlToken.Not) + from operand in Superpower.Parse.Ref(() => UnaryExpr) + select (SqlExpression)new UnaryExpression(UnaryOp.Not, operand)) + .Or( + from op in Token.EqualTo(CosmosSqlToken.Minus) + from operand in Primary + select (SqlExpression)new UnaryExpression(UnaryOp.Negate, operand)) + .Or( + from op in Token.EqualTo(CosmosSqlToken.Tilde) + from operand in Primary + select (SqlExpression)new UnaryExpression(UnaryOp.BitwiseNot, operand)) + .Or(Primary); + + // ── Multiplicative: *, /, % ── + + private static readonly TokenListParser Multiplicative = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.Star).Select(_ => BinaryOp.Multiply) + .Or(Token.EqualTo(CosmosSqlToken.Slash).Select(_ => BinaryOp.Divide)) + .Or(Token.EqualTo(CosmosSqlToken.Percent).Select(_ => BinaryOp.Modulo)), + UnaryExpr, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Additive: +, - ── + + private static readonly TokenListParser Additive = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.Plus).Select(_ => BinaryOp.Add) + .Or(Token.EqualTo(CosmosSqlToken.Minus).Select(_ => BinaryOp.Subtract)), + Multiplicative, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Bitwise AND: & ── + + private static readonly TokenListParser BitwiseAndExpr = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.Ampersand).Select(_ => BinaryOp.BitwiseAnd), + Additive, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Bitwise XOR: ^ ── + + private static readonly TokenListParser BitwiseXorExpr = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.Caret).Select(_ => BinaryOp.BitwiseXor), + BitwiseAndExpr, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Bitwise OR: | ── + + private static readonly TokenListParser BitwiseOrExpr = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.Pipe).Select(_ => BinaryOp.BitwiseOr), + BitwiseXorExpr, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── String concat: || ── + + private static readonly TokenListParser StringConcatExpr = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.DoublePipe).Select(_ => BinaryOp.StringConcat), + BitwiseOrExpr, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Comparison: =, !=, <, >, <=, >=, LIKE ── + + private static readonly TokenListParser CompOp = + Token.EqualTo(CosmosSqlToken.Equals).Select(_ => BinaryOp.Equal) + .Or(Token.EqualTo(CosmosSqlToken.NotEquals).Select(_ => BinaryOp.NotEqual)) + .Or(Token.EqualTo(CosmosSqlToken.LessThanOrEqual).Select(_ => BinaryOp.LessThanOrEqual)) + .Or(Token.EqualTo(CosmosSqlToken.GreaterThanOrEqual).Select(_ => BinaryOp.GreaterThanOrEqual)) + .Or(Token.EqualTo(CosmosSqlToken.LessThan).Select(_ => BinaryOp.LessThan)) + .Or(Token.EqualTo(CosmosSqlToken.GreaterThan).Select(_ => BinaryOp.GreaterThan)) + .Or(Token.EqualTo(CosmosSqlToken.Like).Select(_ => BinaryOp.Like)); + + private static readonly TokenListParser Comparison = + from left in StringConcatExpr + from rest in ( + // BETWEEN low AND high + (from kw in Token.EqualTo(CosmosSqlToken.Between) + from low in StringConcatExpr + from and in Token.EqualTo(CosmosSqlToken.And) + from high in StringConcatExpr + select (Func)(l => new BetweenExpression(l, low, high))) + // NOT BETWEEN low AND high + .Or( + (from not in Token.EqualTo(CosmosSqlToken.Not) + from kw in Token.EqualTo(CosmosSqlToken.Between) + from low in StringConcatExpr + from and in Token.EqualTo(CosmosSqlToken.And) + from high in StringConcatExpr + select (Func)(l => new UnaryExpression(UnaryOp.Not, new BetweenExpression(l, low, high)))) + .Try()) + // IN (val, val, ...) + .Or( + from kw in Token.EqualTo(CosmosSqlToken.In) + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from vals in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (Func)(l => new InExpression(l, vals))) + // NOT IN (val, val, ...) + .Or( + (from not in Token.EqualTo(CosmosSqlToken.Not) + from kw in Token.EqualTo(CosmosSqlToken.In) + from open in Token.EqualTo(CosmosSqlToken.OpenParen) + from vals in Superpower.Parse.Ref(() => Expr).ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from close in Token.EqualTo(CosmosSqlToken.CloseParen) + select (Func)(l => new UnaryExpression(UnaryOp.Not, new InExpression(l, vals)))) + .Try()) + // LIKE pattern ESCAPE char + .Or( + (from kw in Token.EqualTo(CosmosSqlToken.Like) + from pattern in StringConcatExpr + from esc in Token.EqualTo(CosmosSqlToken.Escape) + from escChar in Token.EqualTo(CosmosSqlToken.StringLiteral) + select (Func)(l => new LikeExpression(l, pattern, escChar.Span.ToStringValue()[1..^1]))) + .Try()) + // NOT LIKE pattern ESCAPE char + .Or( + (from not in Token.EqualTo(CosmosSqlToken.Not) + from kw in Token.EqualTo(CosmosSqlToken.Like) + from pattern in StringConcatExpr + from esc in Token.EqualTo(CosmosSqlToken.Escape) + from escChar in Token.EqualTo(CosmosSqlToken.StringLiteral) + select (Func)(l => new UnaryExpression(UnaryOp.Not, new LikeExpression(l, pattern, escChar.Span.ToStringValue()[1..^1])))) + .Try()) + // NOT LIKE pattern (without ESCAPE) + .Or( + (from not in Token.EqualTo(CosmosSqlToken.Not) + from kw in Token.EqualTo(CosmosSqlToken.Like) + from pattern in StringConcatExpr + select (Func)(l => new UnaryExpression(UnaryOp.Not, new BinaryExpression(l, BinaryOp.Like, pattern)))) + .Try()) + // IS NOT NULL (must come before IS NULL to avoid consuming IS and failing on NOT) + .Or( + (from is_ in Token.EqualTo(CosmosSqlToken.Is) + from not_ in Token.EqualTo(CosmosSqlToken.Not) + from null_ in Token.EqualTo(CosmosSqlToken.Null) + select (Func)(l => new BinaryExpression(l, BinaryOp.NotEqual, new LiteralExpression(null)))) + .Try()) + // IS NULL + .Or( + from is_ in Token.EqualTo(CosmosSqlToken.Is) + from null_ in Token.EqualTo(CosmosSqlToken.Null) + select (Func)(l => new BinaryExpression(l, BinaryOp.Equal, new LiteralExpression(null)))) + // op right + .Or( + from op in CompOp + from right in StringConcatExpr + select (Func)(l => new BinaryExpression(l, op, right))) + ).OptionalOrDefault(null) + select rest != null ? rest(left) : left; + + // ── Logical AND ── + + private static readonly TokenListParser AndExpr = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.And).Select(_ => BinaryOp.And), + Comparison, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Logical OR ── + + private static readonly TokenListParser OrExpr = + Superpower.Parse.Chain( + Token.EqualTo(CosmosSqlToken.Or).Select(_ => BinaryOp.Or), + AndExpr, + (op, left, right) => new BinaryExpression(left, op, right)); + + // ── Null coalesce: ?? ── + + private static readonly TokenListParser CoalesceExpr = + from left in OrExpr + from rest in ( + from op in Token.EqualTo(CosmosSqlToken.QuestionQuestion) + from right in Superpower.Parse.Ref(() => CoalesceExpr) + select right + ).OptionalOrDefault(null) + select rest != null ? new CoalesceExpression(left, rest) : left; + + // ── Ternary: cond ? then : else ── + + private static readonly TokenListParser TernaryExpr = + from cond in CoalesceExpr + from rest in ( + from q in Token.EqualTo(CosmosSqlToken.Question) + from ifTrue in Superpower.Parse.Ref(() => Expr) + from colon in Token.EqualTo(CosmosSqlToken.Colon) + from ifFalse in Superpower.Parse.Ref(() => Expr) + select new { ifTrue, ifFalse } + ).OptionalOrDefault(null) + select rest != null ? (SqlExpression)new TernaryExpression(cond, rest.ifTrue, rest.ifFalse) : cond; + + // ── Top-level expression ── + + private static readonly TokenListParser Expr = TernaryExpr; + + // ────────────────────────────────────────────── + // SELECT field parsing + // ────────────────────────────────────────────── + + private static readonly TokenListParser StarField = + Token.EqualTo(CosmosSqlToken.Star).Select(_ => new SelectField("*", null)); + + // Handle "c.*" (alias DOT STAR) as equivalent to "*" + private static readonly TokenListParser AliasDotStarField = + from ident in AnyIdentifierOrKeyword + from dot in Token.EqualTo(CosmosSqlToken.Dot) + from star in Token.EqualTo(CosmosSqlToken.Star) + select new SelectField("*", null); + + private static readonly TokenListParser ExpressionField = + from expr in Expr + from alias in ( + from as_ in Token.EqualTo(CosmosSqlToken.As) + from name in AnyIdentifierOrKeyword + select name + ).OptionalOrDefault(null) + select new SelectField(ExprToString(expr), alias, expr); + + private static readonly TokenListParser SingleSelectField = + StarField.Try().Or(AliasDotStarField.Try()).Or(ExpressionField); + + // ────────────────────────────────────────────── + // JOIN clause parsing + // ────────────────────────────────────────────── + + private static readonly TokenListParser JoinParser = + from kw in Token.EqualTo(CosmosSqlToken.Join) + from alias in AnyIdentifierOrKeyword + from in_ in Token.EqualTo(CosmosSqlToken.In) + from source in DottedPathWithIndex + select ParseJoinSource(alias, source); + + // ────────────────────────────────────────────── + // ORDER BY parsing + // ────────────────────────────────────────────── + + private static readonly TokenListParser OrderByFieldParser = + (from expr in Superpower.Parse.Ref(() => Expr) + from dir in Token.EqualTo(CosmosSqlToken.Asc).Select(_ => true) + .Or(Token.EqualTo(CosmosSqlToken.Desc).Select(_ => false)) + .OptionalOrDefault(true) + select expr is IdentifierExpression ident + ? new OrderByField(ident.Name, dir) + : new OrderByField(null, dir, expr)).Try() + .Or( + from field in DottedPathWithIndex + from dir in Token.EqualTo(CosmosSqlToken.Asc).Select(_ => true) + .Or(Token.EqualTo(CosmosSqlToken.Desc).Select(_ => false)) + .OptionalOrDefault(true) + select new OrderByField(field, dir)); + + // ────────────────────────────────────────────── + // Full query parsing + // ────────────────────────────────────────────── + + private static readonly TokenListParser QueryParser = + from select_ in Token.EqualTo(CosmosSqlToken.Select) + from distinct in Token.EqualTo(CosmosSqlToken.Distinct).OptionalOrDefault() + from top in ( + from topKw in Token.EqualTo(CosmosSqlToken.Top) + from n in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(t => int.Parse(t.Span.ToStringValue())) + select (int?)n + ).OptionalOrDefault(null) + from value_ in Token.EqualTo(CosmosSqlToken.Value).OptionalOrDefault() + from fields in SingleSelectField.ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + from fromKw in Token.EqualTo(CosmosSqlToken.From) + from fromFirstIdent in AnyIdentifierOrKeyword + from fromClause in ( + // FROM alias IN source.path + from in_ in Token.EqualTo(CosmosSqlToken.In) + from source in DottedPath + select (Alias: fromFirstIdent, Source: (string)source) + ).Try().Or( + // FROM source AS alias + from as_ in Token.EqualTo(CosmosSqlToken.As) + from alias in AnyIdentifierOrKeyword + select (Alias: alias, Source: (string)null) + ).Try().Or( + // FROM source alias (implicit alias without AS keyword — only plain identifiers) + from alias in Token.EqualTo(CosmosSqlToken.Identifier).Select(TokenSpanToString) + select (Alias: alias, Source: (string)null) + ).Try().OptionalOrDefault((Alias: fromFirstIdent, Source: (string)null)) + from joins in JoinParser.Many() + from where_ in ( + from whereKw in Token.EqualTo(CosmosSqlToken.Where) + from expr in Expr + select expr + ).OptionalOrDefault(null) + from groupBy in ( + from groupKw in Token.EqualTo(CosmosSqlToken.Group) + from byKw in Token.EqualTo(CosmosSqlToken.By) + from groupFields in Expr.ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + select groupFields + ).OptionalOrDefault(null) + from having in ( + from havingKw in Token.EqualTo(CosmosSqlToken.Having) + from expr in Expr + select expr + ).OptionalOrDefault(null) + from orderByResult in ( + from orderKw in Token.EqualTo(CosmosSqlToken.Order) + from byKw in Token.EqualTo(CosmosSqlToken.By) + from result in ( + from rankKw in Token.EqualTo(CosmosSqlToken.Identifier) + .Where(t => string.Equals(t.Span.ToStringValue(), "RANK", StringComparison.OrdinalIgnoreCase)) + from expr in Expr + select (Fields: (OrderByField[])null, RankExpr: (SqlExpression)expr) + ).Try().Or( + from orderFields in OrderByFieldParser.ManyDelimitedBy(Token.EqualTo(CosmosSqlToken.Comma)) + select (Fields: orderFields, RankExpr: (SqlExpression)null) + ) + select result + ).OptionalOrDefault(default) + from offset in ( + from offsetKw in Token.EqualTo(CosmosSqlToken.Offset) + from n in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(t => int.Parse(t.Span.ToStringValue())) + select (int?)n + ).OptionalOrDefault(null) + from limit in ( + from limitKw in Token.EqualTo(CosmosSqlToken.Limit) + from n in Token.EqualTo(CosmosSqlToken.NumberLiteral).Select(t => int.Parse(t.Span.ToStringValue())) + select (int?)n + ).OptionalOrDefault(null) + select BuildQuery( + distinct.HasValue, top, value_.HasValue, fields, + fromClause.Alias, fromClause.Source, joins, where_, + groupBy?.Select(ExprToString).ToArray(), + groupBy?.ToArray(), + having, orderByResult.Fields, offset, limit, orderByResult.RankExpr); + + // ────────────────────────────────────────────── + // Public API + // ────────────────────────────────────────────── + + private const int ParseCacheMaxSize = 5000; + private static readonly ConcurrentDictionary ParseCache = new(); + + public static CosmosSqlQuery Parse(string sql) + { + if (ParseCache.TryGetValue(sql, out var cached)) + { + return cached; + } + + CosmosSqlQuery parsed; + try + { + var tokens = CosmosSqlTokenizer.Tokenize(sql); + parsed = QueryParser.Parse(tokens); + } + catch (Exception ex) + { + throw new NotSupportedException($"Failed to parse Cosmos SQL query: {sql}", ex); + } + + if (ParseCache.Count < ParseCacheMaxSize) + { + ParseCache.TryAdd(sql, parsed); + } + + return parsed; + } + + public static bool TryParse(string sql, out CosmosSqlQuery result) + { + try + { + result = Parse(sql); + return true; + } + catch + { + result = null; + return false; + } + } + + // Backward-compatible: convert SqlExpression tree to WhereExpression tree + public static WhereExpression ToWhereExpression(SqlExpression expr) + { + if (expr is null) + { + return null; + } + + switch (expr) + { + case BinaryExpression bin when bin.Operator == BinaryOp.And: + return new AndCondition(ToWhereExpression(bin.Left), ToWhereExpression(bin.Right)); + + case BinaryExpression bin when bin.Operator == BinaryOp.Or: + return new OrCondition(ToWhereExpression(bin.Left), ToWhereExpression(bin.Right)); + + case UnaryExpression { Operator: UnaryOp.Not } unary: + return new NotCondition(ToWhereExpression(unary.Operand)); + + case ExistsExpression exists: + return new ExistsCondition(exists.RawSubquery); + + case FunctionCallExpression func: + if (func.FunctionName.StartsWith("UDF.", StringComparison.OrdinalIgnoreCase) || + func.FunctionName.StartsWith("ST_", StringComparison.OrdinalIgnoreCase)) + { + return new SqlExpressionCondition(func); + } + + var hasComplexArgs = func.Arguments.Any(a => a is ObjectLiteralExpression or ArrayLiteralExpression or FunctionCallExpression); + if (hasComplexArgs) + { + return new SqlExpressionCondition(func); + } + + if (LegacyFunctionNames.Contains(func.FunctionName)) + { + var args = func.Arguments.Select(ExprToString).ToArray(); + return new FunctionCondition(func.FunctionName, args); + } + + return new SqlExpressionCondition(func); + + case BinaryExpression bin when IsComparisonOp(bin.Operator): + if (ContainsFunctionCall(bin.Left) || ContainsFunctionCall(bin.Right) || + ContainsArithmetic(bin.Left) || ContainsArithmetic(bin.Right) || + ContainsComplexExpression(bin.Left) || ContainsComplexExpression(bin.Right)) + { + return new SqlExpressionCondition(bin); + } + + var compOp = bin.Operator switch + { + BinaryOp.Equal => ComparisonOp.Equal, + BinaryOp.NotEqual => ComparisonOp.NotEqual, + BinaryOp.LessThan => ComparisonOp.LessThan, + BinaryOp.GreaterThan => ComparisonOp.GreaterThan, + BinaryOp.LessThanOrEqual => ComparisonOp.LessThanOrEqual, + BinaryOp.GreaterThanOrEqual => ComparisonOp.GreaterThanOrEqual, + BinaryOp.Like => ComparisonOp.Like, + _ => ComparisonOp.Equal, + }; + return new ComparisonCondition(ExprToString(bin.Left), compOp, ExprToString(bin.Right)); + + default: + return new SqlExpressionCondition(expr); + } + } + + /// + /// Removes SDK-injected IS_DEFINED(alias) and literal true nodes from + /// AND chains in a WHERE expression AST, returning the simplified user condition. + /// Only strips IS_DEFINED() calls whose argument is exactly the FROM alias + /// (the SDK injects IS_DEFINED(root) for ORDER BY, never IS_DEFINED(root.field)), + /// preserving any legitimate IS_DEFINED() from user code. + /// Returns null when the entire expression reduces to nothing. + /// + public static SqlExpression SimplifySdkWhereExpression( + SqlExpression expr, string fromAlias = null) + { + return SimplifyCore(expr, fromAlias); + } + + private static SqlExpression SimplifyCore(SqlExpression expr, string fromAlias) + { + if (expr is null) + { + return null; + } + + if (expr is LiteralExpression { Value: true }) + { + return null; + } + + if (expr is FunctionCallExpression func && + string.Equals(func.FunctionName, "IS_DEFINED", StringComparison.OrdinalIgnoreCase) && + func.Arguments.Length == 1) + { + var argStr = ExprToString(func.Arguments[0]); + // Only strip when: no alias specified (backward-compat), or arg IS the FROM alias exactly. + // The SDK injects IS_DEFINED(root) — never IS_DEFINED(root.field) — so this + // preserves any user-written IS_DEFINED on specific field paths. + if (fromAlias is null || string.Equals(argStr, fromAlias, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + } + + if (expr is BinaryExpression { Operator: BinaryOp.And } and) + { + var left = SimplifyCore(and.Left, fromAlias); + var right = SimplifyCore(and.Right, fromAlias); + if (left is null && right is null) + { + return null; + } + + if (left is null) + { + return right; + } + + if (right is null) + { + return left; + } + + return new BinaryExpression(left, BinaryOp.And, right); + } + + return expr; + } + + /// + /// Rebuilds a Superpower-parsed into clean SQL that + /// can execute. Strips SDK-injected WHERE clauses + /// (IS_DEFINED(alias), literal true), normalises bracket notation + /// (root["name"]) to dot notation (root.name) via the AST, and + /// reconstructs SELECT, WHERE, ORDER BY, TOP, OFFSET/LIMIT, DISTINCT, and GROUP BY + /// clauses from the parsed structure. + /// + /// For ORDER BY queries where the SDK rewrites the SELECT to include orderByItems + /// and payload, this emits SELECT VALUE alias to return full documents. + /// For all other queries, the original SELECT expressions are emitted from the AST. + /// + /// + public static string SimplifySdkQuery(CosmosSqlQuery parsed) + { + var fromAlias = parsed.FromAlias; + + var isOrderByQuery = parsed.SelectFields.Any(field => + string.Equals(field.Alias, "orderByItems", StringComparison.OrdinalIgnoreCase)); + + // SELECT clause + var sb = new System.Text.StringBuilder("SELECT "); + if (parsed.IsDistinct) + { + sb.Append("DISTINCT "); + } + + if (parsed.TopCount.HasValue) + { + sb.Append($"TOP {parsed.TopCount.Value} "); + } + + if (isOrderByQuery) + { + sb.Append($"VALUE {fromAlias}"); + } + else if (parsed.IsValueSelect) + { + var selectExprs = parsed.SelectFields + .Select(field => ExprToString(field.SqlExpr ?? new IdentifierExpression(field.Expression))) + .ToArray(); + sb.Append("VALUE "); + sb.Append(string.Join(", ", selectExprs)); + } + else if (parsed.IsSelectAll) + { + sb.Append('*'); + } + else + { + var selectParts = parsed.SelectFields.Select(field => + { + var expr = field.SqlExpr is not null ? ExprToString(field.SqlExpr) : field.Expression; + return field.Alias is not null ? $"{expr} AS {field.Alias}" : expr; + }); + sb.Append(string.Join(", ", selectParts)); + } + + // FROM + if (parsed.FromSource is not null) + { + sb.Append($" FROM {fromAlias} IN {parsed.FromSource}"); + } + else + { + sb.Append($" FROM {fromAlias}"); + } + + // JOINs + if (parsed.Joins is { Length: > 0 }) + { + foreach (var join in parsed.Joins) + { + sb.Append($" JOIN {join.Alias} IN {join.SourceAlias}.{join.ArrayField}"); + } + } + + // WHERE — strip SDK-injected nodes + var simplifiedWhere = SimplifySdkWhereExpression(parsed.WhereExpr, fromAlias); + if (simplifiedWhere is not null) + { + sb.Append($" WHERE {ExprToString(simplifiedWhere)}"); + } + + // GROUP BY + if (parsed.GroupByFields is { Length: > 0 }) + { + sb.Append($" GROUP BY {string.Join(", ", parsed.GroupByFields)}"); + } + + // HAVING + if (parsed.HavingExpr is not null) + { + sb.Append($" HAVING {ExprToString(parsed.HavingExpr)}"); + } + + // ORDER BY + if (parsed.OrderByFields is { Length: > 0 }) + { + var orderByStr = string.Join(", ", parsed.OrderByFields.Select(field => + $"{field.Field ?? ExprToString(field.Expression)} {(field.Ascending ? "ASC" : "DESC")}")); + sb.Append($" ORDER BY {orderByStr}"); + } + else if (parsed.RankExpression is not null) + { + sb.Append($" ORDER BY RANK {ExprToString(parsed.RankExpression)}"); + } + + // OFFSET / LIMIT + if (parsed.Offset.HasValue) + { + sb.Append($" OFFSET {parsed.Offset.Value}"); + } + + if (parsed.Limit.HasValue) + { + sb.Append($" LIMIT {parsed.Limit.Value}"); + } + + return sb.ToString(); + } + + // ────────────────────────────────────────────── + // Internal helpers + // ────────────────────────────────────────────── + + internal static WhereExpression ParseWhereExpression(string expression) + { + var wrappedSql = $"SELECT * FROM c WHERE {expression}"; + var query = Parse(wrappedSql); + return query.Where; + } + + private static bool IsComparisonOp(BinaryOp op) => + op is BinaryOp.Equal or BinaryOp.NotEqual or BinaryOp.LessThan or BinaryOp.GreaterThan + or BinaryOp.LessThanOrEqual or BinaryOp.GreaterThanOrEqual or BinaryOp.Like; + + private static bool ContainsFunctionCall(SqlExpression expr) => + expr switch + { + FunctionCallExpression => true, + BinaryExpression bin => ContainsFunctionCall(bin.Left) || ContainsFunctionCall(bin.Right), + UnaryExpression unary => ContainsFunctionCall(unary.Operand), + _ => false + }; + + private static bool ContainsArithmetic(SqlExpression expr) => + expr switch + { + BinaryExpression bin when !IsComparisonOp(bin.Operator) => true, + BinaryExpression bin => ContainsArithmetic(bin.Left) || ContainsArithmetic(bin.Right), + UnaryExpression unary => ContainsArithmetic(unary.Operand), + _ => false + }; + + /// + /// Returns true when the expression contains a subquery, coalesce, or ternary — + /// any of which require full expression evaluation rather than string-based ResolveValue. + /// + private static bool ContainsComplexExpression(SqlExpression expr) => + expr switch + { + SubqueryExpression => true, + CoalesceExpression => true, + TernaryExpression => true, + BinaryExpression bin => ContainsComplexExpression(bin.Left) || ContainsComplexExpression(bin.Right), + UnaryExpression unary => ContainsComplexExpression(unary.Operand), + _ => false + }; + + public static string ExprToString(SqlExpression expr) + { + return expr switch + { + LiteralExpression { Value: null } => "null", + LiteralExpression { Value: string s } => $"'{s}'", + LiteralExpression { Value: bool b } => b ? "true" : "false", + LiteralExpression lit => lit.Value?.ToString() ?? "null", + UndefinedLiteralExpression => "undefined", + IdentifierExpression ident => ident.Name, + ParameterExpression param => param.Name, + PropertyAccessExpression prop => $"{ExprToString(prop.Object)}.{prop.Property}", + IndexAccessExpression idx => $"{ExprToString(idx.Object)}[{ExprToString(idx.Index)}]", + FunctionCallExpression func => $"{func.FunctionName}({string.Join(", ", func.Arguments.Select(ExprToString))})", + BinaryExpression { Operator: BinaryOp.And or BinaryOp.Or } bin => + $"({WrapIfLowPrecedence(bin.Left)} {BinaryOpToString(bin.Operator)} {WrapIfLowPrecedence(bin.Right)})", + BinaryExpression bin => $"{WrapIfLowPrecedence(bin.Left)} {BinaryOpToString(bin.Operator)} {WrapIfLowPrecedence(bin.Right)}", + UnaryExpression unary => $"{UnaryOpToString(unary.Operator)} {WrapIfLowPrecedence(unary.Operand)}", + BetweenExpression betw => $"{WrapIfLowPrecedence(betw.Value)} BETWEEN {ExprToString(betw.Low)} AND {ExprToString(betw.High)}", + InExpression inExpr => $"{WrapIfLowPrecedence(inExpr.Value)} IN ({string.Join(", ", inExpr.List.Select(ExprToString))})", + LikeExpression like => like.EscapeChar is not null + ? $"{WrapIfLowPrecedence(like.Value)} LIKE {ExprToString(like.Pattern)} ESCAPE '{like.EscapeChar}'" + : $"{WrapIfLowPrecedence(like.Value)} LIKE {ExprToString(like.Pattern)}", + ExistsExpression exists => $"EXISTS({exists.RawSubquery})", + SubqueryExpression sub => $"({SubqueryToString(sub.Subquery)})", + TernaryExpression tern => $"{ExprToString(tern.Condition)} ? {ExprToString(tern.IfTrue)} : {ExprToString(tern.IfFalse)}", + CoalesceExpression coal => $"{ExprToString(coal.Left)} ?? {ExprToString(coal.Right)}", + ObjectLiteralExpression obj => "{" + string.Join(", ", obj.Properties.Select(p => $"{p.Key}: {ExprToString(p.Value)}")) + "}", + ArrayLiteralExpression arr => "[" + string.Join(", ", arr.Elements.Select(ExprToString)) + "]", + _ => expr.ToString(), + }; + } + + /// + /// Wraps the expression in parentheses if it has lower precedence than binary/unary operators + /// (i.e., ternary or coalesce expressions). This ensures that re-parsing the serialized string + /// produces the same AST. + /// + // Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/ternary-coalesce-operators + // Ternary (?) and Coalesce (??) have the lowest operator precedence in Cosmos DB SQL. + private static string WrapIfLowPrecedence(SqlExpression expr) => + expr is TernaryExpression or CoalesceExpression + ? $"({ExprToString(expr)})" + : ExprToString(expr); + + private static string SubqueryToString(CosmosSqlQuery sub) + { + var sb = new System.Text.StringBuilder("SELECT "); + if (sub.IsDistinct) + { + sb.Append("DISTINCT "); + } + + if (sub.TopCount.HasValue) + { + sb.Append($"TOP {sub.TopCount.Value} "); + } + + if (sub.IsValueSelect) + { + sb.Append("VALUE "); + } + + if (sub.IsSelectAll) + { + sb.Append('*'); + } + else + { + var selectParts = sub.SelectFields.Select(field => + { + var expr = field.SqlExpr is not null ? ExprToString(field.SqlExpr) : field.Expression; + return field.Alias is not null ? $"{expr} AS {field.Alias}" : expr; + }); + sb.Append(string.Join(", ", selectParts)); + } + + if (sub.FromSource is not null) + { + sb.Append($" FROM {sub.FromAlias} IN {sub.FromSource}"); + } + else + { + sb.Append($" FROM {sub.FromAlias}"); + } + + if (sub.Joins is { Length: > 0 }) + { + foreach (var join in sub.Joins) + { + sb.Append($" JOIN {join.Alias} IN {join.SourceAlias}.{join.ArrayField}"); + } + } + + if (sub.WhereExpr is not null) + { + sb.Append($" WHERE {ExprToString(sub.WhereExpr)}"); + } + + if (sub.GroupByFields is { Length: > 0 }) + { + sb.Append($" GROUP BY {string.Join(", ", sub.GroupByFields)}"); + } + + if (sub.HavingExpr is not null) + { + sb.Append($" HAVING {ExprToString(sub.HavingExpr)}"); + } + + if (sub.OrderByFields is { Length: > 0 }) + { + var orderByStr = string.Join(", ", sub.OrderByFields.Select(field => + $"{field.Field} {(field.Ascending ? "ASC" : "DESC")}")); + sb.Append($" ORDER BY {orderByStr}"); + } + + if (sub.Offset.HasValue) + { + sb.Append($" OFFSET {sub.Offset.Value}"); + } + + if (sub.Limit.HasValue) + { + sb.Append($" LIMIT {sub.Limit.Value}"); + } + + return sb.ToString(); + } + + private static string BinaryOpToString(BinaryOp op) => op switch + { + BinaryOp.Equal => "=", + BinaryOp.NotEqual => "!=", + BinaryOp.LessThan => "<", + BinaryOp.GreaterThan => ">", + BinaryOp.LessThanOrEqual => "<=", + BinaryOp.GreaterThanOrEqual => ">=", + BinaryOp.And => "AND", + BinaryOp.Or => "OR", + BinaryOp.Add => "+", + BinaryOp.Subtract => "-", + BinaryOp.Multiply => "*", + BinaryOp.Divide => "/", + BinaryOp.Modulo => "%", + BinaryOp.Like => "LIKE", + BinaryOp.StringConcat => "||", + BinaryOp.BitwiseAnd => "&", + BinaryOp.BitwiseOr => "|", + BinaryOp.BitwiseXor => "^", + _ => op.ToString(), + }; + + private static string UnaryOpToString(UnaryOp op) => op switch + { + UnaryOp.Not => "NOT", + UnaryOp.Negate => "-", + UnaryOp.BitwiseNot => "~", + _ => op.ToString(), + }; + + private static JoinClause ParseJoinSource(string alias, string source) + { + var dotIdx = source.IndexOf('.'); + if (dotIdx < 0) + { + return new JoinClause(alias, source, source); + } + + return new JoinClause(alias, source[..dotIdx], source[(dotIdx + 1)..]); + } + + private static CosmosSqlQuery BuildQuery( + bool isDistinct, int? top, bool isValue, + SelectField[] fields, string fromAlias, string fromSource, + JoinClause[] joins, SqlExpression whereExpr, + string[] groupBy, SqlExpression[] groupByExpressions, SqlExpression havingExpr, + OrderByField[] orderByFields, + int? offset, int? limit, SqlExpression rankExpr = null) + { + var isSelectAll = fields.Length == 1 && fields[0].Expression == "*"; + var where = whereExpr != null ? ToWhereExpression(whereExpr) : null; + // HAVING always uses SqlExpressionCondition to preserve aggregate function expressions + // (ToWhereExpression would decompose AND/OR into AndCondition/OrCondition with string-based + // comparison operands, losing the ability to evaluate aggregate functions like COUNT/SUM). + var having = havingExpr != null ? new SqlExpressionCondition(havingExpr) : null; + var rawHavingExpr = havingExpr; + + // Backward compat: first join, first order by + var firstJoin = joins.Length > 0 ? joins[0] : null; + OrderByClause legacyOrderBy = null; + if (orderByFields is { Length: > 0 }) + { + legacyOrderBy = new OrderByClause(orderByFields[0].Field, orderByFields[0].Ascending); + } + + return new CosmosSqlQuery( + SelectFields: fields, + IsSelectAll: isSelectAll, + TopCount: top, + FromAlias: fromAlias, + Join: firstJoin, + Where: where, + Offset: offset, + Limit: limit, + OrderBy: legacyOrderBy, + IsDistinct: isDistinct, + IsValueSelect: isValue, + Joins: joins, + OrderByFields: orderByFields, + GroupByFields: groupBy, + GroupByExpressions: groupByExpressions, + Having: having, + WhereExpr: whereExpr, + HavingExpr: rawHavingExpr, + FromSource: fromSource, + RankExpression: rankExpr); + } + + // Captures tokens inside balanced parentheses as a raw string + private static TokenListParser CaptureBalancedParens() + { + return input => + { + var depth = 0; + var startPos = input.Position; + var current = input; + var parts = new List(); + + while (!current.IsAtEnd) + { + var token = current.ConsumeToken(); + if (!token.HasValue) + { + break; + } + + if (token.Value.Kind == CosmosSqlToken.OpenParen) + { + depth++; + parts.Add("("); + current = token.Remainder; + } + else if (token.Value.Kind == CosmosSqlToken.CloseParen) + { + if (depth == 0) + { + return TokenListParserResult.Value(string.Join(" ", parts), input, current); + } + depth--; + parts.Add(")"); + current = token.Remainder; + } + else + { + parts.Add(token.Value.Span.ToStringValue()); + current = token.Remainder; + } + } + + return TokenListParserResult.Value(string.Join(" ", parts), input, current); + }; + } } diff --git a/src/Directory.Build.props b/src/Directory.Build.props index fdf6b25..711e662 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,7 +5,7 @@ true - 4.0.17 + 4.0.18 lemonlion Copyright (c) 2026 lemonlion MIT diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue64CountTernaryUndefinedTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue64CountTernaryUndefinedTests.cs new file mode 100644 index 0000000..204b5b1 --- /dev/null +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue64CountTernaryUndefinedTests.cs @@ -0,0 +1,219 @@ +using AwesomeAssertions; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace CosmosDB.InMemoryEmulator.Tests; + +/// +/// Regression tests for GitHub Issue #64 — COUNT(expr > 0 ? 1 : undefined) +/// incorrectly counts documents where the expression evaluates to undefined. +/// +/// Root cause: ExprToString in CosmosSqlParser did not parenthesise +/// ternary/coalesce sub-expressions within higher-precedence binary operators, so the +/// round-tripped SQL was re-parsed into a different AST. +/// +/// Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/ternary-coalesce-operators +/// Ternary (?) and Coalesce (??) have the lowest operator precedence in Cosmos DB SQL. +/// +[Collection(IntegrationCollection.Name)] +public class Issue64CountTernaryUndefinedTests(EmulatorSession session) : IAsyncLifetime +{ + private readonly ITestContainerFixture _fixture = TestFixtureFactory.Create(session); + private Container _container = null!; + + public async ValueTask InitializeAsync() + { + _container = await _fixture.CreateContainerAsync("issue64", "/partitionKey"); + } + + public async ValueTask DisposeAsync() + { + await _fixture.DisposeAsync(); + } + + private async Task> DrainQuery(string sql) + { + var iterator = _container.GetItemQueryIterator(sql); + var results = new List(); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + return results; + } + + // ═══════════════════════════════════════════════════════════════════════════ + // SELECT VALUE with ternary returning undefined + // ═══════════════════════════════════════════════════════════════════════════ + + [Fact] + public async Task SelectValue_SimpleTernary_WhenConditionFalse_ReturnsEmpty() + { + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", amount = 0 }, new PartitionKey("pk1")); + + // amount = 0, so (0 > 0) is false → ternary returns undefined → excluded from results + var results = await DrainQuery( + "SELECT VALUE c.amount > 0 ? 1 : undefined FROM c"); + + results.Should().BeEmpty(); + } + + [Fact] + public async Task SelectValue_SimpleTernary_WhenConditionTrue_ReturnsValue() + { + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", amount = 5 }, new PartitionKey("pk1")); + + // amount = 5, so (5 > 0) is true → ternary returns 1 + var results = await DrainQuery( + "SELECT VALUE c.amount > 0 ? 1 : undefined FROM c"); + + results.Should().ContainSingle().Which.Should().Be(1); + } + + [Fact] + public async Task SelectValue_NestedTernary_WhenInnerResolvesAndComparisonFails_ReturnsEmpty() + { + // creditValue.amount = 0, so inner ternary resolves to 0; 0 > 0 is false → undefined + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", creditValue = new { amount = 0 }, grossValue = new { amount = 100 } }, + new PartitionKey("pk1")); + + var results = await DrainQuery( + "SELECT VALUE (IS_DEFINED(c.creditValue) ? c.creditValue.amount : c.grossValue.amount) > 0 ? 1 : undefined FROM c"); + + results.Should().BeEmpty(); + } + + [Fact] + public async Task SelectValue_NestedTernary_WhenInnerResolvesAndComparisonSucceeds_ReturnsValue() + { + // creditValue.amount = 50, so inner ternary resolves to 50; 50 > 0 is true → 1 + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", creditValue = new { amount = 50 }, grossValue = new { amount = 100 } }, + new PartitionKey("pk1")); + + var results = await DrainQuery( + "SELECT VALUE (IS_DEFINED(c.creditValue) ? c.creditValue.amount : c.grossValue.amount) > 0 ? 1 : undefined FROM c"); + + results.Should().ContainSingle().Which.Should().Be(1); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // COUNT with ternary returning undefined — the core bug scenario + // ═══════════════════════════════════════════════════════════════════════════ + + [Fact] + public async Task Count_SimpleTernary_WhenConditionFalse_ShouldNotCount() + { + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", amount = 0 }, new PartitionKey("pk1")); + + // amount = 0, so (0 > 0) is false → undefined → COUNT should skip + var results = await DrainQuery( + "SELECT VALUE COUNT(c.amount > 0 ? 1 : undefined) FROM c"); + + results.Should().ContainSingle().Which.Should().Be(0); + } + + [Fact] + public async Task Count_SimpleTernary_MixedDocuments_CountsOnlyMatching() + { + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", amount = 0 }, new PartitionKey("pk1")); + await _container.CreateItemAsync( + new { id = "2", partitionKey = "pk1", amount = 5 }, new PartitionKey("pk1")); + await _container.CreateItemAsync( + new { id = "3", partitionKey = "pk1", amount = -1 }, new PartitionKey("pk1")); + + // Only id=2 (amount=5) satisfies amount > 0 + var results = await DrainQuery( + "SELECT VALUE COUNT(c.amount > 0 ? 1 : undefined) FROM c"); + + results.Should().ContainSingle().Which.Should().Be(1); + } + + [Fact] + public async Task Count_NestedTernary_WhenInnerResolvesToZero_ShouldNotCount() + { + // This is the exact bug scenario from the issue: + // creditValue.amount = 0, IS_DEFINED(c.creditValue) is true, so inner ternary → 0 + // 0 > 0 is false → outer ternary returns undefined → COUNT should NOT count this doc + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", creditValue = new { amount = 0 }, grossValue = new { amount = 100 } }, + new PartitionKey("pk1")); + + var results = await DrainQuery( + "SELECT VALUE COUNT((IS_DEFINED(c.creditValue) ? c.creditValue.amount : c.grossValue.amount) > 0 ? 1 : undefined) FROM c"); + + results.Should().ContainSingle().Which.Should().Be(0); + } + + [Fact] + public async Task Count_NestedTernary_WhenInnerResolvesToPositive_ShouldCount() + { + // creditValue.amount = 50, IS_DEFINED(c.creditValue) is true, so inner ternary → 50 + // 50 > 0 is true → outer ternary returns 1 → COUNT SHOULD count this doc + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", creditValue = new { amount = 50 }, grossValue = new { amount = 100 } }, + new PartitionKey("pk1")); + + var results = await DrainQuery( + "SELECT VALUE COUNT((IS_DEFINED(c.creditValue) ? c.creditValue.amount : c.grossValue.amount) > 0 ? 1 : undefined) FROM c"); + + results.Should().ContainSingle().Which.Should().Be(1); + } + + [Fact] + public async Task Count_NestedTernary_FallsBackToGrossValue_WhenCreditUndefined() + { + // creditValue is NOT defined, so IS_DEFINED is false → inner ternary → grossValue.amount = 200 + // 200 > 0 is true → outer ternary returns 1 → COUNT SHOULD count + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", grossValue = new { amount = 200 } }, + new PartitionKey("pk1")); + + var results = await DrainQuery( + "SELECT VALUE COUNT((IS_DEFINED(c.creditValue) ? c.creditValue.amount : c.grossValue.amount) > 0 ? 1 : undefined) FROM c"); + + results.Should().ContainSingle().Which.Should().Be(1); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Multi-aggregate query (mirrors the original issue's query pattern) + // ═══════════════════════════════════════════════════════════════════════════ + + [Fact] + public async Task Count_MultipleAggregatesWithTernary_IssueScenario() + { + // Mirrors the exact pattern from the bug report + await _container.CreateItemAsync(new + { + id = "doc1", + partitionKey = "pk1", + merchantId = "m1", + transactionType = "Settlement", + settlementDate = "2023-09-07", + creditValue = new { amount = 0 }, + grossSettlementValue = new { amount = 1000.25 }, + upfrontPaymentValue = new { amount = 1000.25 } + }, new PartitionKey("pk1")); + + var query = + "SELECT " + + "COUNT((IS_DEFINED(c.creditValue) ? c.creditValue.amount : c.grossSettlementValue.amount) > 0 ? 1 : undefined) AS NumberTransactions, " + + "COUNT(c.upfrontPaymentValue.amount > 0 ? 1 : undefined) AS UpfrontPaymentCount " + + "FROM c WHERE c.transactionType = 'Settlement' AND c.settlementDate = '2023-09-07' AND c.merchantId = 'm1'"; + + var results = await DrainQuery(query); + + var row = results.Should().ContainSingle().Which; + // creditValue.amount = 0 → inner ternary resolves to 0 → 0 > 0 is false → undefined → NOT counted + row["NumberTransactions"]!.Value().Should().Be(0); + // upfrontPaymentValue.amount = 1000.25 → 1000.25 > 0 is true → 1 → counted + row["UpfrontPaymentCount"]!.Value().Should().Be(1); + } +} From c0c9c62064c05b66091db6650082f38a01a870e2 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 08:12:09 +0100 Subject: [PATCH 02/16] =?UTF-8?q?Remove=20EmulatorFlaky=20trait=20?= =?UTF-8?q?=E2=80=94=20use=20adaptive=20concurrency=20instead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EmulatorFlaky trait was blanket-excluding 12 tests from all emulator runs, creating a regression gap. The root cause was a single test (ConcurrentReadsOfNonExistent) overwhelming the emulator with 50 parallel requests. Instead of excluding the entire class, make the concurrency adaptive: - 50 concurrent reads for in-memory (fast, no resource constraints) - 10 concurrent reads for emulator targets (same assertion, lower volume) Changes: - Remove [Trait(TestTraits.Target, TestTraits.EmulatorFlaky)] from Issue18EdgeCaseIntegrationTests - Remove EmulatorFlaky constant from TestTraits.cs - Remove Target!=EmulatorFlaky filter from scripts/run-tests.ps1 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/run-tests.ps1 | 7 ++----- .../Issue18EdgeCaseIntegrationTests.cs | 15 +++++++-------- .../Infrastructure/TestTraits.cs | 9 --------- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 index eb3b828..47aa50c 100644 --- a/scripts/run-tests.ps1 +++ b/scripts/run-tests.ps1 @@ -44,13 +44,10 @@ if ($Target -ne 'inmemory') { Remove-Item Env:COSMOS_EMULATOR_ENDPOINT -ErrorAction SilentlyContinue } -# Build filter: exclude InMemoryOnly and EmulatorFlaky tests when targeting an emulator. -# EmulatorFlaky tags tests that pass on in-memory but fail reproducibly on the Linux -# Docker / Windows Cosmos DB emulators due to emulator-side instability — the behaviour -# is still validated on the inmemory target, so excluding here just keeps CI signal clean. +# Build filter: exclude InMemoryOnly tests when targeting an emulator. $filterExpr = '' if ($Target -ne 'inmemory') { - $filterExpr = 'Target!=InMemoryOnly&Target!=EmulatorFlaky' + $filterExpr = 'Target!=InMemoryOnly' } if ($Filter) { if ($filterExpr -and $Filter -match '\|') { diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue18EdgeCaseIntegrationTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue18EdgeCaseIntegrationTests.cs index 44864d7..cc3424e 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue18EdgeCaseIntegrationTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue18EdgeCaseIntegrationTests.cs @@ -11,15 +11,11 @@ namespace CosmosDB.InMemoryEmulator.Tests; /// verifying that exceptions are exactly and carry the /// expected status codes. /// -/// Tagged EmulatorFlaky: the entire class has been reproducibly failing on -/// emulator-linux / emulator-windows targets in CI — the emulators return 503 -/// ("high demand in this region") under the concurrent error-path load this class -/// exercises, where the in-memory backend returns the expected 4xx/5xx. The -/// behaviour under test is still validated on the in-memory target; emulator -/// parity would be nice-to-have but isn't load-bearing. +/// The concurrent test uses adaptive concurrency: lower volume when targeting a real +/// emulator to avoid overwhelming its partition service with 503 "high demand" errors, +/// while still validating the same thread-safety behaviour. /// [Collection(IntegrationCollection.Name)] -[Trait(TestTraits.Target, TestTraits.EmulatorFlaky)] public class Issue18EdgeCaseIntegrationTests(EmulatorSession session) : IAsyncLifetime { private readonly ITestContainerFixture _fixture = TestFixtureFactory.Create(session); @@ -188,7 +184,10 @@ public async Task BatchConflict_ReturnsFailedResponse() [Fact] public async Task ConcurrentReadsOfNonExistent_AllThrowCosmosException() { - const int concurrency = 50; + // Use lower concurrency on emulators to avoid overwhelming the partition + // service (503 "high demand in this region"). The behavioural assertion is + // the same — all reads of non-existent items throw CosmosException(404). + var concurrency = session.IsEmulator ? 10 : 50; var tasks = Enumerable.Range(0, concurrency).Select(async i => { var ex = await Assert.ThrowsAsync(async () => diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/TestTraits.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/TestTraits.cs index f76690b..c21fa2f 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/TestTraits.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/TestTraits.cs @@ -17,13 +17,4 @@ public static class TestTraits /// Documents a known divergence between in-memory and emulator. public const string KnownDivergence = "KnownDivergence"; - - /// - /// Test is reproducibly flaky against the Linux Docker / Windows Cosmos DB - /// emulators due to emulator-side instability (typically 503 responses where - /// the in-memory backend returns the expected status). Excluded from - /// emulator-target runs in scripts/run-tests.ps1 to keep CI signal clean. - /// In-memory runs are unaffected — these tests still validate behaviour there. - /// - public const string EmulatorFlaky = "EmulatorFlaky"; } From 8b795dbde799b0f1ea7e56ad9348dc51fcd81d8b Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 08:24:42 +0100 Subject: [PATCH 03/16] Treat 404/0 as transient during emulator warmup The Windows emulator can return 404 with substatus 0 during an intermediate startup state (between 403/1008 and fully ready). This was causing the EmulatorWarmup tool and EmulatorRetry to treat it as fatal, failing the parity workflow before tests even ran. Add 404/0 to the transient error set in both: - tests/EmulatorWarmup/Program.cs (warmup tool) - tests/.../EmulatorSession.cs (shared retry helper) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Infrastructure/EmulatorSession.cs | 4 ++++ tests/EmulatorWarmup/Program.cs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/EmulatorSession.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/EmulatorSession.cs index bead7ac..5c65a1f 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/EmulatorSession.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Shared/Infrastructure/EmulatorSession.cs @@ -282,6 +282,10 @@ public static async Task RunAsync( // can become reachable before its account is fully initialised. Retry until ready. CosmosException ce when ce.StatusCode == System.Net.HttpStatusCode.Forbidden && ce.SubStatusCode == 1008 => true, + // 404/0 = Windows emulator intermediate startup state: HTTP server is reachable + // but internal metadata services haven't fully materialised yet. + CosmosException ce when ce.StatusCode == System.Net.HttpStatusCode.NotFound + && ce.SubStatusCode == 0 => true, CosmosException ce => ce.StatusCode is System.Net.HttpStatusCode.ServiceUnavailable or System.Net.HttpStatusCode.InternalServerError or diff --git a/tests/EmulatorWarmup/Program.cs b/tests/EmulatorWarmup/Program.cs index c3be686..770cf51 100644 --- a/tests/EmulatorWarmup/Program.cs +++ b/tests/EmulatorWarmup/Program.cs @@ -239,6 +239,10 @@ private static async Task RetryAsync( // 404/1013 — "Collection is not yet available for read". CosmosException ce when ce.StatusCode == HttpStatusCode.NotFound && ce.SubStatusCode == 1013 => true, + // 404/0 — Windows emulator intermediate startup state: HTTP server is + // reachable but internal metadata services haven't fully materialised. + CosmosException ce when ce.StatusCode == HttpStatusCode.NotFound + && ce.SubStatusCode == 0 => true, CosmosException ce => ce.StatusCode is HttpStatusCode.ServiceUnavailable or HttpStatusCode.InternalServerError or From 2ea4370427c8ceed522eec11d312bc27520a2e21 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 06:17:30 +0100 Subject: [PATCH 04/16] docs: add Publishing & Releases section to AGENTS.md Documents how to publish beta/prerelease and stable packages via the existing release.yml workflow (tag push convention) for AI/LLM agent discoverability. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 37 +++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 047847f..2d719de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,43 @@ Tests are split into two projects. When creating or moving tests, follow these r ### Key constraint The Integration project does **not** have `InternalsVisibleTo` access. If a test needs internal APIs, it belongs in Unit. +## Publishing & Releases + +Packages are published to NuGet via the `release.yml` GitHub Actions workflow, triggered by pushing a `v*` tag. The workflow runs the full test suite before publishing — if tests fail, nothing is published. + +### Beta / Prerelease + +To publish a prerelease package for testing before merging: + +1. Ensure your fix is committed and pushed to a branch. +2. Create and push a tag with a prerelease suffix: + ```bash + git tag v4.0.18-beta.1 + git push origin v4.0.18-beta.1 + ``` +3. The workflow extracts the version from the tag (strips the `v` prefix), passes it to `dotnet pack -p:Version=...`, and publishes to NuGet as a prerelease package. +4. Consumers install with: `dotnet add package CosmosDB.InMemoryEmulator --version 4.0.18-beta.1` + +The tag can be on any branch — the workflow checks out the tagged commit. + +### Stable Release + +After merging to `main`: + +1. Ensure `src/Directory.Build.props` has the correct `` (e.g. `4.0.18`). +2. Commit, create a tag, and push: + ```bash + git tag v4.0.18 + git push origin v4.0.18 + ``` +3. The workflow publishes stable packages and creates a GitHub Release with auto-generated release notes. + +### Version Conventions + +- `Directory.Build.props` `` is the target stable version — increment the patch after each release. +- The CI workflow **overrides** the version from the tag, so the `.props` value doesn't need to match beta suffixes. +- Prerelease format: `X.Y.Z-beta.N` (increment N for successive betas of the same version). + ## Documentation After any changes are made that might effect the public API or functionality, documentation must be updated to reflect those changes. The documentation should be clear and comprehensive, covering all new features, changes to existing features, and any deprecations or removals. This includes updating README file (if relevant), but mainly the wiki which can be found in a sister folder to the main repository - ../CosmosDB.InMemoryEmulator.wiki. \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ea8bd9..a23590e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [4.0.18] - 2026-05-15 +## [4.0.18] - 2026-05-19 ### Fixed - `COUNT(expr)` with nested ternary expressions no longer miscounts documents (Issue #64). `ExprToString` in `CosmosSqlParser` did not parenthesise ternary/coalesce sub-expressions when they appeared as operands of higher-precedence binary operators. When the SDK's transformed query was round-tripped through `SimplifySdkQuery`, the missing parentheses caused re-parsing to produce a different AST — e.g. `(innerTernary > 0) ? 1 : undefined` became `innerTernary ? val : (otherVal > 0 ? 1 : undefined)` — making `COUNT` evaluate the wrong condition. The fix wraps ternary and coalesce expressions in parentheses whenever they appear inside binary, unary, BETWEEN, IN, or LIKE operators. From 520abfb257d04de4ab6793a976d6880f05582476 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 19:09:28 +0100 Subject: [PATCH 05/16] fix: evaluate literal expressions in ProjectAggregateFields (#67) String literal aliases (e.g. SELECT 'Settlement' AS Label, COUNT(1) ...) returned null on Linux because ProjectAggregateFields treated them as property paths via SelectToken. On Windows the bug was masked by ServiceInterop's native aggregate pipeline. The fix checks for non-identifier SqlExpr nodes in the else branch and evaluates them via EvaluateSqlExpression, matching the existing pattern used in the GROUP BY aggregate path. Covers string, numeric, boolean, and null literal aliases with integration tests. Bumps version to 4.0.19. Closes #67 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DefaultQueryPlanStrategy.cs | 12 +- .../InMemoryContainer.cs | 14 ++ src/Directory.Build.props | 2 +- .../Issue67StringLiteralAliasTests.cs | 144 ++++++++++++++++++ 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue67StringLiteralAliasTests.cs diff --git a/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs b/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs index a6ed3c6..3ab997b 100644 --- a/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs +++ b/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs @@ -147,6 +147,10 @@ private static void BuildFromParsedQuery(JObject queryInfo, CosmosSqlQuery parse // doesn't recognize COUNTIF and tries to apply its built-in aggregate // accumulator, which expects a {payload: {alias: {item: value}}} envelope // the handler only produces for GROUP BY responses. + // Additionally bypass when literal expressions appear alongside aggregates + // (e.g. SELECT 42 AS X, COUNT(1) AS N) — the SDK's AggregateQueryPipelineStage + // doesn't handle non-aggregate expression fields and crashes with + // "Underlying object does not have an 'payload' field". var isGroupByBypass = parsed.GroupByFields is { Length: > 0 }; var aggregateFieldCount = parsed.SelectFields.Count(f => ContainsAggregate(f.SqlExpr)); var isMultiAggregateBypass = !isGroupByBypass && !parsed.IsValueSelect @@ -154,8 +158,14 @@ private static void BuildFromParsedQuery(JObject queryInfo, CosmosSqlQuery parse var isValueAggregateBypass = !isGroupByBypass && parsed.IsValueSelect && aggregateFieldCount > 0; var isCountIfBypass = !isGroupByBypass && parsed.SelectFields.Any(f => ContainsCountIf(f.SqlExpr)); + // Bypass when there are non-aggregate expression fields (literals, function calls) + // alongside at least one aggregate — the SDK pipeline can't project these. + var isLiteralWithAggregateBypass = !isGroupByBypass && !parsed.IsValueSelect + && aggregateFieldCount > 0 + && parsed.SelectFields.Any(f => !ContainsAggregate(f.SqlExpr) + && f.SqlExpr is not null and not IdentifierExpression and not PropertyAccessExpression); - if (isGroupByBypass || isMultiAggregateBypass || isValueAggregateBypass || isCountIfBypass) + if (isGroupByBypass || isMultiAggregateBypass || isValueAggregateBypass || isCountIfBypass || isLiteralWithAggregateBypass) { queryInfo["groupByExpressions"] = new JArray(); queryInfo["groupByAliases"] = new JArray(); diff --git a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs index 07447d3..1d574e0 100644 --- a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs +++ b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs @@ -5352,6 +5352,20 @@ private static List ProjectAggregateFields(IEnumerable itemsEnum if (val is not null and not UndefinedValue) resultObj[outputName] = val is JToken jt ? jt.DeepClone() : JToken.FromObject(val); } + // Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select + // "The SELECT clause supports arbitrary expressions including literal values." + // Handle non-aggregate, non-trivial expressions (literals, function calls, etc.) + // by evaluating the SqlExpr directly instead of treating as a property path. + else if (field.SqlExpr is not null and not IdentifierExpression) + { + var jObj = items.Count > 0 ? JsonParseHelpers.ParseJson(items[0]) : new JObject(); + var val = EvaluateSqlExpression(field.SqlExpr, jObj, parsed.FromAlias, + parameters ?? new Dictionary()); + if (val is not null and not UndefinedValue) + resultObj[outputName] = val is JToken jt ? jt.DeepClone() : JToken.FromObject(val); + else if (val is null) + resultObj[outputName] = JValue.CreateNull(); + } else { var path = field.Expression; diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 711e662..4482e58 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,7 +5,7 @@ true - 4.0.18 + 4.0.19 lemonlion Copyright (c) 2026 lemonlion MIT diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue67StringLiteralAliasTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue67StringLiteralAliasTests.cs new file mode 100644 index 0000000..311b3f5 --- /dev/null +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue67StringLiteralAliasTests.cs @@ -0,0 +1,144 @@ +using AwesomeAssertions; +using CosmosDB.InMemoryEmulator.Tests.Infrastructure; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace CosmosDB.InMemoryEmulator.Tests; + +/// +/// Regression tests for GitHub Issue #67 — String literal aliases in aggregate queries +/// return null on Linux (where ServiceInterop is unavailable). +/// +/// Root cause: ProjectAggregateFields did not evaluate non-aggregate literal +/// expressions (like 'Settlement' AS Label). Instead it fell through to a +/// path-lookup branch that called SelectToken("'Settlement'") → null. +/// +/// Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select +/// "The SELECT clause supports arbitrary expressions including literal values." +/// +[Collection(IntegrationCollection.Name)] +public class Issue67StringLiteralAliasTests(EmulatorSession session) : IAsyncLifetime +{ + private readonly ITestContainerFixture _fixture = TestFixtureFactory.Create(session); + private Container _container = null!; + + public async ValueTask InitializeAsync() + { + _container = await _fixture.CreateContainerAsync("issue67", "/partitionKey"); + + await _container.CreateItemAsync( + new { id = "1", partitionKey = "pk1", type = "Settlement", amount = 100.0m }, + new PartitionKey("pk1")); + await _container.CreateItemAsync( + new { id = "2", partitionKey = "pk1", type = "Settlement", amount = 200.0m }, + new PartitionKey("pk1")); + await _container.CreateItemAsync( + new { id = "3", partitionKey = "pk1", type = "Refund", amount = 50.0m }, + new PartitionKey("pk1")); + } + + public async ValueTask DisposeAsync() + { + await _fixture.DisposeAsync(); + } + + private async Task> DrainQuery(string sql, PartitionKey? pk = null) + { + var opts = pk is not null ? new QueryRequestOptions { PartitionKey = pk } : null; + var iterator = _container.GetItemQueryIterator(sql, requestOptions: opts); + var results = new List(); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + return results; + } + + // ═══════════════════════════════════════════════════════════════════════════ + // String literal in aggregate SELECT projection + // ═══════════════════════════════════════════════════════════════════════════ + + [Fact] + public async Task StringLiteralAlias_InAggregateQuery_ShouldDeserializeCorrectly() + { + // Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select + // "The SELECT clause supports arbitrary expressions including literal values." + var results = await DrainQuery( + "SELECT 'Settlement' AS Label, COUNT(1) AS ItemCount, SUM(c.amount) AS Total FROM c WHERE c.type = 'Settlement'", + new PartitionKey("pk1")); + + results.Should().ContainSingle(); + var result = results[0]; + result["Label"]!.Value().Should().Be("Settlement"); + result["ItemCount"]!.Value().Should().Be(2); + result["Total"]!.Value().Should().Be(300.0m); + } + + [Fact] + public async Task StringLiteralAlias_WithMultipleQueries_ShouldReturnDistinctLiterals() + { + var settlements = await DrainQuery( + "SELECT 'Settlement' AS Label, COUNT(1) AS ItemCount, SUM(c.amount) AS Total FROM c WHERE c.type = 'Settlement'", + new PartitionKey("pk1")); + + var refunds = await DrainQuery( + "SELECT 'Refund' AS Label, COUNT(1) AS ItemCount, SUM(c.amount) AS Total FROM c WHERE c.type = 'Refund'", + new PartitionKey("pk1")); + + settlements.Should().ContainSingle(); + settlements[0]["Label"]!.Value().Should().Be("Settlement"); + settlements[0]["ItemCount"]!.Value().Should().Be(2); + settlements[0]["Total"]!.Value().Should().Be(300.0m); + + refunds.Should().ContainSingle(); + refunds[0]["Label"]!.Value().Should().Be("Refund"); + refunds[0]["ItemCount"]!.Value().Should().Be(1); + refunds[0]["Total"]!.Value().Should().Be(50.0m); + } + + [Fact] + public async Task NumericLiteralAlias_InAggregateQuery_ShouldDeserializeCorrectly() + { + // Also test numeric and boolean literal aliases for completeness + var results = await DrainQuery( + "SELECT 42 AS MagicNumber, COUNT(1) AS ItemCount FROM c WHERE c.type = 'Settlement'", + new PartitionKey("pk1")); + + results.Should().ContainSingle(); + results[0]["MagicNumber"]!.Value().Should().Be(42); + results[0]["ItemCount"]!.Value().Should().Be(2); + } + + [Fact] + public async Task BooleanLiteralAlias_InAggregateQuery_ShouldDeserializeCorrectly() + { + var results = await DrainQuery( + "SELECT true AS IsActive, COUNT(1) AS ItemCount FROM c WHERE c.type = 'Settlement'", + new PartitionKey("pk1")); + + results.Should().ContainSingle(); + results[0]["IsActive"]!.Value().Should().BeTrue(); + results[0]["ItemCount"]!.Value().Should().Be(2); + } + + [Fact] + public async Task NullLiteralAlias_InAggregateQuery_ShouldDeserializeCorrectly() + { + var results = await DrainQuery( + "SELECT null AS Nothing, COUNT(1) AS ItemCount FROM c WHERE c.type = 'Settlement'", + new PartitionKey("pk1")); + + results.Should().ContainSingle(); + results[0]["ItemCount"]!.Value().Should().Be(2); + + // On Linux (no ServiceInterop), our ProjectAggregateFields correctly returns null. + // On Windows, ServiceInterop's pipeline may omit null literal fields from aggregates. + var nothingToken = results[0]["Nothing"]; + if (nothingToken is not null) + { + nothingToken.Type.Should().Be(JTokenType.Null); + } + } +} From 4c875b46d0774273f728c9c7419aae6c72528596 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 20:24:27 +0100 Subject: [PATCH 06/16] ci: add Windows integration test job to catch platform-specific divergences ServiceInterop.dll is only available on Windows, causing fundamentally different code paths to execute. Adding a windows-latest integration test job ensures both paths are tested on every PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/_build-and-test.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/_build-and-test.yml b/.github/workflows/_build-and-test.yml index cde30eb..9822502 100644 --- a/.github/workflows/_build-and-test.yml +++ b/.github/workflows/_build-and-test.yml @@ -85,3 +85,20 @@ jobs: - name: Integration Tests run: dotnet test tests/CosmosDB.InMemoryEmulator.Tests.Integration --configuration Release --no-build --verbosity normal --framework ${{ matrix.framework }} + + test-integration-windows: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + framework: [net8.0, net10.0] + name: integration-windows (${{ matrix.framework }}) + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup-dotnet-build + with: + build-args: --framework ${{ matrix.framework }} + + - name: Integration Tests + shell: bash + run: dotnet test tests/CosmosDB.InMemoryEmulator.Tests.Integration --configuration Release --no-build --verbosity normal --framework ${{ matrix.framework }} From 2825cc3bfefd200e85e67b754082bfd8d5b1c6c0 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 20:28:11 +0100 Subject: [PATCH 07/16] ci: rename integration job to integration-linux for clarity Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/_build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_build-and-test.yml b/.github/workflows/_build-and-test.yml index 9822502..1482eb0 100644 --- a/.github/workflows/_build-and-test.yml +++ b/.github/workflows/_build-and-test.yml @@ -68,7 +68,7 @@ jobs: fail-fast: false matrix: framework: [net8.0, net10.0] - name: integration (${{ matrix.framework }}) + name: integration-linux (${{ matrix.framework }}) steps: - uses: actions/checkout@v5 - uses: ./.github/actions/setup-dotnet-build From 038e994b03bdb8778653d04ea9a53937dc6d26c3 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Tue, 19 May 2026 23:49:04 +0100 Subject: [PATCH 08/16] fix: treat missing properties as null in FilterPredicate evaluation (#70) In real Cosmos DB, FilterPredicate on patch operations treats missing properties as null, so 'FROM c WHERE c.prop = null' matches documents where the property is absent (e.g. due to NullValueHandling.Ignore). Add treatUndefinedAsNull parameter to the WHERE evaluation chain, passed as true only from the two FilterPredicate call sites. This preserves existing query semantics (undefined != null) while fixing the FilterPredicate behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../InMemoryContainer.cs | 58 ++++-- src/Directory.Build.props | 2 +- ...ue70FilterPredicateMissingPropertyTests.cs | 189 ++++++++++++++++++ .../PatchDeepDiveTests.cs | 110 ++++++++++ 4 files changed, 339 insertions(+), 20 deletions(-) create mode 100644 tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue70FilterPredicateMissingPropertyTests.cs diff --git a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs index 1d574e0..546fd40 100644 --- a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs +++ b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs @@ -1130,7 +1130,7 @@ private ItemResponse PatchItemCore( if (predicateParsed.Where is not null) { var matches = EvaluateWhereExpression(predicateParsed.Where, jObj, predicateParsed.FromAlias, - new Dictionary(), null); + new Dictionary(), null, treatUndefinedAsNull: true); if (!matches) { throw InMemoryCosmosException.Create("Precondition Failed", @@ -1614,7 +1614,7 @@ private ResponseMessage PatchItemStreamCore( if (predicateParsed.Where is not null) { var matches = EvaluateWhereExpression(predicateParsed.Where, jObj, predicateParsed.FromAlias, - new Dictionary(), null); + new Dictionary(), null, treatUndefinedAsNull: true); if (!matches) { return CreateResponseMessage(HttpStatusCode.PreconditionFailed); @@ -5543,17 +5543,17 @@ private static Dictionary ExtractQueryParameters(QueryDefinition private static bool EvaluateWhereExpression( WhereExpression expression, JObject item, string fromAlias, - IDictionary parameters, JoinClause join) + IDictionary parameters, JoinClause join, bool treatUndefinedAsNull = false) { return expression switch { - ComparisonCondition c => EvaluateComparison(c, item, fromAlias, parameters), - AndCondition a => EvaluateWhereExpression(a.Left, item, fromAlias, parameters, join) - && EvaluateWhereExpression(a.Right, item, fromAlias, parameters, join), - OrCondition o => EvaluateWhereExpression(o.Left, item, fromAlias, parameters, join) - || EvaluateWhereExpression(o.Right, item, fromAlias, parameters, join), - NotCondition n => !EvaluateWhereExpressionIncludesUndefined(n.Inner, item, fromAlias, parameters, join) - && !EvaluateWhereExpression(n.Inner, item, fromAlias, parameters, join), + ComparisonCondition c => EvaluateComparison(c, item, fromAlias, parameters, treatUndefinedAsNull), + AndCondition a => EvaluateWhereExpression(a.Left, item, fromAlias, parameters, join, treatUndefinedAsNull) + && EvaluateWhereExpression(a.Right, item, fromAlias, parameters, join, treatUndefinedAsNull), + OrCondition o => EvaluateWhereExpression(o.Left, item, fromAlias, parameters, join, treatUndefinedAsNull) + || EvaluateWhereExpression(o.Right, item, fromAlias, parameters, join, treatUndefinedAsNull), + NotCondition n => !EvaluateWhereExpressionIncludesUndefined(n.Inner, item, fromAlias, parameters, join, treatUndefinedAsNull) + && !EvaluateWhereExpression(n.Inner, item, fromAlias, parameters, join, treatUndefinedAsNull), FunctionCondition f => EvaluateFunction(f, item, fromAlias, parameters), ExistsCondition e => EvaluateExists(e, item, fromAlias, parameters, join), SqlExpressionCondition s => IsTruthy(EvaluateSqlExpression(s.Expression, item, fromAlias, parameters)), @@ -5562,10 +5562,20 @@ private static bool EvaluateWhereExpression( } private static bool EvaluateComparison( - ComparisonCondition comparison, JObject item, string fromAlias, IDictionary parameters) + ComparisonCondition comparison, JObject item, string fromAlias, IDictionary parameters, + bool treatUndefinedAsNull = false) { var leftValue = ResolveValue(comparison.Left, item, fromAlias, parameters); var rightValue = ResolveValue(comparison.Right, item, fromAlias, parameters); + + // Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/partial-document-update#filter-predicate + // In FilterPredicate context, missing properties are treated as null. + if (treatUndefinedAsNull) + { + if (leftValue is UndefinedValue) leftValue = null; + if (rightValue is UndefinedValue) rightValue = null; + } + if (leftValue is UndefinedValue || rightValue is UndefinedValue) return false; return comparison.Operator switch @@ -5587,19 +5597,19 @@ private static bool EvaluateComparison( /// private static bool EvaluateWhereExpressionIncludesUndefined( WhereExpression expression, JObject item, string fromAlias, - IDictionary parameters, JoinClause join) + IDictionary parameters, JoinClause join, bool treatUndefinedAsNull = false) { return expression switch { - ComparisonCondition c => ComparisonIncludesUndefined(c, item, fromAlias, parameters), + ComparisonCondition c => ComparisonIncludesUndefined(c, item, fromAlias, parameters, treatUndefinedAsNull), AndCondition a => - EvaluateWhereExpressionIncludesUndefined(a.Left, item, fromAlias, parameters, join) || - EvaluateWhereExpressionIncludesUndefined(a.Right, item, fromAlias, parameters, join), + EvaluateWhereExpressionIncludesUndefined(a.Left, item, fromAlias, parameters, join, treatUndefinedAsNull) || + EvaluateWhereExpressionIncludesUndefined(a.Right, item, fromAlias, parameters, join, treatUndefinedAsNull), OrCondition o => - EvaluateWhereExpressionIncludesUndefined(o.Left, item, fromAlias, parameters, join) || - EvaluateWhereExpressionIncludesUndefined(o.Right, item, fromAlias, parameters, join), + EvaluateWhereExpressionIncludesUndefined(o.Left, item, fromAlias, parameters, join, treatUndefinedAsNull) || + EvaluateWhereExpressionIncludesUndefined(o.Right, item, fromAlias, parameters, join, treatUndefinedAsNull), NotCondition n => - EvaluateWhereExpressionIncludesUndefined(n.Inner, item, fromAlias, parameters, join), + EvaluateWhereExpressionIncludesUndefined(n.Inner, item, fromAlias, parameters, join, treatUndefinedAsNull), SqlExpressionCondition s => EvaluateSqlExpression(s.Expression, item, fromAlias, parameters) is UndefinedValue, _ => false, @@ -5609,12 +5619,22 @@ private static bool EvaluateWhereExpressionIncludesUndefined( /// /// Checks if a comparison involves undefined semantics. For most operators, only /// UndefinedValue counts. For LIKE, null also produces undefined (three-value logic). + /// When treatUndefinedAsNull is true, undefined is treated as null (FilterPredicate context). /// private static bool ComparisonIncludesUndefined( - ComparisonCondition c, JObject item, string fromAlias, IDictionary parameters) + ComparisonCondition c, JObject item, string fromAlias, IDictionary parameters, + bool treatUndefinedAsNull = false) { var left = ResolveValue(c.Left, item, fromAlias, parameters); var right = ResolveValue(c.Right, item, fromAlias, parameters); + + // In FilterPredicate context, undefined is treated as null — not "undefined" + if (treatUndefinedAsNull) + { + if (left is UndefinedValue) left = null; + if (right is UndefinedValue) right = null; + } + if (left is UndefinedValue || right is UndefinedValue) return true; // LIKE with null operand(s) produces undefined per three-value logic diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 4482e58..cc5f590 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,7 +5,7 @@ true - 4.0.19 + 4.0.20 lemonlion Copyright (c) 2026 lemonlion MIT diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue70FilterPredicateMissingPropertyTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue70FilterPredicateMissingPropertyTests.cs new file mode 100644 index 0000000..2031de6 --- /dev/null +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/Issue70FilterPredicateMissingPropertyTests.cs @@ -0,0 +1,189 @@ +using System.Net; +using AwesomeAssertions; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using Xunit; + +namespace CosmosDB.InMemoryEmulator.Tests; + +/// +/// Regression tests for GitHub Issue #70 — FilterPredicate does not treat missing +/// properties as null when NullValueHandling.Ignore is used. +/// +/// When a document is serialized with NullValueHandling.Ignore, null properties are +/// omitted entirely. Real Cosmos DB treats these missing properties as null during +/// FilterPredicate evaluation, so "FROM c WHERE c.prop = null" should match. +/// +/// Tagged InMemoryOnly because the Windows Cosmos DB Emulator (v2.14.0) does not +/// support FilterPredicate syntax — it returns 400 BadRequest (tracked: #53). +/// +/// Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/partial-document-update#filter-predicate +/// "The filter predicate is evaluated against the existing state of the document." +/// Missing properties are semantically equivalent to null in filter predicate evaluation. +/// +public class Issue70FilterPredicateMissingPropertyTests : IAsyncLifetime +{ + private InMemoryCosmosResult _cosmos = null!; + private Container _container = null!; + + public ValueTask InitializeAsync() + { + _cosmos = InMemoryCosmos.Create("issue70", "/partitionKey", + configureOptions: opts => opts.Serializer = new CosmosJsonDotNetSerializer(new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy() + }, + NullValueHandling = NullValueHandling.Ignore + })); + _container = _cosmos.Container; + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cosmos.Dispose(); + return ValueTask.CompletedTask; + } + + [Trait(TestTraits.Target, TestTraits.InMemoryOnly)] + [Fact] + public async Task PatchWithFilterPredicate_MissingPropertyEqualsNull_Succeeds() + { + // linkedId is null → will NOT be serialized due to NullValueHandling.Ignore + var document = new + { + id = Guid.NewGuid().ToString(), + partitionKey = "pk-1", + linkedId = (string?)null, + name = "test" + }; + await _container.CreateItemAsync(document, new PartitionKey("pk-1")); + + // This FilterPredicate should match because the missing property should be treated as null + var patchOperations = new[] { PatchOperation.Set("/linkedId", Guid.NewGuid().ToString()) }; + var options = new PatchItemRequestOptions + { + FilterPredicate = "FROM c WHERE c.linkedId = null" + }; + + var response = await _container.PatchItemAsync( + document.id, new PartitionKey("pk-1"), patchOperations, options); + + response.StatusCode.Should().Be(HttpStatusCode.OK); + } + + [Trait(TestTraits.Target, TestTraits.InMemoryOnly)] + [Fact] + public async Task PatchWithFilterPredicate_MissingPropertyNotEqualNull_FailsPrecondition() + { + // linkedId is null → will NOT be serialized due to NullValueHandling.Ignore + var document = new + { + id = Guid.NewGuid().ToString(), + partitionKey = "pk-1", + linkedId = (string?)null, + name = "test" + }; + await _container.CreateItemAsync(document, new PartitionKey("pk-1")); + + // WHERE c.linkedId != null should NOT match when property is missing (treated as null) + var patchOperations = new[] { PatchOperation.Set("/name", "updated") }; + var options = new PatchItemRequestOptions + { + FilterPredicate = "FROM c WHERE c.linkedId != null" + }; + + var act = () => _container.PatchItemAsync( + document.id, new PartitionKey("pk-1"), patchOperations, options); + + var ex = await act.Should().ThrowAsync(); + ex.Which.StatusCode.Should().Be(HttpStatusCode.PreconditionFailed); + } + + [Trait(TestTraits.Target, TestTraits.InMemoryOnly)] + [Fact] + public async Task PatchWithFilterPredicate_ExplicitNullPropertyEqualsNull_Succeeds() + { + // Use stream to explicitly write null (bypassing NullValueHandling.Ignore) + var id = Guid.NewGuid().ToString(); + var json = $$"""{"id":"{{id}}","partitionKey":"pk-1","linkedId":null,"name":"test"}"""; + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + await _container.CreateItemStreamAsync(stream, new PartitionKey("pk-1")); + + var patchOperations = new[] { PatchOperation.Set("/linkedId", "new-value") }; + var options = new PatchItemRequestOptions + { + FilterPredicate = "FROM c WHERE c.linkedId = null" + }; + + var response = await _container.PatchItemAsync( + id, new PartitionKey("pk-1"), patchOperations, options); + + response.StatusCode.Should().Be(HttpStatusCode.OK); + } + + [Trait(TestTraits.Target, TestTraits.InMemoryOnly)] + [Fact] + public async Task PatchWithFilterPredicate_MissingPropertyEqualsValue_FailsPrecondition() + { + // linkedId is null → will NOT be serialized due to NullValueHandling.Ignore + var document = new + { + id = Guid.NewGuid().ToString(), + partitionKey = "pk-1", + linkedId = (string?)null, + name = "test" + }; + await _container.CreateItemAsync(document, new PartitionKey("pk-1")); + + // WHERE c.linkedId = 'some-value' should NOT match — missing property treated as null ≠ 'some-value' + var patchOperations = new[] { PatchOperation.Set("/name", "updated") }; + var options = new PatchItemRequestOptions + { + FilterPredicate = "FROM c WHERE c.linkedId = 'some-value'" + }; + + var act = () => _container.PatchItemAsync( + document.id, new PartitionKey("pk-1"), patchOperations, options); + + var ex = await act.Should().ThrowAsync(); + ex.Which.StatusCode.Should().Be(HttpStatusCode.PreconditionFailed); + } +} + +/// +/// A simple Newtonsoft.Json-based CosmosSerializer for test use. +/// The SDK's built-in one is internal, so we provide a minimal implementation. +/// +internal sealed class CosmosJsonDotNetSerializer : CosmosSerializer +{ + private readonly JsonSerializer _serializer; + + public CosmosJsonDotNetSerializer(JsonSerializerSettings settings) + { + _serializer = JsonSerializer.Create(settings); + } + + public override T FromStream(Stream stream) + { + using var sr = new StreamReader(stream); + using var jr = new JsonTextReader(sr); + return _serializer.Deserialize(jr)!; + } + + public override Stream ToStream(T input) + { + var ms = new MemoryStream(); + using (var sw = new StreamWriter(ms, leaveOpen: true)) + using (var jw = new JsonTextWriter(sw)) + { + _serializer.Serialize(jw, input); + jw.Flush(); + } + ms.Position = 0; + return ms; + } +} diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/PatchDeepDiveTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/PatchDeepDiveTests.cs index 0cee615..e4a9321 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/PatchDeepDiveTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/PatchDeepDiveTests.cs @@ -594,6 +594,116 @@ public async Task Patch_FilterPredicate_CombinedWithETag() } +// ── Category K2: FilterPredicate Missing Property = null (Issue #70) ───────── + +/// +/// Regression tests for Issue #70: FilterPredicate should treat missing properties as null. +/// Uses InMemoryContainer directly with raw JSON streams to control property presence. +/// +/// Ref: https://learn.microsoft.com/en-us/azure/cosmos-db/partial-document-update#filter-predicate +/// "The filter predicate is evaluated against the existing state of the document." +/// Missing properties are semantically equivalent to null in filter predicate evaluation. +/// +public class PatchFilterPredicateMissingPropertyTests +{ + [Fact] + public async Task Patch_FilterPredicate_MissingPropertyEqualsNull_Succeeds() + { + var container = new InMemoryContainer("test", "/partitionKey"); + // Document without linkedId property at all + await container.CreateItemStreamAsync( + new MemoryStream(Encoding.UTF8.GetBytes( + """{"id":"1","partitionKey":"pk1","name":"test"}""")), + new PartitionKey("pk1")); + + var result = await container.PatchItemAsync( + "1", new PartitionKey("pk1"), + [PatchOperation.Set("/linkedId", "new-value")], + new PatchItemRequestOptions { FilterPredicate = "FROM c WHERE c.linkedId = null" }); + + result.Resource["linkedId"]!.Value().Should().Be("new-value"); + } + + [Fact] + public async Task Patch_FilterPredicate_MissingPropertyNotEqualNull_ThrowsPreconditionFailed() + { + var container = new InMemoryContainer("test", "/partitionKey"); + // Document without linkedId property at all + await container.CreateItemStreamAsync( + new MemoryStream(Encoding.UTF8.GetBytes( + """{"id":"1","partitionKey":"pk1","name":"test"}""")), + new PartitionKey("pk1")); + + var act = () => container.PatchItemAsync( + "1", new PartitionKey("pk1"), + [PatchOperation.Set("/name", "updated")], + new PatchItemRequestOptions { FilterPredicate = "FROM c WHERE c.linkedId != null" }); + + await act.Should().ThrowAsync() + .Where(e => e.StatusCode == HttpStatusCode.PreconditionFailed); + } + + [Fact] + public async Task Patch_FilterPredicate_MissingPropertyEqualsString_ThrowsPreconditionFailed() + { + var container = new InMemoryContainer("test", "/partitionKey"); + // Document without linkedId property at all + await container.CreateItemStreamAsync( + new MemoryStream(Encoding.UTF8.GetBytes( + """{"id":"1","partitionKey":"pk1","name":"test"}""")), + new PartitionKey("pk1")); + + // undefined treated as null — null ≠ 'some-value' + var act = () => container.PatchItemAsync( + "1", new PartitionKey("pk1"), + [PatchOperation.Set("/name", "updated")], + new PatchItemRequestOptions { FilterPredicate = "FROM c WHERE c.linkedId = 'some-value'" }); + + await act.Should().ThrowAsync() + .Where(e => e.StatusCode == HttpStatusCode.PreconditionFailed); + } + + [Fact] + public async Task Patch_FilterPredicate_ExplicitNullEqualsNull_Succeeds() + { + var container = new InMemoryContainer("test", "/partitionKey"); + // Document WITH explicit null for linkedId + await container.CreateItemStreamAsync( + new MemoryStream(Encoding.UTF8.GetBytes( + """{"id":"1","partitionKey":"pk1","linkedId":null,"name":"test"}""")), + new PartitionKey("pk1")); + + var result = await container.PatchItemAsync( + "1", new PartitionKey("pk1"), + [PatchOperation.Set("/linkedId", "new-value")], + new PatchItemRequestOptions { FilterPredicate = "FROM c WHERE c.linkedId = null" }); + + result.Resource["linkedId"]!.Value().Should().Be("new-value"); + } + + [Fact] + public async Task Patch_FilterPredicate_MissingPropertyWithAndCondition_Succeeds() + { + var container = new InMemoryContainer("test", "/partitionKey"); + // Document without linkedId but with name + await container.CreateItemStreamAsync( + new MemoryStream(Encoding.UTF8.GetBytes( + """{"id":"1","partitionKey":"pk1","name":"test"}""")), + new PartitionKey("pk1")); + + var result = await container.PatchItemAsync( + "1", new PartitionKey("pk1"), + [PatchOperation.Set("/linkedId", "new-value")], + new PatchItemRequestOptions + { + FilterPredicate = "FROM c WHERE c.linkedId = null AND c.name = 'test'" + }); + + result.Resource["linkedId"]!.Value().Should().Be("new-value"); + } +} + + // ── Category L: FakeCosmosHandler Patch Bugs ───────────────────────────────── public class FakeCosmosHandlerPatchBugTests From 28a22c3fef4976aa05cabc9b63263947dc4f17cf Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 20 May 2026 14:47:27 +0100 Subject: [PATCH 09/16] fix: return documents in insertion order for queries without ORDER BY Queries without ORDER BY now return documents in insertion order, matching real Cosmos DB behavior. Previously documents were returned in hash-map order due to ConcurrentDictionary enumeration. Added insertion-order tracking list to InMemoryContainer that maintains document position across create, replace, upsert, and delete operations. GetAllItemsForPartition() now iterates this ordered list instead of the dictionary directly. Fixes #72 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 5 + .../InMemoryContainer.cs | 102 +++++++- src/Directory.Build.props | 2 +- .../FakeCosmosHandlerInsertionOrderTests.cs | 245 ++++++++++++++++++ 4 files changed, 345 insertions(+), 9 deletions(-) create mode 100644 tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6da297f..cbd462a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.0.18] - 2026-05-15 + +### Fixed +- Queries without `ORDER BY` now return documents in insertion order, matching real Cosmos DB behavior (Issue #72). Previously documents were returned in hash-map order due to `ConcurrentDictionary` enumeration. Added insertion-order tracking to `InMemoryContainer` that maintains document position across create, replace, upsert, and delete operations. + ## [4.0.17] - 2026-05-14 ### Fixed diff --git a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs index 07447d3..a3158f3 100644 --- a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs +++ b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs @@ -74,6 +74,11 @@ internal class InMemoryContainer : Container, IContainerTestSetup private readonly ConcurrentDictionary<(string Id, string PartitionKey), string> _items = new(); private readonly ConcurrentDictionary<(string Id, string PartitionKey), string> _etags = new(); private readonly ConcurrentDictionary<(string Id, string PartitionKey), DateTimeOffset> _timestamps = new(); + // Ref: Observed behavior on Windows Cosmos DB Emulator — documents return in + // insertion order when no ORDER BY is applied. Maintains creation-time ordering + // so GetAllItemsForPartition can enumerate deterministically. + private readonly List<(string Id, string PartitionKey)> _insertionOrder = new(); + private readonly object _insertionOrderLock = new(); private readonly List<(DateTimeOffset Timestamp, string Id, string PartitionKey, string Json, bool IsDelete)> _changeFeed = new(); private readonly object _changeFeedLock = new(); private long _changeFeedLsnCounter; @@ -357,6 +362,7 @@ public void ClearItems() _items.Clear(); _etags.Clear(); _timestamps.Clear(); + lock (_insertionOrderLock) { _insertionOrder.Clear(); } lock (_changeFeedLock) { _changeFeed.Clear(); } } @@ -402,9 +408,11 @@ public void LoadPersistedState() /// public string ExportState() { - var items = _items - .Where(kvp => !IsExpired(kvp.Key)) - .Select(kvp => JsonParseHelpers.ParseJson(kvp.Value)).ToList(); + List<(string Id, string PartitionKey)> orderedKeys; + lock (_insertionOrderLock) { orderedKeys = _insertionOrder.ToList(); } + var items = orderedKeys + .Where(key => _items.ContainsKey(key) && !IsExpired(key)) + .Select(key => JsonParseHelpers.ParseJson(_items[key])).ToList(); var state = new JObject { ["items"] = new JArray(items) }; return state.ToString(Formatting.Indented); } @@ -437,6 +445,7 @@ public void ImportState(string json) _etags[key] = importEtag; _timestamps[key] = DateTimeOffset.UtcNow; _items[key] = EnrichWithSystemProperties(itemJson, importEtag, _timestamps[key]); + lock (_insertionOrderLock) { _insertionOrder.Add(key); } } } @@ -489,6 +498,25 @@ public void RestoreToPointInTime(DateTimeOffset pointInTime) _etags.Clear(); _timestamps.Clear(); _itemLocks.Clear(); + lock (_insertionOrderLock) { _insertionOrder.Clear(); } + + // Rebuild insertion order from the change feed replay sequence. + // Track which keys were first created (not deleted) to preserve original insertion order. + var insertionOrderKeys = new List<(string Id, string PartitionKey)>(); + foreach (var entry in feedSnapshot) + { + if (entry.IsDelete && entry.Json.Contains("\"_ttlEviction\":true", StringComparison.Ordinal)) + continue; + var entryKey = (entry.Id, entry.PartitionKey); + if (entry.IsDelete) + { + insertionOrderKeys.Remove(entryKey); + } + else if (!insertionOrderKeys.Contains(entryKey)) + { + insertionOrderKeys.Add(entryKey); + } + } foreach (var kvp in lastPerKey) { @@ -500,6 +528,15 @@ public void RestoreToPointInTime(DateTimeOffset pointInTime) _etags[key] = etag; _timestamps[key] = pointInTime; } + + lock (_insertionOrderLock) + { + foreach (var key in insertionOrderKeys) + { + if (_items.ContainsKey(key)) + _insertionOrder.Add(key); + } + } } // ─── IndexingPolicy ─────────────────────────────────────────────────────── @@ -738,6 +775,7 @@ public override async Task> CreateItemAsync( } TrackBatchWrite(key); + lock (_insertionOrderLock) { _insertionOrder.Add(key); } var etag = GenerateETag(); _etags[key] = etag; _timestamps[key] = DateTimeOffset.UtcNow; @@ -773,6 +811,7 @@ public override async Task> CreateItemAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } throw; } } @@ -867,6 +906,7 @@ public override async Task> UpsertItemAsync( } TrackBatchWrite(key); + if (!existed) { lock (_insertionOrderLock) { _insertionOrder.Add(key); } } try { @@ -893,6 +933,7 @@ public override async Task> UpsertItemAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } } throw; } @@ -1045,6 +1086,7 @@ public override async Task> DeleteItemAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } TrackBatchWrite(key); try @@ -1057,6 +1099,7 @@ public override async Task> DeleteItemAsync( _items[key] = existingJson; if (previousEtag is not null) _etags[key] = previousEtag; if (previousTimestamp.HasValue) _timestamps[key] = previousTimestamp.Value; + lock (_insertionOrderLock) { _insertionOrder.Add(key); } throw; } @@ -1236,6 +1279,7 @@ public override async Task CreateItemStreamAsync( } TrackBatchWrite(key); + lock (_insertionOrderLock) { _insertionOrder.Add(key); } var etag = GenerateETag(); _etags[key] = etag; _timestamps[key] = DateTimeOffset.UtcNow; @@ -1252,6 +1296,7 @@ public override async Task CreateItemStreamAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } throw; } @@ -1360,6 +1405,7 @@ public override async Task UpsertItemStreamAsync( } TrackBatchWrite(key); + if (!existed) { lock (_insertionOrderLock) { _insertionOrder.Add(key); } } try { ExecutePostTriggers(requestOptions, JsonParseHelpers.ParseJson(enrichedJson), "Upsert"); @@ -1378,6 +1424,7 @@ public override async Task UpsertItemStreamAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } } throw; } @@ -1525,6 +1572,7 @@ public override async Task DeleteItemStreamAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } TrackBatchWrite(key); try @@ -1537,6 +1585,7 @@ public override async Task DeleteItemStreamAsync( _items[key] = existingJson; if (previousEtag is not null) _etags[key] = previousEtag; if (previousTimestamp.HasValue) _timestamps[key] = previousTimestamp.Value; + lock (_insertionOrderLock) { _insertionOrder.Add(key); } throw; } @@ -2364,6 +2413,7 @@ public override Task DeleteContainerAsync( _items.Clear(); _etags.Clear(); _timestamps.Clear(); + lock (_insertionOrderLock) { _insertionOrder.Clear(); } lock (_changeFeedLock) { _changeFeed.Clear(); } _storedProcedures.Clear(); _userDefinedFunctions.Clear(); @@ -2386,6 +2436,7 @@ public override Task DeleteContainerStreamAsync( _items.Clear(); _etags.Clear(); _timestamps.Clear(); + lock (_insertionOrderLock) { _insertionOrder.Clear(); } lock (_changeFeedLock) { _changeFeed.Clear(); } _storedProcedures.Clear(); _userDefinedFunctions.Clear(); @@ -2449,6 +2500,7 @@ public override Task DeleteAllItemsByPartitionKeyStreamAsync( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } RecordDeleteTombstone(key.Id, pk, partitionKey); } return Task.FromResult(CreateResponseMessage(HttpStatusCode.OK)); @@ -3408,6 +3460,7 @@ private void EvictIfExpired((string Id, string PartitionKey) key) _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } // Record a delete tombstone in the change feed so consumers see TTL evictions RecordDeleteTombstone(key.Id, key.PartitionKey, isTtlEviction: true); @@ -3467,6 +3520,27 @@ internal void RestoreSnapshot( _items.TryRemove(key, out _); _etags.TryRemove(key, out _); _timestamps.TryRemove(key, out _); + lock (_insertionOrderLock) { _insertionOrder.Remove(key); } + } + } + + // Restore insertion order for keys that were newly added during the batch + // but didn't exist in the snapshot — they need to be removed from insertion order. + // Keys that existed in the snapshot should already be in insertion order. + lock (_insertionOrderLock) + { + foreach (var key in touchedKeys) + { + if (!itemsSnapshot.ContainsKey(key)) + { + // Already removed above + } + else if (!_insertionOrder.Contains(key)) + { + // Item existed in snapshot but is missing from insertion order + // (shouldn't normally happen, but be safe) + _insertionOrder.Add(key); + } } } @@ -4061,8 +4135,16 @@ private static void EnrichStoredProcedureSystemProperties(StoredProcedurePropert // Private helpers — Query execution pipeline // ═══════════════════════════════════════════════════════════════════════════ + // Ref: Observed behavior on the Windows Cosmos DB Emulator (priority 6): + // Documents returned by queries without ORDER BY are in insertion order. private IEnumerable GetAllItemsForPartition(QueryRequestOptions requestOptions) { + List<(string Id, string PartitionKey)> orderedKeys; + lock (_insertionOrderLock) + { + orderedKeys = _insertionOrder.ToList(); + } + if (requestOptions?.PartitionKey is not null && requestOptions.PartitionKey != PartitionKey.None) { @@ -4076,15 +4158,19 @@ private IEnumerable GetAllItemsForPartition(QueryRequestOptions requestO if (queryComponents > 0 && queryComponents < PartitionKeyPaths.Count) { var prefix = pk + "|"; - return _items - .Where(kvp => (kvp.Key.PartitionKey?.StartsWith(prefix, StringComparison.Ordinal) ?? false) && !IsExpired(kvp.Key)) - .Select(kvp => kvp.Value); + return orderedKeys + .Where(key => (key.PartitionKey?.StartsWith(prefix, StringComparison.Ordinal) ?? false) && !IsExpired(key) && _items.ContainsKey(key)) + .Select(key => _items[key]); } } - return _items.Where(kvp => kvp.Key.PartitionKey == pk && !IsExpired(kvp.Key)).Select(kvp => kvp.Value); + return orderedKeys + .Where(key => key.PartitionKey == pk && !IsExpired(key) && _items.ContainsKey(key)) + .Select(key => _items[key]); } - return _items.Where(kvp => !IsExpired(kvp.Key)).Select(kvp => kvp.Value); + return orderedKeys + .Where(key => !IsExpired(key) && _items.ContainsKey(key)) + .Select(key => _items[key]); } private static int CountPartitionKeyComponents(PartitionKey partitionKey) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index fdf6b25..711e662 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,7 +5,7 @@ true - 4.0.17 + 4.0.18 lemonlion Copyright (c) 2026 lemonlion MIT diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderTests.cs new file mode 100644 index 0000000..e931af5 --- /dev/null +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderTests.cs @@ -0,0 +1,245 @@ +using AwesomeAssertions; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; +using Xunit; + +namespace CosmosDB.InMemoryEmulator.Tests; + +/// +/// Tests that queries without an ORDER BY clause return documents in insertion order, +/// matching the observed behavior of real Cosmos DB and the Windows Cosmos Emulator. +/// +[Collection(IntegrationCollection.Name)] +public class FakeCosmosHandlerInsertionOrderTests(EmulatorSession session) : IAsyncLifetime +{ + private readonly ITestContainerFixture _fixture = TestFixtureFactory.Create(session); + private Container _container = null!; + + public async ValueTask InitializeAsync() + { + _container = await _fixture.CreateContainerAsync("test-insertion-order", "/partitionKey"); + } + + public async ValueTask DisposeAsync() + { + await _fixture.DisposeAsync(); + } + + private async Task> DrainQuery(string sql, string? partitionKey = null) + { + var options = partitionKey is not null + ? new QueryRequestOptions { PartitionKey = new PartitionKey(partitionKey) } + : null; + var iterator = _container.GetItemQueryIterator(sql, requestOptions: options); + var results = new List(); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + return results; + } + + private async Task> DrainQuery(QueryDefinition queryDef, string? partitionKey = null) + { + var options = partitionKey is not null + ? new QueryRequestOptions { PartitionKey = new PartitionKey(partitionKey) } + : null; + var iterator = _container.GetItemQueryIterator(queryDef, requestOptions: options); + var results = new List(); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + return results; + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Insertion order — queries without ORDER BY + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Verifies that documents queried within a single partition are returned in + /// insertion order when no ORDER BY clause is specified. + /// Ref: Observed behavior on Windows Cosmos DB Emulator — documents return + /// in the order they were created when no ORDER BY is applied. + /// + [Fact] + public async Task Query_WithoutOrderBy_ReturnsSinglePartitionDocsInInsertionOrder() + { + // Arrange — insert 10 documents sequentially + var insertedIds = new List(); + for (var i = 1; i <= 10; i++) + { + var id = $"order-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = "pk-order", Name = $"Doc {i}", Value = i }, + new PartitionKey("pk-order")); + } + + // Act — query without ORDER BY + var results = await DrainQuery( + "SELECT * FROM c WHERE c.partitionKey = 'pk-order'", + "pk-order"); + + // Assert — returned IDs match insertion order + var returnedIds = results.Select(r => r.Id).ToList(); + returnedIds.Should().Equal(insertedIds); + } + + /// + /// Verifies that a cross-partition query without ORDER BY returns documents + /// in insertion order across all partitions. + /// + [Fact] + public async Task Query_WithoutOrderBy_ReturnsCrossPartitionDocsInInsertionOrder() + { + // Arrange — insert documents across two partitions, interleaved + var insertedIds = new List(); + for (var i = 1; i <= 8; i++) + { + var pk = i % 2 == 0 ? "pk-even" : "pk-odd"; + var id = $"cross-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + // Act — cross-partition query without ORDER BY + var results = await DrainQuery("SELECT * FROM c"); + + // Assert — returned IDs match insertion order + var returnedIds = results.Select(r => r.Id).ToList(); + returnedIds.Should().Equal(insertedIds); + } + + /// + /// Verifies that replacing a document does not change its position in the + /// insertion order (replace preserves original creation position). + /// + [Fact] + public async Task Query_AfterReplace_PreservesInsertionOrder() + { + // Arrange — insert 5 documents + var insertedIds = new List(); + for (var i = 1; i <= 5; i++) + { + var id = $"replace-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = "pk-replace", Name = $"Doc {i}", Value = i }, + new PartitionKey("pk-replace")); + } + + // Replace the 2nd document + await _container.ReplaceItemAsync( + new TestDocument { Id = "replace-0002", PartitionKey = "pk-replace", Name = "Updated", Value = 99 }, + "replace-0002", + new PartitionKey("pk-replace")); + + // Act + var results = await DrainQuery( + "SELECT * FROM c WHERE c.partitionKey = 'pk-replace'", + "pk-replace"); + + // Assert — order unchanged + var returnedIds = results.Select(r => r.Id).ToList(); + returnedIds.Should().Equal(insertedIds); + } + + /// + /// Verifies that deleting a document and re-creating it places the new + /// document at the end (new insertion position). + /// + [Fact] + public async Task Query_AfterDeleteAndRecreate_NewDocAppearsAtEnd() + { + // Arrange — insert 5 documents + for (var i = 1; i <= 5; i++) + { + await _container.CreateItemAsync( + new TestDocument { Id = $"delrec-{i:D4}", PartitionKey = "pk-delrec", Name = $"Doc {i}", Value = i }, + new PartitionKey("pk-delrec")); + } + + // Delete the 2nd document + await _container.DeleteItemAsync("delrec-0002", new PartitionKey("pk-delrec")); + + // Re-create with same ID + await _container.CreateItemAsync( + new TestDocument { Id = "delrec-0002", PartitionKey = "pk-delrec", Name = "Recreated", Value = 200 }, + new PartitionKey("pk-delrec")); + + // Act + var results = await DrainQuery( + "SELECT * FROM c WHERE c.partitionKey = 'pk-delrec'", + "pk-delrec"); + + // Assert — recreated doc is now at the end + var returnedIds = results.Select(r => r.Id).ToList(); + returnedIds.Should().Equal(["delrec-0001", "delrec-0003", "delrec-0004", "delrec-0005", "delrec-0002"]); + } + + /// + /// Verifies that upsert of a new document places it at the end of + /// the insertion order (same as create). + /// + [Fact] + public async Task Query_UpsertNewDoc_AppearsAtEnd() + { + // Arrange — insert 3 documents + for (var i = 1; i <= 3; i++) + { + await _container.CreateItemAsync( + new TestDocument { Id = $"upsert-{i:D4}", PartitionKey = "pk-upsert", Name = $"Doc {i}", Value = i }, + new PartitionKey("pk-upsert")); + } + + // Upsert a new document + await _container.UpsertItemAsync( + new TestDocument { Id = "upsert-0004", PartitionKey = "pk-upsert", Name = "New", Value = 4 }, + new PartitionKey("pk-upsert")); + + // Act + var results = await DrainQuery( + "SELECT * FROM c WHERE c.partitionKey = 'pk-upsert'", + "pk-upsert"); + + // Assert — upserted new doc at end + var returnedIds = results.Select(r => r.Id).ToList(); + returnedIds.Should().Equal(["upsert-0001", "upsert-0002", "upsert-0003", "upsert-0004"]); + } + + /// + /// Verifies that upsert of an existing document preserves its original + /// insertion position (same as replace). + /// + [Fact] + public async Task Query_UpsertExistingDoc_PreservesInsertionOrder() + { + // Arrange — insert 3 documents + for (var i = 1; i <= 3; i++) + { + await _container.CreateItemAsync( + new TestDocument { Id = $"upsert-ex-{i:D4}", PartitionKey = "pk-upsert-ex", Name = $"Doc {i}", Value = i }, + new PartitionKey("pk-upsert-ex")); + } + + // Upsert an existing document (update) + await _container.UpsertItemAsync( + new TestDocument { Id = "upsert-ex-0002", PartitionKey = "pk-upsert-ex", Name = "Updated", Value = 99 }, + new PartitionKey("pk-upsert-ex")); + + // Act + var results = await DrainQuery( + "SELECT * FROM c WHERE c.partitionKey = 'pk-upsert-ex'", + "pk-upsert-ex"); + + // Assert — order preserved + var returnedIds = results.Select(r => r.Id).ToList(); + returnedIds.Should().Equal(["upsert-ex-0001", "upsert-ex-0002", "upsert-ex-0003"]); + } +} From fbd7997b1c71211693473248bd3c2ad7aede4909 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 20 May 2026 14:56:45 +0100 Subject: [PATCH 10/16] Add hardening integration tests for insertion order behavior Comprehensive edge case tests covering: - Large batch (55 docs) insertion order - Multi-partition interleaved insertion order - Repeated replaces preserving position - Mixed operations sequence (create/delete/replace/upsert) - SELECT projections maintaining order - WHERE filter preserving relative order - TOP and OFFSET/LIMIT pagination order - Empty container and single document edge cases - Delete all + recreate ordering - Upsert-only creates ordering - DISTINCT VALUE preserving first occurrence order - COUNT and SUM aggregates not crashing - Cross-partition vs single-partition order - Rapid sequential creates order - ORDER BY overriding insertion order - Range filter preserving relative order - Multiple non-consecutive deletes preserving remaining order Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...smosHandlerInsertionOrderHardeningTests.cs | 718 ++++++++++++++++++ 1 file changed, 718 insertions(+) create mode 100644 tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderHardeningTests.cs diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderHardeningTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderHardeningTests.cs new file mode 100644 index 0000000..042f90e --- /dev/null +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/FakeCosmosHandlerInsertionOrderHardeningTests.cs @@ -0,0 +1,718 @@ +using AwesomeAssertions; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; +using Xunit; + +namespace CosmosDB.InMemoryEmulator.Tests; + +/// +/// Hardening tests for insertion order behavior in queries without ORDER BY. +/// Covers edge cases: large batches, multi-partition interleaving, repeated replaces, +/// mixed operation sequences, projections, filters, pagination, and more. +/// Ref: Observed behavior on Windows Cosmos DB Emulator — documents return +/// in the order they were created when no ORDER BY is applied. +/// +[Collection(IntegrationCollection.Name)] +public class FakeCosmosHandlerInsertionOrderHardeningTests(EmulatorSession session) : IAsyncLifetime +{ + private readonly ITestContainerFixture _fixture = TestFixtureFactory.Create(session); + private Container _container = null!; + + public async ValueTask InitializeAsync() + { + _container = await _fixture.CreateContainerAsync("test-insertion-order-hardening", "/partitionKey"); + } + + public async ValueTask DisposeAsync() + { + await _fixture.DisposeAsync(); + } + + private async Task> DrainQuery(string sql, string? partitionKey = null) + { + var options = partitionKey is not null + ? new QueryRequestOptions { PartitionKey = new PartitionKey(partitionKey) } + : null; + var iterator = _container.GetItemQueryIterator(sql, requestOptions: options); + var results = new List(); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + return results; + } + + private async Task> DrainQuery(QueryDefinition queryDef, string? partitionKey = null) + { + var options = partitionKey is not null + ? new QueryRequestOptions { PartitionKey = new PartitionKey(partitionKey) } + : null; + var iterator = _container.GetItemQueryIterator(queryDef, requestOptions: options); + var results = new List(); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + return results; + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 1. Large batch insertion order + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Inserting 50+ documents and querying without ORDER BY must return them + /// in exact insertion order. + /// + [Fact] + public async Task Query_LargeBatch_ReturnsDocsInInsertionOrder() + { + const string pk = "pk-large-batch"; + var insertedIds = new List(); + + for (var i = 1; i <= 55; i++) + { + var id = $"large-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 2. Multiple partitions interleaved + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Documents inserted across 3+ partitions in an interleaved pattern must + /// appear in global insertion order when queried cross-partition. + /// + [Fact] + public async Task Query_MultiplePKsInterleaved_ReturnsCrossPartitionInsertionOrder() + { + var partitions = new[] { "pk-interleave-a", "pk-interleave-b", "pk-interleave-c" }; + var insertedIds = new List(); + + for (var i = 1; i <= 15; i++) + { + var pk = partitions[(i - 1) % 3]; + var id = $"interleave-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery("SELECT * FROM c WHERE STARTSWITH(c.partitionKey, 'pk-interleave-')"); + + results.Select(r => r.Id).Should().Equal(insertedIds); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 3. Replace multiple times + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Replacing the same document 5+ times must never change its position in + /// the insertion order. + /// + [Fact] + public async Task Query_AfterMultipleReplacesOnSameDoc_PositionNeverChanges() + { + const string pk = "pk-multi-replace"; + var insertedIds = new List(); + + for (var i = 1; i <= 5; i++) + { + var id = $"mr-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + // Replace the 3rd document 6 times with different content each time + for (var r = 1; r <= 6; r++) + { + await _container.ReplaceItemAsync( + new TestDocument { Id = "mr-0003", PartitionKey = pk, Name = $"Replaced-{r}", Value = 100 + r }, + "mr-0003", + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds); + // Verify the content was actually updated + results.Single(r => r.Id == "mr-0003").Name.Should().Be("Replaced-6"); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 4. Mixed operations sequence + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Complex mixed operation sequence: + /// create A, B, C, D, E → delete C → replace A → create F → upsert B (existing) → upsert G (new) + /// Expected order: A, B, D, E, F, G + /// + [Fact] + public async Task Query_MixedOperationSequence_ReturnsExpectedInsertionOrder() + { + const string pk = "pk-mixed-ops"; + + // Create A, B, C, D, E + await _container.CreateItemAsync( + new TestDocument { Id = "mixed-A", PartitionKey = pk, Name = "A", Value = 1 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "mixed-B", PartitionKey = pk, Name = "B", Value = 2 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "mixed-C", PartitionKey = pk, Name = "C", Value = 3 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "mixed-D", PartitionKey = pk, Name = "D", Value = 4 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "mixed-E", PartitionKey = pk, Name = "E", Value = 5 }, + new PartitionKey(pk)); + + // Delete C + await _container.DeleteItemAsync("mixed-C", new PartitionKey(pk)); + + // Replace A (should not change position) + await _container.ReplaceItemAsync( + new TestDocument { Id = "mixed-A", PartitionKey = pk, Name = "A-replaced", Value = 100 }, + "mixed-A", + new PartitionKey(pk)); + + // Create F (should appear at end) + await _container.CreateItemAsync( + new TestDocument { Id = "mixed-F", PartitionKey = pk, Name = "F", Value = 6 }, + new PartitionKey(pk)); + + // Upsert B (existing — should not change position) + await _container.UpsertItemAsync( + new TestDocument { Id = "mixed-B", PartitionKey = pk, Name = "B-upserted", Value = 200 }, + new PartitionKey(pk)); + + // Upsert G (new — should appear at end) + await _container.UpsertItemAsync( + new TestDocument { Id = "mixed-G", PartitionKey = pk, Name = "G", Value = 7 }, + new PartitionKey(pk)); + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal( + ["mixed-A", "mixed-B", "mixed-D", "mixed-E", "mixed-F", "mixed-G"]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 5. Query with SELECT specific fields (projection) + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Projecting specific fields (SELECT c.id, c.name) must still maintain + /// insertion order. + /// + [Fact] + public async Task Query_WithProjection_MaintainsInsertionOrder() + { + const string pk = "pk-projection"; + var insertedIds = new List(); + + for (var i = 1; i <= 8; i++) + { + var id = $"proj-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Name-{i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT c.id, c.name FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 6. Query with WHERE filter + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Filtering with a WHERE clause must preserve relative insertion order of + /// the matched documents. + /// + [Fact] + public async Task Query_WithWhereFilter_MaintainsRelativeInsertionOrder() + { + const string pk = "pk-where-filter"; + + // Insert with alternating isActive values: true, false, true, false, true... + for (var i = 1; i <= 10; i++) + { + await _container.CreateItemAsync( + new TestDocument + { + Id = $"filter-{i:D4}", + PartitionKey = pk, + Name = $"Doc {i}", + Value = i, + IsActive = i % 2 != 0 + }, + new PartitionKey(pk)); + } + + // Query only active documents + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}' AND c.isActive = true", pk); + + // Expect odd-numbered docs only, in original order + results.Select(r => r.Id).Should().Equal( + ["filter-0001", "filter-0003", "filter-0005", "filter-0007", "filter-0009"]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 7. Query with TOP/OFFSET pagination + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Using TOP N returns the first N documents in insertion order. + /// + [Fact] + public async Task Query_WithTop_ReturnsFirstNInInsertionOrder() + { + const string pk = "pk-top"; + var insertedIds = new List(); + + for (var i = 1; i <= 20; i++) + { + var id = $"top-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT TOP 5 * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds.Take(5)); + } + + /// + /// Using OFFSET/LIMIT returns the correct page of documents in insertion order. + /// + [Fact] + public async Task Query_WithOffsetLimit_ReturnsCorrectPageInInsertionOrder() + { + const string pk = "pk-offset"; + var insertedIds = new List(); + + for (var i = 1; i <= 20; i++) + { + var id = $"offset-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + // Get page 2 (items 6-10) + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}' OFFSET 5 LIMIT 5", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds.Skip(5).Take(5)); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 8. Empty container query + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Querying an empty container must return an empty result without errors. + /// + [Fact] + public async Task Query_EmptyContainer_ReturnsEmptyResults() + { + var results = await DrainQuery( + "SELECT * FROM c WHERE c.partitionKey = 'pk-nonexistent'", "pk-nonexistent"); + + results.Should().BeEmpty(); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 9. Single document + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// A container with a single document must return just that document. + /// + [Fact] + public async Task Query_SingleDocument_ReturnsThatDocument() + { + const string pk = "pk-single-doc"; + + await _container.CreateItemAsync( + new TestDocument { Id = "only-one", PartitionKey = pk, Name = "Solo", Value = 42 }, + new PartitionKey(pk)); + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Should().HaveCount(1); + results[0].Id.Should().Be("only-one"); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 10. Delete all then recreate + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// After deleting all documents and inserting new ones, the new documents + /// should appear in their own insertion order with no ghost entries. + /// + [Fact] + public async Task Query_DeleteAllThenRecreate_ShowsOnlyNewDocsInOrder() + { + const string pk = "pk-del-all-recreate"; + + // Insert A, B, C + await _container.CreateItemAsync( + new TestDocument { Id = "orig-A", PartitionKey = pk, Name = "A", Value = 1 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "orig-B", PartitionKey = pk, Name = "B", Value = 2 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "orig-C", PartitionKey = pk, Name = "C", Value = 3 }, + new PartitionKey(pk)); + + // Delete all + await _container.DeleteItemAsync("orig-A", new PartitionKey(pk)); + await _container.DeleteItemAsync("orig-B", new PartitionKey(pk)); + await _container.DeleteItemAsync("orig-C", new PartitionKey(pk)); + + // Insert D, E + await _container.CreateItemAsync( + new TestDocument { Id = "new-D", PartitionKey = pk, Name = "D", Value = 4 }, + new PartitionKey(pk)); + await _container.CreateItemAsync( + new TestDocument { Id = "new-E", PartitionKey = pk, Name = "E", Value = 5 }, + new PartitionKey(pk)); + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(["new-D", "new-E"]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 11. Upsert-only to create multiple documents + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Using only UpsertItemAsync to create new documents must produce results + /// in the order the upserts were called. + /// + [Fact] + public async Task Query_UpsertOnlyCreates_ReturnsInUpsertCallOrder() + { + const string pk = "pk-upsert-create"; + var insertedIds = new List(); + + for (var i = 1; i <= 10; i++) + { + var id = $"upsert-new-{i:D4}"; + insertedIds.Add(id); + await _container.UpsertItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 12. Query with DISTINCT + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// DISTINCT on a field with duplicates must preserve the insertion order of + /// the first occurrence of each distinct value. + /// + [Fact] + public async Task Query_WithDistinctValue_MaintainsFirstOccurrenceInsertionOrder() + { + const string pk = "pk-distinct"; + + // Insert documents with duplicate Value fields + // Values: 10, 20, 10, 30, 20, 40 + var docs = new[] + { + ("dist-01", 10), ("dist-02", 20), ("dist-03", 10), + ("dist-04", 30), ("dist-05", 20), ("dist-06", 40) + }; + foreach (var (id, value) in docs) + { + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc-{value}", Value = value }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT DISTINCT VALUE c[\"value\"] FROM c WHERE c.partitionKey = '{pk}'", pk); + + // First occurrences in insertion order: 10, 20, 30, 40 + results.Should().Equal([10, 20, 30, 40]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 13. Aggregate functions don't crash + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// COUNT aggregate should return the correct count without crashing. + /// + [Fact] + public async Task Query_CountAggregate_ReturnsCorrectCount() + { + const string pk = "pk-agg-count"; + + for (var i = 1; i <= 7; i++) + { + await _container.CreateItemAsync( + new TestDocument { Id = $"agg-{i:D4}", PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT VALUE COUNT(1) FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Should().HaveCount(1); + results[0].Should().Be(7); + } + + /// + /// SUM aggregate should return the correct sum without crashing. + /// + [Fact] + public async Task Query_SumAggregate_ReturnsCorrectSum() + { + const string pk = "pk-agg-sum"; + + for (var i = 1; i <= 5; i++) + { + await _container.CreateItemAsync( + new TestDocument { Id = $"sum-{i:D4}", PartitionKey = pk, Name = $"Doc {i}", Value = i * 10 }, + new PartitionKey(pk)); + } + + // SUM of 10+20+30+40+50 = 150 + var results = await DrainQuery( + $"SELECT VALUE SUM(c[\"value\"]) FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Should().HaveCount(1); + results[0].Should().Be(150); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 14. Cross-partition: with vs without partition key filter + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// When querying within a single partition, insertion order within that + /// partition is preserved regardless of documents in other partitions. + /// + [Fact] + public async Task Query_SinglePartitionFilter_PreservesOrderWithinPartition() + { + var pkTarget = "pk-xpart-target"; + var pkOther = "pk-xpart-other"; + + // Interleave: target-1, other-1, target-2, other-2, target-3 + await _container.CreateItemAsync( + new TestDocument { Id = "xpart-t1", PartitionKey = pkTarget, Name = "T1", Value = 1 }, + new PartitionKey(pkTarget)); + await _container.CreateItemAsync( + new TestDocument { Id = "xpart-o1", PartitionKey = pkOther, Name = "O1", Value = 2 }, + new PartitionKey(pkOther)); + await _container.CreateItemAsync( + new TestDocument { Id = "xpart-t2", PartitionKey = pkTarget, Name = "T2", Value = 3 }, + new PartitionKey(pkTarget)); + await _container.CreateItemAsync( + new TestDocument { Id = "xpart-o2", PartitionKey = pkOther, Name = "O2", Value = 4 }, + new PartitionKey(pkOther)); + await _container.CreateItemAsync( + new TestDocument { Id = "xpart-t3", PartitionKey = pkTarget, Name = "T3", Value = 5 }, + new PartitionKey(pkTarget)); + + // Query with partition key filter — should give target docs in their insertion order + var filteredResults = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pkTarget}'", pkTarget); + filteredResults.Select(r => r.Id).Should().Equal(["xpart-t1", "xpart-t2", "xpart-t3"]); + + // Cross-partition query — should give all docs in global insertion order + var allResults = await DrainQuery( + "SELECT * FROM c WHERE STARTSWITH(c.partitionKey, 'pk-xpart-')"); + allResults.Select(r => r.Id).Should().Equal( + ["xpart-t1", "xpart-o1", "xpart-t2", "xpart-o2", "xpart-t3"]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 15. Rapid sequential creates + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Rapidly creating 20 documents sequentially (each awaited) must produce + /// results in the exact call order. + /// + [Fact] + public async Task Query_RapidSequentialCreates_MaintainsCallOrder() + { + const string pk = "pk-rapid"; + var insertedIds = new List(); + + for (var i = 1; i <= 20; i++) + { + var id = $"rapid-{i:D4}"; + insertedIds.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Rapid {i}", Value = i }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal(insertedIds); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Additional hardening: ORDER BY overrides insertion order + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// A query with ORDER BY must return documents sorted by the specified field, + /// overriding the natural insertion order. + /// + [Fact] + public async Task Query_WithOrderBy_OverridesInsertionOrder() + { + const string pk = "pk-orderby"; + + // Insert in non-sorted order: values 50, 10, 40, 20, 30 + var values = new[] { 50, 10, 40, 20, 30 }; + for (var i = 0; i < values.Length; i++) + { + await _container.CreateItemAsync( + new TestDocument + { + Id = $"orderby-{i + 1:D4}", + PartitionKey = pk, + Name = $"Doc {values[i]}", + Value = values[i] + }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}' ORDER BY c[\"value\"] ASC", pk); + + results.Select(r => r.Value).Should().Equal([10, 20, 30, 40, 50]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Additional hardening: WHERE + VALUE filter preserves relative order + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// A range filter (WHERE c.value > X) must preserve relative insertion order + /// of the matching documents. + /// + [Fact] + public async Task Query_WithRangeFilter_MaintainsRelativeInsertionOrder() + { + const string pk = "pk-range-filter"; + + // Insert values: 5, 15, 3, 25, 8, 30, 2 + var values = new[] { 5, 15, 3, 25, 8, 30, 2 }; + var expectedIdsAbove10 = new List(); + + for (var i = 0; i < values.Length; i++) + { + var id = $"range-{i + 1:D4}"; + if (values[i] > 10) expectedIdsAbove10.Add(id); + await _container.CreateItemAsync( + new TestDocument { Id = id, PartitionKey = pk, Name = $"Doc-{values[i]}", Value = values[i] }, + new PartitionKey(pk)); + } + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}' AND c[\"value\"] > 10", pk); + + // Expected: range-0002 (15), range-0004 (25), range-0006 (30) — in insertion order + results.Select(r => r.Id).Should().Equal(expectedIdsAbove10); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Additional hardening: Delete from middle, verify remaining order + // ═══════════════════════════════════════════════════════════════════════════ + + /// + /// Deleting multiple non-consecutive documents must not disturb the relative + /// order of the remaining documents. + /// + [Fact] + public async Task Query_AfterMultipleNonConsecutiveDeletes_PreservesRemainingOrder() + { + const string pk = "pk-multi-del"; + + for (var i = 1; i <= 10; i++) + { + await _container.CreateItemAsync( + new TestDocument { Id = $"mdel-{i:D4}", PartitionKey = pk, Name = $"Doc {i}", Value = i }, + new PartitionKey(pk)); + } + + // Delete positions 2, 5, 8 (non-consecutive) + await _container.DeleteItemAsync("mdel-0002", new PartitionKey(pk)); + await _container.DeleteItemAsync("mdel-0005", new PartitionKey(pk)); + await _container.DeleteItemAsync("mdel-0008", new PartitionKey(pk)); + + var results = await DrainQuery( + $"SELECT * FROM c WHERE c.partitionKey = '{pk}'", pk); + + results.Select(r => r.Id).Should().Equal( + ["mdel-0001", "mdel-0003", "mdel-0004", "mdel-0006", "mdel-0007", "mdel-0009", "mdel-0010"]); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Projection helper models + // ═══════════════════════════════════════════════════════════════════════════ + + private class ProjectedDocument + { + [JsonProperty("id")] + public string Id { get; set; } = default!; + + [JsonProperty("name")] + public string Name { get; set; } = default!; + } + +} From a3727ef59d58ac4547fceae97a0b3df4f5b540a2 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 20 May 2026 15:38:40 +0100 Subject: [PATCH 11/16] docs: update CHANGELOG to cover all merged fixes for v4.0.20 --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0689aa1..cba28ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [4.0.18] - 2026-05-19 +## [4.0.20] - 2026-05-20 ### Fixed - `COUNT(expr)` with nested ternary expressions no longer miscounts documents (Issue #64). `ExprToString` in `CosmosSqlParser` did not parenthesise ternary/coalesce sub-expressions when they appeared as operands of higher-precedence binary operators. When the SDK's transformed query was round-tripped through `SimplifySdkQuery`, the missing parentheses caused re-parsing to produce a different AST — e.g. `(innerTernary > 0) ? 1 : undefined` became `innerTernary ? val : (otherVal > 0 ? 1 : undefined)` — making `COUNT` evaluate the wrong condition. The fix wraps ternary and coalesce expressions in parentheses whenever they appear inside binary, unary, BETWEEN, IN, or LIKE operators. +- String literal aliases in aggregate queries (e.g. `SELECT 'Settlement' AS Label, COUNT(1) AS ItemCount FROM c`) no longer return `null` on Linux (Issue #67). `ProjectAggregateFields` now evaluates literal/expression fields via `EvaluateSqlExpression` instead of treating them as property paths. `DefaultQueryPlanStrategy` also bypasses the SDK aggregate pipeline when literals appear alongside aggregates. +- Patch operations with filter predicates referencing non-existent properties no longer throw (Issue #70). Missing properties are now treated as `null` in `FilterPredicate` evaluation, matching real Cosmos DB behavior. - Queries without `ORDER BY` now return documents in insertion order, matching real Cosmos DB behavior (Issue #72). Previously documents were returned in hash-map order due to `ConcurrentDictionary` enumeration. Added insertion-order tracking to `InMemoryContainer` that maintains document position across create, replace, upsert, and delete operations. +### Changed +- Removed `EmulatorFlaky` trait. The single flaky test (`ConcurrentReadsOfNonExistent`) now uses adaptive concurrency — 50 parallel reads for in-memory, 10 for emulator targets — eliminating the need for blanket test exclusions. + +### CI +- Integration tests now run on **both** `ubuntu-latest` and `windows-latest` to catch platform-specific divergences (e.g. ServiceInterop vs non-ServiceInterop code paths). + ## [4.0.17] - 2026-05-14 ### Fixed From be4653b35356b8af7b7956afcabfa254ba09535c Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 27 May 2026 08:46:38 +0100 Subject: [PATCH 12/16] Add repro tests for decimal scale preservation (issue #75) Bug 1 (cross-platform): Round-trip scale modification - whole numbers gain trailing .0 and trailing zeros are stripped from fractional values. Bug 2 (Linux-specific): SUM aggregate returns different decimal scales on Windows vs Linux - whole number sums gain trailing .0 on Linux only. Test results: - Windows: 13 fail (Bug 1 round-trip), 10 pass (SUM + significant digits) - Linux: 19 fail (Bug 1 + Bug 2), 4 pass (significant digits only) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DecimalScaleTests.cs | 324 ++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs new file mode 100644 index 0000000..09264f7 --- /dev/null +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs @@ -0,0 +1,324 @@ +using AwesomeAssertions; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; +using Xunit; + +namespace CosmosDB.InMemoryEmulator.Tests; + +/// +/// Reproduction tests for decimal scale inconsistencies (GitHub issue #75). +/// +/// Bug 1 - Round-trip scale modification (cross-platform, Windows AND Linux): +/// Whole numbers gain a trailing .0 (e.g., 1500m becomes 1500.0m). +/// Trailing zeros are stripped from fractional values (e.g., 100.50m becomes 100.5m). +/// +/// Bug 2 - SUM aggregate platform inconsistency (Linux-specific): +/// On Windows, SUM(375, 375) returns 750m (ToString => "750"). +/// On Linux, SUM(375, 375) returns 750.0m (ToString => "750.0"). +/// +/// Parity-validated: runs against both FakeCosmosHandler (in-memory) and real emulator. +/// +[Collection(IntegrationCollection.Name)] +public class DecimalScaleTests(EmulatorSession session, ITestOutputHelper output) : IAsyncLifetime +{ + private readonly ITestContainerFixture _fixture = TestFixtureFactory.Create(session); + private Container _container = null!; + + public async ValueTask InitializeAsync() + { + _container = await _fixture.CreateContainerAsync("test-decimal-scale", "/partitionKey"); + } + + public async ValueTask DisposeAsync() + { + await _fixture.DisposeAsync(); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Bug 1: Round-trip scale modification (cross-platform) + // ═══════════════════════════════════════════════════════════════════════════ + + [Theory] + [InlineData(0, "0")] + [InlineData(1, "1")] + [InlineData(42, "42")] + [InlineData(750, "750")] + [InlineData(1500, "1500")] + [InlineData(999999, "999999")] + public async Task RoundTrip_WholeNumber_ShouldNotGainDecimalPlaces(int inputValue, string expectedToString) + { + var id = Guid.NewGuid().ToString(); + var document = new DecimalDoc { Id = id, PartitionKey = "pk", Amount = inputValue }; + + await _container.CreateItemAsync(document, new PartitionKey("pk")); + var response = await _container.ReadItemAsync(id, new PartitionKey("pk")); + + output.WriteLine($"Input: {inputValue}m -> Round-trip: {response.Resource.Amount}m (ToString: \"{response.Resource.Amount}\")"); + + response.Resource.Amount.Should().Be(inputValue, "the arithmetic value must be preserved"); + response.Resource.Amount.ToString().Should().Be(expectedToString, + $"whole number {inputValue}m should not gain trailing decimal places after round-trip"); + } + + [Theory] + [InlineData("100.50", "100.50")] + [InlineData("25.00", "25.00")] + [InlineData("0.10", "0.10")] + [InlineData("1.00", "1.00")] + [InlineData("999.990", "999.990")] + public async Task RoundTrip_TrailingZeros_ShouldBePreserved(string inputValue, string expectedToString) + { + var id = Guid.NewGuid().ToString(); + var amount = decimal.Parse(inputValue); + var document = new DecimalDoc { Id = id, PartitionKey = "pk", Amount = amount }; + + await _container.CreateItemAsync(document, new PartitionKey("pk")); + var response = await _container.ReadItemAsync(id, new PartitionKey("pk")); + + output.WriteLine($"Input: {inputValue} -> Round-trip: {response.Resource.Amount} (ToString: \"{response.Resource.Amount}\")"); + + response.Resource.Amount.Should().Be(amount, "the arithmetic value must be preserved"); + response.Resource.Amount.ToString().Should().Be(expectedToString, + $"decimal {inputValue} should preserve trailing zeros after round-trip (scale must be maintained)"); + } + + [Theory] + [InlineData("0.01", "0.01")] + [InlineData("123.456", "123.456")] + [InlineData("999.99", "999.99")] + [InlineData("0.001", "0.001")] + public async Task RoundTrip_SignificantFractionalDigits_ShouldBePreserved(string inputValue, string expectedToString) + { + var id = Guid.NewGuid().ToString(); + var amount = decimal.Parse(inputValue); + var document = new DecimalDoc { Id = id, PartitionKey = "pk", Amount = amount }; + + await _container.CreateItemAsync(document, new PartitionKey("pk")); + var response = await _container.ReadItemAsync(id, new PartitionKey("pk")); + + output.WriteLine($"Input: {inputValue} -> Round-trip: {response.Resource.Amount} (ToString: \"{response.Resource.Amount}\")"); + + response.Resource.Amount.Should().Be(amount, "the arithmetic value must be preserved"); + response.Resource.Amount.ToString().Should().Be(expectedToString, + $"decimal {inputValue} should preserve all significant digits after round-trip"); + } + + [Fact] + public async Task RoundTrip_MultipleProperties_ShouldAllPreserveScale() + { + var id = Guid.NewGuid().ToString(); + var document = new MultiDecimalDoc + { + Id = id, + PartitionKey = "pk", + WholeNumber = 1500m, + WithTrailingZero = 100.50m, + PureZero = 0m, + SmallFraction = 0.01m + }; + + await _container.CreateItemAsync(document, new PartitionKey("pk")); + var response = await _container.ReadItemAsync(id, new PartitionKey("pk")); + + output.WriteLine($"WholeNumber: {document.WholeNumber} -> {response.Resource.WholeNumber}"); + output.WriteLine($"WithTrailingZero: {document.WithTrailingZero} -> {response.Resource.WithTrailingZero}"); + output.WriteLine($"PureZero: {document.PureZero} -> {response.Resource.PureZero}"); + output.WriteLine($"SmallFraction: {document.SmallFraction} -> {response.Resource.SmallFraction}"); + + response.Resource.WholeNumber.ToString().Should().Be("1500", "whole number should not gain .0"); + response.Resource.WithTrailingZero.ToString().Should().Be("100.50", "trailing zero should be preserved"); + response.Resource.PureZero.ToString().Should().Be("0", "zero should remain 0 without .0"); + response.Resource.SmallFraction.ToString().Should().Be("0.01", "significant fractional digits should be preserved"); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Bug 2: SUM aggregate platform inconsistency (Linux-specific) + // ═══════════════════════════════════════════════════════════════════════════ + + [Fact] + public async Task SumAggregate_TwoWholeNumbers_ShouldReturnWholeNumber() + { + await SeedDocuments(("a", 375m), ("b", 375m)); + + var result = await RunSumQuery(); + + output.WriteLine($"SUM(375 + 375) = {result}, ToString: \"{result}\""); + + result.Should().Be(750m, "arithmetic must be correct"); + result.ToString().Should().Be("750", + "SUM of whole numbers should return a whole number without trailing .0"); + } + + [Fact] + public async Task SumAggregate_MultipleWholeNumbers_ShouldReturnWholeNumber() + { + await SeedDocuments(("a", 100m), ("b", 200m), ("c", 300m), ("d", 400m), ("e", 500m)); + + var result = await RunSumQuery(); + + output.WriteLine($"SUM(100+200+300+400+500) = {result}, ToString: \"{result}\""); + + result.Should().Be(1500m, "arithmetic must be correct"); + result.ToString().Should().Be("1500", + "SUM of multiple whole numbers should return a whole number"); + } + + [Fact] + public async Task SumAggregate_SingleWholeNumber_ShouldReturnWholeNumber() + { + await SeedDocuments(("a", 1500m)); + + var result = await RunSumQuery(); + + output.WriteLine($"SUM(1500) = {result}, ToString: \"{result}\""); + + result.Should().Be(1500m, "arithmetic must be correct"); + result.ToString().Should().Be("1500", + "SUM of a single whole number should not gain decimal places"); + } + + [Fact] + public async Task SumAggregate_FractionalNumbers_ShouldPreserveMinimumScale() + { + await SeedDocuments(("a", 25.50m), ("b", 25.00m)); + + var result = await RunSumQuery(); + + output.WriteLine($"SUM(25.50 + 25.00) = {result}, ToString: \"{result}\""); + + result.Should().Be(50.5m, "arithmetic must be correct"); + result.ToString().Should().BeOneOf("50.5", "50.50", + "SUM of fractional numbers should not add unexpected trailing zeros"); + } + + [Fact] + public async Task SumAggregate_MixedWholeAndFractional_ShouldReturnFractionalResult() + { + await SeedDocuments(("a", 100m), ("b", 50.25m)); + + var result = await RunSumQuery(); + + output.WriteLine($"SUM(100 + 50.25) = {result}, ToString: \"{result}\""); + + result.Should().Be(150.25m, "arithmetic must be correct"); + result.ToString().Should().Be("150.25", + "SUM with a fractional addend should produce a fractional result"); + } + + [Fact] + public async Task SumAggregate_ResultUsedInStringInterpolation_ShouldProduceConsistentOutput() + { + await SeedDocuments(("a", 750m), ("b", 750m)); + + var transactionDebitTotal = await RunSumQuery(); + var clearingFileDebitTotal = 1650m; + + var message = $"Clearing file debit total: {clearingFileDebitTotal}, transaction debit total: {transactionDebitTotal}"; + + output.WriteLine($"Message: \"{message}\""); + + message.Should().Be("Clearing file debit total: 1650, transaction debit total: 1500", + "string interpolation of SUM result should produce platform-consistent output without trailing .0"); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Compound effect: LINQ Sum on round-tripped values + // ═══════════════════════════════════════════════════════════════════════════ + + [Fact] + public async Task LinqSum_OnRoundTrippedWholeNumbers_ShouldReturnWholeNumber() + { + await SeedDocuments(("a", 750m), ("b", 750m)); + + var query = _container.GetItemQueryIterator( + new QueryDefinition("SELECT * FROM c WHERE c.partitionKey = 'pk'")); + + var results = new List(); + while (query.HasMoreResults) + { + var feed = await query.ReadNextAsync(); + results.AddRange(feed); + } + + var sum = results.Sum(r => r.Amount); + + output.WriteLine($"Values after round-trip: {string.Join(", ", results.Select(r => $"\"{r.Amount}\""))}"); + output.WriteLine($"LINQ Sum: {sum}, ToString: \"{sum}\""); + + sum.Should().Be(1500m, "arithmetic must be correct"); + sum.ToString().Should().Be("1500", + "LINQ Sum of whole-number decimals should not produce trailing decimal places"); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Helpers + // ═══════════════════════════════════════════════════════════════════════════ + + private async Task SeedDocuments(params (string id, decimal value)[] items) + { + foreach (var (id, value) in items) + { + await _container.CreateItemAsync( + new DecimalDoc { Id = id, PartitionKey = "pk", Amount = value }, + new PartitionKey("pk")); + } + } + + private async Task RunSumQuery() + { + var query = _container.GetItemQueryIterator( + new QueryDefinition("SELECT SUM(c.amount) as total FROM c WHERE c.partitionKey = 'pk'")); + + while (query.HasMoreResults) + { + var feed = await query.ReadNextAsync(); + var result = feed.FirstOrDefault(); + if (result is not null) return result.Total; + } + + throw new InvalidOperationException("SUM query returned no results"); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Models + // ═══════════════════════════════════════════════════════════════════════════ + + private class DecimalDoc + { + [JsonProperty("id")] + public string Id { get; set; } = string.Empty; + + [JsonProperty("partitionKey")] + public string PartitionKey { get; set; } = string.Empty; + + [JsonProperty("amount")] + public decimal Amount { get; set; } + } + + private class MultiDecimalDoc + { + [JsonProperty("id")] + public string Id { get; set; } = string.Empty; + + [JsonProperty("partitionKey")] + public string PartitionKey { get; set; } = string.Empty; + + [JsonProperty("wholeNumber")] + public decimal WholeNumber { get; set; } + + [JsonProperty("withTrailingZero")] + public decimal WithTrailingZero { get; set; } + + [JsonProperty("pureZero")] + public decimal PureZero { get; set; } + + [JsonProperty("smallFraction")] + public decimal SmallFraction { get; set; } + } + + private class SumResult + { + [JsonProperty("total")] + public decimal Total { get; set; } + } +} From 44bfb1d0d0510225d5b3b40cd45924950b3cff3c Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 27 May 2026 10:43:48 +0100 Subject: [PATCH 13/16] Add test-decimal-scale to emulator container manifest Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/emulator-containers.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/emulator-containers.json b/tests/emulator-containers.json index f0e8c4d..00c442e 100644 --- a/tests/emulator-containers.json +++ b/tests/emulator-containers.json @@ -7,6 +7,7 @@ { "name": "pknone-test", "partitionKeyPath": "/partitionKey" }, { "name": "num-pk-test", "partitionKeyPath": "/numericPk" }, { "name": "bool-pk-test", "partitionKeyPath": "/boolPk" }, + { "name": "test-decimal-scale", "partitionKeyPath": "/partitionKey" }, { "name": "test-batch", "partitionKeyPath": "/partitionKey" }, { "name": "test-bulk", "partitionKeyPath": "/partitionKey" }, { "name": "test-pk", "partitionKeyPath": "/partitionKey" }, From b65e3bf5fe1eb180c5ee0875d32d6144b298a587 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 27 May 2026 11:16:51 +0100 Subject: [PATCH 14/16] Fix trailing-zero test expectations and dev-env NuGetAudit issue - Update trailing-zero tests to expect stripping (real Cosmos behavior): 100.50 -> 100.5, 25.00 -> 25, 0.10 -> 0.1, etc. - Update multi-property test to expect 100.5 not 100.50 - Tighten SUM fractional assertion to expect 50.5 only - Fix run-tests.ps1: pass -p:NuGetAudit=false to warmup dotnet run so it works in offline Docker containers (linux+emulator-linux) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/run-tests.ps1 | 5 +++- .../DecimalScaleTests.cs | 24 +++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 index 47aa50c..eb7859f 100644 --- a/scripts/run-tests.ps1 +++ b/scripts/run-tests.ps1 @@ -78,7 +78,10 @@ if ($needsIntegrationWarmup) { Write-Host "`nPre-creating emulator containers from manifest..." -ForegroundColor Cyan # `dotnet run` self-builds if needed — the warmup tool isn't in the solution, # so it's not covered by the workflow's solution-level build step. - & dotnet run --project $warmupProject --configuration Release -- $manifestPath + # NuGetAudit is disabled because the dev container may not have internet access + # (e.g. linux+emulator-linux uses network_mode: service:emulator) and NU1900 + # becomes a build error via TreatWarningsAsErrors. + & dotnet run --project $warmupProject --configuration Release -p:NuGetAudit=false -- $manifestPath if ($LASTEXITCODE -ne 0) { Write-Host "Emulator warmup failed with exit code $LASTEXITCODE" -ForegroundColor Red exit $LASTEXITCODE diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs index 09264f7..695a35d 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Integration/DecimalScaleTests.cs @@ -60,13 +60,17 @@ public async Task RoundTrip_WholeNumber_ShouldNotGainDecimalPlaces(int inputValu $"whole number {inputValue}m should not gain trailing decimal places after round-trip"); } + // Ref: Observed behavior on both Windows and Linux Cosmos DB emulators. + // Cosmos DB stores numbers in JSON as raw numeric literals. Trailing zeros + // (which are JSON-insignificant) are stripped on round-trip: 100.50 → 100.5, + // 25.00 → 25, etc. This is correct Cosmos behavior, not a bug. [Theory] - [InlineData("100.50", "100.50")] - [InlineData("25.00", "25.00")] - [InlineData("0.10", "0.10")] - [InlineData("1.00", "1.00")] - [InlineData("999.990", "999.990")] - public async Task RoundTrip_TrailingZeros_ShouldBePreserved(string inputValue, string expectedToString) + [InlineData("100.50", "100.5")] + [InlineData("25.00", "25")] + [InlineData("0.10", "0.1")] + [InlineData("1.00", "1")] + [InlineData("999.990", "999.99")] + public async Task RoundTrip_TrailingZeros_ShouldBeStripped(string inputValue, string expectedToString) { var id = Guid.NewGuid().ToString(); var amount = decimal.Parse(inputValue); @@ -79,7 +83,7 @@ public async Task RoundTrip_TrailingZeros_ShouldBePreserved(string inputValue, s response.Resource.Amount.Should().Be(amount, "the arithmetic value must be preserved"); response.Resource.Amount.ToString().Should().Be(expectedToString, - $"decimal {inputValue} should preserve trailing zeros after round-trip (scale must be maintained)"); + $"decimal {inputValue} should have trailing zeros stripped after round-trip (Cosmos stores minimal JSON representation)"); } [Theory] @@ -126,7 +130,7 @@ public async Task RoundTrip_MultipleProperties_ShouldAllPreserveScale() output.WriteLine($"SmallFraction: {document.SmallFraction} -> {response.Resource.SmallFraction}"); response.Resource.WholeNumber.ToString().Should().Be("1500", "whole number should not gain .0"); - response.Resource.WithTrailingZero.ToString().Should().Be("100.50", "trailing zero should be preserved"); + response.Resource.WithTrailingZero.ToString().Should().Be("100.5", "trailing zero should be stripped (Cosmos stores minimal JSON)"); response.Resource.PureZero.ToString().Should().Be("0", "zero should remain 0 without .0"); response.Resource.SmallFraction.ToString().Should().Be("0.01", "significant fractional digits should be preserved"); } @@ -187,8 +191,8 @@ public async Task SumAggregate_FractionalNumbers_ShouldPreserveMinimumScale() output.WriteLine($"SUM(25.50 + 25.00) = {result}, ToString: \"{result}\""); result.Should().Be(50.5m, "arithmetic must be correct"); - result.ToString().Should().BeOneOf("50.5", "50.50", - "SUM of fractional numbers should not add unexpected trailing zeros"); + result.ToString().Should().Be("50.5", + "SUM of fractional numbers should return the minimal representation"); } [Fact] From f1910926927aa3b19f0f6142d489a64a4f96257b Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 27 May 2026 13:51:48 +0100 Subject: [PATCH 15/16] Fix decimal scale preservation and SUM aggregate cross-platform (issue #75) - Add CosmosJsonWriter that normalizes numbers during JSON serialization: whole-number doubles/decimals written as integers (matching real Cosmos DB) - Apply NormalizeNumericResult to SUM/AVG aggregate assignment points - Fix double-to-long overflow: use strict < comparison at long.MaxValue boundary - Bypass SDK AggregateQueryPipelineStage for all aggregate queries in query plan (fixes Linux where ServiceInterop is unavailable and SDK expects payload envelope) - Remove unnecessary async from test methods (fixes CS1998 on .NET 8 SDK) - Bump version to 4.0.18 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 8 ++ scripts/run-tests.ps1 | 5 +- .../DefaultQueryPlanStrategy.cs | 22 ++-- .../InMemoryContainer.cs | 16 +-- .../JsonParseHelpers.cs | 100 +++++++++++++++++- src/Directory.Build.props | 2 +- .../FakeCosmosHandlerAdvancedFeatureTests.cs | 2 +- .../InMemoryCosmosTests.cs | 6 +- .../QueryPlanDeepDiveTests.cs | 16 +-- .../QueryPlanTests.cs | 24 +++-- .../QueryTests.cs | 8 +- .../StoredProcedureTests.cs | 8 +- 12 files changed, 164 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cba28ba..da0038f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.0.21] - 2026-05-27 + +### Fixed +- Decimal scale not preserved on round-trip (Issue #75). Whole numbers stored as decimals (e.g. `1500m`) gained a trailing `.0` after retrieval. A custom `CosmosJsonWriter` now normalizes JSON numeric representations during `ParseJson` to match real Cosmos DB behavior: whole numbers serialize as integers, fractional values use minimal representation (trailing zeros stripped). +- SUM aggregate returns inconsistent results across platforms (Issue #75). On Linux, `SUM` of whole numbers returned `750.0` instead of `750` because `List.Sum()` always returns `double`. Aggregate assignment points now normalize whole-number doubles to longs before serialization. +- Aggregate queries fail on Linux with "Underlying object does not have an 'payload' field" error. On non-Windows platforms the Cosmos SDK's native `ServiceInterop` DLL is unavailable, causing the SDK to request a query plan and activate its `AggregateQueryPipelineStage`. The query plan strategy now suppresses all aggregate info so the SDK never enters this pipeline — our handler already computes aggregates directly. +- Double-to-long overflow when normalizing values at the `long.MaxValue` boundary. `(double)long.MaxValue` rounds up to 2^63 which overflows on cast; comparison changed from `<=` to strict `<`. + ## [4.0.20] - 2026-05-20 ### Fixed diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 index eb7859f..47aa50c 100644 --- a/scripts/run-tests.ps1 +++ b/scripts/run-tests.ps1 @@ -78,10 +78,7 @@ if ($needsIntegrationWarmup) { Write-Host "`nPre-creating emulator containers from manifest..." -ForegroundColor Cyan # `dotnet run` self-builds if needed — the warmup tool isn't in the solution, # so it's not covered by the workflow's solution-level build step. - # NuGetAudit is disabled because the dev container may not have internet access - # (e.g. linux+emulator-linux uses network_mode: service:emulator) and NU1900 - # becomes a build error via TreatWarningsAsErrors. - & dotnet run --project $warmupProject --configuration Release -p:NuGetAudit=false -- $manifestPath + & dotnet run --project $warmupProject --configuration Release -- $manifestPath if ($LASTEXITCODE -ne 0) { Write-Host "Emulator warmup failed with exit code $LASTEXITCODE" -ForegroundColor Red exit $LASTEXITCODE diff --git a/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs b/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs index 3ab997b..09e03d7 100644 --- a/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs +++ b/src/CosmosDB.InMemoryEmulator/DefaultQueryPlanStrategy.cs @@ -142,20 +142,18 @@ private static void BuildFromParsedQuery(JObject queryInfo, CosmosSqlQuery parse queryInfo["hasSelectValue"] = true; } - // Suppress SDK pipeline for GROUP BY, multi-aggregate, and VALUE aggregate - // queries. Also bypass any query referencing COUNTIF — the SDK pipeline - // doesn't recognize COUNTIF and tries to apply its built-in aggregate - // accumulator, which expects a {payload: {alias: {item: value}}} envelope - // the handler only produces for GROUP BY responses. - // Additionally bypass when literal expressions appear alongside aggregates - // (e.g. SELECT 42 AS X, COUNT(1) AS N) — the SDK's AggregateQueryPipelineStage - // doesn't handle non-aggregate expression fields and crashes with + // Suppress SDK pipeline for GROUP BY and any aggregate queries. The SDK's + // AggregateQueryPipelineStage expects a {payload: {alias: {item: value}}} envelope + // the handler only produces for GROUP BY responses. On non-Windows platforms + // (where ServiceInterop is unavailable), this plan drives the SDK pipeline — + // if we report aggregates, the SDK enters its aggregate accumulator which + // fails because our handler returns pre-computed results directly. + // Also bypass COUNTIF (not recognized by SDK pipeline) and literal+aggregate + // combinations (e.g. SELECT 42 AS X, COUNT(1) AS N) which crash with // "Underlying object does not have an 'payload' field". var isGroupByBypass = parsed.GroupByFields is { Length: > 0 }; var aggregateFieldCount = parsed.SelectFields.Count(f => ContainsAggregate(f.SqlExpr)); - var isMultiAggregateBypass = !isGroupByBypass && !parsed.IsValueSelect - && (aggregateFieldCount > 1 || aggregates.Count > 1); - var isValueAggregateBypass = !isGroupByBypass && parsed.IsValueSelect && aggregateFieldCount > 0; + var isAggregateBypass = !isGroupByBypass && aggregateFieldCount > 0; var isCountIfBypass = !isGroupByBypass && parsed.SelectFields.Any(f => ContainsCountIf(f.SqlExpr)); // Bypass when there are non-aggregate expression fields (literals, function calls) @@ -165,7 +163,7 @@ private static void BuildFromParsedQuery(JObject queryInfo, CosmosSqlQuery parse && parsed.SelectFields.Any(f => !ContainsAggregate(f.SqlExpr) && f.SqlExpr is not null and not IdentifierExpression and not PropertyAccessExpression); - if (isGroupByBypass || isMultiAggregateBypass || isValueAggregateBypass || isCountIfBypass || isLiteralWithAggregateBypass) + if (isGroupByBypass || isAggregateBypass || isCountIfBypass || isLiteralWithAggregateBypass) { queryInfo["groupByExpressions"] = new JArray(); queryInfo["groupByAliases"] = new JArray(); diff --git a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs index 8e3c30a..be197b9 100644 --- a/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs +++ b/src/CosmosDB.InMemoryEmulator/InMemoryContainer.cs @@ -4695,13 +4695,13 @@ private List ApplyGroupBy(IEnumerable items, CosmosSqlQuery pars if (funcName == "SUM") { if (values.Count > 0) - resultObj[outputName] = values.Sum(); + resultObj[outputName] = JToken.FromObject(JsonParseHelpers.NormalizeNumericResult(values.Sum())); // else: omit field entirely (undefined) — matches Cosmos DB } else // AVG { if (values.Count > 0) - resultObj[outputName] = values.Average(); + resultObj[outputName] = JToken.FromObject(JsonParseHelpers.NormalizeNumericResult(values.Average())); } } else if (funcName is "MIN" or "MAX" && innerArg != null) @@ -5077,8 +5077,8 @@ private static object EvaluateHavingAggregate( } return func.FunctionName switch { - "SUM" => values.Count > 0 ? (object)values.Sum() : UndefinedValue.Instance, - "AVG" => values.Count > 0 ? (object)values.Average() : UndefinedValue.Instance, + "SUM" => values.Count > 0 ? JsonParseHelpers.NormalizeNumericResult(values.Sum()) : UndefinedValue.Instance, + "AVG" => values.Count > 0 ? JsonParseHelpers.NormalizeNumericResult(values.Average()) : UndefinedValue.Instance, _ => 0.0 }; } @@ -5414,13 +5414,13 @@ private static List ProjectAggregateFields(IEnumerable itemsEnum if (funcName == "SUM") { if (values.Count > 0) - resultObj[outputName] = values.Sum(); + resultObj[outputName] = JToken.FromObject(JsonParseHelpers.NormalizeNumericResult(values.Sum())); // else: omit field entirely (undefined) — matches Cosmos DB } else // AVG { if (values.Count > 0) - resultObj[outputName] = values.Average(); + resultObj[outputName] = JToken.FromObject(JsonParseHelpers.NormalizeNumericResult(values.Average())); // else: omit field entirely (undefined) } } @@ -8006,8 +8006,8 @@ private static object EvaluateSubqueryAggregate( return name switch { - "SUM" => values.Count > 0 ? values.Sum(v => v.Value()) : (object)UndefinedValue.Instance, - "AVG" => values.Count > 0 ? values.Average(v => v.Value()) : (object)UndefinedValue.Instance, + "SUM" => values.Count > 0 ? JsonParseHelpers.NormalizeNumericResult(values.Sum(v => v.Value())) : (object)UndefinedValue.Instance, + "AVG" => values.Count > 0 ? JsonParseHelpers.NormalizeNumericResult(values.Average(v => v.Value())) : (object)UndefinedValue.Instance, "MIN" => AggregateMinMax(values, true), "MAX" => AggregateMinMax(values, false), _ => null diff --git a/src/CosmosDB.InMemoryEmulator/JsonParseHelpers.cs b/src/CosmosDB.InMemoryEmulator/JsonParseHelpers.cs index 1a4c24e..be8c928 100644 --- a/src/CosmosDB.InMemoryEmulator/JsonParseHelpers.cs +++ b/src/CosmosDB.InMemoryEmulator/JsonParseHelpers.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -11,7 +12,18 @@ internal static JObject ParseJson(string json) { DateParseHandling = DateParseHandling.None }; - return JObject.Load(reader); + var obj = JObject.Load(reader); + + // Normalize numeric values to match Cosmos DB's minimal JSON representation. + // Cosmos DB stores numbers without trailing zeros: 1500.0 → 1500, 100.50 → 100.5. + // We serialize with a custom writer that strips trailing zeros, then reparse so the + // internal JValue representations have the correct scale for subsequent ToString() calls. + var normalized = ToCosmosJson(obj); + using var reader2 = new JsonTextReader(new StringReader(normalized)) + { + DateParseHandling = DateParseHandling.None + }; + return JObject.Load(reader2); } internal static JToken ParseJsonToken(string json) @@ -20,6 +32,90 @@ internal static JToken ParseJsonToken(string json) { DateParseHandling = DateParseHandling.None }; - return JToken.Load(reader); + var token = JToken.Load(reader); + + if (token is JObject or JArray) + { + var normalized = ToCosmosJson(token); + using var reader2 = new JsonTextReader(new StringReader(normalized)) + { + DateParseHandling = DateParseHandling.None + }; + return JToken.Load(reader2); + } + + return token; + } + + /// + /// Serializes a JToken to JSON using Cosmos DB's minimal numeric representation. + /// Whole numbers are written without .0, trailing fractional zeros are stripped. + /// + /// + /// Ref: https://learn.microsoft.com/en-us/rest/api/cosmos-db/documents + /// Cosmos DB uses JSON (RFC 8259) for document storage. JSON numbers have no + /// concept of "scale" — 1500.0 and 1500 are the same value. Cosmos normalizes + /// to the minimal representation on storage. + /// + internal static string ToCosmosJson(JToken token) + { + using var sw = new StringWriter(); + using var writer = new CosmosJsonWriter(sw) { Formatting = Formatting.None }; + token.WriteTo(writer); + return sw.ToString(); + } + + /// + /// Normalizes a numeric aggregate result to match Cosmos DB's output format. + /// Whole-number results are returned as long; fractional results as double + /// with trailing zeros stripped. + /// + internal static object NormalizeNumericResult(double value) + { + // Ref: Observed Cosmos DB emulator behavior — SUM/AVG of whole numbers + // returns integers (no .0), e.g. SUM(375, 375) = 750 not 750.0. + // We use strict < for MaxValue because (double)long.MaxValue rounds up to 2^63, + // which overflows the long cast on .NET 8 (wraps to long.MinValue). + if (value == Math.Truncate(value) && value >= long.MinValue && value < (double)long.MaxValue) + return (long)value; + return value; + } + + /// + /// Custom JsonTextWriter that writes numbers in their minimal JSON representation, + /// matching how Cosmos DB stores numeric values. + /// + private sealed class CosmosJsonWriter : JsonTextWriter + { + public CosmosJsonWriter(TextWriter writer) : base(writer) { } + + public override void WriteValue(decimal value) + { + if (value == decimal.Truncate(value) && value >= long.MinValue && value <= long.MaxValue) + { + // Whole number — write as integer (no .0 suffix) + WriteValue((long)value); + } + else + { + // Fractional — write minimal representation (strip trailing zeros) + WriteRawValue(value.ToString("G29", CultureInfo.InvariantCulture)); + } + } + + public override void WriteValue(double value) + { + // Strict < for MaxValue: (double)long.MaxValue rounds up to 2^63 which + // cannot be cast to long without overflow on .NET 8. + if (value == Math.Truncate(value) && value >= long.MinValue && value < (double)long.MaxValue) + { + // Whole number — write as integer + WriteValue((long)value); + } + else + { + base.WriteValue(value); + } + } } } diff --git a/src/Directory.Build.props b/src/Directory.Build.props index cc5f590..751adc4 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,7 +5,7 @@ true - 4.0.20 + 4.0.21 lemonlion Copyright (c) 2026 lemonlion MIT diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/FakeCosmosHandlerAdvancedFeatureTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/FakeCosmosHandlerAdvancedFeatureTests.cs index c21cf93..7064f34 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/FakeCosmosHandlerAdvancedFeatureTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/FakeCosmosHandlerAdvancedFeatureTests.cs @@ -75,7 +75,7 @@ private async Task> DrainQuery(QueryDefinition queryDef) [Fact] public async Task UDF_RegisteredOnBackingContainer_UsableInQueryThroughHandler() { - _inMemoryContainer.RegisterUdf("doubleIt", args => (double)args[0] * 2); + _inMemoryContainer.RegisterUdf("doubleIt", args => Convert.ToDouble(args[0]) * 2); await _container.CreateItemAsync( new TestDocument { Id = "1", PartitionKey = "pk1", Name = "Alice", Value = 5, diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/InMemoryCosmosTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/InMemoryCosmosTests.cs index 8f9cbd7..499df8d 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/InMemoryCosmosTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/InMemoryCosmosTests.cs @@ -125,7 +125,7 @@ await cosmos.Container.CreateItemAsync( } [Fact] - public async Task Create_WithConfigureContainer_ConfiguresContainer() + public void Create_WithConfigureContainer_ConfiguresContainer() { using var cosmos = InMemoryCosmos.Create("orders", "/partitionKey", configureContainer: c => c.DefaultTimeToLive = 3600); @@ -250,7 +250,7 @@ public void Builder_WithHierarchicalPk_Works() } [Fact] - public async Task Builder_WrapHandler_WrapsAllHandlers() + public void Builder_WrapHandler_WrapsAllHandlers() { var callCount = 0; using var cosmos = InMemoryCosmos.Builder() @@ -278,7 +278,7 @@ public void Builder_ConfigureOptions_AppliesOptions() } [Fact] - public async Task Builder_AddContainerWithConfigure_ConfiguresContainer() + public void Builder_AddContainerWithConfigure_ConfiguresContainer() { using var cosmos = InMemoryCosmos.Builder() .AddContainer("orders", "/partitionKey", container => diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanDeepDiveTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanDeepDiveTests.cs index 185c7e7..135cd91 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanDeepDiveTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanDeepDiveTests.cs @@ -207,26 +207,30 @@ public async Task QueryPlan_CountDistinct_CustomAlias() } [Fact] - public async Task QueryPlan_SingleAggregateNoGroupByNoValue_PreservedInPlan() + public async Task QueryPlan_SingleAggregateNoGroupByNoValue_SuppressedForLinuxCompatibility() { + // Single-aggregate bypass: aggregates are suppressed so the SDK + // doesn't activate AggregateQueryPipelineStage on non-Windows platforms. var plan = await GetQueryPlanAsync("SELECT COUNT(1) FROM c"); var info = plan["queryInfo"]!; var aggs = info["aggregates"]!.ToObject()!; - aggs.Should().Contain("Count"); + aggs.Should().BeEmpty(); } [Fact] - public async Task QueryPlan_SingleAggregateWithAlias_MappedCorrectly() + public async Task QueryPlan_SingleAggregateWithAlias_SuppressedForLinuxCompatibility() { + // Single-aggregate bypass: aggregates are suppressed so the SDK + // doesn't activate AggregateQueryPipelineStage on non-Windows platforms. var plan = await GetQueryPlanAsync("SELECT COUNT(1) AS cnt FROM c"); var info = plan["queryInfo"]!; var aggs = info["aggregates"]!.ToObject()!; - aggs.Should().Contain("Count"); + aggs.Should().BeEmpty(); - var aliasMap = info["groupByAliasToAggregateType"]!; - aliasMap["cnt"]!.ToString().Should().Be("Count"); + var aliasMap = (JObject)info["groupByAliasToAggregateType"]!; + aliasMap.Should().BeEmpty(); } [Fact] diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanTests.cs index 0f6573b..0e4b5df 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryPlanTests.cs @@ -154,23 +154,27 @@ public async Task QueryPlan_DistinctWithOrderBy_SetsDistinctTypeUnordered() } [Fact] - public async Task QueryPlan_CountAggregate_DetectsCount() + public async Task QueryPlan_CountAggregate_SuppressedForLinuxCompatibility() { + // Single-aggregate bypass: aggregates are suppressed so the SDK + // doesn't activate AggregateQueryPipelineStage on non-Windows platforms. var plan = await GetQueryPlanAsync("SELECT COUNT(1) FROM c"); var info = plan["queryInfo"]!; var aggregates = (JArray)info["aggregates"]!; - aggregates.Should().Contain(t => t.ToString() == "Count"); + aggregates.Should().BeEmpty(); } [Fact] - public async Task QueryPlan_SumAggregate_DetectsSum() + public async Task QueryPlan_SumAggregate_SuppressedForLinuxCompatibility() { + // Single-aggregate bypass: aggregates are suppressed so the SDK + // doesn't activate AggregateQueryPipelineStage on non-Windows platforms. var plan = await GetQueryPlanAsync("SELECT SUM(c.value) FROM c"); var info = plan["queryInfo"]!; var aggregates = (JArray)info["aggregates"]!; - aggregates.Should().Contain(t => t.ToString() == "Sum"); + aggregates.Should().BeEmpty(); } [Fact] @@ -426,23 +430,27 @@ public async Task QueryPlan_DuplicateAggregateType_DeduplicatesInArray() } [Fact] - public async Task QueryPlan_CountWithoutAlias_StillDetectsAggregate() + public async Task QueryPlan_CountWithoutAlias_SuppressedForLinuxCompatibility() { + // Single-aggregate bypass: aggregates are suppressed so the SDK + // doesn't activate AggregateQueryPipelineStage on non-Windows platforms. var plan = await GetQueryPlanAsync("SELECT COUNT(1) FROM c"); var info = plan["queryInfo"]!; var aggregates = (JArray)info["aggregates"]!; - aggregates.Should().Contain(t => t.ToString() == "Count"); + aggregates.Should().BeEmpty(); } [Fact] - public async Task QueryPlan_AggregateFunctionCaseInsensitive_Detected() + public async Task QueryPlan_AggregateFunctionCaseInsensitive_SuppressedForLinuxCompatibility() { + // Single-aggregate bypass: aggregates are suppressed so the SDK + // doesn't activate AggregateQueryPipelineStage on non-Windows platforms. var plan = await GetQueryPlanAsync("SELECT count(1) AS cnt FROM c"); var info = plan["queryInfo"]!; var aggregates = (JArray)info["aggregates"]!; - aggregates.Should().Contain(t => t.ToString() == "Count"); + aggregates.Should().BeEmpty(); } [Fact] diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryTests.cs index f6508e4..abd146f 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/QueryTests.cs @@ -28462,16 +28462,16 @@ public async Task Round_IntegerInput_ReturnsInteger() } [Fact] - public async Task Floor_DoubleInput_ReturnsFloat() + public async Task Floor_DoubleInput_ReturnsInteger() { await _container.CreateItemStreamAsync( new MemoryStream(Encoding.UTF8.GetBytes("""{"id":"1","partitionKey":"pk1","dblVal":5.7}""")), new PartitionKey("pk1")); var results = await RunQuery("SELECT VALUE FLOOR(c.dblVal) FROM c"); results.Should().ContainSingle(); - // FLOOR(5.7) = 5, but since input was double, result is double - results[0].Type.Should().Be(JTokenType.Float); - results[0].Value().Should().Be(5.0); + // Ref: Cosmos DB normalizes all JSON numbers — FLOOR(5.7) = 5 (integer in JSON) + results[0].Type.Should().Be(JTokenType.Integer); + results[0].Value().Should().Be(5); } [Fact] diff --git a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/StoredProcedureTests.cs b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/StoredProcedureTests.cs index 135e434..d2c528f 100644 --- a/tests/CosmosDB.InMemoryEmulator.Tests.Unit/StoredProcedureTests.cs +++ b/tests/CosmosDB.InMemoryEmulator.Tests.Unit/StoredProcedureTests.cs @@ -1161,7 +1161,7 @@ public class UdfGapTests [Fact] public async Task Udf_RegisterAndUseInQuery() { - _container.RegisterUdf("double", args => ((double)args[0]) * 2); + _container.RegisterUdf("double", args => Convert.ToDouble(args[0]) * 2); await _container.CreateItemAsync( new UdfDocument { Id = "1", PartitionKey = "pk1", Value = 21 }, @@ -1183,7 +1183,7 @@ await _container.CreateItemAsync( [Fact] public async Task Udf_MultipleArgs() { - _container.RegisterUdf("add", args => (double)args[0] + (double)args[1]); + _container.RegisterUdf("add", args => Convert.ToDouble(args[0]) + Convert.ToDouble(args[1])); await _container.CreateItemAsync( new UdfDocument { Id = "1", PartitionKey = "pk1", X = 10, Y = 20 }, @@ -1227,7 +1227,7 @@ await _container.CreateItemAsync( [Fact] public async Task DeregisterUdf_RemovesHandler() { - _container.RegisterUdf("removable", args => (double)args[0] * 10); + _container.RegisterUdf("removable", args => Convert.ToDouble(args[0]) * 10); await _container.CreateItemAsync( new UdfDocument { Id = "1", PartitionKey = "pk1", Value = 5 }, @@ -1315,7 +1315,7 @@ await _container.CreateItemAsync( [Fact] public async Task Udf_InWhereClause_FiltersCorrectly() { - _container.RegisterUdf("isEven", args => ((double)args[0]) % 2 == 0); + _container.RegisterUdf("isEven", args => Convert.ToDouble(args[0]) % 2 == 0); await _container.CreateItemAsync( new UdfDocument { Id = "1", PartitionKey = "pk1", Value = 2 }, new PartitionKey("pk1")); From 496a00ca6a631dd81591cd7353fe11c3c2ad21e6 Mon Sep 17 00:00:00 2001 From: McNultyyy Date: Wed, 27 May 2026 16:07:56 +0100 Subject: [PATCH 16/16] chore: consolidate all fixes under v4.0.20 --- CHANGELOG.md | 14 +++++--------- src/Directory.Build.props | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da0038f..c772adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,21 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [4.0.21] - 2026-05-27 - -### Fixed -- Decimal scale not preserved on round-trip (Issue #75). Whole numbers stored as decimals (e.g. `1500m`) gained a trailing `.0` after retrieval. A custom `CosmosJsonWriter` now normalizes JSON numeric representations during `ParseJson` to match real Cosmos DB behavior: whole numbers serialize as integers, fractional values use minimal representation (trailing zeros stripped). -- SUM aggregate returns inconsistent results across platforms (Issue #75). On Linux, `SUM` of whole numbers returned `750.0` instead of `750` because `List.Sum()` always returns `double`. Aggregate assignment points now normalize whole-number doubles to longs before serialization. -- Aggregate queries fail on Linux with "Underlying object does not have an 'payload' field" error. On non-Windows platforms the Cosmos SDK's native `ServiceInterop` DLL is unavailable, causing the SDK to request a query plan and activate its `AggregateQueryPipelineStage`. The query plan strategy now suppresses all aggregate info so the SDK never enters this pipeline — our handler already computes aggregates directly. -- Double-to-long overflow when normalizing values at the `long.MaxValue` boundary. `(double)long.MaxValue` rounds up to 2^63 which overflows on cast; comparison changed from `<=` to strict `<`. - -## [4.0.20] - 2026-05-20 +## [4.0.20] - 2026-05-27 ### Fixed - `COUNT(expr)` with nested ternary expressions no longer miscounts documents (Issue #64). `ExprToString` in `CosmosSqlParser` did not parenthesise ternary/coalesce sub-expressions when they appeared as operands of higher-precedence binary operators. When the SDK's transformed query was round-tripped through `SimplifySdkQuery`, the missing parentheses caused re-parsing to produce a different AST — e.g. `(innerTernary > 0) ? 1 : undefined` became `innerTernary ? val : (otherVal > 0 ? 1 : undefined)` — making `COUNT` evaluate the wrong condition. The fix wraps ternary and coalesce expressions in parentheses whenever they appear inside binary, unary, BETWEEN, IN, or LIKE operators. - String literal aliases in aggregate queries (e.g. `SELECT 'Settlement' AS Label, COUNT(1) AS ItemCount FROM c`) no longer return `null` on Linux (Issue #67). `ProjectAggregateFields` now evaluates literal/expression fields via `EvaluateSqlExpression` instead of treating them as property paths. `DefaultQueryPlanStrategy` also bypasses the SDK aggregate pipeline when literals appear alongside aggregates. - Patch operations with filter predicates referencing non-existent properties no longer throw (Issue #70). Missing properties are now treated as `null` in `FilterPredicate` evaluation, matching real Cosmos DB behavior. - Queries without `ORDER BY` now return documents in insertion order, matching real Cosmos DB behavior (Issue #72). Previously documents were returned in hash-map order due to `ConcurrentDictionary` enumeration. Added insertion-order tracking to `InMemoryContainer` that maintains document position across create, replace, upsert, and delete operations. +- Decimal scale not preserved on round-trip (Issue #75). Whole numbers stored as decimals (e.g. `1500m`) gained a trailing `.0` after retrieval. A custom `CosmosJsonWriter` now normalizes JSON numeric representations during `ParseJson` to match real Cosmos DB behavior: whole numbers serialize as integers, fractional values use minimal representation (trailing zeros stripped). +- SUM aggregate returns inconsistent results across platforms (Issue #75). On Linux, `SUM` of whole numbers returned `750.0` instead of `750` because `List.Sum()` always returns `double`. Aggregate assignment points now normalize whole-number doubles to longs before serialization. +- Aggregate queries fail on Linux with "Underlying object does not have an 'payload' field" error. On non-Windows platforms the Cosmos SDK's native `ServiceInterop` DLL is unavailable, causing the SDK to request a query plan and activate its `AggregateQueryPipelineStage`. The query plan strategy now suppresses all aggregate info so the SDK never enters this pipeline — our handler already computes aggregates directly. +- Double-to-long overflow when normalizing values at the `long.MaxValue` boundary. `(double)long.MaxValue` rounds up to 2^63 which overflows on cast; comparison changed from `<=` to strict `<`. ### Changed - Removed `EmulatorFlaky` trait. The single flaky test (`ConcurrentReadsOfNonExistent`) now uses adaptive concurrency — 50 parallel reads for in-memory, 10 for emulator targets — eliminating the need for blanket test exclusions. diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 751adc4..cc5f590 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,7 +5,7 @@ true - 4.0.21 + 4.0.20 lemonlion Copyright (c) 2026 lemonlion MIT