diff --git a/.agents/skills/foundatio-repositories/SKILL.md b/.agents/skills/foundatio-repositories/SKILL.md index b9f4ce80..feb80812 100644 --- a/.agents/skills/foundatio-repositories/SKILL.md +++ b/.agents/skills/foundatio-repositories/SKILL.md @@ -6,13 +6,13 @@ description: > aggregation queries, partial and script patches, and search-after pagination. Apply when working with any IRepository, ISearchableRepository, FindAsync, CountAsync, PatchAsync, PatchAllAsync, or RemoveAllAsync method. Never use - raw IElasticClient directly -- always use repository methods. Use context7 + raw ElasticsearchClient directly -- always use repository methods. Use context7 MCP to fetch current API docs and examples. --- # Foundatio Repositories -High-level Elasticsearch repository pattern for .NET. Interface-first, with built-in caching, messaging, patch operations, and soft deletes. **Never use raw `IElasticClient` directly** -- always use repository methods. +High-level Elasticsearch repository pattern for .NET. Interface-first, with built-in caching, messaging, patch operations, and soft deletes. **Never use raw `ElasticsearchClient` directly** -- always use repository methods. ## Documentation via context7 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c10208e6..ea5e0ff6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,3 +7,4 @@ jobs: secrets: inherit with: new-test-runner: true + minimum-major-minor: '8.0' diff --git a/.gitignore b/.gitignore index b92df279..b2fe1fcb 100644 --- a/.gitignore +++ b/.gitignore @@ -39,24 +39,4 @@ _NCrunch_* .DS_Store # Rider - -# User specific -**/.idea/**/workspace.xml -**/.idea/**/tasks.xml -**/.idea/shelf/* -**/.idea/dictionaries - -# Sensitive or high-churn files -**/.idea/**/dataSources/ -**/.idea/**/dataSources.ids -**/.idea/**/dataSources.xml -**/.idea/**/dataSources.local.xml -**/.idea/**/sqlDataSources.xml -**/.idea/**/dynamic.xml - -# Rider -# Rider auto-generates .iml files, and contentModel.xml -**/.idea/**/*.iml -**/.idea/**/contentModel.xml -**/.idea/**/modules.xml -**/.idea/copilot/chatSessions/ +.idea/ diff --git a/AGENTS.md b/AGENTS.md index 46ee1794..59529850 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,7 +88,7 @@ samples - **Guard clauses**: Use `ArgumentNullException.ThrowIfNull(param)` instead of manual `if (param == null) throw` - **Lambda bodies**: Prefer multi-line bodies for lambda expressions—no single-line `{ action(); return true; }` patterns - **Raw string literals**: Use `"""..."""` for multi-line strings. Never use string concatenation (`+`) to build script or query strings -- **Domain-correct syntax in scripts**: Verify operator syntax for the target language (e.g., Painless uses `==` not `===`; NEST uses specific query DSL methods) +- **Domain-correct syntax in scripts**: Verify operator syntax for the target language (e.g., Painless uses `==` not `===`; Elasticsearch client uses specific query DSL methods) ### Architecture Patterns diff --git a/README.md b/README.md index b39fb452..a5ad24fc 100644 --- a/README.md +++ b/README.md @@ -46,16 +46,15 @@ public sealed class EmployeeIndex : VersionedIndex public EmployeeIndex(IElasticConfiguration configuration) : base(configuration, "employees", version: 1) { } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Text(f => f.Name(e => e.Email).AddKeywordAndSortFields()) - .Number(f => f.Name(e => e.Age).Type(NumberType.Integer)) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + .Text(e => e.Email, t => t.AddKeywordAndSortFields()) + .IntegerNumber(e => e.Age) ); } } diff --git a/build/common.props b/build/common.props index e571c759..bd94482e 100644 --- a/build/common.props +++ b/build/common.props @@ -6,10 +6,11 @@ Generic Repository implementations for Elasticsearch. https://github.com/FoundatioFx/Foundatio.Repositories https://github.com/FoundatioFx/Foundatio.Repositories/releases + 8.0 true v true - false + true $(ProjectDir)..\..\..\ Copyright © $([System.DateTime]::Now.ToString(yyyy)) Foundatio. All rights reserved. diff --git a/docker-compose.yml b/docker-compose.yml index a927bdce..bc81e864 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.11 + image: docker.elastic.co/elasticsearch/elasticsearch:9.3.1 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -20,7 +20,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:8.19.11 + image: docker.elastic.co/kibana/kibana:9.3.1 environment: xpack.security.enabled: "false" ports: diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index ccfcce23..51209bbe 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -75,6 +75,7 @@ export default withMermaid( { text: 'Jobs', link: '/guide/jobs' }, { text: 'Custom Fields', link: '/guide/custom-fields' }, { text: 'Troubleshooting', link: '/guide/troubleshooting' }, + { text: 'Upgrading to ES9', link: '/guide/upgrading-to-es9' }, ], }, ], diff --git a/docs/guide/custom-fields.md b/docs/guide/custom-fields.md index 15b1b56d..69a91a97 100644 --- a/docs/guide/custom-fields.md +++ b/docs/guide/custom-fields.md @@ -177,16 +177,15 @@ public sealed class EmployeeIndex : VersionedIndex AddStandardCustomFieldTypes(); } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Text(f => f.Name(e => e.Name)) + .Keyword(e => e.Id) + .Keyword(e => e.CompanyId) + .Text(e => e.Name) ); } } @@ -417,7 +416,7 @@ public interface ICustomFieldType string Type { get; } Task ProcessValueAsync( T document, object value, CustomFieldDefinition fieldDefinition) where T : class; - IProperty ConfigureMapping(SingleMappingSelector map) where T : class; + Func, IProperty> ConfigureMapping() where T : class; } public class ProcessFieldValueResult @@ -453,9 +452,9 @@ public class PercentFieldType : ICustomFieldType return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public Func, IProperty> ConfigureMapping() where T : class { - return map.Number(n => n.Type(NumberType.Integer)); + return factory => factory.IntegerNumber(); } } ``` diff --git a/docs/guide/elasticsearch-setup.md b/docs/guide/elasticsearch-setup.md index d8dd34af..feddd547 100644 --- a/docs/guide/elasticsearch-setup.md +++ b/docs/guide/elasticsearch-setup.md @@ -9,10 +9,9 @@ The `ElasticConfiguration` class manages your Elasticsearch connection and index ### Basic Configuration ```csharp -using Elasticsearch.Net; +using Elastic.Transport; using Foundatio.Repositories.Elasticsearch.Configuration; using Microsoft.Extensions.Logging; -using Nest; public class MyElasticConfiguration : ElasticConfiguration { @@ -24,9 +23,9 @@ public class MyElasticConfiguration : ElasticConfiguration AddIndex(Projects = new ProjectIndex(this)); } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { - return new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + return new SingleNodePool(new Uri("http://localhost:9200")); } public EmployeeIndex Employees { get; } @@ -41,9 +40,9 @@ public class MyElasticConfiguration : ElasticConfiguration For development or single-node clusters: ```csharp -protected override IConnectionPool CreateConnectionPool() +protected override NodePool CreateConnectionPool() { - return new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + return new SingleNodePool(new Uri("http://localhost:9200")); } ``` @@ -52,7 +51,7 @@ protected override IConnectionPool CreateConnectionPool() For production clusters with multiple nodes: ```csharp -protected override IConnectionPool CreateConnectionPool() +protected override NodePool CreateConnectionPool() { var nodes = new[] { @@ -60,7 +59,7 @@ protected override IConnectionPool CreateConnectionPool() new Uri("http://es-node2:9200"), new Uri("http://es-node3:9200") }; - return new StaticConnectionPool(nodes); + return new StaticNodePool(nodes); } ``` @@ -69,152 +68,43 @@ protected override IConnectionPool CreateConnectionPool() Automatically discovers cluster nodes: ```csharp -protected override IConnectionPool CreateConnectionPool() +protected override NodePool CreateConnectionPool() { var nodes = new[] { new Uri("http://es-node1:9200") }; - return new SniffingConnectionPool(nodes); + return new SniffingNodePool(nodes); } ``` ### Connection Settings -Override `ConfigureSettings` to customize the NEST client: +Override `ConfigureSettings` to customize the client: ```csharp -protected override void ConfigureSettings(ConnectionSettings settings) +protected override void ConfigureSettings(ElasticsearchClientSettings settings) { base.ConfigureSettings(settings); - + // Enable detailed logging in development if (_environment.IsDevelopment()) { settings.DisableDirectStreaming(); settings.PrettyJson(); - settings.EnableDebugMode(); } - + // Set default timeout settings.RequestTimeout(TimeSpan.FromSeconds(30)); - - // Configure authentication - settings.BasicAuthentication("username", "password"); - - // Or use API key - settings.ApiKeyAuthentication("api-key-id", "api-key"); -} -``` - -### Serializer Configuration - -NEST 7.x uses its own internal serializer by default. For custom JSON serialization, you have several options: - -#### Option 1: Configure Field Name Inference - -```csharp -protected override void ConfigureSettings(ConnectionSettings settings) -{ - base.ConfigureSettings(settings); - - // Use property names as-is (no camelCase conversion) - settings.DefaultFieldNameInferrer(p => p); -} -``` - -#### Option 2: Use JSON.NET Serializer (Separate Package) - -If you need full control over JSON serialization, install the `NEST.JsonNetSerializer` NuGet package: - -```bash -dotnet add package NEST.JsonNetSerializer -``` - -Then configure it in `CreateElasticClient`: - -```csharp -using Nest.JsonNetSerializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -protected override IElasticClient CreateElasticClient() -{ - var pool = CreateConnectionPool() ?? new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - // Use JSON.NET serializer with custom settings - var settings = new ConnectionSettings(pool, sourceSerializer: (builtin, connectionSettings) => - new JsonNetSerializer(builtin, connectionSettings, () => new JsonSerializerSettings - { - NullValueHandling = NullValueHandling.Ignore, - DateTimeZoneHandling = DateTimeZoneHandling.Utc - }, - resolver => resolver.NamingStrategy = new CamelCaseNamingStrategy())); - - settings.EnableApiVersioningHeader(); - ConfigureSettings(settings); - - foreach (var index in Indexes) - index.ConfigureSettings(settings); - - return new ElasticClient(settings); -} -``` - -#### Option 3: Custom Serializer Class -For more complex scenarios, create a custom serializer class: - -```csharp -using Elasticsearch.Net; -using Nest; -using Nest.JsonNetSerializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -public class CustomJsonNetSerializer : ConnectionSettingsAwareSerializerBase -{ - public CustomJsonNetSerializer( - IElasticsearchSerializer builtinSerializer, - IConnectionSettingsValues connectionSettings) - : base(builtinSerializer, connectionSettings) { } - - protected override JsonSerializerSettings CreateJsonSerializerSettings() => - new JsonSerializerSettings - { - NullValueHandling = NullValueHandling.Include, - DateTimeZoneHandling = DateTimeZoneHandling.Utc - }; + // Configure basic authentication + settings.Authentication(new BasicAuthentication("username", "password")); - protected override void ModifyContractResolver(ConnectionSettingsAwareContractResolver resolver) - { - resolver.NamingStrategy = new CamelCaseNamingStrategy(); - } + // Or use API key + settings.Authentication(new ApiKey("encoded-api-key")); } ``` -Then use it: +### Serialization -```csharp -protected override IElasticClient CreateElasticClient() -{ - var pool = CreateConnectionPool() ?? new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - var settings = new ConnectionSettings(pool, - sourceSerializer: (builtin, connSettings) => new CustomJsonNetSerializer(builtin, connSettings)); - - settings.EnableApiVersioningHeader(); - ConfigureSettings(settings); - - foreach (var index in Indexes) - index.ConfigureSettings(settings); - - return new ElasticClient(settings); -} -``` - -::: tip When to Use JSON.NET Serializer -- You need specific `JsonSerializerSettings` (null handling, date formatting, etc.) -- You have custom `JsonConverter` implementations -- You need to match serialization behavior with other parts of your application -- You're migrating from an older codebase that relies on Newtonsoft.Json behavior -::: +The new `Elastic.Clients.Elasticsearch` client uses **System.Text.Json** by default. Custom serialization is configured via `SourceSerializerFactory` if needed. ### Configuration with Dependency Injection @@ -232,21 +122,21 @@ public class MyElasticConfiguration : ElasticConfiguration { _config = config; _env = env; - + AddIndex(Employees = new EmployeeIndex(this)); } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { var connectionString = _config.GetConnectionString("Elasticsearch") ?? "http://localhost:9200"; - return new SingleNodeConnectionPool(new Uri(connectionString)); + return new SingleNodePool(new Uri(connectionString)); } - protected override void ConfigureSettings(ConnectionSettings settings) + protected override void ConfigureSettings(ElasticsearchClientSettings settings) { base.ConfigureSettings(settings); - + if (_env.IsDevelopment()) { settings.DisableDirectStreaming(); @@ -265,7 +155,7 @@ public class MyElasticConfiguration : ElasticConfiguration ```csharp public interface IElasticConfiguration : IDisposable { - IElasticClient Client { get; } + ElasticsearchClient Client { get; } ICacheClient Cache { get; } IMessageBus MessageBus { get; } ILoggerFactory LoggerFactory { get; } @@ -316,16 +206,15 @@ public sealed class EmployeeIndex : VersionedIndex public EmployeeIndex(IElasticConfiguration configuration) : base(configuration, "employees", version: 1) { } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) // Disable dynamic mapping + map + .Dynamic(DynamicMapping.False) // Disable dynamic mapping .Properties(p => p .SetupDefaults() // Configure Id, CreatedUtc, UpdatedUtc, IsDeleted - .Keyword(f => f.Name(e => e.CompanyId)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Number(f => f.Name(e => e.Age).Type(NumberType.Integer)) + .Keyword(e => e.CompanyId) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + .IntegerNumber(e => e.Age) ); } } @@ -353,7 +242,7 @@ This configures: For exact matching and aggregations: ```csharp -.Keyword(f => f.Name(e => e.Status)) +.Keyword(e => e.Status) ``` #### Text Fields with Keywords @@ -361,7 +250,7 @@ For exact matching and aggregations: For full-text search with exact matching: ```csharp -.Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) +.Text(e => e.Name, t => t.AddKeywordAndSortFields()) ``` This creates: @@ -372,11 +261,10 @@ This creates: #### Nested Objects ```csharp -.Nested
(n => n - .Name(e => e.Addresses) +.Nested(e => e.Addresses, n => n .Properties(ap => ap - .Keyword(f => f.Name(a => a.City)) - .Keyword(f => f.Name(a => a.Country)) + .Keyword(a => a.City) + .Keyword(a => a.Country) )) ``` @@ -385,19 +273,13 @@ Fields mapped as `nested` are automatically wrapped in Elasticsearch `nested` qu ### Index Settings ```csharp -public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) +public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s + base.ConfigureIndex(idx.Settings(s => s .NumberOfShards(3) .NumberOfReplicas(1) - .RefreshInterval(TimeSpan.FromSeconds(5)) .Analysis(a => a .AddSortNormalizer() - .Analyzers(an => an - .Custom("my_analyzer", ca => ca - .Tokenizer("standard") - .Filters("lowercase", "asciifolding") - )) ))); } ``` @@ -467,34 +349,34 @@ public class MyElasticConfiguration : ElasticConfiguration AddIndex(AuditLogs = new AuditLogIndex(this)); } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { var connectionString = _config.GetConnectionString("Elasticsearch"); if (string.IsNullOrEmpty(connectionString)) connectionString = "http://localhost:9200"; - + var uris = connectionString.Split(',').Select(s => new Uri(s.Trim())); - + if (uris.Count() == 1) - return new SingleNodeConnectionPool(uris.First()); - - return new StaticConnectionPool(uris); + return new SingleNodePool(uris.First()); + + return new StaticNodePool(uris); } - protected override void ConfigureSettings(ConnectionSettings settings) + protected override void ConfigureSettings(ElasticsearchClientSettings settings) { base.ConfigureSettings(settings); - + if (_env.IsDevelopment()) { settings.DisableDirectStreaming(); settings.PrettyJson(); } - + var username = _config["Elasticsearch:Username"]; var password = _config["Elasticsearch:Password"]; if (!string.IsNullOrEmpty(username)) - settings.BasicAuthentication(username, password); + settings.Authentication(new BasicAuthentication(username, password)); } public EmployeeIndex Employees { get; } @@ -555,8 +437,9 @@ Implement `IParentChildDocument` for both parent and child entities: ```csharp using Foundatio.Repositories.Elasticsearch; +using Elastic.Clients.Elasticsearch; +using Foundatio.Repositories.Elasticsearch.Repositories; using Foundatio.Repositories.Models; -using Nest; // Parent document public class Organization : IParentChildDocument, IHaveDates, ISupportSoftDeletes @@ -600,21 +483,17 @@ public sealed class OrganizationIndex : VersionedIndex public OrganizationIndex(IElasticConfiguration configuration) : base(configuration, "organizations", version: 1) { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx + base.ConfigureIndex(idx .Settings(s => s.NumberOfReplicas(0).NumberOfShards(1)) - .Map(m => m - .AutoMap() - .AutoMap() + .Mappings(m => m .Properties(p => p .SetupDefaults() - .Keyword(k => k.Name(o => ((Organization)o).Name)) - .Keyword(k => k.Name(e => ((Employee)e).Email)) - // Configure the join field - .Join(j => j - .Name(n => n.Discriminator) - .Relations(r => r.Join()) + .Keyword(o => ((Organization)o).Name) + .Keyword(e => ((Employee)e).Email) + .Join(d => d.Discriminator, j => j + .Relations(r => r.Add("organization", new[] { "employee" })) ) ))); } @@ -718,7 +597,7 @@ services.AddHealthChecks() var config = services.BuildServiceProvider() .GetRequiredService(); var response = config.Client.Ping(); - return response.IsValid + return response.IsValidResponse ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy("Elasticsearch is not responding"); }); diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 69f2c79a..de0f5f96 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -66,34 +66,34 @@ public class Employee : IIdentity, IHaveDates Define how your entity is indexed in Elasticsearch: ```csharp +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; public sealed class EmployeeIndex : VersionedIndex { public EmployeeIndex(IElasticConfiguration configuration) : base(configuration, "employees", version: 1) { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s + base.ConfigureIndex(idx.Settings(s => s .NumberOfReplicas(0) .NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.CompanyId)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Text(f => f.Name(e => e.Email).AddKeywordAndSortFields()) - .Number(f => f.Name(e => e.Age).Type(NumberType.Integer)) + .Keyword(e => e.CompanyId) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + .Text(e => e.Email, t => t.AddKeywordAndSortFields()) + .IntegerNumber(e => e.Age) ); } } @@ -104,10 +104,9 @@ public sealed class EmployeeIndex : VersionedIndex Set up the connection to Elasticsearch: ```csharp -using Elasticsearch.Net; using Foundatio.Repositories.Elasticsearch.Configuration; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Transport; public class MyElasticConfiguration : ElasticConfiguration { @@ -117,9 +116,9 @@ public class MyElasticConfiguration : ElasticConfiguration AddIndex(Employees = new EmployeeIndex(this)); } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { - return new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + return new SingleNodePool(new Uri("http://localhost:9200")); } public EmployeeIndex Employees { get; } diff --git a/docs/guide/index-management.md b/docs/guide/index-management.md index 0d6541f9..831a2689 100644 --- a/docs/guide/index-management.md +++ b/docs/guide/index-management.md @@ -14,15 +14,14 @@ public sealed class EmployeeIndex : Index public EmployeeIndex(IElasticConfiguration configuration) : base(configuration, "employees") { } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.CompanyId)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) + .Keyword(e => e.CompanyId) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) ); } } @@ -38,16 +37,15 @@ public sealed class EmployeeIndex : VersionedIndex public EmployeeIndex(IElasticConfiguration configuration) : base(configuration, "employees", version: 2) { } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.CompanyId)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Keyword(f => f.Name(e => e.Department)) // Added in v2 + .Keyword(e => e.CompanyId) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + .Keyword(e => e.Department) // Added in v2 ); } } @@ -72,15 +70,14 @@ public sealed class LogEventIndex : DailyIndex DiscardExpiredIndexes = true; } - public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Level)) - .Text(f => f.Name(e => e.Message)) + .Keyword(e => e.Level) + .Text(e => e.Message) ); } } @@ -179,19 +176,14 @@ var results = await repository.FindAsync(q => q.Index("logs-last-7-days")); ### Index Settings ```csharp -public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) +public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s + base.ConfigureIndex(idx.Settings(s => s .NumberOfShards(3) .NumberOfReplicas(1) - .RefreshInterval(TimeSpan.FromSeconds(5)) + .RefreshInterval(new Duration(TimeSpan.FromSeconds(5))) .Analysis(a => a .AddSortNormalizer() - .Analyzers(an => an - .Custom("my_analyzer", ca => ca - .Tokenizer("standard") - .Filters("lowercase", "asciifolding") - )) ))); } ``` @@ -199,38 +191,36 @@ public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) ### Index Mapping ```csharp -public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) +public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) // Disable dynamic mapping + map + .Dynamic(DynamicMapping.False) // Disable dynamic mapping .Properties(p => p .SetupDefaults() // Configure Id, CreatedUtc, UpdatedUtc, IsDeleted - + // Keyword fields (exact match, aggregations) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.Status)) - + .Keyword(e => e.CompanyId) + .Keyword(e => e.Status) + // Text fields with keywords (full-text + exact match) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Text(f => f.Name(e => e.Email).AddKeywordAndSortFields()) - + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + .Text(e => e.Email, t => t.AddKeywordAndSortFields()) + // Numeric fields - .Number(f => f.Name(e => e.Age).Type(NumberType.Integer)) - .Number(f => f.Name(e => e.Salary).Type(NumberType.Double)) + .IntegerNumber(e => e.Age) + .DoubleNumber(e => e.Salary) // Date fields - .Date(f => f.Name(e => e.HireDate)) + .Date(e => e.HireDate) // Boolean fields - .Boolean(f => f.Name(e => e.IsActive)) + .Boolean(e => e.IsActive) // Nested objects - .Nested
(n => n - .Name(e => e.Addresses) + .Nested(e => e.Addresses, n => n .Properties(ap => ap - .Keyword(f => f.Name(a => a.City)) - .Keyword(f => f.Name(a => a.Country)) + .Keyword(a => a.City) + .Keyword(a => a.Country) )) ); } diff --git a/docs/guide/jobs.md b/docs/guide/jobs.md index ff7dc2d3..5fa4313a 100644 --- a/docs/guide/jobs.md +++ b/docs/guide/jobs.md @@ -53,10 +53,10 @@ Creates Elasticsearch snapshots for backup: ```csharp public class SnapshotJob : IJob { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly string _repositoryName; - public SnapshotJob(IElasticClient client, string repositoryName = "backups") + public SnapshotJob(ElasticsearchClient client, string repositoryName = "backups") { _client = client; _repositoryName = repositoryName; @@ -66,13 +66,13 @@ public class SnapshotJob : IJob { var snapshotName = $"snapshot-{DateTime.UtcNow:yyyy-MM-dd-HH-mm-ss}"; - var response = await _client.Snapshot.SnapshotAsync( + var response = await _client.Snapshot.CreateAsync( _repositoryName, snapshotName, s => s.WaitForCompletion(false)); - if (!response.IsValid) - return JobResult.FromException(response.OriginalException); + if (!response.IsValidResponse) + return JobResult.FromException(response.OriginalException(), response.GetErrorMessage()); return JobResult.Success; } @@ -100,11 +100,11 @@ Cleans up old snapshots: ```csharp public class CleanupSnapshotJob : IJob { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly string _repositoryName; private readonly TimeSpan _maxAge; - public CleanupSnapshotJob(IElasticClient client, string repositoryName, TimeSpan maxAge) + public CleanupSnapshotJob(ElasticsearchClient client, string repositoryName, TimeSpan maxAge) { _client = client; _repositoryName = repositoryName; @@ -137,11 +137,11 @@ Deletes old indexes based on patterns and age: ```csharp public class CleanupIndexesJob : IJob { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly string _indexPattern; private readonly TimeSpan _maxAge; - public CleanupIndexesJob(IElasticClient client, string indexPattern, TimeSpan maxAge) + public CleanupIndexesJob(ElasticsearchClient client, string indexPattern, TimeSpan maxAge) { _client = client; _indexPattern = indexPattern; @@ -413,10 +413,10 @@ services.AddCronJob("0 0 * * *"); // Daily at midnight ```csharp public class IndexStatisticsJob : IJob { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ILogger _logger; - public IndexStatisticsJob(IElasticClient client, ILogger logger) + public IndexStatisticsJob(ElasticsearchClient client, ILogger logger) { _client = client; _logger = logger; @@ -445,10 +445,10 @@ public class IndexStatisticsJob : IJob ```csharp public class ElasticsearchHealthJob : IJob { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ILogger _logger; - public ElasticsearchHealthJob(IElasticClient client, ILogger logger) + public ElasticsearchHealthJob(ElasticsearchClient client, ILogger logger) { _client = client; _logger = logger; @@ -464,7 +464,7 @@ public class ElasticsearchHealthJob : IJob health.NumberOfNodes, health.ActiveShards); - if (health.Status == Health.Red) + if (health.Status == HealthStatus.Red) { _logger.LogError("Cluster is in RED status!"); return JobResult.FromException(new Exception("Cluster health is RED")); diff --git a/docs/guide/patch-operations.md b/docs/guide/patch-operations.md index fa5d2930..d07bfa43 100644 --- a/docs/guide/patch-operations.md +++ b/docs/guide/patch-operations.md @@ -70,7 +70,7 @@ public class MyRepository : ElasticRepositoryBase | `GetUpdatedUtcFieldPath()` | Returns the Elasticsearch field path for the updated timestamp | Returns the inferred `UpdatedUtc` field name. Throws `RepositoryException` if `HasDateTracking` is `true` but no field path is available | | `SetDocumentDates(T, TimeProvider)` | Sets date properties on the C# object (used by Add, Save, ActionPatch, JsonPatch bulk) | Sets `CreatedUtc` and `UpdatedUtc` on `IHaveDates` models | -The `ApplyDateTracking` overloads for `ScriptPatch`, `PartialPatch`, and `JToken` are also virtual and can be overridden for full control over how dates are injected into each patch type. For nested fields, the script parameter key uses the last segment of the field path (e.g., `dateUpdatedUtc` for `metaData.dateUpdatedUtc`). +The `ApplyDateTracking` overloads for `ScriptPatch`, `PartialPatch`, and `JsonNode` are also virtual and can be overridden for full control over how dates are injected into each patch type. For nested fields, the script parameter key uses the last segment of the field path (e.g., `dateUpdatedUtc` for `metaData.dateUpdatedUtc`). ## Patch Types diff --git a/docs/guide/querying.md b/docs/guide/querying.md index d265260d..e4687113 100644 --- a/docs/guide/querying.md +++ b/docs/guide/querying.md @@ -370,18 +370,17 @@ When querying fields inside [nested objects](https://www.elastic.co/guide/en/ela The field must be mapped as `nested` in your index configuration: ```csharp -public override TypeMappingDescriptor ConfigureIndexMapping( - TypeMappingDescriptor map) +public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Nested(f => f.Name(e => e.PeerReviews).Properties(p1 => p1 - .Keyword(f2 => f2.Name(p2 => p2.ReviewerEmployeeId)) - .Scalar(p3 => p3.Rating, f2 => f2.Name(p3 => p3.Rating)))) + .Keyword(e => e.Id) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + .Nested(e => e.PeerReviews, n => n.Properties(p1 => p1 + .Keyword("reviewerEmployeeId") + .IntegerNumber("rating"))) ); } ``` diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 7271e98d..c231b2f1 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -22,10 +22,10 @@ curl http://localhost:9200 2. **Check connection string:** ```csharp -protected override IConnectionPool CreateConnectionPool() +protected override NodePool CreateConnectionPool() { // Ensure URL is correct - return new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + return new SingleNodePool(new Uri("http://localhost:9200")); } ``` @@ -39,11 +39,10 @@ telnet localhost 9200 4. **Enable debug logging:** ```csharp -protected override void ConfigureSettings(ConnectionSettings settings) +protected override void ConfigureSettings(ElasticsearchClientSettings settings) { settings.DisableDirectStreaming(); settings.PrettyJson(); - settings.EnableDebugMode(); } ``` @@ -56,13 +55,13 @@ protected override void ConfigureSettings(ConnectionSettings settings) **Solutions:** ```csharp -protected override void ConfigureSettings(ConnectionSettings settings) +protected override void ConfigureSettings(ElasticsearchClientSettings settings) { // Basic authentication - settings.BasicAuthentication("username", "password"); + settings.Authentication(new BasicAuthentication("username", "password")); // Or API key - settings.ApiKeyAuthentication("api-key-id", "api-key"); + settings.Authentication(new ApiKey("api-key")); } ``` @@ -122,7 +121,7 @@ await configuration.ConfigureIndexesAsync(); ```csharp // Ensure mapping matches data types -.Number(f => f.Name(e => e.Age).Type(NumberType.Integer)) +.IntegerNumber(e => e.Age) ``` ## Query Issues @@ -410,18 +409,10 @@ public class EmployeeRepository : ElasticRepositoryBase ```csharp // In configuration -protected override void ConfigureSettings(ConnectionSettings settings) +protected override void ConfigureSettings(ElasticsearchClientSettings settings) { settings.DisableDirectStreaming(); settings.PrettyJson(); - settings.OnRequestCompleted(details => - { - _logger.LogDebug("Request: {Method} {Uri}", - details.HttpMethod, details.Uri); - if (details.RequestBodyInBytes != null) - _logger.LogDebug("Body: {Body}", - Encoding.UTF8.GetString(details.RequestBodyInBytes)); - }); } // Per query diff --git a/docs/guide/upgrading-to-es9.md b/docs/guide/upgrading-to-es9.md new file mode 100644 index 00000000..8c4685d9 --- /dev/null +++ b/docs/guide/upgrading-to-es9.md @@ -0,0 +1,482 @@ +# Migrating to Elastic.Clients.Elasticsearch (ES9) + +This guide covers breaking changes when upgrading from `NEST` (ES7) to `Elastic.Clients.Elasticsearch` (ES8/ES9). The new Elasticsearch .NET client is a complete rewrite with a new API surface, so most code that interacts with Elasticsearch directly will need changes. + +> **Query syntax changes**: If you use [Foundatio.Parsers](https://github.com/FoundatioFx/Foundatio.Parsers) for query parsing (e.g., `ElasticQueryParser`, `ElasticMappingResolver`, aggregation parsing), refer to the [Foundatio.Parsers documentation](https://github.com/FoundatioFx/Foundatio.Parsers) for its own ES9-related migration notes. The query parser APIs have been updated to work with the new `Elastic.Clients.Elasticsearch` types. + +## Package Changes + +**Before:** +```xml + +``` + +**After:** +```xml + + +``` + +## Namespace Changes + +Remove old NEST namespaces and add new ones: + +```csharp +// Remove: +using Elasticsearch.Net; +using Nest; + +// Add: +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Transport; +``` + +Additional namespaces you may need depending on usage: + +| Feature | Namespace | +|---------|-----------| +| Aggregations | `Elastic.Clients.Elasticsearch.Aggregations` | +| Bulk operations | `Elastic.Clients.Elasticsearch.Core.Bulk` | +| Search types | `Elastic.Clients.Elasticsearch.Core.Search` | +| Async search | `Elastic.Clients.Elasticsearch.AsyncSearch` | +| Analysis (analyzers, tokenizers) | `Elastic.Clients.Elasticsearch.Analysis` | +| Fluent helpers | `Elastic.Clients.Elasticsearch.Fluent` | + +## ElasticConfiguration Changes + +### Client Type + +| Before | After | +|--------|-------| +| `IElasticClient Client` | `ElasticsearchClient Client` | +| `new ElasticClient(settings)` | `new ElasticsearchClient(settings)` | + +### Connection Pool + +| Before | After | +|--------|-------| +| `IConnectionPool` | `NodePool` | +| `new SingleNodeConnectionPool(uri)` | `new SingleNodePool(uri)` | +| `new StaticConnectionPool(nodes)` | `new StaticNodePool(nodes)` | +| `new SniffingConnectionPool(nodes)` | `new SniffingNodePool(nodes)` | + +### Settings + +| Before | After | +|--------|-------| +| `ConnectionSettings` | `ElasticsearchClientSettings` | +| `settings.BasicAuthentication(u, p)` | `settings.Authentication(new BasicAuthentication(u, p))` | +| `settings.ApiKeyAuthentication(id, key)` | `settings.Authentication(new ApiKey(encoded))` | + +**Before:** +```csharp +protected override IConnectionPool CreateConnectionPool() +{ + return new SingleNodeConnectionPool(new Uri("http://localhost:9200")); +} + +protected override void ConfigureSettings(ConnectionSettings settings) +{ + base.ConfigureSettings(settings); + settings.BasicAuthentication("user", "pass"); +} +``` + +**After:** +```csharp +protected override NodePool CreateConnectionPool() +{ + return new SingleNodePool(new Uri("http://localhost:9200")); +} + +protected override void ConfigureSettings(ElasticsearchClientSettings settings) +{ + base.ConfigureSettings(settings); + settings.Authentication(new BasicAuthentication("user", "pass")); +} +``` + +### Constructor: Serializer Parameter + +`ElasticConfiguration` now accepts an `ITextSerializer` parameter. If you don't provide one, a default `SystemTextJsonSerializer` is created with `ConfigureFoundatioRepositoryDefaults()`. If you have custom serialization needs, pass your own serializer: + +```csharp +var serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + +var config = new MyElasticConfiguration( + serializer: serializer, + cacheClient: cache, + messageBus: bus); +``` + +### Client Disposal + +`ElasticsearchClientSettings` implements `IDisposable` internally but doesn't expose it on its public API. The `ElasticConfiguration.Dispose()` method now handles this by casting to `IDisposable`. If you manage the client lifecycle yourself, be aware of this. + +## Serialization Changes (Newtonsoft.Json to System.Text.Json) + +This is one of the largest breaking changes. The new Elasticsearch client uses **System.Text.Json** instead of Newtonsoft.Json for all serialization. + +### What Changed + +| Before | After | +|--------|-------| +| `NEST.JsonNetSerializer` package | **Removed** — no longer needed or supported | +| `Newtonsoft.Json.JsonConverter` | `System.Text.Json.Serialization.JsonConverter` | +| `[JsonProperty("name")]` | `[JsonPropertyName("name")]` | +| `[JsonIgnore]` (Newtonsoft) | `[JsonIgnore]` (System.Text.Json — same name, different namespace) | +| `[JsonConverter(typeof(...))]` (Newtonsoft) | `[JsonConverter(typeof(...))]` (System.Text.Json) | +| `JsonConvert.SerializeObject(obj)` | `JsonSerializer.Serialize(obj, options)` | +| `JsonConvert.DeserializeObject(json)` | `JsonSerializer.Deserialize(json, options)` | + +### ConfigureFoundatioRepositoryDefaults + +Foundatio.Repositories provides a `ConfigureFoundatioRepositoryDefaults()` extension method on `JsonSerializerOptions` that registers converters needed for correct round-tripping of repository documents: + +```csharp +using Foundatio.Repositories.Serialization; + +var options = new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults(); +``` + +This registers: +- `JsonStringEnumConverter` with camelCase naming and integer fallback +- `DoubleSystemTextJsonConverter` to preserve decimal points on whole-number doubles +- `ObjectToInferredTypesConverter` to deserialize `object`-typed properties as CLR primitives instead of `JsonElement` +- Case-insensitive property matching + +### LazyDocument Serializer Requirement + +`LazyDocument` no longer falls back to a default Newtonsoft serializer. The `ITextSerializer` parameter is now **required**: + +**Before:** +```csharp +new LazyDocument(data, serializer: null); // fell back to JsonNetSerializer +``` + +**After:** +```csharp +new LazyDocument(data, serializer); // serializer is required, throws if null +``` + +### Migration Tips for Custom Converters + +If you have custom Newtonsoft `JsonConverter` implementations: + +1. Create a new class inheriting from `System.Text.Json.Serialization.JsonConverter` +2. Implement `Read` and `Write` methods using `Utf8JsonReader`/`Utf8JsonWriter` +3. Register converters via `JsonSerializerOptions.Converters.Add(...)` or the `[JsonConverter]` attribute +4. Be aware that System.Text.Json is stricter by default (no comments, trailing commas, or unquoted property names) + +## Index Configuration Changes + +### ConfigureIndex: Return Type and Descriptor + +`ConfigureIndex` changed from returning a descriptor (fluent chaining) to `void` (mutating the descriptor in place). The descriptor type also changed: + +| Before | After | +|--------|-------| +| `CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx)` | `void ConfigureIndex(CreateIndexRequestDescriptor idx)` | +| Returns the descriptor | Mutates the descriptor in place | + +**Before:** +```csharp +public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) +{ + return base.ConfigureIndex(idx + .Settings(s => s.NumberOfReplicas(0)) + .Map(m => m.AutoMap().Properties(p => p.SetupDefaults()))); +} +``` + +**After:** +```csharp +public override void ConfigureIndex(CreateIndexRequestDescriptor idx) +{ + base.ConfigureIndex(idx + .Settings(s => s.NumberOfReplicas(0)) + .Mappings(m => m.Properties(p => p.SetupDefaults()))); +} +``` + +> **Note:** `AutoMap()` has been removed from the new client. Define all property mappings explicitly via `.Properties(...)`. + +### ConfigureIndexMapping: Return Type and API + +`ConfigureIndexMapping` changed from returning `TypeMappingDescriptor` to `void`: + +| Before | After | +|--------|-------| +| `TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map)` | `void ConfigureIndexMapping(TypeMappingDescriptor map)` | +| Returns the descriptor | Mutates the descriptor in place | + +**Before:** +```csharp +public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) +{ + return map + .Dynamic(false) + .Properties(p => p + .SetupDefaults() + .Keyword(f => f.Name(e => e.Id)) + .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) + ); +} +``` + +**After:** +```csharp +public override void ConfigureIndexMapping(TypeMappingDescriptor map) +{ + map + .Dynamic(DynamicMapping.False) + .Properties(p => p + .SetupDefaults() + .Keyword(e => e.Id) + .Text(e => e.Name, t => t.AddKeywordAndSortFields()) + ); +} +``` + +### ConfigureIndexAliases Signature + +**Before:** +```csharp +public override IPromise ConfigureIndexAliases(AliasesDescriptor aliases) +{ + return aliases.Alias("my-alias"); +} +``` + +**After:** +```csharp +public override void ConfigureIndexAliases(FluentDictionaryOfNameAlias aliases) +{ + aliases.Add("my-alias", a => a); +} +``` + +### CreateIndexAsync and UpdateIndexAsync + +Internal methods that create or update indexes changed from `Func` (fluent return) to `Action` (void mutation): + +| Before | After | +|--------|-------| +| `Func` | `Action` | +| `Func` | `Action` | + +### ConfigureSettings on Index + +**Before:** +```csharp +public override void ConfigureSettings(ConnectionSettings settings) { } +``` + +**After:** +```csharp +public override void ConfigureSettings(ElasticsearchClientSettings settings) { } +``` + +## Property Mapping (TypeMappingDescriptor) Changes + +The new client uses a simpler expression syntax for property mappings. The `.Name(e => e.Prop)` wrapper is gone — property name inference comes directly from the expression. Configuration lambdas are now a second parameter: + +| Before | After | +|--------|-------| +| `.Keyword(f => f.Name(e => e.Id))` | `.Keyword(e => e.Id)` | +| `.Text(f => f.Name(e => e.Name))` | `.Text(e => e.Name)` | +| `.Text(f => f.Name(e => e.Name).Analyzer("my_analyzer"))` | `.Text(e => e.Name, t => t.Analyzer("my_analyzer"))` | +| `.Number(f => f.Name(e => e.Age).Type(NumberType.Integer))` | `.IntegerNumber(e => e.Age)` | +| `.Number(f => f.Name(e => e.Score).Type(NumberType.Double))` | `.DoubleNumber(e => e.Score)` | +| `.Date(f => f.Name(e => e.CreatedUtc))` | `.Date(e => e.CreatedUtc)` | +| `.Boolean(f => f.Name(e => e.IsActive))` | `.Boolean(e => e.IsActive)` | +| `.Object(f => f.Name(e => e.Address).Properties(...))` | `.Object(e => e.Address, o => o.Properties(...))` | +| `.Nested(f => f.Name(e => e.Items).Properties(...))` | `.Nested(e => e.Items, n => n.Properties(...))` | +| `.Dynamic(false)` | `.Dynamic(DynamicMapping.False)` | +| `.Map(m => m.Properties(...))` | `.Mappings(m => m.Properties(...))` | + +### Number Type Mapping + +The generic `.Number()` with `NumberType` enum is replaced by specific typed methods: + +| Before | After | +|--------|-------| +| `.Number(f => f.Type(NumberType.Integer))` | `.IntegerNumber(e => e.Field)` | +| `.Number(f => f.Type(NumberType.Long))` | `.LongNumber(e => e.Field)` | +| `.Number(f => f.Type(NumberType.Float))` | `.FloatNumber(e => e.Field)` | +| `.Number(f => f.Type(NumberType.Double))` | `.DoubleNumber(e => e.Field)` | + +## Response Validation + +The `IsValid` property on responses was renamed to `IsValidResponse`: + +| Before | After | +|--------|-------| +| `response.IsValid` | `response.IsValidResponse` | +| `response.OriginalException` | `response.OriginalException()` (method call) | +| `response.ServerError?.Status` | `response.ElasticsearchServerError?.Status` | +| `response.ServerError.Error.Type` | `response.ElasticsearchServerError.Error.Type` | + +## Custom Field Type Mapping (ICustomFieldType) + +`ICustomFieldType.ConfigureMapping` changed from accepting a `SingleMappingSelector` parameter and returning `IProperty` to a parameterless method returning a factory function: + +**Before:** +```csharp +public IProperty ConfigureMapping(SingleMappingSelector map) where T : class +{ + return map.Number(n => n.Type(NumberType.Integer)); +} +``` + +**After:** +```csharp +public Func, IProperty> ConfigureMapping() where T : class +{ + return factory => factory.IntegerNumber(); +} +``` + +All standard field types (`IntegerFieldType`, `StringFieldType`, `BooleanFieldType`, `DateFieldType`, `KeywordFieldType`, `LongFieldType`, `FloatFieldType`, `DoubleFieldType`) have been updated to this pattern. If you have custom `ICustomFieldType` implementations, update them to match. + +## Ingest Pipeline on Update + +The old client supported `Pipeline` on bulk update operations via a custom extension. **This feature is not supported by the Elasticsearch Update API** and has been removed. Use the Ingest pipeline on index (PUT) operations only. + +## Snapshot API + +The `Snapshot.SnapshotAsync` method was renamed to `Snapshot.CreateAsync` in the new client. + +## Counting with Index Filtering + +**Before:** +```csharp +await client.CountAsync(d => d.Index(indexName), cancellationToken); +``` + +**After:** +```csharp +await client.CountAsync(d => d.Indices(indexName)); +``` + +## Parent-Child Documents + +The `RoutingField` configuration on `TypeMappingDescriptor` is no longer available as a direct mapping property. Routing is now handled at the index settings level or through query routing parameters. + +## RefreshInterval + +**Before:** +```csharp +settings.RefreshInterval(TimeSpan.FromSeconds(30)); +``` + +**After:** +```csharp +settings.RefreshInterval(Duration.FromSeconds(30)); +``` + +## TopHits Aggregation Round-Trip + +The `TopHitsAggregate` now serializes the raw document JSON in its `Hits` property, enabling round-trip serialization for caching purposes. The `Documents()` method checks both the in-memory `ILazyDocument` list (from a live ES response) and the serialized `Hits` list (from cache deserialization). + +## Known Bugs and Workarounds + +### ResolveIndexAsync Is Broken in ES 9.x Client + +The `Indices.ResolveIndexAsync` method in the Elastic.Clients.Elasticsearch 9.x client is broken — it does not correctly resolve wildcard index patterns. Foundatio.Repositories works around this by using `Indices.GetAsync` with `IgnoreUnavailable()` instead: + +```csharp +// DON'T use ResolveIndexAsync — it's broken in the ES 9.x client +// var resolved = await client.Indices.ResolveIndexAsync(pattern); + +// DO use GetAsync to resolve wildcard patterns +var getResponse = await client.Indices.GetAsync( + Indices.Parse("my-index-*"), + d => d.IgnoreUnavailable()); + +if (getResponse.IsValidResponse && getResponse.Indices is not null) +{ + foreach (var kvp in getResponse.Indices) + Console.WriteLine(kvp.Key); // actual index name +} +``` + +If you have code that calls `ResolveIndexAsync` directly, switch to `GetAsync`. + +### EnableApiVersioningHeader Removed + +The `settings.EnableApiVersioningHeader()` call from NEST is no longer needed and does not exist in the new client. Remove it. + +## Common Gotchas + +1. **Fluent return vs void**: The most pervasive change is that descriptor-based methods (`ConfigureIndex`, `ConfigureIndexMapping`, `ConfigureIndexAliases`) no longer return the descriptor. Remove all `return` statements and change return types to `void`. + +2. **AutoMap is gone**: The new client does not support `AutoMap()`. You must define every property mapping explicitly. This is actually safer — it prevents accidental mapping of fields you don't want indexed. + +3. **Serializer mismatch**: If documents were serialized with Newtonsoft.Json (e.g., stored in a cache) and you try to deserialize with System.Text.Json, you may get errors or silent data loss. Ensure cached data is invalidated or re-serialized during migration. + +4. **Enum serialization**: Newtonsoft.Json serialized enums as integers by default; System.Text.Json does not. The `ConfigureFoundatioRepositoryDefaults()` helper registers a `JsonStringEnumConverter` with camelCase naming to handle this, but verify your existing data is compatible. + +5. **Double precision**: System.Text.Json may round whole-number doubles (e.g., `1.0` becomes `1`). The `DoubleSystemTextJsonConverter` registered by `ConfigureFoundatioRepositoryDefaults()` preserves the decimal point, but only for `double` typed properties. + +6. **object-typed properties**: Without `ObjectToInferredTypesConverter`, System.Text.Json deserializes `object` properties as `JsonElement` instead of CLR primitives. This converter is registered by `ConfigureFoundatioRepositoryDefaults()` but if you're using your own `JsonSerializerOptions`, you must add it manually. + +7. **Indices.Parse vs IndexName cast**: When passing index names to API calls, use `(IndexName)name` for single names or `Indices.Parse(name)` for comma-separated or wildcard patterns. + +8. **CancellationToken parameter changes**: Some API methods that previously accepted `CancellationToken` as a direct parameter now use it differently. Check each call site. + +9. **OriginalException is a method**: `response.OriginalException` changed from a property to a method call `response.OriginalException()`. This will be a compile error, but it's easy to miss in string interpolation. + +10. **ElasticsearchClientSettings is IDisposable**: The settings object implements `IDisposable` but hides it behind an explicit interface implementation. If you manage the client lifecycle yourself, cast to `IDisposable` and dispose it. + +## Migration Checklist + +### Packages and Namespaces +- [ ] Replace `using Elasticsearch.Net;` and `using Nest;` with `using Elastic.Clients.Elasticsearch;` +- [ ] Add additional namespaces as needed (`Mapping`, `IndexManagement`, `Aggregations`, etc.) +- [ ] Remove `NEST.JsonNetSerializer` dependency + +### Configuration +- [ ] Update `CreateConnectionPool()` return type from `IConnectionPool` to `NodePool` +- [ ] Update pool class names (`SingleNodeConnectionPool` → `SingleNodePool`, etc.) +- [ ] Update `ConfigureSettings` parameter from `ConnectionSettings` to `ElasticsearchClientSettings` +- [ ] Update authentication calls (`.BasicAuthentication` → `.Authentication(new BasicAuthentication(...))`) +- [ ] Remove `settings.EnableApiVersioningHeader()` calls +- [ ] Pass an `ITextSerializer` to `ElasticConfiguration` if you need custom serialization + +### Index Configuration +- [ ] Change `ConfigureIndex` return type from `CreateIndexDescriptor` to `void` (remove `return`) +- [ ] Change `ConfigureIndex` parameter from `CreateIndexDescriptor` to `CreateIndexRequestDescriptor` +- [ ] Change `ConfigureIndexMapping` return type to `void` (remove `return`) +- [ ] Change `ConfigureIndexAliases` to use `FluentDictionaryOfNameAlias` and `void` return +- [ ] Replace `.Map(...)` with `.Mappings(...)` +- [ ] Remove `AutoMap()` calls; define all mappings explicitly + +### Property Mappings +- [ ] Update property mapping syntax (remove `.Name(e => e.Prop)` wrapper) +- [ ] Replace `.Number(f => f.Type(NumberType.Integer))` with `.IntegerNumber(e => e.Field)` +- [ ] Replace `.Dynamic(false)` with `.Dynamic(DynamicMapping.False)` +- [ ] Update `.Text()`, `.Object()`, `.Nested()` to use two-parameter form for configuration + +### Serialization +- [ ] Replace `[JsonProperty]` (Newtonsoft) with `[JsonPropertyName]` (System.Text.Json) +- [ ] Rewrite custom `JsonConverter` classes for System.Text.Json +- [ ] Use `ConfigureFoundatioRepositoryDefaults()` on your `JsonSerializerOptions` +- [ ] Update `LazyDocument` construction to provide a required `ITextSerializer` +- [ ] Invalidate caches that may contain Newtonsoft-serialized data + +### Response Handling +- [ ] Replace `response.IsValid` with `response.IsValidResponse` +- [ ] Replace `response.OriginalException` with `response.OriginalException()` (method call) +- [ ] Replace `response.ServerError` with `response.ElasticsearchServerError` + +### Custom Field Types +- [ ] Update `ICustomFieldType.ConfigureMapping` to new `Func, IProperty>` signature + +### Known Issues +- [ ] Replace any `ResolveIndexAsync` calls with `Indices.GetAsync` (broken in ES 9.x client) +- [ ] Verify enum serialization compatibility with existing Elasticsearch data +- [ ] Test document round-tripping with System.Text.Json diff --git a/samples/Foundatio.SampleApp/Server/Repositories/Configuration/Indexes/GameReviewIndex.cs b/samples/Foundatio.SampleApp/Server/Repositories/Configuration/Indexes/GameReviewIndex.cs index b046da5e..982409fe 100644 --- a/samples/Foundatio.SampleApp/Server/Repositories/Configuration/Indexes/GameReviewIndex.cs +++ b/samples/Foundatio.SampleApp/Server/Repositories/Configuration/Indexes/GameReviewIndex.cs @@ -1,7 +1,9 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Parsers.ElasticQueries.Extensions; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.SampleApp.Shared; -using Nest; namespace Foundatio.SampleApp.Server.Repositories.Indexes; @@ -9,26 +11,27 @@ public sealed class GameReviewIndex : VersionedIndex { public GameReviewIndex(IElasticConfiguration configuration) : base(configuration, "gamereview", version: 1) { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s + base.ConfigureIndex(idx); + idx.Settings(s => s .Analysis(a => a.AddSortNormalizer()) .NumberOfReplicas(0) - .NumberOfShards(1))); + .NumberOfShards(1)); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { // adding new fields will automatically update the index mapping // changing existing fields requires a new index version and a reindex - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields()) - .Text(f => f.Name(e => e.Description)) - .Text(f => f.Name(e => e.Category).AddKeywordAndSortFields()) - .Text(f => f.Name(e => e.Tags).AddKeywordAndSortFields()) + .Text(e => e.Name, f => f.AddKeywordAndSortFields()) + .Text(e => e.Description) + .Text(e => e.Category, f => f.AddKeywordAndSortFields()) + .Text(e => e.Tags, f => f.AddKeywordAndSortFields()) ); } } diff --git a/samples/Foundatio.SampleApp/Server/Repositories/Configuration/SampleAppElasticConfiguration.cs b/samples/Foundatio.SampleApp/Server/Repositories/Configuration/SampleAppElasticConfiguration.cs index d47fbd8c..e0d15bc1 100644 --- a/samples/Foundatio.SampleApp/Server/Repositories/Configuration/SampleAppElasticConfiguration.cs +++ b/samples/Foundatio.SampleApp/Server/Repositories/Configuration/SampleAppElasticConfiguration.cs @@ -1,7 +1,7 @@ -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.SampleApp.Server.Repositories.Indexes; -using Nest; namespace Foundatio.SampleApp.Server.Repositories; @@ -17,12 +17,12 @@ public SampleAppElasticConfiguration(IConfiguration config, IWebHostEnvironment AddIndex(GameReviews = new GameReviewIndex(this)); } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { - return new SingleNodeConnectionPool(new Uri(_connectionString)); + return new SingleNodePool(new Uri(_connectionString)); } - protected override void ConfigureSettings(ConnectionSettings settings) + protected override void ConfigureSettings(ElasticsearchClientSettings settings) { // only do this in test and dev mode to enable better debug logging if (_env.IsDevelopment()) diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/DailyIndex.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/DailyIndex.cs index a8760729..225049e1 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/DailyIndex.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/DailyIndex.cs @@ -6,6 +6,9 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.DateTimeExtensions; using Foundatio.Caching; using Foundatio.Parsers.ElasticQueries; @@ -18,7 +21,6 @@ using Foundatio.Repositories.Options; using Foundatio.Repositories.Utility; using Microsoft.Extensions.Logging; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -162,11 +164,12 @@ protected async Task EnsureDateIndexAsync(DateTime utcDate) string index = GetVersionedIndex(utcDate); await CreateIndexAsync(index, descriptor => { - var aliasesDescriptor = new AliasesDescriptor().Alias(unversionedIndexAlias); - foreach (var a in Aliases.Where(a => ShouldCreateAlias(utcDate, a))) - aliasesDescriptor.Alias(a.Name); - - return ConfigureIndex(descriptor).Aliases(a => aliasesDescriptor); + ConfigureIndex(descriptor); + // Add dated alias (e.g., daily-logevents-2025.12.01) + descriptor.AddAlias(unversionedIndexAlias, a => { }); + // Add configured time-based aliases (includes base Name alias and any configured aliases like daily-logevents-today) + foreach (var alias in Aliases.Where(a => ShouldCreateAlias(utcDate, a))) + descriptor.AddAlias(alias.Name, a => { }); }).AnyContext(); _ensuredDates[utcDate] = null; @@ -187,13 +190,27 @@ public override async Task GetCurrentVersionAsync() if (indexes.Count == 0) return Version; - var currentIndexes = indexes + var nonExpiredIndexes = indexes .Where(i => Configuration.TimeProvider.GetUtcNow().UtcDateTime <= GetIndexExpirationDate(i.DateUtc)) - .Select(i => i.CurrentVersion >= 0 ? i.CurrentVersion : i.Version) + .ToList(); + + // First try to find indexes that have a valid dated alias (CurrentVersion >= 0) + var indexesWithAlias = nonExpiredIndexes + .Where(i => i.CurrentVersion >= 0) + .Select(i => i.CurrentVersion) + .OrderBy(v => v) + .ToList(); + + if (indexesWithAlias.Count > 0) + return indexesWithAlias.First(); + + // Fall back to oldest index version if no aliases exist + var allVersions = nonExpiredIndexes + .Select(i => i.Version) .OrderBy(v => v) .ToList(); - return currentIndexes.Count > 0 ? currentIndexes.First() : Version; + return allVersions.Count > 0 ? allVersions.First() : Version; } public virtual string[] GetIndexes(DateTime? utcStart, DateTime? utcEnd) @@ -233,7 +250,7 @@ public override async Task ReindexAsync(Func progressCallback if (indexes.Count == 0) return; - var reindexer = new ElasticReindexer(Configuration.Client, _logger); + var reindexer = new ElasticReindexer(Configuration.Client, Configuration.Serializer, _logger); foreach (var index in indexes) { if (Configuration.TimeProvider.GetUtcNow().UtcDateTime > GetIndexExpirationDate(index.DateUtc)) @@ -278,7 +295,8 @@ protected virtual async Task UpdateAliasesAsync(IList indexes) if (indexes.Count == 0) return; - var aliasDescriptor = new BulkAliasDescriptor(); + var aliasActions = new List(indexes.Count * (Aliases.Count + 1)); + foreach (var indexGroup in indexes.OrderBy(i => i.Version).GroupBy(i => i.DateUtc)) { var indexExpirationDate = GetIndexExpirationDate(indexGroup.Key); @@ -308,7 +326,9 @@ protected virtual async Task UpdateAliasesAsync(IList indexes) if (Configuration.TimeProvider.GetUtcNow().UtcDateTime >= indexExpirationDate || index.Version != index.CurrentVersion) { foreach (var alias in Aliases) - aliasDescriptor = aliasDescriptor.Remove(r => r.Index(index.Index).Alias(alias.Name)); + { + aliasActions.Add(new IndexUpdateAliasesAction { Remove = new RemoveAction { Index = index.Index, Alias = alias.Name } }); + } continue; } @@ -316,25 +336,28 @@ protected virtual async Task UpdateAliasesAsync(IList indexes) foreach (var alias in Aliases) { if (ShouldCreateAlias(indexGroup.Key, alias)) - aliasDescriptor = aliasDescriptor.Add(r => r.Index(index.Index).Alias(alias.Name)); + aliasActions.Add(new IndexUpdateAliasesAction { Add = new AddAction { Index = index.Index, Alias = alias.Name } }); else - aliasDescriptor = aliasDescriptor.Remove(r => r.Index(index.Index).Alias(alias.Name)); + aliasActions.Add(new IndexUpdateAliasesAction { Remove = new RemoveAction { Index = index.Index, Alias = alias.Name } }); } } } - var response = await Configuration.Client.Indices.BulkAliasAsync(aliasDescriptor).AnyContext(); + if (aliasActions.Count == 0) + return; + + var response = await Configuration.Client.Indices.UpdateAliasesAsync(u => u.Actions(aliasActions)).AnyContext(); - if (response.IsValid) + if (response.IsValidResponse) { _logger.LogRequest(response); } else { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return; - throw new DocumentException(response.GetErrorMessage("Error updating aliases"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error updating aliases"), response.OriginalException()); } } @@ -368,9 +391,14 @@ protected override async Task DeleteIndexAsync(string name) await base.DeleteIndexAsync(name).AnyContext(); if (name.EndsWith("*")) + { await _aliasCache.RemoveAllAsync().AnyContext(); + _ensuredDates.Clear(); + } else + { await _aliasCache.RemoveAsync(GetIndexByDate(GetIndexDate(name))).AnyContext(); + } } public override string[] GetIndexesByQuery(IRepositoryQuery query) @@ -405,18 +433,25 @@ protected override ElasticMappingResolver CreateMappingResolver() return ElasticMappingResolver.Create(GetLatestIndexMapping, Configuration.Client.Infer, _logger); } - protected ITypeMapping GetLatestIndexMapping() + protected TypeMapping GetLatestIndexMapping() { string filter = $"{Name}-v{Version}-*"; - var catResponse = Configuration.Client.Cat.Indices(i => i.Pri().Index(Indices.Index((IndexName)filter))); - if (!catResponse.IsValid) + var indicesResponse = Configuration.Client.Indices.Get((Indices)(IndexName)filter); + if (!indicesResponse.IsValidResponse) { - throw new RepositoryException(catResponse.GetErrorMessage($"Error getting latest index mapping {filter}"), catResponse.OriginalException); + if (indicesResponse.ElasticsearchServerError?.Status == 404) + return null; + + throw new RepositoryException(indicesResponse.GetErrorMessage($"Error getting latest index mapping {filter}"), indicesResponse.OriginalException()); } - var latestIndex = catResponse.Records - .Where(i => GetIndexVersion(i.Index) == Version) - .Select(i => new IndexInfo { DateUtc = GetIndexDate(i.Index), Index = i.Index, Version = GetIndexVersion(i.Index) }) + var latestIndex = indicesResponse.Indices.Keys + .Where(i => GetIndexVersion(i.ToString()) == Version) + .Select(i => + { + string indexName = i.ToString(); + return new IndexInfo { DateUtc = GetIndexDate(indexName), Index = indexName, Version = GetIndexVersion(indexName) }; + }) .OrderByDescending(i => i.DateUtc) .FirstOrDefault(); @@ -426,20 +461,20 @@ protected ITypeMapping GetLatestIndexMapping() var mappingResponse = Configuration.Client.Indices.GetMapping(new GetMappingRequest(latestIndex.Index)); _logger.LogTrace("GetMapping: {Request}", mappingResponse.GetRequest(false, true)); - if (!mappingResponse.IsValid) + if (!mappingResponse.IsValidResponse) { - if (mappingResponse.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (mappingResponse.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) { _logger.LogWarning("Index {Index} not found when getting mapping", latestIndex.Index); return null; } - _logger.LogError("Error getting mapping for {Index}: {Error}", latestIndex.Index, mappingResponse.ServerError); + _logger.LogError("Error getting mapping for {Index}: {Error}", latestIndex.Index, mappingResponse.ElasticsearchServerError); return null; } // use first returned mapping because index could have been an index alias - var mapping = mappingResponse.Indices.Values.FirstOrDefault()?.Mappings; + var mapping = mappingResponse.Mappings.Values.FirstOrDefault()?.Mappings; return mapping; } @@ -543,32 +578,30 @@ protected override ElasticMappingResolver CreateMappingResolver() return ElasticMappingResolver.Create(ConfigureIndexMapping, Configuration.Client.Infer, GetLatestIndexMapping, _logger); } - public virtual TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public virtual void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map.AutoMap().Properties(p => p.SetupDefaults()); + map.Properties(p => p.SetupDefaults()); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - idx = base.ConfigureIndex(idx); - return idx.Map(f => + base.ConfigureIndex(idx); + idx.Mappings(f => { if (CustomFieldTypes.Count > 0) { f.DynamicTemplates(d => { foreach (var customFieldType in CustomFieldTypes.Values) - d.DynamicTemplate($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping)); - - return d; + d.Add($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping())); }); } - return ConfigureIndexMapping(f); + ConfigureIndexMapping(f); }); } - public override void ConfigureSettings(ConnectionSettings settings) + public override void ConfigureSettings(ElasticsearchClientSettings settings) { settings.DefaultMappingFor(d => d.IndexName(Name)); } diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/DynamicIndex.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/DynamicIndex.cs index 45844bb6..12568f06 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/DynamicIndex.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/DynamicIndex.cs @@ -1,6 +1,6 @@ -using Foundatio.Parsers.ElasticQueries; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -13,8 +13,8 @@ protected override ElasticMappingResolver CreateMappingResolver() return ElasticMappingResolver.Create(ConfigureIndexMapping, Configuration.Client, Name, _logger); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map.Dynamic().AutoMap().Properties(p => p.SetupDefaults()); + map.Dynamic(DynamicMapping.True).Properties(p => p.SetupDefaults()); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/ElasticConfiguration.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/ElasticConfiguration.cs index 1f406948..592c72c6 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/ElasticConfiguration.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/ElasticConfiguration.cs @@ -3,7 +3,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Serialization; +using Elastic.Transport; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; @@ -13,11 +15,12 @@ using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Extensions; +using Foundatio.Repositories.Serialization; using Foundatio.Resilience; +using Foundatio.Serializer; using Foundatio.Utility; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -29,18 +32,27 @@ public class ElasticConfiguration : IElasticConfiguration protected readonly ILockProvider _lockProvider; private readonly List _indexes = new(); private readonly Lazy> _frozenIndexes; - private readonly Lazy _client; + private readonly Lazy _client; private readonly Lazy _customFieldDefinitionRepository; protected readonly bool _shouldDisposeCache; private readonly bool _shouldDisposeMessageBus; private int _disposed; - public ElasticConfiguration(IQueue workItemQueue = null, ICacheClient cacheClient = null, IMessageBus messageBus = null, TimeProvider timeProvider = null, IResiliencePolicyProvider resiliencePolicyProvider = null, ILoggerFactory loggerFactory = null) + public ElasticConfiguration(IQueue workItemQueue = null, ICacheClient cacheClient = null, IMessageBus messageBus = null, ITextSerializer serializer = null, TimeProvider timeProvider = null, IResiliencePolicyProvider resiliencePolicyProvider = null, ILoggerFactory loggerFactory = null) { _workItemQueue = workItemQueue; TimeProvider = timeProvider ?? TimeProvider.System; LoggerFactory = loggerFactory ?? NullLoggerFactory.Instance; _logger = LoggerFactory.CreateLogger(GetType()); + + if (serializer is null) + { + _logger.LogWarning("No serializer configured, using default System.Text.Json serializer"); + serializer = new Foundatio.Serializer.SystemTextJsonSerializer( + new System.Text.Json.JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + } + + Serializer = serializer; ResiliencePolicyProvider = resiliencePolicyProvider ?? cacheClient?.GetResiliencePolicyProvider() ?? new ResiliencePolicyProvider(); ResiliencePolicy = ResiliencePolicyProvider.GetPolicy(_logger, TimeProvider); Cache = cacheClient ?? new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, ResiliencePolicyProvider = ResiliencePolicyProvider, TimeProvider = TimeProvider, LoggerFactory = LoggerFactory }); @@ -51,37 +63,40 @@ public ElasticConfiguration(IQueue workItemQueue = null, ICacheCli MessageBus = messageBus ?? new InMemoryMessageBus(new InMemoryMessageBusOptions { ResiliencePolicyProvider = ResiliencePolicyProvider, TimeProvider = TimeProvider, LoggerFactory = LoggerFactory }); _frozenIndexes = new Lazy>(() => _indexes.AsReadOnly()); _customFieldDefinitionRepository = new Lazy(CreateCustomFieldDefinitionRepository); - _client = new Lazy(CreateElasticClient); + _client = new Lazy(CreateElasticClient); } - protected virtual IElasticClient CreateElasticClient() + protected virtual ElasticsearchClient CreateElasticClient() { - var settings = new ConnectionSettings(CreateConnectionPool() ?? new SingleNodeConnectionPool(new Uri("http://localhost:9200"))); - settings.EnableApiVersioningHeader(); + var settings = new ElasticsearchClientSettings( + CreateConnectionPool() ?? new SingleNodePool(new Uri("http://localhost:9200")), + sourceSerializer: (_, clientSettings) => + new DefaultSourceSerializer(clientSettings, options => options.ConfigureFoundatioRepositoryDefaults())); ConfigureSettings(settings); foreach (var index in Indexes) index.ConfigureSettings(settings); - return new ElasticClient(settings); + return new ElasticsearchClient(settings); } public virtual void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) { } public virtual void ConfigureGlobalQueryParsers(ElasticQueryParserConfiguration config) { } - protected virtual void ConfigureSettings(ConnectionSettings settings) + protected virtual void ConfigureSettings(ElasticsearchClientSettings settings) { settings.EnableTcpKeepAlive(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(2)); } - protected virtual IConnectionPool CreateConnectionPool() + protected virtual NodePool CreateConnectionPool() { return null; } - public IElasticClient Client => _client.Value; + public ElasticsearchClient Client => _client.Value; public ICacheClient Cache { get; } public IMessageBus MessageBus { get; } + public ITextSerializer Serializer { get; } public ILoggerFactory LoggerFactory { get; } public IResiliencePolicyProvider ResiliencePolicyProvider { get; } public IResiliencePolicy ResiliencePolicy { get; } @@ -228,6 +243,11 @@ public virtual void Dispose() if (Interlocked.Exchange(ref _disposed, 1) != 0) return; + // ElasticsearchClientSettings implements IDisposable internally but doesn't expose it + // on its public API, so we must cast to IDisposable to release its underlying resources. + if (_client.IsValueCreated) + (_client.Value.ElasticsearchClientSettings as IDisposable)?.Dispose(); + if (_shouldDisposeCache) Cache.Dispose(); diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/IElasticConfiguration.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/IElasticConfiguration.cs index 1930ca8e..b4b7ff57 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/IElasticConfiguration.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/IElasticConfiguration.cs @@ -1,33 +1,69 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Caching; using Foundatio.Messaging; using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; +/// +/// Provides configuration and lifecycle management for Elasticsearch indexes, including +/// client access, caching, messaging, serialization, and index maintenance operations. +/// public interface IElasticConfiguration : IDisposable { - IElasticClient Client { get; } + /// The configured Elasticsearch client used for all cluster operations. + ElasticsearchClient Client { get; } + + /// Cache client used for repository-level caching and lock coordination. ICacheClient Cache { get; } + + /// Message bus used for entity change notifications and cache invalidation. IMessageBus MessageBus { get; } + + /// Serializer used for document serialization and deserialization. + ITextSerializer Serializer { get; } + + /// Logger factory for creating loggers within the configuration and its indexes. ILoggerFactory LoggerFactory { get; } + + /// Provider for resilience policies (retry, circuit breaker) applied to Elasticsearch operations. IResiliencePolicyProvider ResiliencePolicyProvider { get; } + + /// Time provider used for time-dependent operations such as daily index naming. TimeProvider TimeProvider { get; set; } + + /// The collection of all registered index definitions. IReadOnlyCollection Indexes { get; } + + /// Repository for managing custom field definitions, or null if no custom field index is configured. ICustomFieldDefinitionRepository CustomFieldDefinitionRepository { get; } + /// Returns the index with the specified , or null if not found. IIndex GetIndex(string name); + + /// Configures global query builders applied to every query across all indexes. void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder); + + /// Configures global query parser settings applied to every query across all indexes. void ConfigureGlobalQueryParsers(ElasticQueryParserConfiguration config); + + /// Creates and configures the specified indexes (or all registered indexes) in Elasticsearch. Task ConfigureIndexesAsync(IEnumerable indexes = null, bool beginReindexingOutdated = true); + + /// Runs maintenance tasks (e.g., alias management, cleanup) on the specified or all indexes. Task MaintainIndexesAsync(IEnumerable indexes = null); + + /// Deletes the specified indexes (or all registered indexes) from Elasticsearch. Task DeleteIndexesAsync(IEnumerable indexes = null); + + /// Reindexes outdated versioned indexes to their latest version. Task ReindexAsync(IEnumerable indexes = null, Func progressCallbackAsync = null); } diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/IIndex.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/IIndex.cs index 14f2e33c..63597f4c 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/IIndex.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/IIndex.cs @@ -2,10 +2,11 @@ using System.Collections.Generic; using System.Linq.Expressions; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Elasticsearch.Queries.Builders; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -19,7 +20,7 @@ public interface IIndex : IDisposable IElasticConfiguration Configuration { get; } IDictionary CustomFieldTypes { get; } - void ConfigureSettings(ConnectionSettings settings); + void ConfigureSettings(ElasticsearchClientSettings settings); Task ConfigureAsync(); Task EnsureIndexAsync(object target); Task MaintainAsync(bool includeOptionalTasks = true); @@ -32,7 +33,7 @@ public interface IIndex : IDisposable public interface IIndex : IIndex where T : class { - TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map); + void ConfigureIndexMapping(TypeMappingDescriptor map); Inferrer Infer { get; } string InferField(Expression> objectPath); string InferPropertyName(Expression> objectPath); diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/Index.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/Index.cs index 6f05ab99..f5fdc786 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/Index.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/Index.cs @@ -5,6 +5,11 @@ using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Analysis; +using Elastic.Clients.Elasticsearch.Fluent; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.AsyncEx; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.ElasticQueries.Extensions; @@ -19,7 +24,6 @@ using Foundatio.Repositories.Utility; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -170,9 +174,8 @@ public virtual Task MaintainAsync(bool includeOptionalTasks = true) return Task.CompletedTask; } - public virtual IPromise ConfigureIndexAliases(AliasesDescriptor aliases) + public virtual void ConfigureIndexAliases(FluentDictionaryOfNameAlias fluentDictionaryOfNameAlias) { - return aliases; } public IElasticQueryBuilder QueryBuilder => _queryBuilder.Value; @@ -191,117 +194,126 @@ public virtual async Task DeleteAsync() } } - protected virtual async Task CreateIndexAsync(string name, Func descriptor = null) + protected virtual async Task CreateIndexAsync(string name, Action descriptor = null) { if (name == null) throw new ArgumentNullException(nameof(name)); - descriptor ??= ConfigureIndex; + descriptor ??= d => ConfigureIndex(d); - var response = await Configuration.Client.Indices.CreateAsync(name, descriptor).AnyContext(); + var response = await Configuration.Client.Indices.CreateAsync((IndexName)name, descriptor).AnyContext(); _logger.LogRequest(response); - // check for valid response or that the index already exists - if (response.IsValid || response.ServerError?.Status == 400 && - response.ServerError.Error.Type is "index_already_exists_exception" or "resource_already_exists_exception") + if (response.IsValidResponse || response.ElasticsearchServerError?.Status == 400 && + response.ElasticsearchServerError.Error.Type is "index_already_exists_exception" or "resource_already_exists_exception") { _isEnsured = true; return; } - throw new RepositoryException(response.GetErrorMessage($"Error creating the index {name}"), response.OriginalException); + _logger.LogErrorRequest(response, "Error creating the index {Name}", name); + throw new RepositoryException(response.GetErrorMessage($"Error creating the index {name}"), response.OriginalException()); } - protected virtual async Task UpdateIndexAsync(string name, Func descriptor = null) + protected virtual async Task UpdateIndexAsync(string name, Action descriptor = null) { - var updateIndexDescriptor = new UpdateIndexSettingsDescriptor(name); if (descriptor != null) { - updateIndexDescriptor = descriptor(updateIndexDescriptor); + var response = await Configuration.Client.Indices.PutSettingsAsync(name, descriptor).AnyContext(); + + if (response.IsValidResponse) + _logger.LogRequest(response); + else + _logger.LogErrorRequest(response, $"Error updating index ({name}) settings"); + return; } - else + + var currentSettings = await Configuration.Client.Indices.GetSettingsAsync((Indices)name).AnyContext(); + if (!currentSettings.IsValidResponse) + { + _logger.LogErrorRequest(currentSettings, "Error getting index settings for {Name}", name); + throw new RepositoryException(currentSettings.GetErrorMessage($"Error getting index settings for {name}"), currentSettings.OriginalException()); + } + + var indexState = currentSettings.Settings.TryGetValue(name, out var indexSettings) ? indexSettings : null; + var currentAnalyzers = indexState?.Settings?.Analysis?.Analyzers ?? new Analyzers(); + var currentTokenizers = indexState?.Settings?.Analysis?.Tokenizers ?? new Tokenizers(); + var currentTokenFilters = indexState?.Settings?.Analysis?.TokenFilters ?? new TokenFilters(); + var currentNormalizers = indexState?.Settings?.Analysis?.Normalizers ?? new Normalizers(); + var currentCharFilters = indexState?.Settings?.Analysis?.CharFilters ?? new CharFilters(); + + // default to update dynamic index settings from the ConfigureIndex method + var createIndexRequestDescriptor = new CreateIndexRequestDescriptor((IndexName)name); + ConfigureIndex(createIndexRequestDescriptor); + CreateIndexRequest createRequest = createIndexRequestDescriptor; + var settings = createRequest.Settings; + + // strip off non-dynamic index settings + settings.Store = null; + settings.NumberOfRoutingShards = null; + settings.NumberOfShards = null; + settings.Queries = null; + settings.RoutingPartitionSize = null; + settings.Hidden = null; + settings.Sort = null; + settings.SoftDeletes = null; + + if (settings.Analysis?.Analyzers is not null && currentAnalyzers is not null) { - var currentSettings = await Configuration.Client.Indices.GetSettingsAsync(name).AnyContext(); - if (!currentSettings.IsValid) - throw new RepositoryException(currentSettings.GetErrorMessage($"Error getting index settings for {name}"), currentSettings.OriginalException); - - var currentAnalyzers = currentSettings.Indices[name]?.Settings?.Analysis?.Analyzers ?? new Analyzers(); - var currentTokenizers = currentSettings.Indices[name]?.Settings?.Analysis?.Tokenizers ?? new Tokenizers(); - var currentTokenFilters = currentSettings.Indices[name]?.Settings?.Analysis?.TokenFilters ?? new TokenFilters(); - var currentNormalizers = currentSettings.Indices[name]?.Settings?.Analysis?.Normalizers ?? new Normalizers(); - var currentCharFilters = currentSettings.Indices[name]?.Settings?.Analysis?.CharFilters ?? new CharFilters(); - - // default to update dynamic index settings from the ConfigureIndex method - var createIndexDescriptor = new CreateIndexDescriptor(name); - createIndexDescriptor = ConfigureIndex(createIndexDescriptor); - var settings = ((IIndexState)createIndexDescriptor).Settings; - - // strip off non-dynamic index settings - settings.FileSystemStorageImplementation = null; - settings.NumberOfRoutingShards = null; - settings.NumberOfShards = null; - settings.Queries = null; - settings.RoutingPartitionSize = null; - settings.Hidden = null; - settings.Sorting = null; - settings.SoftDeletes = null; - - if (settings.Analysis?.Analyzers != null) + var currentKeys = currentAnalyzers.Select(kvp => kvp.Key).ToHashSet(); + foreach (var analyzer in settings.Analysis.Analyzers.ToList()) { - foreach (var analyzer in settings.Analysis.Analyzers.ToList()) - { - if (!currentAnalyzers.ContainsKey(analyzer.Key)) - _logger.LogError("New analyzer {AnalyzerKey} can't be added to existing index", analyzer.Key); - } + if (!currentKeys.Contains(analyzer.Key)) + _logger.LogWarning("Adding new analyzer {AnalyzerKey} to existing index (requires close/reopen)", analyzer.Key); } + } - if (settings.Analysis?.Tokenizers != null) + if (settings.Analysis?.Tokenizers is not null && currentTokenizers is not null) + { + var currentKeys = currentTokenizers.Select(kvp => kvp.Key).ToHashSet(); + foreach (var tokenizer in settings.Analysis.Tokenizers.ToList()) { - foreach (var tokenizer in settings.Analysis.Tokenizers.ToList()) - { - if (!currentTokenizers.ContainsKey(tokenizer.Key)) - _logger.LogError("New tokenizer {TokenizerKey} can't be added to existing index", tokenizer.Key); - } + if (!currentKeys.Contains(tokenizer.Key)) + _logger.LogWarning("Adding new tokenizer {TokenizerKey} to existing index (requires close/reopen)", tokenizer.Key); } + } - if (settings.Analysis?.TokenFilters != null) + if (settings.Analysis?.TokenFilters is not null && currentTokenFilters is not null) + { + var currentKeys = currentTokenFilters.Select(kvp => kvp.Key).ToHashSet(); + foreach (var tokenFilter in settings.Analysis.TokenFilters.ToList()) { - foreach (var tokenFilter in settings.Analysis.TokenFilters.ToList()) - { - if (!currentTokenFilters.ContainsKey(tokenFilter.Key)) - _logger.LogError("New token filter {TokenFilterKey} can't be added to existing index", tokenFilter.Key); - } + if (!currentKeys.Contains(tokenFilter.Key)) + _logger.LogWarning("Adding new token filter {TokenFilterKey} to existing index (requires close/reopen)", tokenFilter.Key); } + } - if (settings.Analysis?.Normalizers != null) + if (settings.Analysis?.Normalizers is not null && currentNormalizers is not null) + { + var currentKeys = currentNormalizers.Select(kvp => kvp.Key).ToHashSet(); + foreach (var normalizer in settings.Analysis.Normalizers.ToList()) { - foreach (var normalizer in settings.Analysis.Normalizers.ToList()) - { - if (!currentNormalizers.ContainsKey(normalizer.Key)) - _logger.LogError("New normalizer {NormalizerKey} can't be added to existing index", normalizer.Key); - } + if (!currentKeys.Contains(normalizer.Key)) + _logger.LogWarning("Adding new normalizer {NormalizerKey} to existing index (requires close/reopen)", normalizer.Key); } + } - if (settings.Analysis?.CharFilters != null) + if (settings.Analysis?.CharFilters is not null && currentCharFilters is not null) + { + var currentKeys = currentCharFilters.Select(kvp => kvp.Key).ToHashSet(); + foreach (var charFilter in settings.Analysis.CharFilters.ToList()) { - foreach (var charFilter in settings.Analysis.CharFilters.ToList()) - { - if (!currentCharFilters.ContainsKey(charFilter.Key)) - _logger.LogError("New char filter {CharFilterKey} can't be added to existing index", charFilter.Key); - } + if (!currentKeys.Contains(charFilter.Key)) + _logger.LogWarning("Adding new char filter {CharFilterKey} to existing index (requires close/reopen)", charFilter.Key); } - - settings.Analysis = null; - - updateIndexDescriptor.IndexSettings(_ => new NestPromise(settings)); } - var response = await Configuration.Client.Indices.UpdateSettingsAsync(name, _ => updateIndexDescriptor).AnyContext(); + var updateResponse = await Configuration.Client.Indices.PutSettingsAsync(name, d => d.Reopen().Settings(settings)).AnyContext(); - if (response.IsValid) - _logger.LogRequest(response); + if (updateResponse.IsValidResponse) + _logger.LogRequest(updateResponse); else - _logger.LogErrorRequest(response, $"Error updating index ({name}) settings"); + _logger.LogErrorRequest(updateResponse, $"Error updating index ({name}) settings"); } protected virtual Task DeleteIndexAsync(string name) @@ -317,15 +329,48 @@ protected virtual async Task DeleteIndexesAsync(string[] names) if (names == null || names.Length == 0) throw new ArgumentNullException(nameof(names)); - var response = await Configuration.Client.Indices.DeleteAsync(Indices.Index(names), i => i.IgnoreUnavailable()).AnyContext(); - - if (response.IsValid) + // Resolve wildcards to actual index names; use GetAsync because ResolveIndexAsync is broken in ES 9.x client. + var indexNames = new List(); + foreach (var name in names) { - _logger.LogRequest(response); - return; + if (name.Contains("*") || name.Contains("?")) + { + var getResponse = await Configuration.Client.Indices.GetAsync(Indices.Parse(name), d => d.IgnoreUnavailable()).AnyContext(); + if (getResponse.IsValidResponse && getResponse.Indices is not null) + { + foreach (var kvp in getResponse.Indices) + indexNames.Add(kvp.Key); + } + else if (getResponse.ElasticsearchServerError?.Status is not 404) + { + _logger.LogErrorRequest(getResponse, "Error resolving wildcard index pattern {Pattern}", name); + } + } + else + { + indexNames.Add(name); + } } - throw new RepositoryException(response.GetErrorMessage($"Error deleting the index {names}"), response.OriginalException); + if (indexNames.Count == 0) + return; + + // Batch delete to avoid HTTP line too long errors (ES default max is 4096 bytes) + // Each index name is roughly 30-50 bytes, so we batch in groups of 50 + const int batchSize = 50; + foreach (var batch in indexNames.Chunk(batchSize)) + { + var response = await Configuration.Client.Indices.DeleteAsync(Indices.Parse(String.Join(",", batch)), i => i.IgnoreUnavailable()).AnyContext(); + + if (response.IsValidResponse) + { + _logger.LogRequest(response); + continue; + } + + _logger.LogErrorRequest(response, "Error deleting the index {Indexes}", String.Join(",", batch)); + throw new RepositoryException(response.GetErrorMessage($"Error deleting the index {String.Join(",", batch)}"), response.OriginalException()); + } } protected async Task IndexExistsAsync(string name) @@ -334,13 +379,14 @@ protected async Task IndexExistsAsync(string name) throw new ArgumentNullException(nameof(name)); var response = await Configuration.Client.Indices.ExistsAsync(name).AnyContext(); - if (response.ApiCall.Success) + if (response.ApiCallDetails.HasSuccessfulStatusCode) { _logger.LogRequest(response); return response.Exists; } - throw new RepositoryException(response.GetErrorMessage($"Error checking to see if index {name} exists"), response.OriginalException); + _logger.LogErrorRequest(response, "Error checking to see if index {Name} exists", name); + throw new RepositoryException(response.GetErrorMessage($"Error checking to see if index {name} exists"), response.OriginalException()); } public virtual Task ReindexAsync(Func progressCallbackAsync = null) @@ -353,7 +399,7 @@ public virtual Task ReindexAsync(Func progressCallbackAsync = TimestampField = GetTimeStampField() }; - var reindexer = new ElasticReindexer(Configuration.Client, _logger); + var reindexer = new ElasticReindexer(Configuration.Client, Configuration.Serializer, _logger); return reindexer.ReindexAsync(reindexWorkItem, progressCallbackAsync); } @@ -362,12 +408,12 @@ protected virtual string GetTimeStampField() return null; } - public virtual CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public virtual void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return idx.Aliases(ConfigureIndexAliases); + idx.Aliases(ConfigureIndexAliases); } - public virtual void ConfigureSettings(ConnectionSettings settings) { } + public virtual void ConfigureSettings(ElasticsearchClientSettings settings) { } public virtual void Dispose() { @@ -393,63 +439,57 @@ protected override ElasticMappingResolver CreateMappingResolver() return ElasticMappingResolver.Create(ConfigureIndexMapping, Configuration.Client, Name, _logger); } - public virtual TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public virtual void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map.AutoMap().Properties(p => p.SetupDefaults()); + map.Properties(p => p.SetupDefaults()); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - idx = base.ConfigureIndex(idx); - return idx.Map(f => + base.ConfigureIndex(idx); + idx.Mappings(f => { if (CustomFieldTypes.Count > 0) { f.DynamicTemplates(d => { foreach (var customFieldType in CustomFieldTypes.Values) - d.DynamicTemplate($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping)); - - return d; + d.Add($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping())); }); } - return ConfigureIndexMapping(f); + ConfigureIndexMapping(f); }); } - protected override async Task UpdateIndexAsync(string name, Func descriptor = null) + protected override async Task UpdateIndexAsync(string name, Action descriptor = null) { await base.UpdateIndexAsync(name, descriptor).AnyContext(); var typeMappingDescriptor = new TypeMappingDescriptor(); - typeMappingDescriptor = ConfigureIndexMapping(typeMappingDescriptor); - var mapping = (ITypeMapping)typeMappingDescriptor; + ConfigureIndexMapping(typeMappingDescriptor); + var mapping = (TypeMapping)typeMappingDescriptor; var response = await Configuration.Client.Indices.PutMappingAsync(m => { - m.Properties(_ => new NestPromise(mapping.Properties)); + m.Properties(mapping.Properties); if (CustomFieldTypes.Count > 0) { m.DynamicTemplates(d => { foreach (var customFieldType in CustomFieldTypes.Values) - d.DynamicTemplate($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping)); - - return d; + d.Add($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping())); }); } - - return m; }).AnyContext(); - if (response.IsValid) + if (response.IsValidResponse) _logger.LogRequest(response); else _logger.LogErrorRequest(response, $"Error updating index ({name}) mappings."); } - public override void ConfigureSettings(ConnectionSettings settings) + public override void ConfigureSettings(ElasticsearchClientSettings settings) { settings.DefaultMappingFor(d => d.IndexName(Name)); } @@ -469,13 +509,3 @@ protected override string GetTimeStampField() public string InferField(Expression> objectPath) => Infer.Field(objectPath); public string InferPropertyName(Expression> objectPath) => Infer.PropertyName(objectPath); } - -internal class NestPromise : IPromise where TValue : class -{ - public NestPromise(TValue value) - { - Value = value; - } - - public TValue Value { get; } -} diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/MonthlyIndex.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/MonthlyIndex.cs index 5f7b131a..581bf430 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/MonthlyIndex.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/MonthlyIndex.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.DateTimeExtensions; using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -66,32 +68,30 @@ protected override ElasticMappingResolver CreateMappingResolver() return ElasticMappingResolver.Create(ConfigureIndexMapping, Configuration.Client.Infer, GetLatestIndexMapping, _logger); } - public virtual TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public virtual void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map.AutoMap().Properties(p => p.SetupDefaults()); + map.Properties(p => p.SetupDefaults()); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - idx = base.ConfigureIndex(idx); - return idx.Map(f => + base.ConfigureIndex(idx); + idx.Mappings(f => { if (CustomFieldTypes.Count > 0) { f.DynamicTemplates(d => { foreach (var customFieldType in CustomFieldTypes.Values) - d.DynamicTemplate($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping)); - - return d; + d.Add($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping())); }); } - return ConfigureIndexMapping(f); + ConfigureIndexMapping(f); }); } - public override void ConfigureSettings(ConnectionSettings settings) + public override void ConfigureSettings(ElasticsearchClientSettings settings) { settings.DefaultMappingFor(d => d.IndexName(Name)); } diff --git a/src/Foundatio.Repositories.Elasticsearch/Configuration/VersionedIndex.cs b/src/Foundatio.Repositories.Elasticsearch/Configuration/VersionedIndex.cs index 6f4c3b9d..672878a3 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Configuration/VersionedIndex.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Configuration/VersionedIndex.cs @@ -5,6 +5,9 @@ using System.Linq.Expressions; using System.Text; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Extensions; @@ -14,7 +17,6 @@ using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Configuration; @@ -153,7 +155,13 @@ public override async Task ConfigureAsync() if (!await IndexExistsAsync(VersionedName).AnyContext()) { if (!await AliasExistsAsync(Name).AnyContext()) - await CreateIndexAsync(VersionedName, d => ConfigureIndex(d).Aliases(ad => ad.Alias(Name))).AnyContext(); + { + await CreateIndexAsync(VersionedName, d => + { + ConfigureIndex(d); + d.Aliases(ad => ad.Add(Name, a => { })); + }).AnyContext(); + } else // new version of an existing index, don't set the alias yet await CreateIndexAsync(VersionedName, ConfigureIndex).AnyContext(); } @@ -173,23 +181,27 @@ protected virtual async Task CreateAliasAsync(string index, string name) if (await AliasExistsAsync(name).AnyContext()) return; - var response = await Configuration.Client.Indices.BulkAliasAsync(a => a.Add(s => s.Index(index).Alias(name))).AnyContext(); - if (response.IsValid) + var response = await Configuration.Client.Indices.UpdateAliasesAsync(a => a.Actions(actions => actions.Add(s => s.Index(index).Alias(name)))).AnyContext(); + if (response.IsValidResponse) + { + _logger.LogRequest(response); return; + } if (await AliasExistsAsync(name).AnyContext()) return; - throw new RepositoryException(response.GetErrorMessage($"Error creating alias {name}"), response.OriginalException); + _logger.LogErrorRequest(response, "Error creating alias {Name}", name); + throw new RepositoryException(response.GetErrorMessage($"Error creating alias {name}"), response.OriginalException()); } protected async Task AliasExistsAsync(string alias) { - var response = await Configuration.Client.Indices.AliasExistsAsync(Names.Parse(alias)).AnyContext(); - if (response.ApiCall.Success) + var response = await Configuration.Client.Indices.ExistsAliasAsync(Names.Parse(alias)).AnyContext(); + if (response.ApiCallDetails.HasSuccessfulStatusCode) return response.Exists; - throw new RepositoryException(response.GetErrorMessage($"Error checking to see if alias {alias} exists"), response.OriginalException); + throw new RepositoryException(response.GetErrorMessage($"Error checking to see if alias {alias} exists"), response.OriginalException()); } public override async Task DeleteAsync() @@ -251,7 +263,7 @@ public override async Task ReindexAsync(Func progressCallback return; var reindexWorkItem = CreateReindexWorkItem(currentVersion); - var reindexer = new ElasticReindexer(Configuration.Client, _logger); + var reindexer = new ElasticReindexer(Configuration.Client, Configuration.Serializer, _logger); await reindexer.ReindexAsync(reindexWorkItem, progressCallbackAsync).AnyContext(); } @@ -286,14 +298,19 @@ public virtual async Task GetCurrentVersionAsync() protected virtual async Task GetVersionFromAliasAsync(string alias) { - var response = await Configuration.Client.Indices.GetAliasAsync(alias).AnyContext(); - if (!response.IsValid && response.ServerError?.Status == 404) + var response = await Configuration.Client.Indices.GetAliasAsync(a => a.Name(Names.Parse(alias))).AnyContext(); + if (!response.IsValidResponse && response.ElasticsearchServerError?.Status == 404) return -1; - if (response.IsValid && response.Indices.Count > 0) +#if ELASTICSEARCH9 + var indices = response.Aliases; +#else + var indices = response.Values; +#endif + if (response.IsValidResponse && indices != null && indices.Count > 0) { _logger.LogRequest(response); - return response.Indices.Keys.Select(i => GetIndexVersion(i.Name)).OrderBy(v => v).First(); + return indices.Keys.Select(i => GetIndexVersion(i.ToString())).OrderBy(v => v).First(); } _logger.LogErrorRequest(response, "Error getting index version from alias"); @@ -327,35 +344,49 @@ protected virtual async Task> GetIndexesAsync(int version = -1) filter += "-*"; var sw = Stopwatch.StartNew(); - var response = await Configuration.Client.Cat.IndicesAsync(i => i.Pri().Index(Indices.Index((IndexName)filter))).AnyContext(); + var response = await Configuration.Client.Indices.GetAsync((Indices)(IndexName)filter).AnyContext(); sw.Stop(); _logger.LogRequest(response); - if (!response.IsValid) - throw new RepositoryException(response.GetErrorMessage($"Error getting indices {filter}"), response.OriginalException); + if (!response.IsValidResponse) + { + if (response.ElasticsearchServerError?.Status == 404) + return new List(); - if (response.Records.Count == 0) + throw new RepositoryException(response.GetErrorMessage($"Error getting indices {filter}"), response.OriginalException()); + } + + if (response.Indices.Count == 0) return new List(); - var aliasResponse = await Configuration.Client.Cat.AliasesAsync(i => i.Name($"{Name}-*")).AnyContext(); + var aliasResponse = await Configuration.Client.Indices.GetAliasAsync(a => a.Name($"{Name}-*")).AnyContext(); _logger.LogRequest(aliasResponse); - if (!aliasResponse.IsValid) - throw new RepositoryException(aliasResponse.GetErrorMessage($"Error getting index aliases for {filter}"), aliasResponse.OriginalException); + if (!aliasResponse.IsValidResponse && aliasResponse.ElasticsearchServerError?.Status != 404) + throw new RepositoryException(aliasResponse.GetErrorMessage($"Error getting index aliases for {filter}"), aliasResponse.OriginalException()); - var indices = response.Records - .Where(i => version < 0 || GetIndexVersion(i.Index) == version) +#if ELASTICSEARCH9 + var aliasIndices = aliasResponse.Aliases; +#else + var aliasIndices = aliasResponse.Values; +#endif + var indices = response.Indices.Keys + .Where(i => version < 0 || GetIndexVersion(i.ToString()) == version) .Select(i => { - var indexDate = GetIndexDate(i.Index); - string indexAliasName = GetIndexByDate(GetIndexDate(i.Index)); - var aliasRecord = aliasResponse.Records.FirstOrDefault(r => r.Alias == indexAliasName); + string indexName = i.ToString(); + var indexDate = GetIndexDate(indexName); + string indexAliasName = GetIndexByDate(GetIndexDate(indexName)); int currentVersion = -1; - if (aliasRecord != null) - currentVersion = GetIndexVersion(aliasRecord.Index); + if (aliasResponse.IsValidResponse && aliasIndices != null && aliasIndices.TryGetValue(i, out var indexAliases)) + { + // Find if any of our aliases point to this index + if (indexAliases.Aliases.ContainsKey(indexAliasName)) + currentVersion = GetIndexVersion(indexName); + } - return new IndexInfo { DateUtc = indexDate, Index = i.Index, Version = GetIndexVersion(i.Index), CurrentVersion = currentVersion }; + return new IndexInfo { DateUtc = indexDate, Index = indexName, Version = GetIndexVersion(indexName), CurrentVersion = currentVersion }; }) .OrderBy(i => i.DateUtc) .ToList(); @@ -398,65 +429,59 @@ protected override ElasticMappingResolver CreateMappingResolver() return ElasticMappingResolver.Create(ConfigureIndexMapping, Configuration.Client, VersionedName, _logger); } - public virtual TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public virtual void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map.AutoMap().Properties(p => p.SetupDefaults()); + map.Properties(p => p.SetupDefaults()); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - idx = base.ConfigureIndex(idx); - return idx.Map(f => + base.ConfigureIndex(idx); + idx.Mappings(f => { if (CustomFieldTypes.Count > 0) { f.DynamicTemplates(d => { foreach (var customFieldType in CustomFieldTypes.Values) - d.DynamicTemplate($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping)); - - return d; + d.Add($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping())); }); } - return ConfigureIndexMapping(f); + ConfigureIndexMapping(f); }); } - protected override async Task UpdateIndexAsync(string name, Func descriptor = null) + protected override async Task UpdateIndexAsync(string name, Action descriptor = null) { await base.UpdateIndexAsync(name, descriptor).AnyContext(); var typeMappingDescriptor = new TypeMappingDescriptor(); - typeMappingDescriptor = ConfigureIndexMapping(typeMappingDescriptor); - var mapping = (ITypeMapping)typeMappingDescriptor; + ConfigureIndexMapping(typeMappingDescriptor); + var mapping = (TypeMapping)typeMappingDescriptor; var response = await Configuration.Client.Indices.PutMappingAsync(m => { - m.Index(name); - m.Properties(_ => new NestPromise(mapping.Properties)); + m.Indices(name); + m.Properties(mapping.Properties); if (CustomFieldTypes.Count > 0) { m.DynamicTemplates(d => { foreach (var customFieldType in CustomFieldTypes.Values) - d.DynamicTemplate($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping)); - - return d; + d.Add($"idx_{customFieldType.Type}", df => df.PathMatch("idx.*").Match($"{customFieldType.Type}-*").Mapping(customFieldType.ConfigureMapping())); }); } - - return m; }).AnyContext(); // TODO: Check for issues with attempting to change existing fields and warn that index version needs to be incremented - if (response.IsValid) + if (response.IsValidResponse) _logger.LogRequest(response); else _logger.LogErrorRequest(response, $"Error updating index ({name}) mappings. Changing existing fields requires a new index version."); } - public override void ConfigureSettings(ConnectionSettings settings) + public override void ConfigureSettings(ElasticsearchClientSettings settings) { settings.DefaultMappingFor(d => d.IndexName(Name)); } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/CustomFieldDefinitionRepository.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/CustomFieldDefinitionRepository.cs index 8a91e63c..cb8c84c6 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/CustomFieldDefinitionRepository.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/CustomFieldDefinitionRepository.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Caching; using Foundatio.Lock; using Foundatio.Repositories.Elasticsearch.Configuration; @@ -11,7 +13,7 @@ using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Microsoft.Extensions.Logging; -using Nest; +using ChangeType = Foundatio.Repositories.Models.ChangeType; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -334,25 +336,23 @@ public CustomFieldDefinitionIndex(IElasticConfiguration configuration, string na _replicas = replicas; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.EntityType)) - .Keyword(f => f.Name(e => e.TenantKey)) - .Keyword(f => f.Name(e => e.IndexType)) - .Number(f => f.Name(e => e.IndexSlot)) - .Date(f => f.Name(e => e.CreatedUtc)) - .Date(f => f.Name(e => e.UpdatedUtc)) + .Keyword(e => e.EntityType) + .Keyword(e => e.TenantKey) + .Keyword(e => e.IndexType) + .IntegerNumber(e => e.IndexSlot) ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx).Settings(s => s + base.ConfigureIndex(idx); + idx.Settings(s => s .NumberOfShards(1) .NumberOfReplicas(_replicas)); } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/ICustomFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/ICustomFieldType.cs index 59b9aa04..adfb70a1 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/ICustomFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/ICustomFieldType.cs @@ -1,5 +1,6 @@ +using System; using System.Threading.Tasks; -using Nest; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -30,7 +31,7 @@ public interface ICustomFieldType /// Configures the Elasticsearch mapping for index slots of this type. /// This is used in dynamic templates to map idx.{type}-* fields. /// - IProperty ConfigureMapping(SingleMappingSelector map) where T : class; + Func, IProperty> ConfigureMapping() where T : class; } /// diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/BooleanFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/BooleanFieldType.cs index 3a06207c..ee4c734d 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/BooleanFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/BooleanFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Boolean(mp => mp); + return factory => factory.Boolean(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DateFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DateFieldType.cs index ce9d5390..0d1888b0 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DateFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DateFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Date(mp => mp); + return factory => factory.Date(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DoubleFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DoubleFieldType.cs index 8fdbef68..a0e9a018 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DoubleFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/DoubleFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Number(mp => mp.Type(NumberType.Double)); + return factory => factory.DoubleNumber(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/FloatFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/FloatFieldType.cs index 139451b3..47c1dd68 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/FloatFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/FloatFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Number(mp => mp.Type(NumberType.Float)); + return factory => factory.FloatNumber(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/IntegerFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/IntegerFieldType.cs index 3efd4e22..0b1a4e8d 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/IntegerFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/IntegerFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Number(mp => mp.Type(NumberType.Integer)); + return factory => factory.IntegerNumber(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/KeywordFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/KeywordFieldType.cs index cb8f4d04..eff6cb37 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/KeywordFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/KeywordFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Keyword(mp => mp); + return factory => factory.Keyword(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/LongFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/LongFieldType.cs index 9ba5edf3..4caf0d1b 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/LongFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/LongFieldType.cs @@ -1,5 +1,6 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +14,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Number(mp => mp.Type(NumberType.Long)); + return factory => factory.LongNumber(); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/StringFieldType.cs b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/StringFieldType.cs index a9fbbe56..18502c31 100644 --- a/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/StringFieldType.cs +++ b/src/Foundatio.Repositories.Elasticsearch/CustomFields/StandardFieldTypes/StringFieldType.cs @@ -1,5 +1,7 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Parsers.ElasticQueries.Extensions; namespace Foundatio.Repositories.Elasticsearch.CustomFields; @@ -13,8 +15,8 @@ public virtual Task ProcessValueAsync(T document, ob return Task.FromResult(new ProcessFieldValueResult { Value = value }); } - public virtual IProperty ConfigureMapping(SingleMappingSelector map) where T : class + public virtual Func, IProperty> ConfigureMapping() where T : class { - return map.Text(mp => mp.AddKeywordField()); + return factory => factory.Text(p => p.AddKeywordField()); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/ElasticUtility.cs b/src/Foundatio.Repositories.Elasticsearch/ElasticUtility.cs index a4a9c4fe..395b8008 100644 --- a/src/Foundatio.Repositories.Elasticsearch/ElasticUtility.cs +++ b/src/Foundatio.Repositories.Elasticsearch/ElasticUtility.cs @@ -2,62 +2,93 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Extensions; +using Foundatio.Resilience; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch; public class ElasticUtility { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; + private readonly IResiliencePolicy _resiliencePolicy; - public ElasticUtility(IElasticClient client, ILogger logger) : this(client, TimeProvider.System, logger) + public ElasticUtility(ElasticsearchClient client, ILogger logger) : this(client, TimeProvider.System, logger) { } - public ElasticUtility(IElasticClient client, TimeProvider timeProvider, ILogger logger) + public ElasticUtility(ElasticsearchClient client, TimeProvider timeProvider, ILogger logger) + : this(client, timeProvider, new ResiliencePolicyProvider(), logger) + { + } + + public ElasticUtility(ElasticsearchClient client, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILogger logger) { _client = client; _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? NullLogger.Instance; + + resiliencePolicyProvider ??= new ResiliencePolicyProvider(); + _resiliencePolicy = resiliencePolicyProvider.GetPolicy( + fallback => fallback.WithMaxAttempts(3).WithDelay(TimeSpan.FromSeconds(2)), + _logger, _timeProvider); } public async Task SnapshotRepositoryExistsAsync(string repository) { - var repositoriesResponse = await _client.Snapshot.GetRepositoryAsync(new GetRepositoryRequest(repository)).AnyContext(); + var repositoriesResponse = await _client.Snapshot.GetRepositoryAsync(r => r.Name(repository)).AnyContext(); _logger.LogRequest(repositoriesResponse); - return repositoriesResponse.Repositories.Count > 0; + return repositoriesResponse.IsValidResponse && repositoriesResponse.Repositories.Count() > 0; } - public async Task SnapshotInProgressAsync() + public async Task SnapshotInProgressAsync(string repository = null) { - var repositoriesResponse = await _client.Snapshot.GetRepositoryAsync().AnyContext(); - _logger.LogRequest(repositoriesResponse); - if (repositoriesResponse.Repositories.Count == 0) - return false; - - foreach (string repo in repositoriesResponse.Repositories.Keys) + if (!String.IsNullOrEmpty(repository)) { - var snapshotsResponse = await _client.Cat.SnapshotsAsync(new CatSnapshotsRequest(repo)).AnyContext(); + var snapshotsResponse = await _client.Snapshot.GetAsync(new Elastic.Clients.Elasticsearch.Snapshot.GetSnapshotRequest(repository, "*")).AnyContext(); _logger.LogRequest(snapshotsResponse); - foreach (var snapshot in snapshotsResponse.Records) + if (snapshotsResponse.IsValidResponse) { - if (snapshot.Status == "IN_PROGRESS") - return true; + foreach (var snapshot in snapshotsResponse.Snapshots) + { + if (snapshot.State == "IN_PROGRESS") + return true; + } + } + } + else + { + var repositoriesResponse = await _client.Snapshot.GetRepositoryAsync().AnyContext(); + _logger.LogRequest(repositoriesResponse); + if (!repositoriesResponse.IsValidResponse || repositoriesResponse.Repositories.Count() == 0) + return false; + + foreach (var repo in repositoriesResponse.Repositories) + { + var snapshotsResponse = await _client.Snapshot.GetAsync(new Elastic.Clients.Elasticsearch.Snapshot.GetSnapshotRequest(repo.Key, "*")).AnyContext(); + _logger.LogRequest(snapshotsResponse); + if (snapshotsResponse.IsValidResponse) + { + foreach (var snapshot in snapshotsResponse.Snapshots) + { + if (snapshot.State == "IN_PROGRESS") + return true; + } + } } } var tasksResponse = await _client.Tasks.ListAsync().AnyContext(); _logger.LogRequest(tasksResponse); - if (!tasksResponse.IsValid) + if (!tasksResponse.IsValidResponse) { - _logger.LogWarning("Failed to list tasks: {Error}", tasksResponse.ServerError); + _logger.LogWarning("Failed to list tasks: {Error}", tasksResponse.ElasticsearchServerError); return false; } @@ -75,40 +106,90 @@ public async Task SnapshotInProgressAsync() public async Task> GetSnapshotListAsync(string repository) { - var snapshotsResponse = await _client.Cat.SnapshotsAsync(new CatSnapshotsRequest(repository)).AnyContext(); + var snapshotsResponse = await _client.Snapshot.GetAsync(new Elastic.Clients.Elasticsearch.Snapshot.GetSnapshotRequest(repository, "*")).AnyContext(); _logger.LogRequest(snapshotsResponse); - if (!snapshotsResponse.IsValid) + if (!snapshotsResponse.IsValidResponse) { - _logger.LogWarning("Failed to get snapshot list for {Repository}: {Error}", repository, snapshotsResponse.ServerError); + _logger.LogWarning("Failed to get snapshot list for {Repository}: {Error}", repository, snapshotsResponse.ElasticsearchServerError); return Array.Empty(); } - return snapshotsResponse.Records.Select(r => r.Id).ToList(); + return snapshotsResponse.Snapshots.Select(s => s.Snapshot).ToList(); } public async Task> GetIndexListAsync() { - var indicesResponse = await _client.Cat.IndicesAsync(new CatIndicesRequest()).AnyContext(); - _logger.LogRequest(indicesResponse); - if (!indicesResponse.IsValid) + // Note: ResolveIndexAsync sends a body in ES 9.x client which ES rejects; use GetAsync instead. + var response = await _client.Indices.GetAsync(Indices.All, d => d.IgnoreUnavailable()).AnyContext(); + _logger.LogRequest(response); + if (!response.IsValidResponse || response.Indices is null) { - _logger.LogWarning("Failed to get index list: {Error}", indicesResponse.ServerError); + _logger.LogWarning("Failed to get index list: {Error}", response.ElasticsearchServerError); return Array.Empty(); } - return indicesResponse.Records.Select(r => r.Index).ToList(); + return response.Indices.Keys.Select(k => k.ToString()).ToList(); } - public Task WaitForTaskAsync(string taskId, TimeSpan? maxWaitTime = null, TimeSpan? waitInterval = null) + /// + /// Waits for an Elasticsearch task to complete, polling at intervals until completion or timeout. + /// + /// The task ID to wait for (format: "nodeId:taskId"). + /// Maximum time to wait before returning false. Defaults to 5 minutes. + /// Interval between polls. Defaults to 2 seconds. + /// True if the task completed successfully; false if it timed out or failed. + public async Task WaitForTaskAsync(string taskId, TimeSpan? maxWaitTime = null, TimeSpan? waitInterval = null) { - // check task is completed in loop - return Task.FromResult(false); + if (String.IsNullOrEmpty(taskId)) + return false; + + var maxWait = maxWaitTime ?? TimeSpan.FromMinutes(5); + var interval = waitInterval ?? TimeSpan.FromSeconds(2); + var started = _timeProvider.GetUtcNow(); + + while (_timeProvider.GetUtcNow() - started < maxWait) + { + var getTaskResponse = await _client.Tasks.GetAsync(taskId).AnyContext(); + _logger.LogRequest(getTaskResponse); + + if (!getTaskResponse.IsValidResponse) + return false; + + if (getTaskResponse.Completed) + return true; + + await Task.Delay(interval, _timeProvider).AnyContext(); + } + + _logger.LogWarning("Timed out waiting for task {TaskId} after {MaxWaitTime}", taskId, maxWait); + return false; } - public Task WaitForSafeToSnapshotAsync(string repository, TimeSpan? maxWaitTime = null, TimeSpan? waitInterval = null) + /// + /// Waits until no snapshots are in progress, polling at intervals. + /// + /// The snapshot repository to check. + /// Maximum time to wait. Defaults to 30 minutes. + /// Interval between polls. Defaults to 5 seconds. + /// True if safe to snapshot; false if timed out. + public async Task WaitForSafeToSnapshotAsync(string repository, TimeSpan? maxWaitTime = null, TimeSpan? waitInterval = null) { - // check SnapshotInProgressAsync in loop - return Task.FromResult(false); + var maxWait = maxWaitTime ?? TimeSpan.FromMinutes(30); + var interval = waitInterval ?? TimeSpan.FromSeconds(5); + var started = _timeProvider.GetUtcNow(); + + while (_timeProvider.GetUtcNow() - started < maxWait) + { + bool inProgress = await SnapshotInProgressAsync(repository).AnyContext(); + if (!inProgress) + return true; + + _logger.LogDebug("Snapshot in progress for repository {Repository}; waiting {Interval}...", repository, interval); + await Task.Delay(interval, _timeProvider).AnyContext(); + } + + _logger.LogWarning("Timed out waiting for safe snapshot window after {MaxWaitTime}", maxWait); + return false; } public async Task CreateSnapshotAsync(CreateSnapshotOptions options) @@ -118,38 +199,94 @@ public async Task CreateSnapshotAsync(CreateSnapshotOptions options) bool repoExists = await SnapshotRepositoryExistsAsync(options.Repository).AnyContext(); if (!repoExists) - throw new RepositoryException(); + throw new RepositoryException($"Snapshot repository '{options.Repository}' does not exist."); - bool success = await WaitForSafeToSnapshotAsync(options.Repository).AnyContext(); - if (!success) - throw new RepositoryException(); + bool safe = await WaitForSafeToSnapshotAsync(options.Repository).AnyContext(); + if (!safe) + throw new RepositoryException($"Timed out waiting for a safe window to create snapshot in repository '{options.Repository}'."); - var snapshotResponse = await _client.Snapshot.SnapshotAsync(new SnapshotRequest(options.Repository, options.Name) - { - Indices = options.Indices != null ? Indices.Index(options.Indices) : Indices.All, - WaitForCompletion = false, - IgnoreUnavailable = options.IgnoreUnavailable, - IncludeGlobalState = options.IncludeGlobalState - }).AnyContext(); + var snapshotResponse = await _client.Snapshot.CreateAsync(options.Repository, options.Name, s => s + .Indices(options.Indices != null ? Indices.Parse(String.Join(",", options.Indices)) : Indices.All) + .WaitForCompletion(false) + .IgnoreUnavailable(options.IgnoreUnavailable) + .IncludeGlobalState(options.IncludeGlobalState) + ).AnyContext(); _logger.LogRequest(snapshotResponse); - // TODO: wait for snapshot to be success in loop + if (!snapshotResponse.IsValidResponse) + { + _logger.LogError("Failed to create snapshot '{SnapshotName}' in repository '{Repository}'", options.Name, options.Repository); + return false; + } - return false; - // TODO: should we use lock provider as well as checking for WaitForSafeToSnapshotAsync? - // TODO: create a new snapshot in the repository with retries + // Wait for the snapshot to complete by polling until it's no longer IN_PROGRESS + return await WaitForSafeToSnapshotAsync(options.Repository, maxWaitTime: TimeSpan.FromHours(2)).AnyContext(); } - public Task DeleteSnapshotsAsync(string repository, ICollection snapshots, int? maxRetries = null, TimeSpan? retryInterval = null) + /// + /// Deletes the specified snapshots using the configured resilience policy. + /// + /// The snapshot repository. + /// The snapshot names to delete. + /// True if all snapshots were deleted; false if any deletion failed after retries. + public async Task DeleteSnapshotsAsync(string repository, ICollection snapshots) { - // TODO: attempt to delete all indices with retries and wait interval - return Task.FromResult(true); + if (snapshots == null || snapshots.Count == 0) + return true; + + bool allSucceeded = true; + + foreach (var snapshot in snapshots) + { + try + { + await _resiliencePolicy.ExecuteAsync(async _ => + { + var response = await _client.Snapshot.DeleteAsync(repository, snapshot).AnyContext(); + _logger.LogRequest(response); + + if (!response.IsValidResponse) + throw response.OriginalException() ?? new ApplicationException($"Failed to delete snapshot '{snapshot}'"); + }).AnyContext(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete snapshot '{Snapshot}'", snapshot); + allSucceeded = false; + } + } + + return allSucceeded; } - public Task DeleteIndicesAsync(ICollection indices, int? maxRetries = null, TimeSpan? retryInterval = null) + /// + /// Deletes the specified indices using the configured resilience policy. + /// + /// The index names to delete. + /// True if all indices were deleted; false if deletion failed after retries. + public async Task DeleteIndicesAsync(ICollection indices) { - // TODO: attempt to delete all indices with retries - return Task.FromResult(true); + if (indices == null || indices.Count == 0) + return true; + + try + { + await _resiliencePolicy.ExecuteAsync(async _ => + { + var response = await _client.Indices.DeleteAsync(Indices.Parse(String.Join(",", indices))).AnyContext(); + _logger.LogRequest(response); + + if (!response.IsValidResponse) + throw response.OriginalException() ?? new ApplicationException($"Failed to delete indices [{String.Join(", ", indices)}]"); + }).AnyContext(); + + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete indices [{Indices}]", String.Join(", ", indices)); + return false; + } } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticIndexExtensions.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticIndexExtensions.cs index 6632e806..37cc1439 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticIndexExtensions.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticIndexExtensions.cs @@ -2,71 +2,73 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; -using System.Reflection; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.AsyncSearch; +using Elastic.Clients.Elasticsearch.Core.Bulk; +using Elastic.Clients.Elasticsearch.Core.Search; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; +using Foundatio.Serializer; using Foundatio.Utility; using Microsoft.Extensions.Logging; +using ElasticAggregations = Elastic.Clients.Elasticsearch.Aggregations; namespace Foundatio.Repositories.Elasticsearch.Extensions; public static class ElasticIndexExtensions { - public static Nest.AsyncSearchSubmitDescriptor ToAsyncSearchSubmitDescriptor(this Nest.SearchDescriptor searchDescriptor) where T : class, new() - { - var asyncSearchDescriptor = new Nest.AsyncSearchSubmitDescriptor(); - - var searchRequest = (Nest.ISearchRequest)searchDescriptor; - var asyncSearchRequest = (Nest.IAsyncSearchSubmitRequest)asyncSearchDescriptor; - - asyncSearchRequest.RequestParameters.QueryString = searchRequest.RequestParameters.QueryString; - asyncSearchRequest.RequestParameters.RequestConfiguration = searchRequest.RequestParameters.RequestConfiguration; - asyncSearchDescriptor.Index(searchRequest.Index); - asyncSearchDescriptor.SequenceNumberPrimaryTerm(searchRequest.RequestParameters.SequenceNumberPrimaryTerm); - asyncSearchDescriptor.IgnoreUnavailable(searchRequest.RequestParameters.IgnoreUnavailable); - asyncSearchDescriptor.TrackTotalHits(searchRequest.RequestParameters.TrackTotalHits); - - asyncSearchRequest.Aggregations = searchRequest.Aggregations; - asyncSearchRequest.Collapse = searchRequest.Collapse; - asyncSearchRequest.DocValueFields = searchRequest.DocValueFields; - asyncSearchRequest.Explain = searchRequest.Explain; - asyncSearchRequest.From = searchRequest.From; - asyncSearchRequest.Highlight = searchRequest.Highlight; - asyncSearchRequest.IndicesBoost = searchRequest.IndicesBoost; - asyncSearchRequest.MinScore = searchRequest.MinScore; - asyncSearchRequest.PostFilter = searchRequest.PostFilter; - asyncSearchRequest.Profile = searchRequest.Profile; - asyncSearchRequest.Query = searchRequest.Query; - asyncSearchRequest.Rescore = searchRequest.Rescore; - asyncSearchRequest.ScriptFields = searchRequest.ScriptFields; - asyncSearchRequest.SearchAfter = searchRequest.SearchAfter; - asyncSearchRequest.Size = searchRequest.Size; - asyncSearchRequest.Sort = searchRequest.Sort; - asyncSearchRequest.Source = searchRequest.Source; - asyncSearchRequest.StoredFields = searchRequest.StoredFields; - asyncSearchRequest.Suggest = searchRequest.Suggest; - asyncSearchRequest.TerminateAfter = searchRequest.TerminateAfter; - asyncSearchRequest.Timeout = searchRequest.Timeout; - asyncSearchRequest.TrackScores = searchRequest.TrackScores; - asyncSearchRequest.TrackTotalHits = searchRequest.TrackTotalHits; - asyncSearchRequest.Version = searchRequest.Version; - asyncSearchRequest.RuntimeFields = searchRequest.RuntimeFields; - - return asyncSearchDescriptor; - } - - public static FindResults ToFindResults(this Nest.ISearchResponse response, ICommandOptions options, ILogger logger = null) where T : class, new() - { - if (!response.IsValid) - { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + public static SubmitAsyncSearchRequest ToAsyncSearchSubmitRequest(this SearchRequest searchRequest) where T : class, new() + { + var asyncSearchRequest = new SubmitAsyncSearchRequest(searchRequest.Indices) + { + Aggregations = searchRequest.Aggregations, + Collapse = searchRequest.Collapse, + DocvalueFields = searchRequest.DocvalueFields, + Explain = searchRequest.Explain, + From = searchRequest.From, + Highlight = searchRequest.Highlight, + IndicesBoost = searchRequest.IndicesBoost, + MinScore = searchRequest.MinScore, + PostFilter = searchRequest.PostFilter, + Profile = searchRequest.Profile, + Query = searchRequest.Query, + Rescore = searchRequest.Rescore, + ScriptFields = searchRequest.ScriptFields, + SearchAfter = searchRequest.SearchAfter, + Size = searchRequest.Size, + Sort = searchRequest.Sort, + Source = searchRequest.Source, + StoredFields = searchRequest.StoredFields, + Suggest = searchRequest.Suggest, + TerminateAfter = searchRequest.TerminateAfter, + Timeout = searchRequest.Timeout, + TrackScores = searchRequest.TrackScores, + TrackTotalHits = searchRequest.TrackTotalHits, + Version = searchRequest.Version, + RuntimeMappings = searchRequest.RuntimeMappings, + SeqNoPrimaryTerm = searchRequest.SeqNoPrimaryTerm + }; + + return asyncSearchRequest; + } + + public static FindResults ToFindResults(this SearchResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() + { + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(serializer); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return new FindResults(); - throw new DocumentException(response.GetErrorMessage("Error while searching"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error while searching"), response.OriginalException()); } int limit = options.GetLimit(); @@ -74,9 +76,9 @@ public static class ElasticIndexExtensions var data = new DataDictionary(); if (response.ScrollId != null) - data.Add(ElasticDataKeys.ScrollId, response.ScrollId); + data.Add(ElasticDataKeys.ScrollId, response.ScrollId.ToString()); - var results = new FindResults(docs, response.Total, response.ToAggregations(logger), null, data); + var results = new FindResults(docs, response.Total, response.ToAggregations(serializer, logger), null, data); var protectedResults = (IFindResults)results; if (options.ShouldUseSnapshotPaging()) protectedResults.HasMore = response.Hits.Count >= limit; @@ -85,21 +87,21 @@ public static class ElasticIndexExtensions if (options.HasSearchAfter()) { - results.SetSearchBeforeToken(); + results.SetSearchBeforeToken(serializer); if (results.HasMore) - results.SetSearchAfterToken(); + results.SetSearchAfterToken(serializer); } else if (options.HasSearchBefore()) { // reverse results protectedResults.Reverse(); - results.SetSearchAfterToken(); + results.SetSearchAfterToken(serializer); if (results.HasMore) - results.SetSearchBeforeToken(); + results.SetSearchBeforeToken(serializer); } else if (results.HasMore) { - results.SetSearchAfterToken(); + results.SetSearchAfterToken(serializer); } protectedResults.Page = options.GetPage(); @@ -107,14 +109,18 @@ public static class ElasticIndexExtensions return results; } - public static FindResults ToFindResults(this Nest.IAsyncSearchResponse response, ICommandOptions options, ILogger logger = null) where T : class, new() + public static FindResults ToFindResults(this SubmitAsyncSearchResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() { - if (!response.IsValid) + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(serializer); + + if (!response.IsValidResponse) { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return new FindResults(); - throw new DocumentException(response.GetErrorMessage("Error while searching"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error while searching"), response.OriginalException()); } int limit = options.GetLimit(); @@ -130,7 +136,7 @@ public static class ElasticIndexExtensions if (options.ShouldAutoDeleteAsyncQuery() && !response.IsRunning) data.Remove(AsyncQueryDataKeys.AsyncQueryId); - var results = new FindResults(docs, response.Response.Total, response.ToAggregations(logger), null, data); + var results = new FindResults(docs, response.Response.Total, response.ToAggregations(serializer, logger), null, data); var protectedResults = (IFindResults)results; if (options.ShouldUseSnapshotPaging()) protectedResults.HasMore = response.Response.Hits.Count >= limit; @@ -139,21 +145,21 @@ public static class ElasticIndexExtensions if (options.HasSearchAfter()) { - results.SetSearchBeforeToken(); + results.SetSearchBeforeToken(serializer); if (results.HasMore) - results.SetSearchAfterToken(); + results.SetSearchAfterToken(serializer); } else if (options.HasSearchBefore()) { // reverse results protectedResults.Reverse(); - results.SetSearchAfterToken(); + results.SetSearchAfterToken(serializer); if (results.HasMore) - results.SetSearchBeforeToken(); + results.SetSearchBeforeToken(serializer); } else if (results.HasMore) { - results.SetSearchAfterToken(); + results.SetSearchAfterToken(serializer); } protectedResults.Page = options.GetPage(); @@ -161,36 +167,36 @@ public static class ElasticIndexExtensions return results; } - public static IEnumerable> ToFindHits(this IEnumerable> hits) where T : class + public static IEnumerable> ToFindHits(this IEnumerable> hits) where T : class { return hits.Select(h => h.ToFindHit()); } - public static CountResult ToCountResult(this Nest.ISearchResponse response, ICommandOptions options, ILogger logger = null) where T : class, new() + public static CountResult ToCountResult(this SearchResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() { - if (!response.IsValid) + if (!response.IsValidResponse) { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return new FindResults(); - throw new DocumentException(response.GetErrorMessage("Error while counting"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error while counting"), response.OriginalException()); } var data = new DataDictionary(); if (response.ScrollId != null) - data.Add(ElasticDataKeys.ScrollId, response.ScrollId); + data.Add(ElasticDataKeys.ScrollId, response.ScrollId.ToString()); - return new CountResult(response.Total, response.ToAggregations(logger), data); + return new CountResult(response.Total, response.ToAggregations(serializer, logger), data); } - public static CountResult ToCountResult(this Nest.IAsyncSearchResponse response, ICommandOptions options, ILogger logger = null) where T : class, new() + public static CountResult ToCountResult(this SubmitAsyncSearchResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() { - if (!response.IsValid) + if (!response.IsValidResponse) { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return new FindResults(); - throw new DocumentException(response.GetErrorMessage("Error while counting"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error while counting"), response.OriginalException()); } var data = new DataDictionary @@ -203,40 +209,149 @@ public static IEnumerable> ToFindHits(this IEnumerable ToFindHit(this Nest.GetResponse hit) where T : class + public static FindResults ToFindResults(this GetAsyncSearchResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() + { + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(serializer); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) + return new FindResults(); + + throw new DocumentException(response.GetErrorMessage("Error while searching"), response.OriginalException()); + } + + int limit = options.GetLimit(); + var docs = response.Response.Hits.Take(limit).ToFindHits().ToList(); + + var data = new DataDictionary + { + { AsyncQueryDataKeys.AsyncQueryId, response.Id }, + { AsyncQueryDataKeys.IsRunning, response.IsRunning }, + { AsyncQueryDataKeys.IsPartial, response.IsPartial } + }; + + if (options.ShouldAutoDeleteAsyncQuery() && !response.IsRunning) + data.Remove(AsyncQueryDataKeys.AsyncQueryId); + + var results = new FindResults(docs, response.Response.Total, response.ToAggregations(serializer, logger), null, data); + var protectedResults = (IFindResults)results; + if (options.ShouldUseSnapshotPaging()) + protectedResults.HasMore = response.Response.Hits.Count >= limit; + else + protectedResults.HasMore = response.Response.Hits.Count > limit || response.Response.Hits.Count >= options.GetMaxLimit(); + + if (options.HasSearchAfter()) + { + results.SetSearchBeforeToken(serializer); + if (results.HasMore) + results.SetSearchAfterToken(serializer); + } + else if (options.HasSearchBefore()) + { + protectedResults.Reverse(); + results.SetSearchAfterToken(serializer); + if (results.HasMore) + results.SetSearchBeforeToken(serializer); + } + else if (results.HasMore) + { + results.SetSearchAfterToken(serializer); + } + + protectedResults.Page = options.GetPage(); + + return results; + } + + public static CountResult ToCountResult(this GetAsyncSearchResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() + { + if (!response.IsValidResponse) + { + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) + return new FindResults(); + + throw new DocumentException(response.GetErrorMessage("Error while counting"), response.OriginalException()); + } + + var data = new DataDictionary + { + { AsyncQueryDataKeys.AsyncQueryId, response.Id }, + { AsyncQueryDataKeys.IsRunning, response.IsRunning }, + { AsyncQueryDataKeys.IsPartial, response.IsPartial } + }; + + if (options.ShouldAutoDeleteAsyncQuery() && !response.IsRunning) + data.Remove(AsyncQueryDataKeys.AsyncQueryId); + + return new CountResult(response.Response.Total, response.ToAggregations(serializer, logger), data); + } + + public static FindResults ToFindResults(this ScrollResponse response, ICommandOptions options, ITextSerializer serializer, ILogger logger = null) where T : class, new() + { + ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(serializer); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) + return new FindResults(); + + throw new DocumentException(response.GetErrorMessage("Error while searching"), response.OriginalException()); + } + + int limit = options.GetLimit(); + var docs = response.Hits.Take(limit).ToFindHits().ToList(); + + var data = new DataDictionary(); + if (response.ScrollId != null) + data.Add(ElasticDataKeys.ScrollId, response.ScrollId.ToString()); + + var results = new FindResults(docs, response.Total, response.ToAggregations(serializer, logger), null, data); + var protectedResults = (IFindResults)results; + protectedResults.HasMore = response.Hits.Count > 0 && response.Hits.Count >= limit; + + protectedResults.Page = options.GetPage(); + + return results; + } + + public static FindHit ToFindHit(this GetResponse hit) where T : class { var data = new DataDictionary { { ElasticDataKeys.Index, hit.Index } }; - var versionedDoc = hit.Source as IVersioned; - if (versionedDoc != null) + if (hit.Source is IVersioned versionedDoc) versionedDoc.Version = hit.GetElasticVersion(); return new FindHit(hit.Id, hit.Source, 0, hit.GetElasticVersion(), hit.Routing, data); } - public static ElasticDocumentVersion GetElasticVersion(this Nest.GetResponse hit) where T : class + public static ElasticDocumentVersion GetElasticVersion(this GetResponse hit) where T : class { - if (!hit.PrimaryTerm.HasValue || !hit.SequenceNumber.HasValue) + if (!hit.PrimaryTerm.HasValue || !hit.SeqNo.HasValue) return ElasticDocumentVersion.Empty; - if (hit.PrimaryTerm.Value == 0 && hit.SequenceNumber.Value == 0) + if (hit.PrimaryTerm.Value == 0 && hit.SeqNo.Value == 0) return ElasticDocumentVersion.Empty; - return new ElasticDocumentVersion(hit.PrimaryTerm.Value, hit.SequenceNumber.Value); + return new ElasticDocumentVersion(hit.PrimaryTerm.Value, hit.SeqNo.Value); } - public static ElasticDocumentVersion GetElasticVersion(this Nest.IHit hit) where T : class + public static ElasticDocumentVersion GetElasticVersion(this Hit hit) where T : class { - if (!hit.PrimaryTerm.HasValue || !hit.SequenceNumber.HasValue) + if (!hit.PrimaryTerm.HasValue || !hit.SeqNo.HasValue) return ElasticDocumentVersion.Empty; - if (hit.PrimaryTerm.Value == 0 && hit.SequenceNumber.Value == 0) + if (hit.PrimaryTerm.Value == 0 && hit.SeqNo.Value == 0) return ElasticDocumentVersion.Empty; - return new ElasticDocumentVersion(hit.PrimaryTerm.Value, hit.SequenceNumber.Value); + return new ElasticDocumentVersion(hit.PrimaryTerm.Value, hit.SeqNo.Value); } public static ElasticDocumentVersion GetElasticVersion(this FindHit hit) where T : class @@ -247,28 +362,20 @@ public static ElasticDocumentVersion GetElasticVersion(this FindHit hit) w return hit.Version; } - public static ElasticDocumentVersion GetElasticVersion(this Nest.IndexResponse hit) + public static ElasticDocumentVersion GetElasticVersion(this IndexResponse hit) { - if (hit.PrimaryTerm == 0 && hit.SequenceNumber == 0) + if (hit.PrimaryTerm.GetValueOrDefault() == 0 && hit.SeqNo.GetValueOrDefault() == 0) return ElasticDocumentVersion.Empty; - return new ElasticDocumentVersion(hit.PrimaryTerm, hit.SequenceNumber); + return new ElasticDocumentVersion(hit.PrimaryTerm.GetValueOrDefault(), hit.SeqNo.GetValueOrDefault()); } - public static ElasticDocumentVersion GetElasticVersion(this Nest.IMultiGetHit hit) where T : class + public static ElasticDocumentVersion GetElasticVersion(this ResponseItem hit) { - if (!hit.PrimaryTerm.HasValue || !hit.SequenceNumber.HasValue) + if (hit.PrimaryTerm.GetValueOrDefault() == 0 && hit.SeqNo.GetValueOrDefault() == 0) return ElasticDocumentVersion.Empty; - return new ElasticDocumentVersion(hit.PrimaryTerm.Value, hit.SequenceNumber.Value); - } - - public static ElasticDocumentVersion GetElasticVersion(this Nest.BulkResponseItemBase hit) - { - if (hit.PrimaryTerm == 0 && hit.SequenceNumber == 0) - return ElasticDocumentVersion.Empty; - - return new ElasticDocumentVersion(hit.PrimaryTerm, hit.SequenceNumber); + return new ElasticDocumentVersion(hit.PrimaryTerm.GetValueOrDefault(), hit.SeqNo.GetValueOrDefault()); } public static ElasticDocumentVersion GetElasticVersion(this IVersioned versioned) @@ -279,160 +386,422 @@ public static ElasticDocumentVersion GetElasticVersion(this IVersioned versioned return versioned.Version; } - public static FindHit ToFindHit(this Nest.IHit hit) where T : class + public static FindHit ToFindHit(this Hit hit) where T : class { var data = new DataDictionary { - { ElasticDataKeys.Index, hit.Index }, - { ElasticDataKeys.Sorts, hit.Sorts } + { ElasticDataKeys.Index, hit.Index } }; - var versionedDoc = hit.Source as IVersioned; - if (versionedDoc != null && hit.PrimaryTerm.HasValue) + // Only add sorts if they exist + if (hit.Sort != null && hit.Sort.Count > 0) + data[ElasticDataKeys.Sorts] = hit.Sort; + + if (hit.Source is IVersioned versionedDoc && hit.PrimaryTerm.HasValue) versionedDoc.Version = hit.GetElasticVersion(); return new FindHit(hit.Id, hit.Source, hit.Score.GetValueOrDefault(), hit.GetElasticVersion(), hit.Routing, data); } - public static FindHit ToFindHit(this Nest.IMultiGetHit hit) where T : class + public static IEnumerable> ToFindHits(this MultiGetResponse response, ILogger logger = null) where T : class { - var data = new DataDictionary { { ElasticDataKeys.Index, hit.Index } }; + foreach (var doc in response.Docs) + { + FindHit findHit = null; + doc.Match( + result => + { + if (result.Found) + { + var data = new DataDictionary { { ElasticDataKeys.Index, result.Index } }; - var versionedDoc = hit.Source as IVersioned; - if (versionedDoc != null) - versionedDoc.Version = hit.GetElasticVersion(); + var version = ElasticDocumentVersion.Empty; + if (result.PrimaryTerm.HasValue && result.SeqNo.HasValue && (result.PrimaryTerm.Value != 0 || result.SeqNo.Value != 0)) + version = new ElasticDocumentVersion(result.PrimaryTerm.Value, result.SeqNo.Value); - return new FindHit(hit.Id, hit.Source, 0, hit.GetElasticVersion(), hit.Routing, data); + if (result.Source is IVersioned versionedDoc) + versionedDoc.Version = version; + + findHit = new FindHit(result.Id, result.Source, 0, version, result.Routing, data); + } + else + { + logger?.LogDebug("MultiGet document not found: index={Index}, id={Id}", result.Index, result.Id); + } + }, + error => + { + logger?.LogWarning("MultiGet document error: index={Index}, id={Id}, reason={Reason}", error.Index, error.Id, error.Error?.Reason); + } + ); + if (findHit is not null) + yield return findHit; + } } private static readonly long _epochTicks = new DateTimeOffset(1970, 1, 1, 0, 0, 0, 0, TimeSpan.Zero).Ticks; + private static readonly IReadOnlyDictionary _stringBucketData = new ReadOnlyDictionary(new Dictionary { { "@type", "string" } }); + private static readonly IReadOnlyDictionary _doubleBucketData = new ReadOnlyDictionary(new Dictionary { { "@type", "double" } }); + private static readonly IReadOnlyDictionary _rangeBucketData = new ReadOnlyDictionary(new Dictionary { { "@type", "range" } }); + private static readonly IReadOnlyDictionary _geohashBucketData = new ReadOnlyDictionary(new Dictionary { { "@type", "geohash" } }); - private static readonly Lazy>> _getHits = - new(() => - { - var hitsField = typeof(Nest.TopHitsAggregate).GetField("_hits", BindingFlags.GetField | BindingFlags.NonPublic | BindingFlags.Instance); - return agg => hitsField?.GetValue(agg) as IList; - }); - - public static IAggregate ToAggregate(this Nest.IAggregate aggregate, string name = null, ILogger logger = null) + public static IAggregate ToAggregate(this ElasticAggregations.IAggregate aggregate, string key, ITextSerializer serializer, ILogger logger = null) { - if (aggregate is Nest.ValueAggregate valueAggregate) + switch (aggregate) { - if (valueAggregate.Meta != null && valueAggregate.Meta.TryGetValue("@field_type", out object value)) - { - string type = value.ToString(); - if (type == "date" && valueAggregate.Value.HasValue) + case ElasticAggregations.AverageAggregate avg: + return new ValueAggregate { Value = avg.Value, Data = avg.Meta.ToReadOnlyData() }; + + case ElasticAggregations.SumAggregate sum: + return new ValueAggregate { Value = sum.Value, Data = sum.Meta.ToReadOnlyData() }; + + case ElasticAggregations.MinAggregate min: + return ToValueAggregate(min.Value, min.Meta); + + case ElasticAggregations.MaxAggregate max: + return ToValueAggregate(max.Value, max.Meta); + + case ElasticAggregations.CardinalityAggregate cardinality: + return new ValueAggregate { Value = cardinality.Value, Data = cardinality.Meta.ToReadOnlyData() }; + + case ElasticAggregations.ValueCountAggregate valueCount: + return new ValueAggregate { Value = valueCount.Value, Data = valueCount.Meta.ToReadOnlyData() }; + + case ElasticAggregations.ScriptedMetricAggregate scripted: + return new ObjectValueAggregate + { + Value = scripted.Value, + Data = scripted.Meta.ToReadOnlyData() + }; + + case ElasticAggregations.ExtendedStatsAggregate extendedStats: + return new ExtendedStatsAggregate { - return new ValueAggregate + Count = extendedStats.Count, + Min = extendedStats.Min, + Max = extendedStats.Max, + Average = extendedStats.Avg, + Sum = extendedStats.Sum, + StdDeviation = extendedStats.StdDeviation, + StdDeviationBounds = extendedStats.StdDeviationBounds != null ? new StandardDeviationBounds { - Value = GetDate(valueAggregate), - Data = valueAggregate.Meta.ToReadOnlyData>() - }; - } - } + Lower = extendedStats.StdDeviationBounds.Lower, + Upper = extendedStats.StdDeviationBounds.Upper + } : null, + SumOfSquares = extendedStats.SumOfSquares, + Variance = extendedStats.Variance, + Data = extendedStats.Meta.ToReadOnlyData() + }; + + case ElasticAggregations.StatsAggregate stats: + return new StatsAggregate + { + Count = stats.Count, + Min = stats.Min, + Max = stats.Max, + Average = stats.Avg, + Sum = stats.Sum, + Data = stats.Meta.ToReadOnlyData() + }; + + case ElasticAggregations.TopHitsAggregate topHits: + var docs = topHits.Hits?.Hits?.Select(h => new ElasticLazyDocument(h, serializer)).Cast().ToList(); + var rawHits = topHits.Hits?.Hits? + .Select(h => h.Source != null ? serializer.SerializeToString(h.Source) : null) + .Where(s => s != null) + .ToList(); + return new TopHitsAggregate(docs) + { + Total = topHits.Hits?.Total?.Match(t => t.Value, l => l) ?? 0, + MaxScore = topHits.Hits?.MaxScore, + Hits = rawHits, + Data = topHits.Meta.ToReadOnlyData() + }; + + case ElasticAggregations.TDigestPercentilesAggregate percentiles: + var items = percentiles.Values?.Select(item => new PercentileItem + { + Percentile = item.Key, + Value = item.Value + }) ?? Enumerable.Empty(); + return new PercentilesAggregate(items) + { + Data = percentiles.Meta.ToReadOnlyData() + }; - return new ValueAggregate { Value = valueAggregate.Value, Data = valueAggregate.Meta.ToReadOnlyData() }; - } + case ElasticAggregations.HdrPercentilesAggregate hdrPercentiles: + var hdrItems = hdrPercentiles.Values?.Select(item => new PercentileItem + { + Percentile = item.Key, + Value = item.Value + }) ?? Enumerable.Empty(); + return new PercentilesAggregate(hdrItems) + { + Data = hdrPercentiles.Meta.ToReadOnlyData() + }; - if (aggregate is Nest.ScriptedMetricAggregate scriptedAggregate) - return new ObjectValueAggregate - { - Value = scriptedAggregate.Value(), - Data = scriptedAggregate.Meta.ToReadOnlyData() - }; + case ElasticAggregations.FilterAggregate filter: + return new SingleBucketAggregate(filter.ToAggregations(serializer, logger)) + { + Data = filter.Meta.ToReadOnlyData(), + Total = filter.DocCount + }; - if (aggregate is Nest.ExtendedStatsAggregate extendedStatsAggregate) - return new ExtendedStatsAggregate - { - Count = extendedStatsAggregate.Count, - Min = extendedStatsAggregate.Min, - Max = extendedStatsAggregate.Max, - Average = extendedStatsAggregate.Average, - Sum = extendedStatsAggregate.Sum, - StdDeviation = extendedStatsAggregate.StdDeviation, - StdDeviationBounds = new StandardDeviationBounds + case ElasticAggregations.GlobalAggregate globalAgg: + return new SingleBucketAggregate(globalAgg.ToAggregations(serializer, logger)) { - Lower = extendedStatsAggregate.StdDeviationBounds.Lower, - Upper = extendedStatsAggregate.StdDeviationBounds.Upper - }, - SumOfSquares = extendedStatsAggregate.SumOfSquares, - Variance = extendedStatsAggregate.Variance, - Data = extendedStatsAggregate.Meta.ToReadOnlyData() - }; + Data = globalAgg.Meta.ToReadOnlyData(), + Total = globalAgg.DocCount + }; - if (aggregate is Nest.StatsAggregate statsAggregate) - return new StatsAggregate - { - Count = statsAggregate.Count, - Min = statsAggregate.Min, - Max = statsAggregate.Max, - Average = statsAggregate.Average, - Sum = statsAggregate.Sum, - Data = statsAggregate.Meta.ToReadOnlyData() - }; + case ElasticAggregations.MissingAggregate missing: + return new SingleBucketAggregate(missing.ToAggregations(serializer, logger)) + { + Data = missing.Meta.ToReadOnlyData(), + Total = missing.DocCount + }; + + case ElasticAggregations.NestedAggregate nested: + return new SingleBucketAggregate(nested.ToAggregations(serializer, logger)) + { + Data = nested.Meta.ToReadOnlyData(), + Total = nested.DocCount + }; + + case ElasticAggregations.ReverseNestedAggregate reverseNested: + return new SingleBucketAggregate(reverseNested.ToAggregations(serializer, logger)) + { + Data = reverseNested.Meta.ToReadOnlyData(), + Total = reverseNested.DocCount + }; + + case ElasticAggregations.DateHistogramAggregate dateHistogram: + return ToDateHistogramBucketAggregate(dateHistogram, serializer, logger); + + case ElasticAggregations.StringTermsAggregate stringTerms: + return ToTermsBucketAggregate(stringTerms, key, serializer, logger); + + case ElasticAggregations.LongTermsAggregate longTerms: + return ToTermsBucketAggregate(longTerms, key, serializer, logger); + + case ElasticAggregations.DoubleTermsAggregate doubleTerms: + return ToTermsBucketAggregate(doubleTerms, key, serializer, logger); + + case ElasticAggregations.DateRangeAggregate dateRange: + return ToRangeBucketAggregate(dateRange, serializer, logger); + + case ElasticAggregations.RangeAggregate range: + return ToRangeBucketAggregate(range, serializer, logger); + + case ElasticAggregations.GeohashGridAggregate geohashGrid: + return ToGeohashGridBucketAggregate(geohashGrid, serializer, logger); - if (aggregate is Nest.TopHitsAggregate topHitsAggregate) + default: + return null; + } + } + + private static BucketAggregate ToDateHistogramBucketAggregate(ElasticAggregations.DateHistogramAggregate aggregate, ITextSerializer serializer, ILogger logger = null) + { + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); + + // Check if there's a timezone offset in the metadata + bool hasTimezone = data.TryGetValue("@timezone", out object timezoneValue) && timezoneValue != null; + + var buckets = aggregate.Buckets.Select(b => { - var hits = _getHits.Value(topHitsAggregate); - var docs = hits?.Select(h => new ElasticLazyDocument(h)).Cast().ToList(); + // When there's a timezone, the bucket key from Elasticsearch already represents the local time boundary + // We use Unspecified kind since the dates are adjusted for the timezone + DateTime date = hasTimezone + ? DateTime.SpecifyKind(b.Key.UtcDateTime, DateTimeKind.Unspecified) + : b.Key.UtcDateTime; + var keyAsLong = b.Key.ToUnixTimeMilliseconds(); + // Propagate timezone metadata to bucket data for round-trip serialization + var bucketData = new Dictionary { { "@type", "datehistogram" } }; + if (hasTimezone) + bucketData["@timezone"] = timezoneValue; - return new TopHitsAggregate(docs) + return (IBucket)new DateHistogramBucket(date, b.ToAggregations(serializer, logger)) { - Total = topHitsAggregate.Total.Value, - MaxScore = topHitsAggregate.MaxScore, - Data = topHitsAggregate.Meta.ToReadOnlyData() + Total = b.DocCount, + Key = keyAsLong, + KeyAsString = b.KeyAsString ?? date.ToString("O"), + Data = bucketData }; + }).ToList(); + + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; + } + + private static BucketAggregate ToTermsBucketAggregate(ElasticAggregations.StringTermsAggregate aggregate, string name, ITextSerializer serializer, ILogger logger = null) + { + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); + if (aggregate.DocCountErrorUpperBound.GetValueOrDefault() > 0) + { + logger?.LogWarning("Terms aggregation {AggregationName} has doc_count_error_upper_bound of {DocCountErrorUpperBound}. Results may be inaccurate. Consider increasing shard_size.", name, aggregate.DocCountErrorUpperBound); + data.Add(nameof(aggregate.DocCountErrorUpperBound), aggregate.DocCountErrorUpperBound); } + if (aggregate.SumOtherDocCount.GetValueOrDefault() > 0) + data.Add(nameof(aggregate.SumOtherDocCount), aggregate.SumOtherDocCount); - if (aggregate is Nest.PercentilesAggregate percentilesAggregate) - return new PercentilesAggregate(percentilesAggregate.Items.Select(i => new PercentileItem { Percentile = i.Percentile, Value = i.Value })) - { - Data = percentilesAggregate.Meta.ToReadOnlyData() - }; + var buckets = aggregate.Buckets.Select(b => (IBucket)new KeyedBucket(b.ToAggregations(serializer, logger)) + { + Total = b.DocCount, + Key = b.Key.ToString(), + KeyAsString = b.Key.ToString(), + Data = _stringBucketData + }).ToList(); - if (aggregate is Nest.SingleBucketAggregate singleBucketAggregate) - return new SingleBucketAggregate(singleBucketAggregate.ToAggregations(logger)) - { - Data = singleBucketAggregate.Meta.ToReadOnlyData(), - Total = singleBucketAggregate.DocCount - }; + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; + } - if (aggregate is Nest.BucketAggregate bucketAggregation) + private static BucketAggregate ToTermsBucketAggregate(ElasticAggregations.LongTermsAggregate aggregate, string name, ITextSerializer serializer, ILogger logger = null) + { + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); + if (aggregate.DocCountErrorUpperBound.GetValueOrDefault() > 0) { - var data = new Dictionary((IDictionary)bucketAggregation.Meta ?? new Dictionary()); - if (bucketAggregation.DocCountErrorUpperBound.GetValueOrDefault() > 0) - { - logger?.LogWarning("Terms aggregation {AggregationName} has doc_count_error_upper_bound of {DocCountErrorUpperBound}. Results may be inaccurate. Consider increasing shard_size.", name, bucketAggregation.DocCountErrorUpperBound); - data.Add(nameof(bucketAggregation.DocCountErrorUpperBound), bucketAggregation.DocCountErrorUpperBound); - } - if (bucketAggregation.SumOtherDocCount.GetValueOrDefault() > 0) - data.Add(nameof(bucketAggregation.SumOtherDocCount), bucketAggregation.SumOtherDocCount); + logger?.LogWarning("Terms aggregation {AggregationName} has doc_count_error_upper_bound of {DocCountErrorUpperBound}. Results may be inaccurate. Consider increasing shard_size.", name, aggregate.DocCountErrorUpperBound); + data.Add(nameof(aggregate.DocCountErrorUpperBound), aggregate.DocCountErrorUpperBound); + } + if (aggregate.SumOtherDocCount.GetValueOrDefault() > 0) + data.Add(nameof(aggregate.SumOtherDocCount), aggregate.SumOtherDocCount); - return new BucketAggregate - { - Items = bucketAggregation.Items.Select(i => i.ToBucket(data, logger)).ToList(), - Data = new ReadOnlyDictionary(data).ToReadOnlyData(), - Total = bucketAggregation.DocCount - }; + var buckets = aggregate.Buckets.Select(b => (IBucket)new KeyedBucket(b.ToAggregations(serializer, logger)) + { + Total = b.DocCount, + Key = b.Key, + KeyAsString = b.KeyAsString ?? b.Key.ToString(), + Data = _doubleBucketData + }).ToList(); + + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; + } + + private static BucketAggregate ToTermsBucketAggregate(ElasticAggregations.DoubleTermsAggregate aggregate, string name, ITextSerializer serializer, ILogger logger = null) + { + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); + if (aggregate.DocCountErrorUpperBound.GetValueOrDefault() > 0) + { + logger?.LogWarning("Terms aggregation {AggregationName} has doc_count_error_upper_bound of {DocCountErrorUpperBound}. Results may be inaccurate. Consider increasing shard_size.", name, aggregate.DocCountErrorUpperBound); + data.Add(nameof(aggregate.DocCountErrorUpperBound), aggregate.DocCountErrorUpperBound); } + if (aggregate.SumOtherDocCount.GetValueOrDefault() > 0) + data.Add(nameof(aggregate.SumOtherDocCount), aggregate.SumOtherDocCount); + + var buckets = aggregate.Buckets.Select(b => (IBucket)new KeyedBucket(b.ToAggregations(serializer, logger)) + { + Total = b.DocCount, + Key = b.Key, + KeyAsString = b.KeyAsString ?? b.Key.ToString(), + Data = _doubleBucketData + }).ToList(); - return null; + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; } - private static DateTime GetDate(Nest.ValueAggregate valueAggregate) + private static BucketAggregate ToRangeBucketAggregate(ElasticAggregations.DateRangeAggregate aggregate, ITextSerializer serializer, ILogger logger = null) { - if (valueAggregate?.Value == null) - throw new ArgumentNullException(nameof(valueAggregate)); + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); + + var buckets = aggregate.Buckets.Select(b => (IBucket)new RangeBucket(b.ToAggregations(serializer, logger)) + { + Total = b.DocCount, + Key = b.Key, + From = b.From, + FromAsString = b.FromAsString, + To = b.To, + ToAsString = b.ToAsString, + Data = _rangeBucketData + }).ToList(); + + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; + } - var kind = DateTimeKind.Utc; - long ticks = _epochTicks + ((long)valueAggregate.Value * TimeSpan.TicksPerMillisecond); + private static BucketAggregate ToRangeBucketAggregate(ElasticAggregations.RangeAggregate aggregate, ITextSerializer serializer, ILogger logger = null) + { + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); - if (valueAggregate.Meta.TryGetValue("@timezone", out object value) && value != null) + var buckets = aggregate.Buckets.Select(b => (IBucket)new RangeBucket(b.ToAggregations(serializer, logger)) { - kind = DateTimeKind.Unspecified; - ticks -= Exceptionless.DateTimeExtensions.TimeUnit.Parse(value.ToString()).Ticks; + Total = b.DocCount, + Key = b.Key, + From = b.From, + FromAsString = b.FromAsString, + To = b.To, + ToAsString = b.ToAsString, + Data = _rangeBucketData + }).ToList(); + + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; + } + + private static BucketAggregate ToGeohashGridBucketAggregate(ElasticAggregations.GeohashGridAggregate aggregate, ITextSerializer serializer, ILogger logger = null) + { + var data = aggregate.Meta != null ? new Dictionary(aggregate.Meta) : new Dictionary(); + + var buckets = aggregate.Buckets.Select(b => (IBucket)new KeyedBucket(b.ToAggregations(serializer, logger)) + { + Total = b.DocCount, + Key = b.Key, + KeyAsString = b.Key, + Data = _geohashBucketData + }).ToList(); + + return new BucketAggregate + { + Items = buckets, + Data = new ReadOnlyDictionary(data).ToReadOnlyData() + }; + } + + private static IAggregate ToValueAggregate(double? value, IReadOnlyDictionary meta) + { + if (meta != null && meta.TryGetValue("@field_type", out object fieldType)) + { + string type = fieldType?.ToString(); + if (type == "date" && value.HasValue) + { + var kind = DateTimeKind.Utc; + long ticks = _epochTicks + ((long)value.Value * TimeSpan.TicksPerMillisecond); + + if (meta.TryGetValue("@timezone", out object timezoneValue) && timezoneValue != null) + { + kind = DateTimeKind.Unspecified; + ticks -= Exceptionless.DateTimeExtensions.TimeUnit.Parse(timezoneValue.ToString()).Ticks; + } + + return new ValueAggregate + { + Value = GetDate(ticks, kind), + Data = meta.ToReadOnlyData>() + }; + } } - return GetDate(ticks, kind); + return new ValueAggregate { Value = value, Data = meta.ToReadOnlyData() }; } private static DateTime GetDate(long ticks, DateTimeKind kind) @@ -446,86 +815,90 @@ private static DateTime GetDate(long ticks, DateTimeKind kind) return new DateTime(ticks, kind); } - public static IBucket ToBucket(this Nest.IBucket bucket, IDictionary parentData = null, ILogger logger = null) + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.AggregateDictionary aggregations, ITextSerializer serializer, ILogger logger = null) { - if (bucket is Nest.DateHistogramBucket dateHistogramBucket) - { - bool hasTimezone = parentData != null && parentData.ContainsKey("@timezone"); - var kind = hasTimezone ? DateTimeKind.Unspecified : DateTimeKind.Utc; - long ticks = _epochTicks + ((long)dateHistogramBucket.Key * TimeSpan.TicksPerMillisecond); - var date = GetDate(ticks, kind); - var data = new Dictionary { { "@type", "datehistogram" } }; + if (aggregations == null) + return null; - if (hasTimezone) - data.Add("@timezone", parentData["@timezone"]); + return aggregations.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } - return new DateHistogramBucket(date, dateHistogramBucket.ToAggregations(logger)) - { - Total = dateHistogramBucket.DocCount, - Key = dateHistogramBucket.Key, - KeyAsString = date.ToString("O"), - Data = data - }; - } + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.DateHistogramBucket bucket, ITextSerializer serializer, ILogger logger = null) + { + return bucket.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } - if (bucket is Nest.RangeBucket rangeBucket) - return new RangeBucket(rangeBucket.ToAggregations(logger)) - { - Total = rangeBucket.DocCount, - Key = rangeBucket.Key, - From = rangeBucket.From, - FromAsString = rangeBucket.FromAsString, - To = rangeBucket.To, - ToAsString = rangeBucket.ToAsString, - Data = new Dictionary { { "@type", "range" } } - }; + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.StringTermsBucket bucket, ITextSerializer serializer, ILogger logger = null) + { + return bucket.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } - if (bucket is Nest.KeyedBucket stringKeyedBucket) - return new KeyedBucket(stringKeyedBucket.ToAggregations(logger)) - { - Total = stringKeyedBucket.DocCount, - Key = stringKeyedBucket.Key, - KeyAsString = stringKeyedBucket.KeyAsString, - Data = new Dictionary { { "@type", "string" } } - }; + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.LongTermsBucket bucket, ITextSerializer serializer, ILogger logger = null) + { + return bucket.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } - if (bucket is Nest.KeyedBucket doubleKeyedBucket) - return new KeyedBucket(doubleKeyedBucket.ToAggregations(logger)) - { - Total = doubleKeyedBucket.DocCount, - Key = doubleKeyedBucket.Key, - KeyAsString = doubleKeyedBucket.KeyAsString, - Data = new Dictionary { { "@type", "double" } } - }; + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.DoubleTermsBucket bucket, ITextSerializer serializer, ILogger logger = null) + { + return bucket.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } - if (bucket is Nest.KeyedBucket objectKeyedBucket) - return new KeyedBucket(objectKeyedBucket.ToAggregations(logger)) - { - Total = objectKeyedBucket.DocCount, - Key = objectKeyedBucket.Key, - KeyAsString = objectKeyedBucket.KeyAsString, - Data = new Dictionary { { "@type", "object" } } - }; + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.RangeBucket bucket, ITextSerializer serializer, ILogger logger = null) + { + return bucket.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } - return null; + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.GeohashGridBucket bucket, ITextSerializer serializer, ILogger logger = null) + { + return bucket.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); } - public static IReadOnlyDictionary ToAggregations(this IReadOnlyDictionary aggregations, ILogger logger = null) + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.FilterAggregate aggregate, ITextSerializer serializer, ILogger logger = null) { - return aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, logger)); + return aggregate.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); } - public static IReadOnlyDictionary ToAggregations(this Nest.ISearchResponse res, ILogger logger = null) where T : class + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.GlobalAggregate aggregate, ITextSerializer serializer, ILogger logger = null) { - return res.Aggregations.ToAggregations(logger); + return aggregate.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); } - public static IReadOnlyDictionary ToAggregations(this Nest.IAsyncSearchResponse res, ILogger logger = null) where T : class + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.MissingAggregate aggregate, ITextSerializer serializer, ILogger logger = null) { - return res.Response.Aggregations.ToAggregations(logger); + return aggregate.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); } - public static Nest.PropertiesDescriptor SetupDefaults(this Nest.PropertiesDescriptor pd) where T : class + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.NestedAggregate aggregate, ITextSerializer serializer, ILogger logger = null) + { + return aggregate.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } + + public static IReadOnlyDictionary ToAggregations(this ElasticAggregations.ReverseNestedAggregate aggregate, ITextSerializer serializer, ILogger logger = null) + { + return aggregate.Aggregations?.ToDictionary(a => a.Key, a => a.Value.ToAggregate(a.Key, serializer, logger)); + } + + public static IReadOnlyDictionary ToAggregations(this SearchResponse res, ITextSerializer serializer, ILogger logger = null) where T : class + { + return res.Aggregations.ToAggregations(serializer, logger); + } + + public static IReadOnlyDictionary ToAggregations(this SubmitAsyncSearchResponse res, ITextSerializer serializer, ILogger logger = null) where T : class + { + return res.Response?.Aggregations.ToAggregations(serializer, logger); + } + + public static IReadOnlyDictionary ToAggregations(this GetAsyncSearchResponse res, ITextSerializer serializer, ILogger logger = null) where T : class + { + return res.Response?.Aggregations.ToAggregations(serializer, logger); + } + + public static IReadOnlyDictionary ToAggregations(this ScrollResponse res, ITextSerializer serializer, ILogger logger = null) where T : class + { + return res.Aggregations.ToAggregations(serializer, logger); + } + + public static PropertiesDescriptor SetupDefaults(this PropertiesDescriptor pd) where T : class { bool hasIdentity = typeof(IIdentity).IsAssignableFrom(typeof(T)); bool hasDates = typeof(IHaveDates).IsAssignableFrom(typeof(T)); @@ -535,19 +908,28 @@ public static Nest.PropertiesDescriptor SetupDefaults(this Nest.Properties bool hasVirtualCustomFields = typeof(IHaveVirtualCustomFields).IsAssignableFrom(typeof(T)); if (hasIdentity) - pd.Keyword(p => p.Name(d => ((IIdentity)d).Id)); + pd.Keyword(d => ((IIdentity)d).Id); if (supportsSoftDeletes) - pd.Boolean(p => p.Name(d => ((ISupportSoftDeletes)d).IsDeleted)).FieldAlias(a => a.Path(p => ((ISupportSoftDeletes)p).IsDeleted).Name("deleted")); + { + pd.Boolean(d => ((ISupportSoftDeletes)d).IsDeleted); + pd.FieldAlias("deleted", a => a.Path(p => ((ISupportSoftDeletes)p).IsDeleted)); + } if (hasCreatedDate) - pd.Date(p => p.Name(d => ((IHaveCreatedDate)d).CreatedUtc)).FieldAlias(a => a.Path(p => ((IHaveCreatedDate)p).CreatedUtc).Name("created")); + { + pd.Date(d => ((IHaveCreatedDate)d).CreatedUtc); + pd.FieldAlias("created", a => a.Path(p => ((IHaveCreatedDate)p).CreatedUtc)); + } if (hasDates) - pd.Date(p => p.Name(d => ((IHaveDates)d).UpdatedUtc)).FieldAlias(a => a.Path(p => ((IHaveDates)p).UpdatedUtc).Name("updated")); + { + pd.Date(d => ((IHaveDates)d).UpdatedUtc); + pd.FieldAlias("updated", a => a.Path(p => ((IHaveDates)p).UpdatedUtc)); + } if (hasCustomFields || hasVirtualCustomFields) - pd.Object(f => f.Name("idx").Dynamic()); + pd.Object("idx", f => f.Dynamic(DynamicMapping.True)); return pd; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticLazyDocument.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticLazyDocument.cs index 97f8a183..ef81a6d8 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticLazyDocument.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/ElasticLazyDocument.cs @@ -1,65 +1,47 @@ -using System; -using System.IO; -using System.Reflection; -using Elasticsearch.Net; -using Nest; +using System; +using System.Text.Json; +using Elastic.Clients.Elasticsearch.Core.Search; +using Foundatio.Serializer; using ILazyDocument = Foundatio.Repositories.Models.ILazyDocument; namespace Foundatio.Repositories.Elasticsearch.Extensions; public class ElasticLazyDocument : ILazyDocument { - private readonly Nest.ILazyDocument _inner; - private IElasticsearchSerializer _requestResponseSerializer; + private readonly Hit _hit; + private readonly ITextSerializer _serializer; - public ElasticLazyDocument(Nest.ILazyDocument inner) + public ElasticLazyDocument(Hit hit, ITextSerializer serializer) { - _inner = inner; + _hit = hit; + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); } - private static readonly Lazy> _getSerializer = - new(() => - { - var serializerField = typeof(Nest.LazyDocument).GetField("_requestResponseSerializer", BindingFlags.GetField | BindingFlags.NonPublic | BindingFlags.Instance); - return lazyDocument => - { - var d = lazyDocument as Nest.LazyDocument; - if (d == null) - return null; - - var serializer = serializerField?.GetValue(d) as IElasticsearchSerializer; - return serializer; - }; - }); - - private static readonly Lazy> _getBytes = - new(() => - { - var bytesProperty = typeof(Nest.LazyDocument).GetProperty("Bytes", BindingFlags.GetProperty | BindingFlags.NonPublic | BindingFlags.Instance); - return lazyDocument => - { - var d = lazyDocument as Nest.LazyDocument; - if (d == null) - return null; - - var bytes = bytesProperty?.GetValue(d) as byte[]; - return bytes; - }; - }); - public T As() where T : class { - if (_requestResponseSerializer == null) - _requestResponseSerializer = _getSerializer.Value(_inner); + if (_hit?.Source is null) + return null; + + if (_hit.Source is T typed) + return typed; - var bytes = _getBytes.Value(_inner); - var hit = _requestResponseSerializer.Deserialize>(new MemoryStream(bytes)); - return hit?.Source; + if (_hit.Source is JsonElement jsonElement) + return _serializer.Deserialize(jsonElement.GetRawText()); + + return _serializer.Deserialize(_serializer.SerializeToString(_hit.Source)); } public object As(Type objectType) { - var hitType = typeof(IHit<>).MakeGenericType(objectType); - return _inner.As(hitType); + if (_hit?.Source is null) + return null; + + if (objectType.IsInstanceOfType(_hit.Source)) + return _hit.Source; + + if (_hit.Source is JsonElement jsonElement) + return _serializer.Deserialize(jsonElement.GetRawText(), objectType); + + return _serializer.Deserialize(_serializer.SerializeToString(_hit.Source), objectType); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/FindHitExtensions.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/FindHitExtensions.cs index 0fad7cd5..abddc7ab 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Extensions/FindHitExtensions.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/FindHitExtensions.cs @@ -1,24 +1,16 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; -using Nest; +using Foundatio.Serializer; namespace Foundatio.Repositories.Elasticsearch.Extensions; public static class FindHitExtensions { - private static JsonSerializerOptions _options; - static FindHitExtensions() - { - _options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; - _options.Converters.Add(new ObjectConverter()); - } - public static string GetIndex(this FindHit hit) { return hit?.Data?.GetString(ElasticDataKeys.Index); @@ -29,8 +21,39 @@ public static object[] GetSorts(this FindHit hit) if (hit == null || !hit.Data.TryGetValue(ElasticDataKeys.Sorts, out object sorts)) return Array.Empty(); - object[] sortsArray = sorts as object[]; - return sortsArray; + // Handle different collection types - new ES client returns IReadOnlyCollection + if (sorts is object[] sortsArray) + return sortsArray; + + if (sorts is IEnumerable fieldValues) + { + // Extract actual values from FieldValue objects + return fieldValues.Select(GetFieldValueAsObject).ToArray(); + } + + if (sorts is IEnumerable sortsList) + return sortsList.ToArray(); + + return Array.Empty(); + } + + private static object GetFieldValueAsObject(FieldValue fv) + { + // FieldValue is a tagged union in the new ES client + // We need to extract the actual value based on the variant + if (fv.TryGetLong(out var longVal)) + return longVal; + if (fv.TryGetDouble(out var doubleVal)) + return doubleVal; + if (fv.TryGetString(out var strVal)) + return strVal; + if (fv.TryGetBool(out var boolVal)) + return boolVal; + if (fv.IsNull) + return null; + + // Fallback - return the FieldValue itself + return fv; } public static string GetSearchBeforeToken(this FindResults results) where T : class @@ -49,45 +72,66 @@ public static string GetSearchAfterToken(this FindResults results) where T return results.Data.GetString(ElasticDataKeys.SearchAfterToken, null); } - internal static void SetSearchBeforeToken(this FindResults results) where T : class + internal static void SetSearchBeforeToken(this FindResults results, ITextSerializer serializer) where T : class { if (results == null || results.Hits.Count == 0) return; - string token = results.Hits.First().GetSortToken(); + string token = results.Hits.First().GetSortToken(serializer); if (!String.IsNullOrEmpty(token)) results.Data[ElasticDataKeys.SearchBeforeToken] = token; } - internal static void SetSearchAfterToken(this FindResults results) where T : class + internal static void SetSearchAfterToken(this FindResults results, ITextSerializer serializer) where T : class { if (results == null || results.Hits.Count == 0) return; - string token = results.Hits.Last().GetSortToken(); + string token = results.Hits.Last().GetSortToken(serializer); if (!String.IsNullOrEmpty(token)) results.Data[ElasticDataKeys.SearchAfterToken] = token; } - public static string GetSortToken(this FindHit hit) + public static string GetSortToken(this FindHit hit, ITextSerializer serializer) { object[] sorts = hit?.GetSorts(); if (sorts == null || sorts.Length == 0) return null; - return Encode(JsonSerializer.Serialize(sorts)); + return Encode(serializer.SerializeToString(sorts)); } - public static ISort ReverseOrder(this ISort sort) + public static SortOptions ReverseOrder(this SortOptions sort) { if (sort == null) return null; - sort.Order = !sort.Order.HasValue || sort.Order == SortOrder.Ascending ? SortOrder.Descending : SortOrder.Ascending; + // SortOptions is a discriminated union - we need to reverse the order on the underlying variant + if (sort.Field != null) + { + sort.Field.Order = !sort.Field.Order.HasValue || sort.Field.Order == SortOrder.Asc ? SortOrder.Desc : SortOrder.Asc; + } + else if (sort.Score != null) + { + sort.Score.Order = !sort.Score.Order.HasValue || sort.Score.Order == SortOrder.Asc ? SortOrder.Desc : SortOrder.Asc; + } + else if (sort.Doc != null) + { + sort.Doc.Order = !sort.Doc.Order.HasValue || sort.Doc.Order == SortOrder.Asc ? SortOrder.Desc : SortOrder.Asc; + } + else if (sort.GeoDistance != null) + { + sort.GeoDistance.Order = !sort.GeoDistance.Order.HasValue || sort.GeoDistance.Order == SortOrder.Asc ? SortOrder.Desc : SortOrder.Asc; + } + else if (sort.Script != null) + { + sort.Script.Order = !sort.Script.Order.HasValue || sort.Script.Order == SortOrder.Asc ? SortOrder.Desc : SortOrder.Asc; + } + return sort; } - public static IEnumerable ReverseOrder(this IEnumerable sorts) + public static IEnumerable ReverseOrder(this IEnumerable sorts) { if (sorts == null) return null; @@ -97,10 +141,9 @@ public static IEnumerable ReverseOrder(this IEnumerable sorts) return sortList; } - public static object[] DecodeSortToken(string sortToken) + public static object[] DecodeSortToken(string sortToken, ITextSerializer serializer) { - object[] tokens = JsonSerializer.Deserialize(Decode(sortToken), _options); - return tokens; + return serializer.Deserialize(Decode(sortToken)); } private static string Encode(string text) @@ -137,33 +180,3 @@ public static class ElasticDataKeys public const string SearchBeforeToken = nameof(SearchBeforeToken); public const string SearchAfterToken = nameof(SearchAfterToken); } - -public class ObjectConverter : JsonConverter -{ - public override object Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - return reader.TokenType switch - { - JsonTokenType.Number => GetNumber(reader), - JsonTokenType.String => reader.GetString(), - JsonTokenType.True => reader.GetBoolean(), - JsonTokenType.False => reader.GetBoolean(), - _ => null - }; - } - - private object GetNumber(Utf8JsonReader reader) - { - if (reader.TryGetInt64(out var l)) - return l; - else if (reader.TryGetDecimal(out var d)) - return d; - else - throw new InvalidOperationException("Value is not a number"); - } - - public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } -} diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/IBodyWithApiCallDetailsExtensions.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/IBodyWithApiCallDetailsExtensions.cs index 5724706c..f71fe69e 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Extensions/IBodyWithApiCallDetailsExtensions.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/IBodyWithApiCallDetailsExtensions.cs @@ -1,19 +1,23 @@ -using System.Text; -using System.Text.Json; -using Nest; +using System; +using Elastic.Transport.Products.Elasticsearch; +using Foundatio.Serializer; namespace Foundatio.Repositories.Elasticsearch.Extensions; internal static class IBodyWithApiCallDetailsExtensions { - private static readonly JsonSerializerOptions _options = new() { PropertyNameCaseInsensitive = true, }; - - public static T DeserializeRaw(this IResponse call) where T : class, new() + public static T DeserializeRaw(this ElasticsearchResponse call, ITextSerializer serializer) where T : class, new() { - if (call?.ApiCall?.ResponseBodyInBytes == null) + ArgumentNullException.ThrowIfNull(serializer); + + if (call?.ApiCallDetails?.ResponseBodyInBytes == null) return default; - string rawResponse = Encoding.UTF8.GetString(call.ApiCall.ResponseBodyInBytes); - return JsonSerializer.Deserialize(rawResponse, _options); + return serializer.Deserialize(call.ApiCallDetails.ResponseBodyInBytes); + } + + public static Exception OriginalException(this ElasticsearchResponse response) + { + return response?.ApiCallDetails?.OriginalException; } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/LoggerExtensions.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/LoggerExtensions.cs index 2611f282..dbc8dc54 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Extensions/LoggerExtensions.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/LoggerExtensions.cs @@ -1,61 +1,57 @@ -using System; +using System; using System.IO; using System.Linq; using System.Text; using System.Text.Encodings.Web; using System.Text.Json; -using Elasticsearch.Net; +using Elastic.Transport.Products.Elasticsearch; using Foundatio.Parsers.ElasticQueries.Extensions; using Microsoft.Extensions.Logging; -using Nest; using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Foundatio.Repositories.Elasticsearch.Extensions; public static class LoggerExtensions { - [Obsolete("Use LogRequest instead")] - public static void LogTraceRequest(this ILogger logger, IElasticsearchResponse elasticResponse, LogLevel logLevel = LogLevel.Trace) - { - LogRequest(logger, elasticResponse, logLevel); - } - - public static void LogRequest(this ILogger logger, IElasticsearchResponse elasticResponse, LogLevel logLevel = LogLevel.Trace) + public static void LogRequest(this ILogger logger, ElasticsearchResponse elasticResponse, LogLevel logLevel = LogLevel.Trace) { if (elasticResponse == null || !logger.IsEnabled(logLevel)) return; - var apiCall = elasticResponse?.ApiCall; + var apiCall = elasticResponse.ApiCallDetails; if (apiCall?.RequestBodyInBytes != null) { - string body = Encoding.UTF8.GetString(apiCall?.RequestBodyInBytes); + string body = Encoding.UTF8.GetString(apiCall.RequestBodyInBytes); body = JsonUtility.Normalize(body); logger.Log(logLevel, "[{HttpStatusCode}] {HttpMethod} {HttpPathAndQuery}\r\n{HttpBody}", apiCall.HttpStatusCode, apiCall.HttpMethod, apiCall.Uri.PathAndQuery, body); } - else + else if (apiCall != null) { logger.Log(logLevel, "[{HttpStatusCode}] {HttpMethod} {HttpPathAndQuery}", apiCall.HttpStatusCode, apiCall.HttpMethod, apiCall.Uri.PathAndQuery); } } - public static void LogErrorRequest(this ILogger logger, IElasticsearchResponse elasticResponse, string message, params object[] args) + public static void LogErrorRequest(this ILogger logger, ElasticsearchResponse elasticResponse, string message, params object[] args) { LogErrorRequest(logger, null, elasticResponse, message, args); } - public static void LogErrorRequest(this ILogger logger, Exception ex, IElasticsearchResponse elasticResponse, string message, params object[] args) + public static void LogErrorRequest(this ILogger logger, Exception ex, ElasticsearchResponse elasticResponse, string message, params object[] args) { if (elasticResponse == null || !logger.IsEnabled(LogLevel.Error)) return; - var response = elasticResponse as IResponse; + var originalException = elasticResponse.ApiCallDetails?.OriginalException; AggregateException aggEx = null; - if (ex != null && response?.OriginalException != null) - aggEx = new AggregateException(ex, response.OriginalException); + if (ex != null && originalException != null) + aggEx = new AggregateException(ex, originalException); - logger.LogError(aggEx ?? response?.OriginalException, elasticResponse.GetErrorMessage(message), args); + var allArgs = new object[args.Length + 1]; + args.CopyTo(allArgs, 0); + allArgs[^1] = elasticResponse.GetErrorMessage(); + logger.LogError(aggEx ?? originalException, message + ": {ElasticError}", allArgs); } } @@ -72,7 +68,7 @@ public static string Normalize(string jsonStr) public static string Normalize(JsonElement element) { - var ms = new MemoryStream(); + using var ms = new MemoryStream(); var opts = new JsonWriterOptions { Indented = true, @@ -134,7 +130,7 @@ private static void Write(JsonElement element, Utf8JsonWriter writer) break; default: - throw new NotImplementedException($"Kind: {element.ValueKind}"); + throw new NotSupportedException($"Unsupported JsonValueKind: {element.ValueKind}"); } } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/ResolverExtensions.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/ResolverExtensions.cs index 931c7186..b8a59f87 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Extensions/ResolverExtensions.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/ResolverExtensions.cs @@ -1,8 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Extensions; @@ -16,7 +16,7 @@ public static ICollection GetResolvedFields(this ElasticMappingResolver r return fields.Select(field => ResolveFieldName(resolver, field)).ToList(); } - public static ICollection GetResolvedFields(this ElasticMappingResolver resolver, ICollection sorts) + public static ICollection GetResolvedFields(this ElasticMappingResolver resolver, ICollection sorts) { if (sorts.Count == 0) return sorts; @@ -29,21 +29,29 @@ public static Field ResolveFieldName(this ElasticMappingResolver resolver, Field if (field is null) throw new ArgumentNullException(nameof(field)); - return new Field(resolver.GetResolvedField(field), field.Boost, field.Format); + return new Field(resolver.GetResolvedField(field), field.Boost); } - public static IFieldSort ResolveFieldSort(this ElasticMappingResolver resolver, IFieldSort sort) + public static SortOptions ResolveFieldSort(this ElasticMappingResolver resolver, SortOptions sort) { - return new FieldSort + // SortOptions is a discriminated union - check if it's a field sort + if (sort?.Field != null) { - Field = resolver.GetSortFieldName(sort.SortKey), - IgnoreUnmappedFields = sort.IgnoreUnmappedFields, - Missing = sort.Missing, - Mode = sort.Mode, - Nested = sort.Nested, - NumericType = sort.NumericType, - Order = sort.Order, - UnmappedType = sort.UnmappedType - }; + var fieldSort = sort.Field; + var resolvedField = resolver.GetSortFieldName(fieldSort.Field); + // Create a new FieldSort with the resolved field name + return new FieldSort + { + Field = resolvedField, + Missing = fieldSort.Missing, + Mode = fieldSort.Mode, + Nested = fieldSort.Nested, + NumericType = fieldSort.NumericType, + Order = fieldSort.Order, + UnmappedType = fieldSort.UnmappedType + }; + } + + return sort; } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Extensions/TimeSpanExtensions.cs b/src/Foundatio.Repositories.Elasticsearch/Extensions/TimeSpanExtensions.cs new file mode 100644 index 00000000..73121234 --- /dev/null +++ b/src/Foundatio.Repositories.Elasticsearch/Extensions/TimeSpanExtensions.cs @@ -0,0 +1,27 @@ +using System; + +namespace Foundatio.Repositories.Elasticsearch.Extensions; + +internal static class TimeSpanExtensions +{ + /// + /// Converts a to an Elasticsearch duration string (e.g., "500ms", "30s", "5m", "2h", "1d"). + /// Uses the largest whole unit that fits: days, hours, minutes, seconds, then milliseconds. + /// + public static string ToElasticDuration(this TimeSpan timeSpan) + { + if (timeSpan.TotalDays >= 1 && timeSpan.TotalDays == Math.Truncate(timeSpan.TotalDays)) + return $"{(int)timeSpan.TotalDays}d"; + + if (timeSpan.TotalHours >= 1 && timeSpan.TotalHours == Math.Truncate(timeSpan.TotalHours)) + return $"{(int)timeSpan.TotalHours}h"; + + if (timeSpan.TotalMinutes >= 1 && timeSpan.TotalMinutes == Math.Truncate(timeSpan.TotalMinutes)) + return $"{(int)timeSpan.TotalMinutes}m"; + + if (timeSpan.TotalSeconds >= 1 && timeSpan.TotalSeconds == Math.Truncate(timeSpan.TotalSeconds)) + return $"{(int)timeSpan.TotalSeconds}s"; + + return $"{(int)timeSpan.TotalMilliseconds}ms"; + } +} diff --git a/src/Foundatio.Repositories.Elasticsearch/Foundatio.Repositories.Elasticsearch.csproj b/src/Foundatio.Repositories.Elasticsearch/Foundatio.Repositories.Elasticsearch.csproj index fadc08af..207c3ea3 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Foundatio.Repositories.Elasticsearch.csproj +++ b/src/Foundatio.Repositories.Elasticsearch/Foundatio.Repositories.Elasticsearch.csproj @@ -1,10 +1,13 @@ + + + - + diff --git a/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupIndexesJob.cs b/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupIndexesJob.cs index a4e610d8..db051f44 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupIndexesJob.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupIndexesJob.cs @@ -1,10 +1,12 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; using Exceptionless.DateTimeExtensions; using Foundatio.Jobs; using Foundatio.Lock; @@ -12,20 +14,19 @@ using Foundatio.Repositories.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Jobs; public class CleanupIndexesJob : IJob { - protected readonly IElasticClient _client; + protected readonly ElasticsearchClient _client; protected readonly ILogger _logger; protected readonly ILockProvider _lockProvider; protected readonly TimeProvider _timeProvider; private static readonly CultureInfo _enUS = new("en-US"); private readonly ICollection _indexes = new List(); - public CleanupIndexesJob(IElasticClient client, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) + public CleanupIndexesJob(ElasticsearchClient client, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) { _client = client; _lockProvider = lockProvider; @@ -54,14 +55,15 @@ public virtual async Task RunAsync(CancellationToken cancellationToke _logger.LogInformation("Starting index cleanup..."); var sw = Stopwatch.StartNew(); - var result = await _client.Cat.IndicesAsync( - d => d.RequestConfiguration(r => r.RequestTimeout(TimeSpan.FromMinutes(5))), cancellationToken).AnyContext(); + // Note: ResolveIndexAsync sends a body in ES 9.x client which ES rejects; use GetAsync instead. + var result = await _client.Indices.GetAsync(Indices.All, + d => d.IgnoreUnavailable().RequestConfiguration(r => r.RequestTimeout(TimeSpan.FromMinutes(5))), cancellationToken).AnyContext(); sw.Stop(); - if (result.IsValid) + if (result.IsValidResponse) { _logger.LogRequest(result); - _logger.LogInformation("Retrieved list of {IndexCount} indexes in {Duration:g}", result.Records?.Count, sw.Elapsed.ToWords(true)); + _logger.LogInformation("Retrieved list of {IndexCount} indexes in {Duration:g}", result.Indices?.Count, sw.Elapsed.ToWords(true)); } else { @@ -69,8 +71,8 @@ public virtual async Task RunAsync(CancellationToken cancellationToke } var indexes = new List(); - if (result.IsValid && result.Records != null) - indexes = result.Records?.Select(r => GetIndexDate(r.Index)).Where(r => r != null).ToList(); + if (result.IsValidResponse && result.Indices != null) + indexes = result.Indices?.Keys.Select(k => GetIndexDate(k.ToString())).Where(r => r != null).ToList(); if (indexes == null || indexes.Count == 0) return JobResult.Success; @@ -107,11 +109,11 @@ await _lockProvider.TryUsingAsync("es-delete-index", async t => { _logger.LogInformation("Got lock to delete index {OldIndex}", oldIndex.Index); sw.Restart(); - var response = await _client.Indices.DeleteAsync(oldIndex.Index, d => d, t).AnyContext(); + var response = await _client.Indices.DeleteAsync(Indices.Index(oldIndex.Index), t).AnyContext(); sw.Stop(); _logger.LogRequest(response); - if (response.IsValid) + if (response.IsValidResponse) await OnIndexDeleted(oldIndex.Index, sw.Elapsed).AnyContext(); else shouldContinue = await OnIndexDeleteFailure(oldIndex.Index, sw.Elapsed, response, null).AnyContext(); diff --git a/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupSnapshotJob.cs b/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupSnapshotJob.cs index 45d3cab0..759b6970 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupSnapshotJob.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Jobs/CleanupSnapshotJob.cs @@ -1,9 +1,11 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Snapshot; using Exceptionless.DateTimeExtensions; using Foundatio.Jobs; using Foundatio.Repositories.Elasticsearch.Extensions; @@ -11,29 +13,28 @@ using Foundatio.Resilience; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Jobs; public class CleanupSnapshotJob : IJob { - protected readonly IElasticClient _client; + protected readonly ElasticsearchClient _client; protected readonly IResiliencePolicyProvider _resiliencePolicyProvider; protected readonly IResiliencePolicy _resiliencePolicy; protected readonly TimeProvider _timeProvider; protected readonly ILogger _logger; private readonly ICollection _repositories = new List(); - public CleanupSnapshotJob(IElasticClient client, ILoggerFactory loggerFactory) : this(client, TimeProvider.System, new ResiliencePolicyProvider(), loggerFactory) + public CleanupSnapshotJob(ElasticsearchClient client, ILoggerFactory loggerFactory) : this(client, TimeProvider.System, new ResiliencePolicyProvider(), loggerFactory) { } - public CleanupSnapshotJob(IElasticClient client, TimeProvider timeProvider, ILoggerFactory loggerFactory) + public CleanupSnapshotJob(ElasticsearchClient client, TimeProvider timeProvider, ILoggerFactory loggerFactory) : this(client, timeProvider, new ResiliencePolicyProvider(), loggerFactory) { } - public CleanupSnapshotJob(IElasticClient client, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) + public CleanupSnapshotJob(ElasticsearchClient client, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) { _client = client; _resiliencePolicyProvider = resiliencePolicyProvider; @@ -74,15 +75,15 @@ private async Task DeleteOldSnapshotsAsync(string repo, TimeSpan maxAge, Cancell _logger.LogRequest(result); var snapshots = new List(); - if (result.IsValid && result.Snapshots != null) + if (result.IsValidResponse && result.Snapshots != null) { snapshots = result.Snapshots .Where(r => !String.Equals(r.State, "IN_PROGRESS")) - .Select(r => new SnapshotDate { Name = r.Name, Date = r.EndTime }) + .Select(r => new SnapshotDate { Name = r.Snapshot, Date = r.EndTime.HasValue ? r.EndTime.Value.UtcDateTime : DateTime.MinValue }) .ToList(); } - if (result.IsValid) + if (result.IsValidResponse) _logger.LogInformation("Retrieved list of {SnapshotCount} snapshots from {Repo} in {Duration:g}", snapshots.Count, repo, sw.Elapsed); else _logger.LogErrorRequest(result, "Failed to retrieve list of snapshots from {Repo} in {Duration:g}", repo, sw.Elapsed); @@ -125,10 +126,10 @@ await _resiliencePolicy.ExecuteAsync(async pct => { _logger.LogInformation("Deleting {SnapshotCount} expired snapshot(s) from {Repo}: {SnapshotNames}", snapshotCount, repo, snapshotNames); - var response = await _client.Snapshot.DeleteAsync(repo, snapshotNames, r => r.RequestConfiguration(c => c.RequestTimeout(TimeSpan.FromMinutes(5))), ct: pct).AnyContext(); + var response = await _client.Snapshot.DeleteAsync(repo, snapshotNames, r => r.RequestConfiguration(c => c.RequestTimeout(TimeSpan.FromMinutes(5))), cancellationToken: pct).AnyContext(); _logger.LogRequest(response); - if (response.IsValid) + if (response.IsValidResponse) { await OnSnapshotDeleted(snapshotNames, sw.Elapsed).AnyContext(); } @@ -136,9 +137,9 @@ await _resiliencePolicy.ExecuteAsync(async pct => { shouldContinue = await OnSnapshotDeleteFailure(snapshotNames, sw.Elapsed, response, null).AnyContext(); if (shouldContinue) - throw response.OriginalException ?? new ApplicationException($"Failed deleting snapshot(s) \"{snapshotNames}\""); + throw response.OriginalException() ?? new ApplicationException($"Failed deleting snapshot(s) \"{snapshotNames}\""); } - }, cancellationToken); + }, cancellationToken).AnyContext(); sw.Stop(); } catch (Exception ex) diff --git a/src/Foundatio.Repositories.Elasticsearch/Jobs/ReindexWorkItemHandler.cs b/src/Foundatio.Repositories.Elasticsearch/Jobs/ReindexWorkItemHandler.cs index fefe1e0f..7c602cc2 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Jobs/ReindexWorkItemHandler.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Jobs/ReindexWorkItemHandler.cs @@ -1,10 +1,11 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Jobs; using Foundatio.Lock; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Jobs; @@ -13,9 +14,10 @@ public class ReindexWorkItemHandler : WorkItemHandlerBase private readonly ElasticReindexer _reindexer; private readonly ILockProvider _lockProvider; - public ReindexWorkItemHandler(IElasticClient client, ILockProvider lockProvider, ILoggerFactory loggerFactory = null) : base(loggerFactory) + public ReindexWorkItemHandler(ElasticsearchClient client, ITextSerializer serializer, ILockProvider lockProvider, ILoggerFactory loggerFactory = null) + : base(loggerFactory) { - _reindexer = new ElasticReindexer(client, loggerFactory?.CreateLogger()); + _reindexer = new ElasticReindexer(client, serializer, loggerFactory?.CreateLogger()); _lockProvider = lockProvider; AutoRenewLockOnProgress = true; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Jobs/SnapshotJob.cs b/src/Foundatio.Repositories.Elasticsearch/Jobs/SnapshotJob.cs index ae23822a..ed9ab5a8 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Jobs/SnapshotJob.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Jobs/SnapshotJob.cs @@ -1,9 +1,11 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Snapshot; using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Parsers.ElasticQueries.Extensions; @@ -13,24 +15,23 @@ using Foundatio.Resilience; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Jobs; public class SnapshotJob : IJob { - protected readonly IElasticClient _client; + protected readonly ElasticsearchClient _client; protected readonly ILockProvider _lockProvider; protected readonly IResiliencePolicyProvider _resiliencePolicyProvider; protected readonly IResiliencePolicy _resiliencePolicy; protected readonly TimeProvider _timeProvider; protected readonly ILogger _logger; - public SnapshotJob(IElasticClient client, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : this(client, lockProvider, timeProvider, new ResiliencePolicyProvider(), loggerFactory) + public SnapshotJob(ElasticsearchClient client, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : this(client, lockProvider, timeProvider, new ResiliencePolicyProvider(), loggerFactory) { } - public SnapshotJob(IElasticClient client, ILockProvider lockProvider, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) + public SnapshotJob(ElasticsearchClient client, ILockProvider lockProvider, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) { _client = client; _lockProvider = lockProvider; @@ -43,14 +44,14 @@ public SnapshotJob(IElasticClient client, ILockProvider lockProvider, TimeProvid public virtual async Task RunAsync(CancellationToken cancellationToken = default) { - var hasSnapshotRepositoryResponse = await _client.Snapshot.GetRepositoryAsync(r => r.RepositoryName(Repository), cancellationToken); + var hasSnapshotRepositoryResponse = await _client.Snapshot.GetRepositoryAsync(r => r.Name(Repository), cancellationToken); _logger.LogRequest(hasSnapshotRepositoryResponse); - if (!hasSnapshotRepositoryResponse.IsValid) + if (!hasSnapshotRepositoryResponse.IsValidResponse) { - if (hasSnapshotRepositoryResponse.ApiCall.HttpStatusCode == 404) + if (hasSnapshotRepositoryResponse.ApiCallDetails.HttpStatusCode == 404) return JobResult.CancelledWithMessage($"Snapshot repository {Repository} has not been configured."); - return JobResult.FromException(hasSnapshotRepositoryResponse.OriginalException, hasSnapshotRepositoryResponse.GetErrorMessage()); + return JobResult.FromException(hasSnapshotRepositoryResponse.OriginalException(), hasSnapshotRepositoryResponse.GetErrorMessage()); } string snapshotName = _timeProvider.GetUtcNow().UtcDateTime.ToString("'" + Repository + "-'yyyy-MM-dd-HH-mm"); @@ -62,34 +63,34 @@ await _lockProvider.TryUsingAsync("es-snapshot", async lockCancellationToken => using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, lockCancellationToken); var result = await _resiliencePolicy.ExecuteAsync(async ct => { - var response = await _client.Snapshot.SnapshotAsync( + var response = await _client.Snapshot.CreateAsync( Repository, snapshotName, d => d .Indices(IncludedIndexes.Count > 0 ? String.Join(",", IncludedIndexes) : "*") - .IgnoreUnavailable() + .IgnoreUnavailable(true) .IncludeGlobalState(false) .WaitForCompletion(false) , ct).AnyContext(); _logger.LogRequest(response); // 400 means the snapshot already exists - if (!response.IsValid && response.ApiCall.HttpStatusCode != 400) - throw new RepositoryException(response.GetErrorMessage("Snapshot failed"), response.OriginalException); + if (!response.IsValidResponse && response.ApiCallDetails.HttpStatusCode != 400) + throw new RepositoryException(response.GetErrorMessage("Snapshot failed"), response.OriginalException()); return response; }, linkedCancellationTokenSource.Token).AnyContext(); - _logger.LogTrace("Started snapshot {SnapshotName} in {Repository}: httpstatus={StatusCode}", snapshotName, Repository, result.ApiCall?.HttpStatusCode); + _logger.LogTrace("Started snapshot {SnapshotName} in {Repository}: httpstatus={StatusCode}", snapshotName, Repository, result.ApiCallDetails?.HttpStatusCode); bool success = false; do { await Task.Delay(TimeSpan.FromSeconds(10), linkedCancellationTokenSource.Token).AnyContext(); - var status = await _client.Snapshot.StatusAsync(s => s.Snapshot(snapshotName).RepositoryName(Repository), linkedCancellationTokenSource.Token).AnyContext(); + var status = await _client.Snapshot.StatusAsync(s => s.Snapshot(snapshotName).Repository(Repository), linkedCancellationTokenSource.Token).AnyContext(); _logger.LogRequest(status); - if (status.IsValid && status.Snapshots.Count > 0) + if (status.IsValidResponse && status.Snapshots.Count > 0) { string state = status.Snapshots.First().State; if (state.Equals("SUCCESS", StringComparison.OrdinalIgnoreCase)) @@ -126,7 +127,7 @@ public virtual Task OnSuccess(string snapshotName, TimeSpan duration) return Task.CompletedTask; } - public virtual Task OnFailure(string snapshotName, SnapshotResponse response, TimeSpan duration) + public virtual Task OnFailure(string snapshotName, CreateSnapshotResponse response, TimeSpan duration) { _logger.LogErrorRequest(response, "Failed snapshot {SnapshotName} in {Repository} after {Duration:g}", snapshotName, Repository, duration); return Task.CompletedTask; diff --git a/src/Foundatio.Repositories.Elasticsearch/Options/ElasticCommandOptions.cs b/src/Foundatio.Repositories.Elasticsearch/Options/ElasticCommandOptions.cs index 84ca8ed4..4b9fffb8 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Options/ElasticCommandOptions.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Options/ElasticCommandOptions.cs @@ -1,5 +1,5 @@ using System; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Repositories.Elasticsearch.Configuration; diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ChildQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ChildQueryBuilder.cs index b2c0b201..776512e2 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ChildQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ChildQueryBuilder.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -63,22 +64,22 @@ public class ChildQueryBuilder : IElasticQueryBuilder childOptions.DocumentType(childQuery.GetDocumentType()); var childContext = new QueryBuilderContext(childQuery, childOptions, null); - await index.QueryBuilder.BuildAsync(childContext); + await index.QueryBuilder.BuildAsync(childContext).AnyContext(); - if (childContext.Filter != null && ((IQueryContainer)childContext.Filter).IsConditionless == false) + if (childContext.Filter != null) ctx.Filter &= new HasChildQuery { - Type = childQuery.GetDocumentType(), + Type = childQuery.GetDocumentType().Name.ToLowerInvariant(), Query = new BoolQuery { Filter = new[] { childContext.Filter } } }; - if (childContext.Query != null && ((IQueryContainer)childContext.Query).IsConditionless == false) + if (childContext.Query != null) ctx.Query &= new HasChildQuery { - Type = childQuery.GetDocumentType(), + Type = childQuery.GetDocumentType().Name.ToLowerInvariant(), Query = new BoolQuery { Must = new[] { childContext.Query } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DateRangeQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DateRangeQueryBuilder.cs index 86ffb876..661a4a68 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DateRangeQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DateRangeQueryBuilder.cs @@ -4,11 +4,12 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Exceptionless.DateTimeExtensions; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -110,9 +111,9 @@ public class DateRangeQueryBuilder : IElasticQueryBuilder { var rangeQuery = new DateRangeQuery { Field = resolver.ResolveFieldName(dateRange.Field) }; if (dateRange.UseStartDate) - rangeQuery.GreaterThanOrEqualTo = dateRange.GetStartDate(); + rangeQuery.Gte = dateRange.GetStartDate(); if (dateRange.UseEndDate) - rangeQuery.LessThanOrEqualTo = dateRange.GetEndDate(); + rangeQuery.Lte = dateRange.GetEndDate(); if (!String.IsNullOrEmpty(dateRange.TimeZone)) rangeQuery.TimeZone = dateRange.TimeZone; diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DefaultSortQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DefaultSortQueryBuilder.cs index bc28a695..31eda133 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DefaultSortQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/DefaultSortQueryBuilder.cs @@ -1,9 +1,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Queries.Builders; @@ -13,18 +13,33 @@ public class DefaultSortQueryBuilder : IElasticQueryBuilder public Task BuildAsync(QueryBuilderContext ctx) where T : class, new() { - if (ctx.Search is not ISearchRequest searchRequest) - return Task.CompletedTask; + // Get existing sorts from context data (set by SortQueryBuilder or ExpressionQueryBuilder) + List sortFields = null; + if (ctx.Data.TryGetValue(SortQueryBuilder.SortFieldsKey, out var sortsObj) && sortsObj is List sorts) + { + sortFields = sorts; + } + + sortFields ??= new List(); var resolver = ctx.GetMappingResolver(); string idField = resolver.GetResolvedField(Id) ?? "_id"; - searchRequest.Sort ??= new List(); - var sortFields = searchRequest.Sort; - // ensure id field is always present as a sort (default or tiebreaker) - if (!sortFields.Any(s => idField.Equals(resolver.GetResolvedField(s.SortKey)))) + bool hasIdField = sortFields.Any(s => + { + if (s?.Field?.Field == null) + return false; + string fieldName = resolver.GetSortFieldName(s.Field.Field); + return fieldName?.Equals(idField) == true; + }); + + if (!hasIdField) + { sortFields.Add(new FieldSort { Field = idField }); + } + + ctx.Data[SortQueryBuilder.SortFieldsKey] = sortFields; return Task.CompletedTask; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ElasticFilterQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ElasticFilterQueryBuilder.cs index 656e8155..bed3df2d 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ElasticFilterQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ElasticFilterQueryBuilder.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -9,7 +9,7 @@ public static class ElasticFilterQueryExtensions { internal const string ElasticFiltersKey = "@ElasticFilters"; - public static T ElasticFilter(this T query, QueryContainer filter) where T : IRepositoryQuery + public static T ElasticFilter(this T query, Query filter) where T : IRepositoryQuery { return query.AddCollectionOptionValue(ElasticFiltersKey, filter); } @@ -20,9 +20,9 @@ namespace Foundatio.Repositories.Options { public static class ReadElasticFilterQueryExtensions { - public static ICollection GetElasticFilters(this IRepositoryQuery query) + public static ICollection GetElasticFilters(this IRepositoryQuery query) { - return query.SafeGetCollection(ElasticFilterQueryExtensions.ElasticFiltersKey); + return query.SafeGetCollection(ElasticFilterQueryExtensions.ElasticFiltersKey); } } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ExpressionQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ExpressionQueryBuilder.cs index 79d48594..514cc699 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ExpressionQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ExpressionQueryBuilder.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.ElasticQueries.Visitors; @@ -8,7 +9,6 @@ using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -144,7 +144,8 @@ public FieldResolverQueryBuilder(QueryFieldResolver aliasMap) var sortFields = GetSortFieldsVisitor.Run(result, ctx).ToList(); - ctx.Search.Sort(sortFields); + // Store sorts in context data - SearchAfterQueryBuilder will apply them + ctx.Data[SortQueryBuilder.SortFieldsKey] = sortFields; } return Task.CompletedTask; @@ -184,7 +185,8 @@ public class ExpressionQueryBuilder : IElasticQueryBuilder var sortFields = GetSortFieldsVisitor.Run(result, ctx).ToList(); - ctx.Search.Sort(sortFields); + // Store sorts in context data - SearchAfterQueryBuilder will apply them + ctx.Data[SortQueryBuilder.SortFieldsKey] = sortFields; } return Task.CompletedTask; @@ -217,7 +219,8 @@ public ParsedExpressionQueryBuilder(ElasticQueryParser parser) { var sortFields = (await _parser.BuildSortAsync(sort, ctx).AnyContext()).ToList(); - ctx.Search.Sort(sortFields); + // Store sorts in context data - SearchAfterQueryBuilder will apply them + ctx.Data[SortQueryBuilder.SortFieldsKey] = sortFields; } } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldConditionsQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldConditionsQueryBuilder.cs index 9f1c514f..dedb0d2f 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldConditionsQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldConditionsQueryBuilder.cs @@ -4,12 +4,14 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Repositories.Elasticsearch.Utility; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -218,32 +220,32 @@ public class FieldConditionsQueryBuilder : IElasticQueryBuilder switch (fieldValue.Operator) { case ComparisonOperator.Equals: - QueryBase eqQuery; + Query eqQuery; if (fieldValue.Value is IEnumerable and not string) { - var values = new List(); + var values = new List(); foreach (var value in (IEnumerable)fieldValue.Value) - values.Add(value); - eqQuery = new TermsQuery { Field = resolvedField, Terms = values }; + values.Add(ToFieldValue(value)); + eqQuery = new TermsQuery { Field = resolvedField, Terms = new TermsQueryField(values) }; } else - eqQuery = new TermQuery { Field = resolvedField, Value = fieldValue.Value }; + eqQuery = new TermQuery { Field = resolvedField, Value = ToFieldValue(fieldValue.Value) }; ctx.Filter &= eqQuery; break; case ComparisonOperator.NotEquals: - QueryBase neQuery; + Query neQuery; if (fieldValue.Value is IEnumerable and not string) { - var values = new List(); + var values = new List(); foreach (var value in (IEnumerable)fieldValue.Value) - values.Add(value); - neQuery = new TermsQuery { Field = resolvedField, Terms = values }; + values.Add(ToFieldValue(value)); + neQuery = new TermsQuery { Field = resolvedField, Terms = new TermsQueryField(values) }; } else - neQuery = new TermQuery { Field = resolvedField, Value = fieldValue.Value }; + neQuery = new TermQuery { Field = resolvedField, Value = ToFieldValue(fieldValue.Value) }; - ctx.Filter &= new BoolQuery { MustNot = new QueryContainer[] { neQuery } }; + ctx.Filter &= new BoolQuery { MustNot = new Query[] { neQuery } }; break; case ComparisonOperator.Contains: if (!resolver.IsPropertyAnalyzed(resolvedField)) @@ -262,11 +264,11 @@ public class FieldConditionsQueryBuilder : IElasticQueryBuilder string notContainsText = fieldValue.Value is IEnumerable and not string ? String.Join(" ", ((IEnumerable)fieldValue.Value).Cast()) : fieldValue.Value?.ToString() ?? String.Empty; - ctx.Filter &= new BoolQuery { MustNot = new QueryContainer[] { new MatchQuery { Field = resolvedField, Query = notContainsText, Operator = Operator.And } } }; + ctx.Filter &= new BoolQuery { MustNot = new Query[] { new MatchQuery { Field = resolvedField, Query = notContainsText, Operator = Operator.And } } }; break; case ComparisonOperator.IsEmpty: - ctx.Filter &= new BoolQuery { MustNot = new QueryContainer[] { new ExistsQuery { Field = resolvedField } } }; + ctx.Filter &= new BoolQuery { MustNot = new Query[] { new ExistsQuery { Field = resolvedField } } }; break; case ComparisonOperator.HasValue: ctx.Filter &= new ExistsQuery { Field = resolvedField }; @@ -291,5 +293,7 @@ public class FieldConditionsQueryBuilder : IElasticQueryBuilder return resolved; } + + private static FieldValue ToFieldValue(object value) => FieldValueHelper.ToFieldValue(value); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldIncludesQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldIncludesQueryBuilder.cs index 962ac0c0..3035ec72 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldIncludesQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/FieldIncludesQueryBuilder.cs @@ -3,12 +3,13 @@ using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.Search; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Utility; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -380,12 +381,16 @@ public class FieldIncludesQueryBuilder : IElasticQueryBuilder resolvedExcludes = resolvedExcludes.Where(f => !resolvedRequiredFields.Contains(f)).ToArray(); } - if (resolvedIncludes.Length > 0 && resolvedExcludes.Length > 0) - ctx.Search.Source(s => s.Includes(i => i.Fields(resolvedIncludes)).Excludes(i => i.Fields(resolvedExcludes))); - else if (resolvedIncludes.Length > 0) - ctx.Search.Source(s => s.Includes(i => i.Fields(resolvedIncludes))); - else if (resolvedExcludes.Length > 0) - ctx.Search.Source(s => s.Excludes(i => i.Fields(resolvedExcludes))); + if (resolvedIncludes.Length > 0 || resolvedExcludes.Length > 0) + { + var filter = new SourceFilter(); + if (resolvedIncludes.Length > 0) + filter.Includes = resolvedIncludes; + if (resolvedExcludes.Length > 0) + filter.Excludes = resolvedExcludes; + + ctx.Search.Source(new SourceConfig(filter)); + } return Task.CompletedTask; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IElasticQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IElasticQueryBuilder.cs index afb39034..39079ad5 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IElasticQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IElasticQueryBuilder.cs @@ -1,6 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.ElasticQueries.Visitors; @@ -9,7 +11,6 @@ using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Queries.Builders; @@ -20,11 +21,11 @@ public interface IElasticQueryBuilder public class QueryBuilderContext : IQueryBuilderContext, IElasticQueryVisitorContext, IQueryVisitorContextWithFieldResolver, IQueryVisitorContextWithIncludeResolver, IQueryVisitorContextWithValidation where T : class, new() { - public QueryBuilderContext(IRepositoryQuery source, ICommandOptions options, SearchDescriptor search = null, IQueryBuilderContext parentContext = null) + public QueryBuilderContext(IRepositoryQuery source, ICommandOptions options, SearchRequestDescriptor? search = null, IQueryBuilderContext parentContext = null) { Source = source; Options = options; - Search = search ?? new SearchDescriptor(); + Search = search ?? new SearchRequestDescriptor(); Parent = parentContext; ((IQueryVisitorContextWithIncludeResolver)this).IncludeResolver = options.GetIncludeResolver(); ((IQueryVisitorContextWithFieldResolver)this).FieldResolver = options.GetQueryFieldResolver(); @@ -41,9 +42,9 @@ public QueryBuilderContext(IRepositoryQuery source, ICommandOptions options, Sea public IQueryBuilderContext Parent { get; } public IRepositoryQuery Source { get; } public ICommandOptions Options { get; } - public QueryContainer Query { get; set; } - public QueryContainer Filter { get; set; } - public SearchDescriptor Search { get; } + public Query Query { get; set; } + public Query Filter { get; set; } + public SearchRequestDescriptor Search { get; } public IDictionary Data { get; } = new Dictionary(); public QueryValidationOptions ValidationOptions { get; set; } public QueryValidationResult ValidationResult { get; set; } @@ -53,6 +54,7 @@ public QueryBuilderContext(IRepositoryQuery source, ICommandOptions options, Sea ICollection IElasticQueryVisitorContext.RuntimeFields { get; } = new List(); bool? IElasticQueryVisitorContext.EnableRuntimeFieldResolver { get; set; } RuntimeFieldResolver IElasticQueryVisitorContext.RuntimeFieldResolver { get; set; } + Func> IElasticQueryVisitorContext.GeoLocationResolver { get; set; } GroupOperator IQueryVisitorContext.DefaultOperator { get; set; } Func> IElasticQueryVisitorContext.DefaultTimeZone { get; set; } @@ -77,8 +79,8 @@ public interface IQueryBuilderContext IQueryBuilderContext Parent { get; } IRepositoryQuery Source { get; } ICommandOptions Options { get; } - QueryContainer Query { get; set; } - QueryContainer Filter { get; set; } + Query Query { get; set; } + Query Filter { get; set; } IDictionary Data { get; } } @@ -112,7 +114,7 @@ public static Task GetTimeZoneAsync(this IQueryBuilderContext context) public static class ElasticQueryBuilderExtensions { - public static async Task BuildQueryAsync(this IElasticQueryBuilder builder, IRepositoryQuery query, ICommandOptions options, SearchDescriptor search) where T : class, new() + public static async Task BuildQueryAsync(this IElasticQueryBuilder builder, IRepositoryQuery query, ICommandOptions options, SearchRequestDescriptor search) where T : class, new() { var ctx = new QueryBuilderContext(query, options, search); await builder.BuildAsync(ctx).AnyContext(); @@ -124,12 +126,9 @@ public static class ElasticQueryBuilderExtensions }; } - public static async Task ConfigureSearchAsync(this IElasticQueryBuilder builder, IRepositoryQuery query, ICommandOptions options, SearchDescriptor search) where T : class, new() + public static async Task ConfigureSearchAsync(this IElasticQueryBuilder builder, IRepositoryQuery query, ICommandOptions options, SearchRequestDescriptor search) where T : class, new() { - if (search == null) - throw new ArgumentNullException(nameof(search)); - var q = await builder.BuildQueryAsync(query, options, search).AnyContext(); - search.Query(d => q); + search.Query(q); } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IdentityQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IdentityQueryBuilder.cs index 5af0c82b..0dff7a11 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IdentityQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/IdentityQueryBuilder.cs @@ -1,7 +1,9 @@ using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Queries; -using Nest; +using ElasticId = Elastic.Clients.Elasticsearch.Id; +using ElasticIds = Elastic.Clients.Elasticsearch.Ids; namespace Foundatio.Repositories.Elasticsearch.Queries.Builders; @@ -11,11 +13,11 @@ public class IdentityQueryBuilder : IElasticQueryBuilder { var ids = ctx.Source.GetIds(); if (ids.Count > 0) - ctx.Filter &= new IdsQuery { Values = ids.Select(id => new Nest.Id(id)) }; + ctx.Filter &= new IdsQuery { Values = new ElasticIds(ids.Select(id => new ElasticId(id))) }; var excludesIds = ctx.Source.GetExcludedIds(); if (excludesIds.Count > 0) - ctx.Filter &= !new IdsQuery { Values = excludesIds.Select(id => new Nest.Id(id)) }; + ctx.Filter &= new BoolQuery { MustNot = new Query[] { new IdsQuery { Values = new ElasticIds(excludesIds.Select(id => new ElasticId(id))) } } }; return Task.CompletedTask; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/PageableQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/PageableQueryBuilder.cs index 9e9b3daa..eb2a3095 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/PageableQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/PageableQueryBuilder.cs @@ -1,4 +1,6 @@ -using System.Threading.Tasks; +using System.Linq; +using System.Threading.Tasks; +using Foundatio.Repositories.Elasticsearch.Utility; using Foundatio.Repositories.Options; namespace Foundatio.Repositories.Elasticsearch.Queries.Builders; @@ -19,12 +21,15 @@ public class PageableQueryBuilder : IElasticQueryBuilder } // can only use search_after or skip + // Note: skip (from) is not allowed in scroll context, so only apply if not snapshot paging if (ctx.Options.HasSearchAfter()) - ctx.Search.SearchAfter(ctx.Options.GetSearchAfter()); + ctx.Search.SearchAfter(ctx.Options.GetSearchAfter().Select(FieldValueHelper.ToFieldValue).ToList()); else if (ctx.Options.HasSearchBefore()) - ctx.Search.SearchAfter(ctx.Options.GetSearchBefore()); - else if (ctx.Options.ShouldUseSkip()) - ctx.Search.Skip(ctx.Options.GetSkip()); + ctx.Search.SearchAfter(ctx.Options.GetSearchBefore().Select(FieldValueHelper.ToFieldValue).ToList()); + // Skip (from) is intentionally ignored during snapshot paging because Elasticsearch + // does not support the 'from' parameter in a point-in-time / scroll context. + else if (ctx.Options.ShouldUseSkip() && !ctx.Options.ShouldUseSnapshotPaging()) + ctx.Search.From(ctx.Options.GetSkip()); return Task.CompletedTask; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ParentQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ParentQueryBuilder.cs index c5bb66d5..94a413ca 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ParentQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/ParentQueryBuilder.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -84,31 +85,31 @@ public class ParentQueryBuilder : IElasticQueryBuilder { foreach (var parentQuery in parentQueries) { + if (parentQuery.GetDocumentType() == typeof(object)) + parentQuery.DocumentType(ctx.Options.ParentDocumentType()); + var parentOptions = ctx.Options.Clone(); parentOptions.DocumentType(parentQuery.GetDocumentType()); parentOptions.ParentDocumentType(null); - if (parentQuery.GetDocumentType() == typeof(object)) - parentQuery.DocumentType(ctx.Options.ParentDocumentType()); - var parentContext = new QueryBuilderContext(parentQuery, parentOptions, null); - await index.QueryBuilder.BuildAsync(parentContext); + await index.QueryBuilder.BuildAsync(parentContext).AnyContext(); - if (parentContext.Filter != null && ((IQueryContainer)parentContext.Filter).IsConditionless == false) + if (parentContext.Filter != null) ctx.Filter &= new HasParentQuery { - ParentType = parentQuery.GetDocumentType(), + ParentType = parentQuery.GetDocumentType().Name.ToLowerInvariant(), Query = new BoolQuery { Filter = new[] { parentContext.Filter } } }; - if (parentContext.Query != null && ((IQueryContainer)parentContext.Query).IsConditionless == false) + if (parentContext.Query != null) ctx.Query &= new HasParentQuery { - ParentType = parentQuery.GetDocumentType(), + ParentType = parentQuery.GetDocumentType().Name.ToLowerInvariant(), Query = new BoolQuery { Must = new[] { parentContext.Query } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/RuntimeFieldsQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/RuntimeFieldsQueryBuilder.cs index ec6e8b76..f3357589 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/RuntimeFieldsQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/RuntimeFieldsQueryBuilder.cs @@ -1,11 +1,12 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Repositories.Options; using Foundatio.Repositories.Queries; -using Nest; namespace Foundatio.Repositories { @@ -105,34 +106,37 @@ public class RuntimeFieldsQueryBuilder : IElasticQueryBuilder // fields need to be added to the context from the query before this if (elasticContext.RuntimeFields.Count > 0) - ctx.Search.RuntimeFields(f => + { + var runtimeMappings = new Dictionary(); + foreach (var field in elasticContext.RuntimeFields) { - foreach (var field in elasticContext.RuntimeFields) - f.RuntimeField(field.Name, GetFieldType(field.FieldType), d => - { - if (!String.IsNullOrEmpty(field.Script)) - d.Script(field.Script); - - return d; - }); - return f; - }); + var runtimeField = new RuntimeField + { + Type = GetFieldType(field.FieldType) + }; + if (!String.IsNullOrEmpty(field.Script)) + runtimeField.Script = new Script { Source = field.Script }; + + runtimeMappings[new Field(field.Name)] = runtimeField; + } + ctx.Search.RuntimeMappings(runtimeMappings); + } return Task.CompletedTask; } - private FieldType GetFieldType(ElasticRuntimeFieldType fieldType) + private RuntimeFieldType GetFieldType(ElasticRuntimeFieldType fieldType) { return fieldType switch { - ElasticRuntimeFieldType.Boolean => FieldType.Boolean, - ElasticRuntimeFieldType.Date => FieldType.Date, - ElasticRuntimeFieldType.Double => FieldType.Double, - ElasticRuntimeFieldType.GeoPoint => FieldType.GeoPoint, - ElasticRuntimeFieldType.Ip => FieldType.Ip, - ElasticRuntimeFieldType.Keyword => FieldType.Keyword, - ElasticRuntimeFieldType.Long => FieldType.Long, - _ => FieldType.Keyword, + ElasticRuntimeFieldType.Boolean => RuntimeFieldType.Boolean, + ElasticRuntimeFieldType.Date => RuntimeFieldType.Date, + ElasticRuntimeFieldType.Double => RuntimeFieldType.Double, + ElasticRuntimeFieldType.GeoPoint => RuntimeFieldType.GeoPoint, + ElasticRuntimeFieldType.Ip => RuntimeFieldType.Ip, + ElasticRuntimeFieldType.Keyword => RuntimeFieldType.Keyword, + ElasticRuntimeFieldType.Long => RuntimeFieldType.Long, + _ => RuntimeFieldType.Keyword, }; } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SearchAfterQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SearchAfterQueryBuilder.cs index aa08f8ee..44c4a85e 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SearchAfterQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SearchAfterQueryBuilder.cs @@ -1,9 +1,13 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; +using Foundatio.Serializer; namespace Foundatio.Repositories { @@ -33,12 +37,12 @@ public static T SearchAfter(this T options, params object[] values) where T : return options; } - public static T SearchAfterToken(this T options, string searchAfterToken) where T : ICommandOptions + public static T SearchAfterToken(this T options, string searchAfterToken, ITextSerializer serializer) where T : ICommandOptions { options.SearchAfterPaging(); if (!String.IsNullOrEmpty(searchAfterToken)) { - object[] values = FindHitExtensions.DecodeSortToken(searchAfterToken); + object[] values = FindHitExtensions.DecodeSortToken(searchAfterToken, serializer); options.Values.Set(SearchAfterKey, values); } else @@ -64,12 +68,12 @@ public static T SearchBefore(this T options, params object[] values) where T return options; } - public static T SearchBeforeToken(this T options, string searchBeforeToken) where T : ICommandOptions + public static T SearchBeforeToken(this T options, string searchBeforeToken, ITextSerializer serializer) where T : ICommandOptions { options.SearchAfterPaging(); if (!String.IsNullOrEmpty(searchBeforeToken)) { - object[] values = FindHitExtensions.DecodeSortToken(searchBeforeToken); + object[] values = FindHitExtensions.DecodeSortToken(searchBeforeToken, serializer); options.Values.Set(SearchBeforeKey, values); } else @@ -117,19 +121,58 @@ public static bool HasSearchBefore(this ICommandOptions options) namespace Foundatio.Repositories.Elasticsearch.Queries.Builders { + /// + /// Handles search_after paging by collecting sorts from context data, + /// adding the ID field for uniqueness, and reversing sorts for SearchBefore. + /// This builder runs last (Int32.MaxValue priority) so it sees all accumulated sorts. + /// public class SearchAfterQueryBuilder : IElasticQueryBuilder { + private const string Id = nameof(IIdentity.Id); + public Task BuildAsync(QueryBuilderContext ctx) where T : class, new() { - if (!ctx.Options.ShouldUseSearchAfterPaging()) - return Task.CompletedTask; + // Get sorts from context data (set by SortQueryBuilder or ExpressionQueryBuilder) + List sortFields = null; + if (ctx.Data.TryGetValue(SortQueryBuilder.SortFieldsKey, out var sortsObj) && sortsObj is List sorts) + { + sortFields = sorts; + } - if (ctx.Search is not ISearchRequest searchRequest) - return Task.CompletedTask; + // For search_after paging, we need to ensure we have at least the ID field for uniqueness + if (ctx.Options.ShouldUseSearchAfterPaging()) + { + sortFields ??= new List(); + + var resolver = ctx.GetMappingResolver(); + string idField = resolver.GetResolvedField(Id) ?? "_id"; + + // Check if id field is already in the sort list + bool hasIdField = sortFields.Any(s => + { + if (s?.Field?.Field == null) + return false; + string fieldName = resolver.GetSortFieldName(s.Field.Field); + return fieldName?.Equals(idField) == true; + }); + + if (!hasIdField) + { + sortFields.Add(new FieldSort { Field = idField }); + } + + // Reverse sort orders if searching before + if (ctx.Options.HasSearchBefore()) + { + sortFields = sortFields.Select(s => s.ReverseOrder()).ToList(); + } + } - // reverse sort orders on all sorts for search before - if (ctx.Options.HasSearchBefore()) - searchRequest.Sort?.ReverseOrder(); + // Apply sorts to search descriptor if we have any + if (sortFields != null && sortFields.Count > 0) + { + ctx.Search.Sort(sortFields); + } return Task.CompletedTask; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SoftDeletesQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SoftDeletesQueryBuilder.cs index f51af6bc..33ce3570 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SoftDeletesQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SoftDeletesQueryBuilder.cs @@ -1,7 +1,8 @@ using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Queries.Builders; @@ -40,10 +41,10 @@ public class SoftDeletesQueryBuilder : IElasticQueryBuilder if (parentType != null && parentType != typeof(object)) ctx.Filter &= new HasParentQuery { - ParentType = parentType, + ParentType = parentType.Name.ToLowerInvariant(), Query = new BoolQuery { - Filter = new[] { new QueryContainer(new TermQuery { Field = fieldName, Value = false }) } + Filter = [new TermQuery { Field = fieldName, Value = false }] } }; diff --git a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SortQueryBuilder.cs b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SortQueryBuilder.cs index 46d125cf..13995873 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SortQueryBuilder.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Queries/Builders/SortQueryBuilder.cs @@ -1,11 +1,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -15,32 +16,32 @@ public static class SortQueryExtensions public static T Sort(this T query, Field field, SortOrder? order = null) where T : IRepositoryQuery { - return query.AddCollectionOptionValue(SortsKey, new FieldSort { Field = field, Order = order }); + return query.AddCollectionOptionValue(SortsKey, new FieldSort { Field = field, Order = order }); } public static T SortDescending(this T query, Field field) where T : IRepositoryQuery { - return query.Sort(field, SortOrder.Descending); + return query.Sort(field, SortOrder.Desc); } public static T SortAscending(this T query, Field field) where T : IRepositoryQuery { - return query.Sort(field, SortOrder.Ascending); + return query.Sort(field, SortOrder.Asc); } public static IRepositoryQuery Sort(this IRepositoryQuery query, Expression> objectPath, SortOrder? order = null) where T : class { - return query.AddCollectionOptionValue, IFieldSort>(SortsKey, new FieldSort { Field = objectPath, Order = order }); + return query.AddCollectionOptionValue, SortOptions>(SortsKey, new FieldSort { Field = objectPath, Order = order }); } public static IRepositoryQuery SortDescending(this IRepositoryQuery query, Expression> objectPath) where T : class { - return query.Sort(objectPath, SortOrder.Descending); + return query.Sort(objectPath, SortOrder.Desc); } public static IRepositoryQuery SortAscending(this IRepositoryQuery query, Expression> objectPath) where T : class { - return query.Sort(objectPath, SortOrder.Ascending); + return query.Sort(objectPath, SortOrder.Asc); } } } @@ -49,9 +50,9 @@ namespace Foundatio.Repositories.Options { public static class ReadSortQueryExtensions { - public static ICollection GetSorts(this IRepositoryQuery query) + public static ICollection GetSorts(this IRepositoryQuery query) { - return query.SafeGetCollection(SortQueryExtensions.SortsKey); + return query.SafeGetCollection(SortQueryExtensions.SortsKey); } } } @@ -60,16 +61,22 @@ namespace Foundatio.Repositories.Elasticsearch.Queries.Builders { public class SortQueryBuilder : IElasticQueryBuilder { + internal const string SortFieldsKey = "__SortFields"; + public Task BuildAsync(QueryBuilderContext ctx) where T : class, new() { - var sortFields = ctx.Source.GetSorts(); + var sortFields = ctx.Source.GetSorts().ToList(); + if (sortFields.Count <= 0) return Task.CompletedTask; var resolver = ctx.GetMappingResolver(); - sortFields = resolver.GetResolvedFields(sortFields); + sortFields = resolver.GetResolvedFields(sortFields).ToList(); + + // Store sorts in context data - SearchAfterQueryBuilder will apply them + // along with any sorts from ExpressionQueryBuilder + ctx.Data[SortFieldsKey] = sortFields; - ctx.Search.Sort(sortFields); return Task.CompletedTask; } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Repositories/BulkResult.cs b/src/Foundatio.Repositories.Elasticsearch/Repositories/BulkResult.cs index 655a1691..cc34b9d0 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Repositories/BulkResult.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Repositories/BulkResult.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; +using Elastic.Clients.Elasticsearch; using Foundatio.Parsers.ElasticQueries.Extensions; -using Nest; +using Foundatio.Repositories.Elasticsearch.Extensions; namespace Foundatio.Repositories.Elasticsearch; @@ -36,12 +37,12 @@ internal sealed record BulkResult internal static BulkResult From(BulkResponse response) { - if (!response.IsValid && !response.ItemsWithErrors.Any()) + if (!response.IsValidResponse && !response.ItemsWithErrors.Any()) { return new BulkResult { TransportError = response.GetErrorMessage("Error processing bulk operation"), - TransportException = response.OriginalException + TransportException = response.OriginalException() }; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs b/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs index 891babb2..092705d7 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs @@ -4,7 +4,11 @@ using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.MGet; +using Elastic.Clients.Elasticsearch.Core.Search; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Elastic.Transport; using Foundatio.Caching; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; @@ -20,7 +24,7 @@ using Foundatio.Resilience; using Foundatio.Utility; using Microsoft.Extensions.Logging; -using Nest; +using ChangeType = Foundatio.Repositories.Models.ChangeType; namespace Foundatio.Repositories.Elasticsearch; @@ -41,8 +45,8 @@ namespace Foundatio.Repositories.Elasticsearch; protected readonly Lazy _updatedUtcField; protected readonly ILogger _logger; - protected readonly Lazy _lazyClient; - protected IElasticClient _client => _lazyClient.Value; + protected readonly Lazy _lazyClient; + protected ElasticsearchClient _client => _lazyClient.Value; protected readonly IResiliencePolicyProvider _resiliencePolicyProvider; protected readonly IResiliencePolicy _resiliencePolicy; @@ -58,7 +62,7 @@ protected ElasticReadOnlyRepositoryBase(IIndex index) _idField = new Lazy(() => InferField(d => ((IIdentity)d).Id) ?? "id"); if (HasDates) _updatedUtcField = new Lazy(() => InferField(d => ((IHaveDates)d).UpdatedUtc)); - _lazyClient = new Lazy(() => index.Configuration.Client); + _lazyClient = new Lazy(() => index.Configuration.Client); SetCacheClient(index.Configuration.Cache); _logger = index.Configuration.LoggerFactory.CreateLogger(GetType()); @@ -93,7 +97,7 @@ public virtual async Task GetByIdAsync(Id id, ICommandOptions options = null) options = ConfigureOptions(options.As()); - await OnBeforeGetAsync(new Ids(id), options, typeof(T)); + await OnBeforeGetAsync(new Ids(id), options, typeof(T)).AnyContext(); // we don't have the parent id so we have to do a query if (HasParent && id.Routing == null) @@ -123,8 +127,8 @@ public virtual async Task GetByIdAsync(Id id, ICommandOptions options = null) var response = await _client.GetAsync(request).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid && response.ApiCall.HttpStatusCode.GetValueOrDefault() != 404) - throw new DocumentException(response.GetErrorMessage($"Error getting document {id.Value}"), response.OriginalException); + if (!response.IsValidResponse && response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() != 404) + throw new DocumentException(response.GetErrorMessage($"Error getting document {id.Value}"), response.OriginalException()); var findHit = response.Found ? response.ToFindHit() : null; @@ -150,7 +154,7 @@ public virtual async Task> GetByIdsAsync(Ids ids, IComman options = ConfigureOptions(options.As()); - await OnBeforeGetAsync(new Ids(idList), options, typeof(T)); + await OnBeforeGetAsync(new Ids(idList), options, typeof(T)).AnyContext(); var hits = new List>(); if (IsCacheEnabled && options.ShouldReadCache()) @@ -160,30 +164,34 @@ public virtual async Task> GetByIdsAsync(Ids ids, IComman if (itemsToFind.Count == 0) return hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly(); - var multiGet = new MultiGetDescriptor(); - foreach (var id in itemsToFind.Where(i => i.Routing != null || !HasParent)) + // Build MultiGetOperation objects for each ID + var itemsForMultiGet = itemsToFind.Where(i => i.Routing != null || !HasParent).ToList(); + if (itemsForMultiGet.Count > 0) { - multiGet.Get(f => + var docOperations = itemsForMultiGet + .Select(id => + { + var op = new MultiGetOperation(id.Value) { Index = ElasticIndex.GetIndex(id) }; + if (id.Routing != null) + op.Routing = id.Routing; + return op; + }) + .ToList(); + + var multiGet = new MultiGetRequestDescriptor().Docs(docOperations); + + ConfigureMultiGetRequest(multiGet, options); + var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext(); + _logger.LogRequest(multiGetResults, options.GetQueryLogLevel()); + + if (!multiGetResults.IsValidResponse) + throw new DocumentException(multiGetResults.GetErrorMessage("Error getting documents"), multiGetResults.OriginalException()); + + foreach (var findHit in multiGetResults.ToFindHits(_logger)) { - f.Id(id.Value).Index(ElasticIndex.GetIndex(id)); - if (id.Routing != null) - f.Routing(id.Routing); - - return f; - }); - } - - ConfigureMultiGetRequest(multiGet, options); - var multiGetResults = await _client.MultiGetAsync(multiGet).AnyContext(); - _logger.LogRequest(multiGetResults, options.GetQueryLogLevel()); - - if (!multiGetResults.IsValid) - throw new DocumentException(multiGetResults.GetErrorMessage("Error getting documents"), multiGetResults.OriginalException); - - foreach (var doc in multiGetResults.Hits) - { - hits.Add(((IMultiGetHit)doc).ToFindHit()); - itemsToFind.Remove(new Id(doc.Id, doc.Routing)); + hits.Add(findHit); + itemsToFind.Remove(new Id(findHit.Id, findHit.Routing)); + } } // fallback to doing a find @@ -193,12 +201,24 @@ public virtual async Task> GetByIdsAsync(Ids ids, IComman do { if (response.Hits.Count > 0) - hits.AddRange(response.Hits.Where(h => h.Document != null)); + { + foreach (var hit in response.Hits.Where(h => h.Document != null)) + { + hits.Add(hit); + itemsToFind.Remove(new Id(hit.Id, hit.Routing)); + } + } } while (await response.NextPageAsync().AnyContext()); } if (IsCacheEnabled && options.ShouldUseCache()) + { + // Add null markers for IDs that were not found (to cache the "not found" result) + foreach (var id in itemsToFind) + hits.Add(new FindHit(id, null, 0)); + await AddDocumentsToCacheAsync(hits, options, false).AnyContext(); + } return hits.Where(h => h.Document != null && ShouldReturnDocument(h.Document, options)).Select(h => h.Document).ToList().AsReadOnly(); } @@ -226,18 +246,15 @@ public virtual async Task ExistsAsync(Id id, ICommandOptions options = nul // documents that use soft deletes or have parents without a routing id need to use search for exists if (!SupportsSoftDeletes && (!HasParent || id.Routing != null)) { - var response = await _client.DocumentExistsAsync(new DocumentPath(id.Value), d => - { - d.Index(ElasticIndex.GetIndex(id)); - if (id.Routing != null) - d.Routing(id.Routing); + var request = new ExistsRequest(ElasticIndex.GetIndex(id), id.Value); + if (id.Routing != null) + request.Routing = id.Routing; - return d; - }).AnyContext(); + var response = await _client.ExistsAsync(request).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid && response.ApiCall.HttpStatusCode.GetValueOrDefault() != 404) - throw new DocumentException(response.GetErrorMessage($"Error checking if document {id.Value} exists"), response.OriginalException); + if (!response.IsValidResponse && response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() != 404) + throw new DocumentException(response.GetErrorMessage($"Error checking if document {id.Value} exists"), response.OriginalException()); return response.Exists; } @@ -376,23 +393,23 @@ public Task> FindAsync(IRepositoryQuery query, ICommandOptions op { if (options.HasAsyncQueryWaitTime()) s.WaitForCompletionTimeout(options.GetAsyncQueryWaitTime()); - return s; }).AnyContext(); if (options.ShouldAutoDeleteAsyncQuery() && !response.IsRunning) - await RemoveQueryAsync(queryId); + await RemoveQueryAsync(queryId).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid && response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (!response.IsValidResponse && response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) throw new AsyncQueryNotFoundException(queryId); - result = response.ToFindResults(options, _logger); + result = response.ToFindResults(options, ElasticIndex.Configuration.Serializer, _logger); } else if (options.HasSnapshotScrollId()) { - var response = await _client.ScrollAsync(options.GetSnapshotLifetime(), options.GetSnapshotScrollId()).AnyContext(); + var scrollRequest = new ScrollRequest(options.GetSnapshotScrollId()) { Scroll = options.GetSnapshotLifetime() }; + var response = await _client.ScrollAsync(scrollRequest).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - result = response.ToFindResults(options, _logger); + result = response.ToFindResults(options, ElasticIndex.Configuration.Serializer, _logger); } else { @@ -405,20 +422,21 @@ public Task> FindAsync(IRepositoryQuery query, ICommandOptions op if (options.ShouldUseAsyncQuery()) { - var asyncSearchDescriptor = searchDescriptor.ToAsyncSearchSubmitDescriptor(); + SearchRequest searchRequest = searchDescriptor; + var asyncSearchRequest = searchRequest.ToAsyncSearchSubmitRequest(); if (options.HasAsyncQueryWaitTime()) - asyncSearchDescriptor.WaitForCompletionTimeout(options.GetAsyncQueryWaitTime()); + asyncSearchRequest.WaitForCompletionTimeout = options.GetAsyncQueryWaitTime(); - var response = await _client.AsyncSearch.SubmitAsync(asyncSearchDescriptor).AnyContext(); + var response = await _client.AsyncSearch.SubmitAsync(asyncSearchRequest).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - result = response.ToFindResults(options, _logger); + result = response.ToFindResults(options, ElasticIndex.Configuration.Serializer, _logger); } else { var response = await _client.SearchAsync(searchDescriptor).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - result = response.ToFindResults(options, _logger); + result = response.ToFindResults(options, ElasticIndex.Configuration.Serializer, _logger); } } @@ -430,7 +448,7 @@ public Task> FindAsync(IRepositoryQuery query, ICommandOptions op string scrollId = result.GetScrollId(); if (!String.IsNullOrEmpty(scrollId)) { - var response = await _client.ClearScrollAsync(s => s.ScrollId(result.GetScrollId())).AnyContext(); + var response = await _client.ClearScrollAsync(s => s.ScrollId(scrollId)).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); } } @@ -456,11 +474,12 @@ public async Task RemoveQueryAsync(string queryId) string scrollId = previousResults.GetScrollId(); if (!String.IsNullOrEmpty(scrollId)) { - var scrollResponse = await _client.ScrollAsync(options.GetSnapshotLifetime(), scrollId).AnyContext(); + var scrollRequest = new ScrollRequest(scrollId) { Scroll = options.GetSnapshotLifetime() }; + var scrollResponse = await _client.ScrollAsync(scrollRequest).AnyContext(); _logger.LogRequest(scrollResponse, options.GetQueryLogLevel()); - var results = scrollResponse.ToFindResults(options, _logger); - ((IFindResults)results).Page = previousResults.Page + 1; + var results = scrollResponse.ToFindResults(options, ElasticIndex.Configuration.Serializer, _logger); + ((IFindResults)results).Page = previousResults.Page + 1; // clear the scroll if (!results.HasMore) @@ -473,12 +492,9 @@ public async Task RemoveQueryAsync(string queryId) } if (options.ShouldUseSearchAfterPaging()) - options.SearchAfterToken(previousResults.GetSearchAfterToken()); + options.SearchAfterToken(previousResults.GetSearchAfterToken(), ElasticIndex.Configuration.Serializer); - if (options == null) - return new FindResults(); - - options?.PageNumber(!options.HasPageNumber() ? 2 : options.GetPage() + 1); + options.PageNumber(!options.HasPageNumber() ? 2 : options.GetPage() + 1); return await FindAsAsync(query, options).AnyContext(); } @@ -506,12 +522,12 @@ public virtual async Task> FindOneAsync(IRepositoryQuery query, IComm var response = await _client.SearchAsync(searchDescriptor).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return FindHit.Empty; - throw new DocumentException(response.GetErrorMessage("Error while finding document"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error while finding document"), response.OriginalException()); } result = response.Hits.Select(h => h.ToFindHit()).ToList(); @@ -555,31 +571,34 @@ public virtual async Task CountAsync(IRepositoryQuery query, IComma { if (options.HasAsyncQueryWaitTime()) s.WaitForCompletionTimeout(options.GetAsyncQueryWaitTime()); - return s; }).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); if (options.ShouldAutoDeleteAsyncQuery() && !response.IsRunning) - await RemoveQueryAsync(queryId); + await RemoveQueryAsync(queryId).AnyContext(); - result = response.ToCountResult(options, _logger); + result = response.ToCountResult(options, ElasticIndex.Configuration.Serializer, _logger); } else if (options.ShouldUseAsyncQuery()) { - var asyncSearchDescriptor = searchDescriptor.ToAsyncSearchSubmitDescriptor(); - - if (options.HasAsyncQueryWaitTime()) - asyncSearchDescriptor.WaitForCompletionTimeout(options.GetAsyncQueryWaitTime()); + var response = await _client.AsyncSearch.SubmitAsync(s => + { + string[] indices = ElasticIndex.GetIndexesByQuery(query); + if (indices?.Length > 0) + s.Indices(String.Join(",", indices)); + s.Size(0); - var response = await _client.AsyncSearch.SubmitAsync(asyncSearchDescriptor).AnyContext(); + if (options.HasAsyncQueryWaitTime()) + s.WaitForCompletionTimeout(options.GetAsyncQueryWaitTime()); + }).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - result = response.ToCountResult(options, _logger); + result = response.ToCountResult(options, ElasticIndex.Configuration.Serializer, _logger); } else { var response = await _client.SearchAsync(searchDescriptor).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - result = response.ToCountResult(options, _logger); + result = response.ToCountResult(options, ElasticIndex.Configuration.Serializer, _logger); } await OnAfterQueryAsync(query, options, result).AnyContext(); @@ -603,16 +622,16 @@ public virtual async Task ExistsAsync(IRepositoryQuery query, ICommandOpti await RefreshForConsistency(query, options).AnyContext(); var searchDescriptor = (await CreateSearchDescriptorAsync(query, options).AnyContext()).Size(0); - searchDescriptor.DocValueFields(_idField.Value); + searchDescriptor.DocvalueFields(new FieldAndFormat[] { new() { Field = _idField.Value } }); var response = await _client.SearchAsync(searchDescriptor).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { - if (response.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (response.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) return false; - throw new DocumentException(response.GetErrorMessage("Error checking if document exists"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error checking if document exists"), response.OriginalException()); } return response.Total > 0; @@ -753,27 +772,29 @@ protected virtual Task InvalidateCacheAsync(IReadOnlyCollection> CreateSearchDescriptorAsync(IRepositoryQuery query, ICommandOptions options) + protected virtual Task> CreateSearchDescriptorAsync(IRepositoryQuery query, ICommandOptions options) { - return ConfigureSearchDescriptorAsync(null, query, options); + return ConfigureSearchDescriptorAsync(new SearchRequestDescriptor(), query, options); } - protected virtual async Task> ConfigureSearchDescriptorAsync(SearchDescriptor search, IRepositoryQuery query, ICommandOptions options) + protected virtual async Task> ConfigureSearchDescriptorAsync(SearchRequestDescriptor search, IRepositoryQuery query, ICommandOptions options) { - search ??= new SearchDescriptor(); query = ConfigureQuery(query.As()).Unwrap(); string[] indices = ElasticIndex.GetIndexesByQuery(query); if (indices?.Length > 0) - search.Index(String.Join(",", indices)); + search.Indices(String.Join(",", indices)); if (HasVersion) - search.SequenceNumberPrimaryTerm(HasVersion); + search.SeqNoPrimaryTerm(HasVersion); if (options.HasQueryTimeout()) - search.Timeout(new Time(options.GetQueryTimeout()).ToString()); + { + var timeout = options.GetQueryTimeout(); + search.Timeout(timeout.ToElasticDuration()); + } search.IgnoreUnavailable(); - search.TrackTotalHits(); + search.TrackTotalHits(new TrackHits(true)); await ElasticIndex.QueryBuilder.ConfigureSearchAsync(query, options, search).AnyContext(); @@ -818,7 +839,7 @@ protected virtual void ConfigureGetRequest(GetRequest request, ICommandOptions o } } - protected virtual void ConfigureMultiGetRequest(MultiGetDescriptor request, ICommandOptions options) + protected virtual void ConfigureMultiGetRequest(MultiGetRequestDescriptor request, ICommandOptions options) { var (resolvedIncludes, resolvedExcludes) = GetResolvedIncludesAndExcludes(options); @@ -898,11 +919,11 @@ protected bool ShouldReturnDocument(T document, ICommandOptions options) protected async Task RefreshForConsistency(IRepositoryQuery query, ICommandOptions options) { - if (options.GetConsistency(DefaultConsistency) != Consistency.Eventual) + if (options.GetConsistency(DefaultConsistency) is not Consistency.Eventual) { string[] indices = ElasticIndex.GetIndexesByQuery(query); - var response = await _client.Indices.RefreshAsync(indices); - if (response.IsValid) + var response = await _client.Indices.RefreshAsync(indices).AnyContext(); + if (response.IsValidResponse) _logger.LogRequest(response); else _logger.LogErrorRequest(response, "Failed to refresh indices for immediate consistency"); @@ -917,9 +938,9 @@ protected async Task GetCachedQueryResultAsync(ICommandOptions if (!IsCacheEnabled || !options.ShouldReadCache() || !options.HasCacheKey()) return default; - string cacheKey = cachePrefix != null ? cachePrefix + ":" + options.GetCacheKey() : options.GetCacheKey(); + string cacheKey = cachePrefix != null ? $"{cachePrefix}:{options.GetCacheKey()}" : options.GetCacheKey(); if (!String.IsNullOrEmpty(cacheSuffix)) - cacheKey += ":" + cacheSuffix; + cacheKey = $"{cacheKey}:{cacheSuffix}"; var result = await Cache.GetAsync(cacheKey, default).AnyContext(); _logger.LogTrace("Cache {HitOrMiss}: type={EntityType} key={CacheKey}", (result != null ? "hit" : "miss"), EntityTypeName, cacheKey); @@ -935,9 +956,9 @@ protected async Task SetCachedQueryResultAsync(ICommandOptions options, if (!options.HasCacheKey()) throw new ArgumentException("Cache key is required when enabling cache.", nameof(options)); - string cacheKey = cachePrefix != null ? cachePrefix + ":" + options.GetCacheKey() : options.GetCacheKey(); + string cacheKey = cachePrefix != null ? $"{cachePrefix}:{options.GetCacheKey()}" : options.GetCacheKey(); if (!String.IsNullOrEmpty(cacheSuffix)) - cacheKey += ":" + cacheSuffix; + cacheKey = $"{cacheKey}:{cacheSuffix}"; await Cache.SetAsync(cacheKey, result, options.GetExpiresIn()).AnyContext(); _logger.LogTrace("Set cache: type={EntityType} key={CacheKey}", EntityTypeName, cacheKey); @@ -1104,39 +1125,3 @@ public virtual void Dispose() _disposedCancellationTokenSource.Dispose(); } } - -internal class SearchResponse : IResponse, IElasticsearchResponse where TDocument : class -{ - public IApiCallDetails ApiCall { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - - public string DebugInformation => throw new NotImplementedException(); - - public bool IsValid => throw new NotImplementedException(); - - public Exception OriginalException => throw new NotImplementedException(); - - public ServerError ServerError => throw new NotImplementedException(); - - AggregateDictionary Aggregations { get; } - bool TimedOut { get; } - bool TerminatedEarly { get; } - ISuggestDictionary Suggest { get; } - ShardStatistics Shards { get; } - string ScrollId { get; } - Profile Profile { get; } - long Took { get; } - string PointInTimeId { get; } - double MaxScore { get; } - IHitsMetadata HitsMetadata { get; } - IReadOnlyCollection> Hits { get; } - IReadOnlyCollection Fields { get; } - IReadOnlyCollection Documents { get; } - ClusterStatistics Clusters { get; } - long NumberOfReducePhases { get; } - long Total { get; } - - public bool TryGetServerErrorReason(out string reason) - { - throw new NotImplementedException(); - } -} diff --git a/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReindexer.cs b/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReindexer.cs index 16b505e8..e75d2958 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReindexer.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReindexer.cs @@ -2,24 +2,30 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.Search; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Elastic.Transport.Products.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Jobs; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Utility; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Nest; -using Newtonsoft.Json; namespace Foundatio.Repositories.Elasticsearch; public class ElasticReindexer { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; + private readonly ITextSerializer _serializer; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; private readonly IResiliencePolicyProvider _resiliencePolicyProvider; @@ -27,17 +33,18 @@ public class ElasticReindexer private const string ID_FIELD = "id"; private const int MAX_STATUS_FAILS = 10; - public ElasticReindexer(IElasticClient client, ILogger logger = null) : this(client, TimeProvider.System, logger) + public ElasticReindexer(ElasticsearchClient client, ITextSerializer serializer, ILogger logger = null) : this(client, serializer, TimeProvider.System, logger) { } - public ElasticReindexer(IElasticClient client, TimeProvider timeProvider, ILogger logger = null) : this(client, timeProvider ?? TimeProvider.System, new ResiliencePolicyProvider(), logger ?? NullLogger.Instance) + public ElasticReindexer(ElasticsearchClient client, ITextSerializer serializer, TimeProvider timeProvider, ILogger logger = null) : this(client, serializer, timeProvider ?? TimeProvider.System, new ResiliencePolicyProvider(), logger ?? NullLogger.Instance) { } - public ElasticReindexer(IElasticClient client, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILogger logger = null) + public ElasticReindexer(ElasticsearchClient client, ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILogger logger = null) { _client = client; + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); _timeProvider = timeProvider ?? TimeProvider.System; _resiliencePolicyProvider = resiliencePolicyProvider ?? new ResiliencePolicyProvider(); _logger = logger ?? NullLogger.Instance; @@ -80,13 +87,25 @@ public async Task ReindexAsync(ReindexWorkItem workItem, Func if (aliases.Count > 0) { - var bulkResponse = await _client.Indices.BulkAliasAsync(x => + // Build list of actions - each action is either an Add or Remove + var aliasActions = new List(); + + foreach (string alias in aliases) + { + // Remove from old index + aliasActions.Add(new IndexUpdateAliasesAction { Remove = new RemoveAction { Alias = alias, Index = workItem.OldIndex } }); + // Add to new index + aliasActions.Add(new IndexUpdateAliasesAction { Add = new AddAction { Alias = alias, Index = workItem.NewIndex } }); + } + + var bulkResponse = await _client.Indices.UpdateAliasesAsync(x => x.Actions(aliasActions)).AnyContext(); + + if (!bulkResponse.IsValidResponse) { - foreach (string alias in aliases) - x = x.Remove(a => a.Alias(alias).Index(workItem.OldIndex)).Add(a => a.Alias(alias).Index(workItem.NewIndex)); + _logger.LogErrorRequest(bulkResponse, "Error updating aliases during reindex"); + return; + } - return x; - }).AnyContext(); _logger.LogRequest(bulkResponse); await progressCallbackAsync(92, $"Updated aliases: {String.Join(", ", aliases)} Remove: {workItem.OldIndex} Add: {workItem.NewIndex}").AnyContext(); @@ -95,8 +114,8 @@ public async Task ReindexAsync(ReindexWorkItem workItem, Func var refreshResponse = await _client.Indices.RefreshAsync(Indices.All).AnyContext(); _logger.LogRequest(refreshResponse); - if (!refreshResponse.IsValid) - _logger.LogWarning("Failed to refresh indices before second reindex pass: {Error}", refreshResponse.ServerError); + if (!refreshResponse.IsValidResponse) + _logger.LogWarning("Failed to refresh indices before second reindex pass: {Error}", refreshResponse.ElasticsearchServerError); ReindexResult secondPassResult = null; if (!String.IsNullOrEmpty(workItem.TimestampField)) @@ -116,31 +135,31 @@ public async Task ReindexAsync(ReindexWorkItem workItem, Func { refreshResponse = await _client.Indices.RefreshAsync(Indices.All).AnyContext(); _logger.LogRequest(refreshResponse); - if (!refreshResponse.IsValid) - _logger.LogWarning("Failed to refresh indices before doc count comparison: {Error}", refreshResponse.ServerError); + if (!refreshResponse.IsValidResponse) + _logger.LogWarning("Failed to refresh indices before doc count comparison: {Error}", refreshResponse.ElasticsearchServerError); - var newDocCountResponse = await _client.CountAsync(d => d.Index(workItem.NewIndex)).AnyContext(); + var newDocCountResponse = await _client.CountAsync(d => d.Indices(workItem.NewIndex)).AnyContext(); _logger.LogRequest(newDocCountResponse); - if (!newDocCountResponse.IsValid) - _logger.LogWarning("Failed to get new index doc count: {Error}", newDocCountResponse.ServerError); + if (!newDocCountResponse.IsValidResponse) + _logger.LogWarning("Failed to get new index doc count: {Error}", newDocCountResponse.ElasticsearchServerError); - var oldDocCountResponse = await _client.CountAsync(d => d.Index(workItem.OldIndex)).AnyContext(); + var oldDocCountResponse = await _client.CountAsync(d => d.Indices(workItem.OldIndex)).AnyContext(); _logger.LogRequest(oldDocCountResponse); - if (!oldDocCountResponse.IsValid) - _logger.LogWarning("Failed to get old index doc count: {Error}", oldDocCountResponse.ServerError); + if (!oldDocCountResponse.IsValidResponse) + _logger.LogWarning("Failed to get old index doc count: {Error}", oldDocCountResponse.ElasticsearchServerError); await progressCallbackAsync(98, $"Old Docs: {oldDocCountResponse.Count} New Docs: {newDocCountResponse.Count}").AnyContext(); - if (newDocCountResponse.IsValid && oldDocCountResponse.IsValid && newDocCountResponse.Count >= oldDocCountResponse.Count) + if (newDocCountResponse.IsValidResponse && oldDocCountResponse.IsValidResponse && newDocCountResponse.Count >= oldDocCountResponse.Count) { var deleteIndexResponse = await _client.Indices.DeleteAsync(Indices.Index(workItem.OldIndex)).AnyContext(); _logger.LogRequest(deleteIndexResponse); - if (!deleteIndexResponse.IsValid) - _logger.LogWarning("Failed to delete old index {OldIndex}: {Error}", workItem.OldIndex, deleteIndexResponse.ServerError); + if (!deleteIndexResponse.IsValidResponse) + _logger.LogWarning("Failed to delete old index {OldIndex}: {Error}", workItem.OldIndex, deleteIndexResponse.ElasticsearchServerError); - if (deleteIndexResponse.IsValid) + if (deleteIndexResponse.IsValidResponse) await progressCallbackAsync(99, $"Deleted index: {workItem.OldIndex}").AnyContext(); else - await progressCallbackAsync(99, $"Failed to delete old index {workItem.OldIndex}: {deleteIndexResponse.ServerError}").AnyContext(); + await progressCallbackAsync(99, $"Failed to delete old index {workItem.OldIndex}: {deleteIndexResponse.ElasticsearchServerError}").AnyContext(); } } @@ -153,29 +172,37 @@ private async Task InternalReindexAsync(ReindexWorkItem workItem, var result = await _resiliencePolicy.ExecuteAsync(async ct => { - var response = await _client.ReindexOnServerAsync(d => + var response = await _client.ReindexAsync(d => { - d.Source(src => src - .Index(workItem.OldIndex) - .Query(q => query)) - .Destination(dest => dest.Index(workItem.NewIndex)) - .Conflicts(Conflicts.Proceed) - .WaitForCompletion(false); - - // NEST client emitting a script if null, inline this when that's fixed - if (!String.IsNullOrWhiteSpace(workItem.Script)) - d.Script(workItem.Script); + d.Source(src => + { + src.Indices(workItem.OldIndex); + if (query != null) + src.Query(query); + }); + d.Dest(dest => dest.Index(workItem.NewIndex)); + d.Conflicts(Conflicts.Proceed); + d.WaitForCompletion(false); - return d; + if (!String.IsNullOrWhiteSpace(workItem.Script)) + d.Script(new Script { Source = workItem.Script }); }, ct).AnyContext(); _logger.LogRequest(response); return response; }, cancellationToken).AnyContext(); - _logger.LogInformation("Reindex Task Id: {TaskId}", result.Task.FullyQualifiedId); + if (result.Task == null) + { + _logger.LogError("Reindex failed to start - no task returned. Response valid: {IsValid}, Reason: {Reason}", + result.IsValidResponse, result.ElasticsearchServerError?.Error?.Reason ?? "Unknown"); + _logger.LogErrorRequest(result, "Reindex failed"); + return new ReindexResult { Total = 0, Completed = 0 }; + } + + _logger.LogInformation("Reindex Task Id: {TaskId}", result.Task.ToString()); _logger.LogRequest(result); - long totalDocs = result.Total; + long totalDocs = result.Total ?? 0; bool taskSuccess = false; TaskReindexResult lastReindexResponse = null; @@ -184,8 +211,8 @@ private async Task InternalReindexAsync(ReindexWorkItem workItem, var sw = Stopwatch.StartNew(); do { - var status = await _client.Tasks.GetTaskAsync(result.Task, null, cancellationToken).AnyContext(); - if (status.IsValid) + var status = await _client.Tasks.GetAsync(result.Task.FullyQualifiedId, cancellationToken).AnyContext(); + if (status.IsValidResponse) { _logger.LogRequest(status); } @@ -205,7 +232,7 @@ private async Task InternalReindexAsync(ReindexWorkItem workItem, statusGetFails = 0; - var response = status.DeserializeRaw(); + var response = status.DeserializeRaw(_serializer); if (response?.Error != null) { _logger.LogError("Error reindex: {Type}, {Reason}, Cause: {CausedBy} Stack: {Stack}", response.Error.Type, response.Error.Reason, response.Error.Caused_By?.Reason, String.Join("\r\n", response.Error.Script_Stack ?? new List())); @@ -214,15 +241,45 @@ private async Task InternalReindexAsync(ReindexWorkItem workItem, lastReindexResponse = response?.Response; - long lastCompleted = status.Task.Status.Created + status.Task.Status.Updated + status.Task.Status.Noops; + // Extract status values from the raw JSON. The Status property is object? and may be + // deserialized as JsonElement or IDictionary depending on serializer config. + TaskStatusValues taskStatus = null; + if (status.Task.Status is JsonElement jsonElement) + { + taskStatus = new TaskStatusValues + { + Total = jsonElement.TryGetProperty("total", out var totalProp) ? totalProp.GetInt64() : 0, + Created = jsonElement.TryGetProperty("created", out var createdProp) ? createdProp.GetInt64() : 0, + Updated = jsonElement.TryGetProperty("updated", out var updatedProp) ? updatedProp.GetInt64() : 0, + Noops = jsonElement.TryGetProperty("noops", out var noopsProp) ? noopsProp.GetInt64() : 0, + VersionConflicts = jsonElement.TryGetProperty("version_conflicts", out var conflictsProp) ? conflictsProp.GetInt64() : 0 + }; + } + else if (status.Task.Status is IDictionary dict) + { + taskStatus = new TaskStatusValues + { + Total = dict.TryGetValue("total", out var totalVal) ? Convert.ToInt64(totalVal) : 0, + Created = dict.TryGetValue("created", out var createdVal) ? Convert.ToInt64(createdVal) : 0, + Updated = dict.TryGetValue("updated", out var updatedVal) ? Convert.ToInt64(updatedVal) : 0, + Noops = dict.TryGetValue("noops", out var noopsVal) ? Convert.ToInt64(noopsVal) : 0, + VersionConflicts = dict.TryGetValue("version_conflicts", out var conflictsVal) ? Convert.ToInt64(conflictsVal) : 0 + }; + } + else if (status.Task.Status != null) + { + _logger.LogWarning("Unexpected task status type {StatusType}: {Status}", status.Task.Status.GetType().Name, status.Task.Status); + } + + long lastCompleted = (taskStatus?.Created ?? 0) + (taskStatus?.Updated ?? 0) + (taskStatus?.Noops ?? 0); // restart the stop watch if there was progress made if (lastCompleted > lastProgress) sw.Restart(); lastProgress = lastCompleted; - string lastMessage = $"Total: {status.Task.Status.Total:N0} Completed: {lastCompleted:N0} VersionConflicts: {status.Task.Status.VersionConflicts:N0}"; - await progressCallbackAsync(CalculateProgress(status.Task.Status.Total, lastCompleted, startProgress, endProgress), lastMessage).AnyContext(); + string lastMessage = $"Total: {taskStatus?.Total:N0} Completed: {lastCompleted:N0} VersionConflicts: {taskStatus?.VersionConflicts:N0}"; + await progressCallbackAsync(CalculateProgress(taskStatus?.Total ?? 0, lastCompleted, startProgress, endProgress), lastMessage).AnyContext(); if (status.Completed && response?.Error == null) { @@ -238,7 +295,7 @@ private async Task InternalReindexAsync(ReindexWorkItem workItem, } var timeToWait = TimeSpan.FromSeconds(totalDocs < 100000 ? 1 : 10); - if (status.Task.Status.Total < 100) + if ((taskStatus?.Total ?? 0) < 100) timeToWait = TimeSpan.FromMilliseconds(100); await _timeProvider.Delay(timeToWait, cancellationToken).AnyContext(); @@ -271,14 +328,14 @@ private async Task InternalReindexAsync(ReindexWorkItem workItem, private async Task CreateFailureIndexAsync(ReindexWorkItem workItem) { - string errorIndex = workItem.NewIndex + "-error"; + string errorIndex = $"{workItem.NewIndex}-error"; var existsResponse = await _client.Indices.ExistsAsync(errorIndex).AnyContext(); _logger.LogRequest(existsResponse); - if (existsResponse.ApiCall.Success && existsResponse.Exists) + if (existsResponse.ApiCallDetails.HasSuccessfulStatusCode && existsResponse.Exists) return true; - var createResponse = await _client.Indices.CreateAsync(errorIndex, d => d.Map(md => md.Dynamic(false))).AnyContext(); - if (!createResponse.IsValid) + var createResponse = await _client.Indices.CreateAsync(errorIndex, d => d.Mappings(md => md.Dynamic(DynamicMapping.False))).AnyContext(); + if (!createResponse.IsValidResponse) { _logger.LogErrorRequest(createResponse, "Unable to create error index"); return false; @@ -293,49 +350,70 @@ private async Task HandleFailureAsync(ReindexWorkItem workItem, BulkIndexByScrol _logger.LogError("Error reindexing document {Index}/{Id}: [{Status}] {Message}", workItem.OldIndex, failure.Id, failure.Status, failure.Cause.Reason); var gr = await _client.GetAsync(request: new GetRequest(workItem.OldIndex, failure.Id)).AnyContext(); - if (!gr.IsValid) + if (!gr.IsValidResponse) { _logger.LogErrorRequest(gr, "Error getting document {Index}/{Id}", workItem.OldIndex, failure.Id); return; } _logger.LogRequest(gr); - string document = JsonConvert.SerializeObject(new + var errorDocument = new { failure.Index, failure.Id, gr.Version, gr.Routing, gr.Source, - failure.Cause, + Cause = new + { + Type = failure.Cause?.Type, + Reason = failure.Cause?.Reason, + StackTrace = failure.Cause?.StackTrace + }, failure.Status, gr.Found, - }); - var indexResponse = await _client.LowLevel.IndexAsync(workItem.NewIndex + "-error", PostData.String(document)); - if (indexResponse.Success) + }; + var indexResponse = await _client.IndexAsync(errorDocument, i => i.Index($"{workItem.NewIndex}-error")); + if (indexResponse.IsValidResponse) _logger.LogRequest(indexResponse); else - _logger.LogErrorRequest(indexResponse, "Error indexing document {Index}/{Id}", workItem.NewIndex + "-error", gr.Id); + _logger.LogErrorRequest(indexResponse, "Error indexing document {Index}/{Id}", $"{workItem.NewIndex}-error", gr.Id); } private async Task> GetIndexAliasesAsync(string index) { - var aliasesResponse = await _client.Indices.GetAliasAsync(index).AnyContext(); + var aliasesResponse = await _client.Indices.GetAliasAsync(Indices.Index(index)).AnyContext(); _logger.LogRequest(aliasesResponse); - if (aliasesResponse.IsValid && aliasesResponse.Indices.Count > 0) + if (aliasesResponse.IsValidResponse) { - var aliases = aliasesResponse.Indices.SingleOrDefault(a => String.Equals(a.Key, index)); - if (aliases.Value?.Aliases != null) - return aliases.Value.Aliases.Select(a => a.Key).ToList(); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + if (indices != null && indices.Count > 0) + { + var aliases = indices.SingleOrDefault(a => String.Equals(a.Key, index)); + if (aliases.Value?.Aliases != null) + return aliases.Value.Aliases.Select(a => a.Key).ToList(); + } + + return []; } - return new List(); + if (aliasesResponse.ApiCallDetails is { HttpStatusCode: 404 }) + return []; + + _logger.LogWarning("Failed to get aliases for index {Index}: {Error}", index, + aliasesResponse.ElasticsearchServerError?.Error?.Reason ?? "Unknown error"); + + return []; } - private async Task GetResumeQueryAsync(string newIndex, string timestampField, DateTime? startTime) + private async Task GetResumeQueryAsync(string newIndex, string timestampField, DateTime? startTime) { - var descriptor = new QueryContainerDescriptor(); + var descriptor = new QueryDescriptor(); if (startTime.HasValue) return CreateRangeQuery(descriptor, timestampField, startTime); @@ -343,32 +421,33 @@ private async Task GetResumeQueryAsync(string newIndex, string t if (startingPoint.HasValue) return CreateRangeQuery(descriptor, timestampField, startingPoint); - return descriptor; + // Return null when no query is needed - reindexing all documents + return null; } - private QueryContainer CreateRangeQuery(QueryContainerDescriptor descriptor, string timestampField, DateTime? startTime) + private Query CreateRangeQuery(QueryDescriptor descriptor, string timestampField, DateTime? startTime) { if (!startTime.HasValue) return descriptor; if (!String.IsNullOrEmpty(timestampField)) - return descriptor.DateRange(dr => dr.Field(timestampField).GreaterThanOrEquals(startTime)); + return descriptor.Range(dr => dr.Date(drr => drr.Field(timestampField).Gte(startTime.Value))); - return descriptor.TermRange(dr => dr.Field(ID_FIELD).GreaterThanOrEquals(ObjectId.GenerateNewId(startTime.Value).ToString())); + return descriptor.Range(dr => dr.Term(tr => tr.Field(ID_FIELD).Gte(ObjectId.GenerateNewId(startTime.Value).ToString()))); } private async Task GetResumeStartingPointAsync(string newIndex, string timestampField) { var newestDocumentResponse = await _client.SearchAsync>(d => d - .Index(newIndex) - .Sort(s => s.Descending(timestampField)) - .DocValueFields(timestampField) - .Source(s => s.ExcludeAll()) + .Indices(newIndex) + .Sort(s => s.Field(timestampField, fs => fs.Order(SortOrder.Desc))) + .DocvalueFields(new FieldAndFormat[] { new() { Field = timestampField } }) + .Source(new SourceConfig(false)) .Size(1) ).AnyContext(); _logger.LogRequest(newestDocumentResponse); - if (!newestDocumentResponse.IsValid || !newestDocumentResponse.Documents.Any()) + if (!newestDocumentResponse.IsValidResponse || !newestDocumentResponse.Documents.Any()) return null; var doc = newestDocumentResponse.Hits.FirstOrDefault(); @@ -387,8 +466,23 @@ private QueryContainer CreateRangeQuery(QueryContainerDescriptor descrip if (value == null) return null; - var datesArray = await value.AsAsync(); - return datesArray?.FirstOrDefault(); + if (value is not JsonElement jsonElement) + return null; + + var target = jsonElement; + if (jsonElement.ValueKind == JsonValueKind.Array) + { + if (jsonElement.GetArrayLength() == 0) + return null; + target = jsonElement[0]; + } + + if (target.TryGetDateTime(out var dateTime)) + return dateTime; + if (target.ValueKind == JsonValueKind.String && DateTime.TryParse(target.GetString(), out dateTime)) + return dateTime; + + return null; } private int CalculateProgress(long total, long completed, int startProgress = 0, int endProgress = 100) @@ -437,6 +531,15 @@ private class TaskReindexResult public IReadOnlyCollection Failures { get; set; } } + private class TaskStatusValues + { + public long Total { get; set; } + public long Created { get; set; } + public long Updated { get; set; } + public long Noops { get; set; } + public long VersionConflicts { get; set; } + } + private class BulkIndexByScrollFailure { public Error Cause { get; set; } diff --git a/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticRepositoryBase.cs b/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticRepositoryBase.cs index 30d613bc..0783d9d4 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticRepositoryBase.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticRepositoryBase.cs @@ -3,8 +3,12 @@ using System.IO; using System.Linq; using System.Linq.Expressions; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.Bulk; +using Elastic.Transport.Extensions; using Foundatio.Messaging; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Visitors; @@ -21,8 +25,7 @@ using Foundatio.Resilience; using Foundatio.Utility; using Microsoft.Extensions.Logging; -using Nest; -using Newtonsoft.Json.Linq; +using Tasks = Elastic.Clients.Elasticsearch.Tasks; namespace Foundatio.Repositories.Elasticsearch; @@ -200,70 +203,168 @@ public virtual async Task PatchAsync(Id id, IPatchOperation operation, ICo { // ScriptPatch: noop detection requires the script to explicitly set ctx.op = 'none'. // Simply reassigning the same value is treated as a modification by Elasticsearch. - // TODO: Figure out how to specify a pipeline here. - var request = new UpdateRequest(ElasticIndex.GetIndex(id), id.Value) + if (!String.IsNullOrEmpty(DefaultPipeline)) { - Script = new InlineScript(scriptOperation.Script) { Params = scriptOperation.Params }, - RetryOnConflict = options.GetRetryCount(), - Refresh = options.GetRefreshMode(DefaultConsistency) - }; - if (id.Routing != null) - request.Routing = id.Routing; + var request = new UpdateByQueryRequest(ElasticIndex.GetIndex(id)) + { + Query = new Elastic.Clients.Elasticsearch.QueryDsl.IdsQuery + { + Values = new Elastic.Clients.Elasticsearch.Ids(new[] { new Elastic.Clients.Elasticsearch.Id(id.Value) }) + }, + Conflicts = Conflicts.Proceed, + Script = new Script { Source = scriptOperation.Script, Params = scriptOperation.Params }, + Pipeline = DefaultPipeline, + Refresh = options.GetRefreshMode(DefaultConsistency) != Refresh.False, + WaitForCompletion = true + }; + if (id.Routing != null) + request.Routing = new Elastic.Clients.Elasticsearch.Routing(id.Routing); - var response = await _client.UpdateAsync(request).AnyContext(); - _logger.LogRequest(response, options.GetQueryLogLevel()); + var response = await _client.UpdateByQueryAsync(request).AnyContext(); + _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) - { - if (response.ApiCall is { HttpStatusCode: 404 }) - throw new DocumentNotFoundException(id); + if (!response.IsValidResponse) + { + if (response.ApiCallDetails is { HttpStatusCode: 404 }) + throw new DocumentNotFoundException(id); - if (response.ApiCall is { HttpStatusCode: 409 }) - throw new VersionConflictDocumentException( - response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), - response.OriginalException); + throw new DocumentException(response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), response.OriginalException()); + } - throw new DocumentException( - response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), - response.OriginalException); + modified = (response.Noops ?? 0) is 0; } + else + { + var request = new UpdateRequest(ElasticIndex.GetIndex(id), id.Value) + { + Script = new Script { Source = scriptOperation.Script, Params = scriptOperation.Params }, + RetryOnConflict = options.GetRetryCount(), + Refresh = options.GetRefreshMode(DefaultConsistency) + }; + if (id.Routing != null) + request.Routing = id.Routing; + + var response = await _client.UpdateAsync(request).AnyContext(); + _logger.LogRequest(response, options.GetQueryLogLevel()); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails is { HttpStatusCode: 404 }) + throw new DocumentNotFoundException(id); - modified = response.Result is not Result.Noop; + if (response.ApiCallDetails is { HttpStatusCode: 409 }) + throw new VersionConflictDocumentException( + response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), + response.OriginalException()); + + throw new DocumentException(response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), response.OriginalException()); + } + + modified = response.Result is not Result.NoOp; + } } else if (operation is PartialPatch partialOperation) { // PartialPatch: Elasticsearch's detect_noop (enabled by default) reports noop when no // field values change. However, ApplyDateTracking injects UpdatedUtc for IHaveDates // models, which typically prevents noop detection since the timestamp always changes. - // TODO: Figure out how to specify a pipeline here. - var request = new UpdateRequest(ElasticIndex.GetIndex(id), id.Value) + if (!String.IsNullOrEmpty(DefaultPipeline)) { - Doc = partialOperation.Document, - RetryOnConflict = options.GetRetryCount() - }; - if (id.Routing != null) - request.Routing = id.Routing; - request.Refresh = options.GetRefreshMode(DefaultConsistency); + // Pipeline path uses get-merge-reindex, so a write always occurs (like JsonPatch). + var policy = _resiliencePolicy; + if (options.HasRetryCount()) + { + if (policy is ResiliencePolicy resiliencePolicy) + policy = resiliencePolicy.Clone(options.GetRetryCount()); + else + _logger.LogWarning("Unable to override resilience policy max attempts"); + } - var response = await _client.UpdateAsync(request).AnyContext(); - _logger.LogRequest(response, options.GetQueryLogLevel()); + await policy.ExecuteAsync(async ct => + { + var getRequest = new GetRequest(ElasticIndex.GetIndex(id), id.Value); + if (id.Routing != null) + getRequest.Routing = id.Routing; - if (!response.IsValid) - { - if (response.ApiCall is { HttpStatusCode: 404 }) - throw new DocumentNotFoundException(id); + var response = await _client.GetAsync(getRequest, ct).AnyContext(); + _logger.LogRequest(response, options.GetQueryLogLevel()); + if (!response.IsValidResponse) + { + if (!response.Found) + throw new DocumentNotFoundException(id); + + throw new DocumentException(response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), response.OriginalException()); + } - if (response.ApiCall is { HttpStatusCode: 409 }) - throw new VersionConflictDocumentException( - response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), - response.OriginalException); + var sourceJson = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(response.Source); + var sourceNode = JsonNode.Parse(sourceJson); + var partialJson = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(partialOperation.Document); + var partialNode = JsonNode.Parse(partialJson); + + if (sourceNode is JsonObject sourceObj && partialNode is JsonObject partialObj) + { + foreach (var prop in partialObj) + sourceObj[prop.Key] = prop.Value?.DeepClone(); + } + + using var mergedStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(sourceNode.ToJsonString())); + var mergedDocument = _client.ElasticsearchClientSettings.SourceSerializer.Deserialize(mergedStream); + + var indexRequest = new IndexRequest(mergedDocument, ElasticIndex.GetIndex(id), id.Value) + { + Pipeline = DefaultPipeline, + Refresh = options.GetRefreshMode(DefaultConsistency) + }; + if (id.Routing != null) + indexRequest.Routing = id.Routing; + + if (HasVersion && !options.ShouldSkipVersionCheck()) + { + indexRequest.IfSeqNo = response.SeqNo; + indexRequest.IfPrimaryTerm = response.PrimaryTerm; + } + + var indexResponse = await _client.IndexAsync(indexRequest, ct).AnyContext(); + _logger.LogRequest(indexResponse, options.GetQueryLogLevel()); + + if (!indexResponse.IsValidResponse) + { + if (indexResponse.ElasticsearchServerError?.Status == 409) + throw new VersionConflictDocumentException(indexResponse.GetErrorMessage("Error saving document"), indexResponse.OriginalException()); - throw new DocumentException( - response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), - response.OriginalException); + throw new DocumentException(indexResponse.GetErrorMessage("Error saving document"), indexResponse.OriginalException()); + } + }).AnyContext(); } + else + { + var request = new UpdateRequest(ElasticIndex.GetIndex(id), id.Value) + { + Doc = partialOperation.Document, + RetryOnConflict = options.GetRetryCount() + }; + if (id.Routing != null) + request.Routing = id.Routing; + request.Refresh = options.GetRefreshMode(DefaultConsistency); + + var response = await _client.UpdateAsync(request).AnyContext(); + _logger.LogRequest(response, options.GetQueryLogLevel()); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails is { HttpStatusCode: 404 }) + throw new DocumentNotFoundException(id); + + if (response.ApiCallDetails is { HttpStatusCode: 409 }) + throw new VersionConflictDocumentException( + response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), + response.OriginalException()); - modified = response.Result is not Result.Noop; + throw new DocumentException(response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), response.OriginalException()); + } + + modified = response.Result is not Result.NoOp; + } } else if (operation is JsonPatch jsonOperation) { @@ -280,49 +381,56 @@ public virtual async Task PatchAsync(Id id, IPatchOperation operation, ICo await policy.ExecuteAsync(async ct => { - var response = await _client.LowLevel.GetAsync>>(ElasticIndex.GetIndex(id), id.Value, new GetRequestParameters { Routing = id.Routing }, ct).AnyContext(); + var request = new GetRequest(ElasticIndex.GetIndex(id), id.Value); + if (id.Routing != null) + request.Routing = id.Routing; + + var response = await _client.GetAsync(request, ct).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { if (!response.Found) throw new DocumentNotFoundException(id); - throw new DocumentException( - response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), - response.OriginalException); + throw new DocumentException(response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), response.OriginalException()); } - var jObject = JObject.FromObject(response.Source); - var target = (JToken)jObject; + // Serialize to JSON string, apply patch, deserialize back + // Using System.Text.Json.Nodes.JsonNode since Elastic.Clients.Elasticsearch uses System.Text.Json exclusively + var json = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(response.Source); + var target = JsonNode.Parse(json); new JsonPatcher().Patch(ref target, jsonOperation.Patch); ApplyDateTracking(target); - var indexParameters = new IndexRequestParameters + using var patchStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(target.ToJsonString())); + var patchedDocument = _client.ElasticsearchClientSettings.SourceSerializer.Deserialize(patchStream); + + var indexRequest = new IndexRequest(patchedDocument, ElasticIndex.GetIndex(id), id.Value) { Pipeline = DefaultPipeline, Refresh = options.GetRefreshMode(DefaultConsistency) }; if (id.Routing != null) - indexParameters.Routing = id.Routing; + indexRequest.Routing = id.Routing; if (HasVersion && !options.ShouldSkipVersionCheck()) { - indexParameters.IfSequenceNumber = response.SequenceNumber; - indexParameters.IfPrimaryTerm = response.PrimaryTerm; + indexRequest.IfSeqNo = response.SeqNo; + indexRequest.IfPrimaryTerm = response.PrimaryTerm; } - var updateResponse = await _client.LowLevel.IndexAsync(ElasticIndex.GetIndex(id), id.Value, PostData.String(target.ToString()), indexParameters, ct).AnyContext(); + var updateResponse = await _client.IndexAsync(indexRequest, ct).AnyContext(); _logger.LogRequest(updateResponse, options.GetQueryLogLevel()); - if (!updateResponse.Success) + if (!updateResponse.IsValidResponse) { - if (updateResponse.HttpStatusCode is 409) - throw new VersionConflictDocumentException(updateResponse.GetErrorMessage("Error saving document"), updateResponse.OriginalException); + if (updateResponse.ElasticsearchServerError?.Status is 409) + throw new VersionConflictDocumentException(updateResponse.GetErrorMessage("Error saving document"), updateResponse.OriginalException()); - throw new DocumentException(updateResponse.GetErrorMessage("Error saving document"), updateResponse.OriginalException); + throw new DocumentException(updateResponse.GetErrorMessage("Error saving document"), updateResponse.OriginalException()); } - }); + }).AnyContext(); } else if (operation is ActionPatch actionPatch) { @@ -346,14 +454,12 @@ await policy.ExecuteAsync(async ct => var response = await _client.GetAsync(request, ct).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { if (!response.Found) throw new DocumentNotFoundException(id); - throw new DocumentException( - response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), - response.OriginalException); + throw new DocumentException(response.GetErrorMessage($"Error patching document {ElasticIndex.GetIndex(id)}/{id.Value}"), response.OriginalException()); } if (response.Source is IVersioned versionedDoc && response.PrimaryTerm.HasValue) @@ -374,7 +480,7 @@ await policy.ExecuteAsync(async ct => modifiedDocument = response.Source; await IndexDocumentsAsync([response.Source], false, options).AnyContext(); - }); + }).AnyContext(); if (modified && modifiedDocument is not null) { @@ -440,13 +546,22 @@ public virtual async Task PatchAsync(Ids ids, IPatchOperation operation, I if (operation is not ScriptPatch and not PartialPatch) throw new ArgumentException("Unknown operation type", nameof(operation)); + if (!String.IsNullOrEmpty(DefaultPipeline)) + { + long pipelineModified = 0; + foreach (var id in ids) + { + if (await PatchAsync(id, operation, options).AnyContext()) + pipelineModified++; + } + return pipelineModified; + } + var bulkResponse = await _client.BulkAsync(b => { b.Refresh(options.GetRefreshMode(DefaultConsistency)); foreach (var id in ids) { - b.Pipeline(DefaultPipeline); - if (operation is ScriptPatch scriptOperation) b.Update(u => { @@ -457,25 +572,21 @@ public virtual async Task PatchAsync(Ids ids, IPatchOperation operation, I if (id.Routing != null) u.Routing(id.Routing); - - return u; }); else if (operation is PartialPatch partialOperation) b.Update(u => { u.Id(id.Value) .Index(ElasticIndex.GetIndex(id)) + // TODO: Null-valued properties are silently dropped by the ES client's SourceSerializer + // (elastic/elasticsearch-net#8763). Consumers must use ScriptPatch or JsonPatch to set fields to null. .Doc(partialOperation.Document) .RetriesOnConflict(options.GetRetryCount()); if (id.Routing != null) u.Routing(id.Routing); - - return u; }); } - - return b; }).AnyContext(); _logger.LogRequest(bulkResponse, options.GetQueryLogLevel()); @@ -601,9 +712,9 @@ public virtual async Task RemoveAsync(IEnumerable documents, ICommandOptions var response = await _client.DeleteAsync(request).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid && response.ApiCall.HttpStatusCode != 404) + if (!response.IsValidResponse && response.ApiCallDetails.HttpStatusCode != 404) { - throw new DocumentException(response.GetErrorMessage($"Error removing document {ElasticIndex.GetIndex(document)}/{document.Id}"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage($"Error removing document {ElasticIndex.GetIndex(document)}/{document.Id}"), response.OriginalException()); } } else @@ -618,11 +729,7 @@ public virtual async Task RemoveAsync(IEnumerable documents, ICommandOptions if (GetParentIdFunc is not null) d.Routing(GetParentIdFunc(doc)); - - return d; }); - - return bulk; }).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); @@ -649,7 +756,7 @@ public Task RemoveAllAsync(CommandOptionsDescriptor options) public virtual async Task RemoveAllAsync(ICommandOptions options = null) { - long count = await RemoveAllAsync(NewQuery(), options); + long count = await RemoveAllAsync(NewQuery(), options).AnyContext(); if (IsCacheEnabled && count > 0) await Cache.RemoveAllAsync().AnyContext(); @@ -700,10 +807,12 @@ await BatchProcessAsync(query, async results => b.Refresh(options.GetRefreshMode(DefaultConsistency)); foreach (var h in results.Hits) { - string json = _client.ConnectionSettings.SourceSerializer.SerializeToString(h.Document); - var target = JToken.Parse(json); + // Using System.Text.Json.Nodes.JsonNode since Elastic.Clients.Elasticsearch uses System.Text.Json exclusively + var json = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(h.Document); + var target = JsonNode.Parse(json); patcher.Patch(ref target, jsonOperation.Patch); - var doc = _client.ConnectionSettings.SourceSerializer.Deserialize(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(target.ToString()))); + using var docStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(target.ToJsonString())); + var doc = _client.ElasticsearchClientSettings.SourceSerializer.Deserialize(docStream); if (HasDateTracking) SetDocumentDates(doc, ElasticIndex.Configuration.TimeProvider); @@ -711,10 +820,9 @@ await BatchProcessAsync(query, async results => processedDocs[h.Id] = doc; var elasticVersion = h.GetElasticVersion(); - b.Index(i => + b.Index(doc, i => { - i.Document(doc) - .Id(h.Id) + i.Id(h.Id) .Routing(h.Routing) .Index(h.GetIndex()) .Pipeline(DefaultPipeline); @@ -724,16 +832,12 @@ await BatchProcessAsync(query, async results => i.IfPrimaryTerm(elasticVersion.PrimaryTerm); i.IfSequenceNumber(elasticVersion.SequenceNumber); } - - return i; }); } - - return b; }).AnyContext(); var retriedIds = new List(); - if (bulkResult.IsValid) + if (bulkResult.IsValidResponse) { _logger.LogRequest(bulkResult, options.GetQueryLogLevel()); } @@ -801,10 +905,9 @@ await BatchProcessAsync(query, async results => { var elasticVersion = h.GetElasticVersion(); - b.Index(i => + b.Index(h.Document, i => { - i.Document(h.Document) - .Id(h.Id) + i.Id(h.Id) .Routing(h.Routing) .Index(h.GetIndex()) .Pipeline(DefaultPipeline); @@ -814,16 +917,12 @@ await BatchProcessAsync(query, async results => i.IfPrimaryTerm(elasticVersion.PrimaryTerm); i.IfSequenceNumber(elasticVersion.SequenceNumber); } - - return i; }); } - - return b; }).AnyContext(); var retriedIds = new List(); - if (bulkResult.IsValid) + if (bulkResult.IsValidResponse) { _logger.LogRequest(bulkResult, options.GetQueryLogLevel()); } @@ -869,9 +968,9 @@ await BatchProcessAsync(query, async results => { var request = new UpdateByQueryRequest(Indices.Index(String.Join(",", ElasticIndex.GetIndexesByQuery(query)))) { - Query = await ElasticIndex.QueryBuilder.BuildQueryAsync(query, options, new SearchDescriptor()).AnyContext(), + Query = await ElasticIndex.QueryBuilder.BuildQueryAsync(query, options, new SearchRequestDescriptor()).AnyContext(), Conflicts = Conflicts.Proceed, - Script = new InlineScript(scriptOperation.Script) { Params = scriptOperation.Params }, + Script = new Script { Source = scriptOperation.Script, Params = scriptOperation.Params }, Pipeline = DefaultPipeline, Version = HasVersion, Refresh = options.GetRefreshMode(DefaultConsistency) != Refresh.False, @@ -881,28 +980,29 @@ await BatchProcessAsync(query, async results => var response = await _client.UpdateByQueryAsync(request).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { - throw new DocumentException(response.GetErrorMessage("Error occurred while patching by query"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error occurred while patching by query"), response.OriginalException()); } - var taskId = response.Task; + var taskId = response.Task.ToString(); int attempts = 0; do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId, t => t.WaitForCompletion(false)).AnyContext(); + var taskRequest = new Tasks.GetTasksRequest(taskId) { WaitForCompletion = false }; + var taskStatus = await _client.Tasks.GetAsync(taskRequest).AnyContext(); _logger.LogRequest(taskStatus, options.GetQueryLogLevel()); - if (!taskStatus.IsValid) + if (!taskStatus.IsValidResponse) { - if (taskStatus.ApiCall.HttpStatusCode.GetValueOrDefault() == 404) + if (taskStatus.ApiCallDetails.HttpStatusCode.GetValueOrDefault() == 404) { _logger.LogWarning("Task {TaskId} not found (404), treating as completed", taskId); break; } - _logger.LogError("Error getting task status for {TaskId}: {Error}", taskId, taskStatus.ServerError); + _logger.LogError("Error getting task status for {TaskId}: {Error}", taskId, taskStatus.ElasticsearchServerError); if (attempts >= 20) throw new DocumentException($"Failed to get task status for {taskId} after {attempts} attempts"); @@ -911,19 +1011,41 @@ await BatchProcessAsync(query, async results => continue; } - var status = taskStatus.Task.Status; + // Extract status values from the raw JSON. The Status property is object? and may be + // deserialized as JsonElement or IDictionary depending on serializer config. + long? created = null, updated = null, deleted = null, versionConflicts = null, total = null; + if (taskStatus.Task.Status is JsonElement jsonElement) + { + total = jsonElement.TryGetProperty("total", out var totalProp) ? totalProp.GetInt64() : 0; + created = jsonElement.TryGetProperty("created", out var createdProp) ? createdProp.GetInt64() : 0; + updated = jsonElement.TryGetProperty("updated", out var updatedProp) ? updatedProp.GetInt64() : 0; + deleted = jsonElement.TryGetProperty("deleted", out var deletedProp) ? deletedProp.GetInt64() : 0; + versionConflicts = jsonElement.TryGetProperty("version_conflicts", out var conflictsProp) ? conflictsProp.GetInt64() : 0; + } + else if (taskStatus.Task.Status is IDictionary dict) + { + total = dict.TryGetValue("total", out var totalVal) ? Convert.ToInt64(totalVal) : 0; + created = dict.TryGetValue("created", out var createdVal) ? Convert.ToInt64(createdVal) : 0; + updated = dict.TryGetValue("updated", out var updatedVal) ? Convert.ToInt64(updatedVal) : 0; + deleted = dict.TryGetValue("deleted", out var deletedVal) ? Convert.ToInt64(deletedVal) : 0; + versionConflicts = dict.TryGetValue("version_conflicts", out var conflictsVal) ? Convert.ToInt64(conflictsVal) : 0; + } + if (taskStatus.Completed) { - if (status.VersionConflicts > 0) - _logger.LogWarning("Script operation task ({TaskId}) completed with {Conflicts} version conflicts", taskId, status.VersionConflicts); + if (taskStatus.Error is not null) + throw new DocumentException($"Script operation task ({taskId}) failed: {taskStatus.Error.Type} - {taskStatus.Error.Reason}", taskStatus.OriginalException()); + + if (versionConflicts > 0) + _logger.LogWarning("Script operation task ({TaskId}) completed with {Conflicts} version conflicts", taskId, versionConflicts); else - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, created, updated, deleted, versionConflicts, total); - affectedRecords += status.Created + status.Updated + status.Deleted; + affectedRecords += (created ?? 0) + (updated ?? 0) + (deleted ?? 0); break; } - _logger.LogDebug("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogDebug("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, created, updated, deleted, versionConflicts, total); var delay = TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5); await ElasticIndex.Configuration.TimeProvider.SafeDelay(delay, DisposedCancellationToken).AnyContext(); } while (!DisposedCancellationToken.IsCancellationRequested); @@ -943,7 +1065,38 @@ await BatchProcessAsync(query, async results => foreach (var h in results.Hits) { - if (operation is ScriptPatch sp) + if (operation is PartialPatch partialOp && !String.IsNullOrEmpty(DefaultPipeline)) + { + var sourceJson = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(h.Document); + var sourceNode = JsonNode.Parse(sourceJson); + var partialJson = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(partialOp.Document); + var partialNode = JsonNode.Parse(partialJson); + + if (sourceNode is JsonObject sourceObj && partialNode is JsonObject partialObj) + { + foreach (var prop in partialObj) + sourceObj[prop.Key] = prop.Value?.DeepClone(); + } + + using var mergedStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(sourceNode.ToJsonString())); + var mergedDoc = _client.ElasticsearchClientSettings.SourceSerializer.Deserialize(mergedStream); + var elasticVersion = h.GetElasticVersion(); + + b.Index(mergedDoc, i => + { + i.Id(h.Id) + .Routing(h.Routing) + .Index(h.GetIndex()) + .Pipeline(DefaultPipeline); + + if (HasVersion) + { + i.IfPrimaryTerm(elasticVersion.PrimaryTerm); + i.IfSequenceNumber(elasticVersion.SequenceNumber); + } + }); + } + else if (operation is ScriptPatch sp) b.Update(u => u .Id(h.Id) .Routing(h.Routing) @@ -951,17 +1104,17 @@ await BatchProcessAsync(query, async results => .Script(s => s.Source(sp.Script).Params(sp.Params)) .RetriesOnConflict(options.GetRetryCount())); else if (operation is PartialPatch pp) + // TODO: Null-valued properties are silently dropped by the ES client's SourceSerializer + // (elastic/elasticsearch-net#8763). Consumers must use ScriptPatch or JsonPatch to set fields to null. b.Update(u => u.Id(h.Id) .Routing(h.Routing) .Index(h.GetIndex()) .Doc(pp.Document) .RetriesOnConflict(options.GetRetryCount())); } - - return b; }).AnyContext(); - if (bulkResult.IsValid) + if (bulkResult.IsValidResponse) { _logger.LogRequest(bulkResult, options.GetQueryLogLevel()); } @@ -1051,16 +1204,16 @@ public virtual async Task RemoveAllAsync(IRepositoryQuery query, ICommandO { Refresh = options.GetRefreshMode(DefaultConsistency) != Refresh.False, Conflicts = Conflicts.Proceed, - Query = await ElasticIndex.QueryBuilder.BuildQueryAsync(query, options, new SearchDescriptor()).AnyContext() + Query = await ElasticIndex.QueryBuilder.BuildQueryAsync(query, options, new SearchRequestDescriptor()).AnyContext() }).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { - throw new DocumentException(response.GetErrorMessage("Error removing documents"), response.OriginalException); + throw new DocumentException(response.GetErrorMessage("Error removing documents"), response.OriginalException()); } - if (response.Deleted > 0) + if (response.Deleted.HasValue && response.Deleted > 0) { if (IsCacheEnabled) await InvalidateCacheByQueryAsync(query.As()).AnyContext(); @@ -1072,7 +1225,7 @@ public virtual async Task RemoveAllAsync(IRepositoryQuery query, ICommandO if (response.Total != response.Deleted) _logger.LogWarning("RemoveAll: {Deleted} of {Total} records were removed ({Conflicts} version conflicts)", response.Deleted, response.Total, response.VersionConflicts); - return response.Deleted; + return response.Deleted ?? 0; } public Task BatchProcessAsync(RepositoryQueryDescriptor query, Func, Task> processFunc, CommandOptionsDescriptor options = null) @@ -1171,7 +1324,7 @@ protected virtual async Task OnCustomFieldsBeforeQuery(object sender, BeforeQuer if (String.IsNullOrEmpty(tenantKey)) return; - var fieldMapping = await ElasticIndex.Configuration.CustomFieldDefinitionRepository.GetFieldMappingAsync(EntityTypeName, tenantKey); + var fieldMapping = await ElasticIndex.Configuration.CustomFieldDefinitionRepository.GetFieldMappingAsync(EntityTypeName, tenantKey).AnyContext(); var mapping = fieldMapping.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.GetIdxName(), StringComparer.OrdinalIgnoreCase); args.Options.QueryFieldResolver(mapping.ToHierarchicalFieldResolver("idx.")); @@ -1183,7 +1336,7 @@ protected virtual async Task OnCustomFieldsDocumentsChanging(object sender, Docu foreach (var tenant in tenantGroups) { - var fieldDefinitions = await ElasticIndex.Configuration.CustomFieldDefinitionRepository.GetFieldMappingAsync(EntityTypeName, tenant.Key); + var fieldDefinitions = await ElasticIndex.Configuration.CustomFieldDefinitionRepository.GetFieldMappingAsync(EntityTypeName, tenant.Key).AnyContext(); foreach (var doc in tenant) { @@ -1201,7 +1354,7 @@ protected virtual async Task OnCustomFieldsDocumentsChanging(object sender, Docu { if (!fieldDefinitions.TryGetValue(customField.Key, out var fieldDefinition)) { - fieldDefinition = await HandleUnmappedCustomField(doc, customField.Key, customField.Value, fieldDefinitions); + fieldDefinition = await HandleUnmappedCustomField(doc, customField.Key, customField.Value, fieldDefinitions).AnyContext(); if (fieldDefinition == null) continue; @@ -1214,7 +1367,7 @@ protected virtual async Task OnCustomFieldsDocumentsChanging(object sender, Docu continue; } - var result = await fieldType.ProcessValueAsync(doc, customField.Value, fieldDefinition); + var result = await fieldType.ProcessValueAsync(doc, customField.Value, fieldDefinition).AnyContext(); SetDocumentCustomField(doc, customField.Key, result.Value); idx[fieldDefinition.GetIdxName()] = result.Idx ?? result.Value; @@ -1230,8 +1383,8 @@ protected virtual async Task OnCustomFieldsDocumentsChanging(object sender, Docu continue; } - object value = GetDocumentCustomField(doc, alwaysProcessField.Name); - var result = await fieldType.ProcessValueAsync(doc, value, alwaysProcessField); + var value = GetDocumentCustomField(doc, alwaysProcessField.Name); + var result = await fieldType.ProcessValueAsync(doc, value, alwaysProcessField).AnyContext(); SetDocumentCustomField(doc, alwaysProcessField.Name, result.Value); idx[alwaysProcessField.GetIdxName()] = result.Idx ?? result.Value; @@ -1251,7 +1404,7 @@ protected virtual async Task HandleUnmappedCustomField(T if (String.IsNullOrEmpty(tenantKey)) return null; - return await ElasticIndex.Configuration.CustomFieldDefinitionRepository.AddFieldAsync(EntityTypeName, GetDocumentTenantKey(document), name, StringFieldType.IndexType); + return await ElasticIndex.Configuration.CustomFieldDefinitionRepository.AddFieldAsync(EntityTypeName, GetDocumentTenantKey(document), name, StringFieldType.IndexType).AnyContext(); } protected string GetDocumentTenantKey(T document) @@ -1506,22 +1659,20 @@ private async Task IndexSingleDocumentAsync(T document, bool isCreat { var elasticVersion = ((IVersioned)document).GetElasticVersion(); i.IfPrimaryTerm(elasticVersion.PrimaryTerm); - i.IfSequenceNumber(elasticVersion.SequenceNumber); + i.IfSeqNo(elasticVersion.SequenceNumber); } - - return i; }).AnyContext(); _logger.LogRequest(response, options.GetQueryLogLevel()); - if (!response.IsValid) + if (!response.IsValidResponse) { string message = $"Error {(isCreateOperation ? "adding" : "saving")} document"; - if (response.ServerError?.Status is 409) + if (response.ElasticsearchServerError?.Status is 409) throw isCreateOperation - ? new DuplicateDocumentException(response.GetErrorMessage(message), response.OriginalException) - : new VersionConflictDocumentException(response.GetErrorMessage(message), response.OriginalException); + ? new DuplicateDocumentException(response.GetErrorMessage(message), response.OriginalException()) + : new VersionConflictDocumentException(response.GetErrorMessage(message), response.OriginalException()); - throw new DocumentException(response.GetErrorMessage(message), response.OriginalException); + throw new DocumentException(response.GetErrorMessage(message), response.OriginalException()); } if (HasVersion) @@ -1547,28 +1698,31 @@ private async Task IndexDocumentsBulkAsync(IReadOnlyCollection do var bulkRequest = new BulkRequest(); var list = docsToIndex.Select(d => { - IBulkOperation baseOperation; + var routing = GetParentIdFunc?.Invoke(d); + var index = ElasticIndex.GetIndex(d); + if (isCreateOperation) { - baseOperation = new BulkCreateOperation(d) { Pipeline = DefaultPipeline }; + var createOperation = new BulkCreateOperation(d) { Pipeline = DefaultPipeline }; + if (routing != null) + createOperation.Routing = routing; + createOperation.Index = index; + return (IBulkOperation)createOperation; } else { var indexOperation = new BulkIndexOperation(d) { Pipeline = DefaultPipeline }; + if (routing != null) + indexOperation.Routing = routing; + indexOperation.Index = index; if (HasVersion && !options.ShouldSkipVersionCheck()) { var elasticVersion = ((IVersioned)d).GetElasticVersion(); indexOperation.IfSequenceNumber = elasticVersion.SequenceNumber; indexOperation.IfPrimaryTerm = elasticVersion.PrimaryTerm; } - baseOperation = indexOperation; + return (IBulkOperation)indexOperation; } - - if (GetParentIdFunc is not null) - baseOperation.Routing = GetParentIdFunc(d); - baseOperation.Index = ElasticIndex.GetIndex(d); - - return baseOperation; }).ToList(); bulkRequest.Operations = list; bulkRequest.Refresh = options.GetRefreshMode(DefaultConsistency); @@ -1940,37 +2094,37 @@ protected virtual PartialPatch ApplyDateTracking(PartialPatch patch) return patch; var fieldPath = GetUpdatedUtcFieldPath(); - var serialized = _client.ConnectionSettings.SourceSerializer.SerializeToString(patch.Document); - var partialDoc = JToken.Parse(serialized); + var serialized = _client.ElasticsearchClientSettings.SourceSerializer.SerializeToString(patch.Document); + var partialDoc = JsonNode.Parse(serialized); - if (partialDoc is not JObject partialObject) + if (partialDoc is not JsonObject partialObject) return patch; - if (GetNestedJToken(partialObject, fieldPath) is not null) + if (GetNestedJsonNode(partialObject, fieldPath) is not null) { _logger.LogDebug("Skipping automatic {FieldPath} injection; caller already provided it", fieldPath); return patch; } - SetNestedJTokenValue(partialObject, fieldPath, - JToken.FromObject(ElasticIndex.Configuration.TimeProvider.GetUtcNow().UtcDateTime)); + SetNestedJsonNodeValue(partialObject, fieldPath, + JsonValue.Create(ElasticIndex.Configuration.TimeProvider.GetUtcNow().UtcDateTime)); return new PartialPatch(ToDictionary(partialObject)); } /// - /// Sets the updated timestamp on a document (used by single-doc JsonPatch). + /// Sets the updated timestamp on a document (used by single-doc JsonPatch). /// Supports nested dot-path fields. /// - protected virtual void ApplyDateTracking(JToken target) + protected virtual void ApplyDateTracking(JsonNode target) { if (!HasDateTracking) return; var fieldPath = GetUpdatedUtcFieldPath(); - SetNestedJTokenValue(target, fieldPath, - JToken.FromObject(ElasticIndex.Configuration.TimeProvider.GetUtcNow().UtcDateTime)); + SetNestedJsonNodeValue(target, fieldPath, + JsonValue.Create(ElasticIndex.Configuration.TimeProvider.GetUtcNow().UtcDateTime)); } private static string BuildNestedAssignmentScript(string fieldPath, string paramKey) @@ -2002,7 +2156,7 @@ private static string BuildNestedAssignmentScript(string fieldPath, string param return sb.ToString(); } - private static JToken GetNestedJToken(JToken token, string dotPath) + private static JsonNode GetNestedJsonNode(JsonNode node, string dotPath) { var remaining = dotPath.AsSpan(); @@ -2011,20 +2165,20 @@ private static JToken GetNestedJToken(JToken token, string dotPath) var dotIndex = remaining.IndexOf('.'); var segment = dotIndex >= 0 ? remaining[..dotIndex] : remaining; - if (token is not JObject obj) + if (node is not JsonObject obj) return null; - token = obj[segment.ToString()]; - if (token is null) + node = obj[segment.ToString()]; + if (node is null) return null; remaining = dotIndex >= 0 ? remaining[(dotIndex + 1)..] : ReadOnlySpan.Empty; } - return token; + return node; } - private static void SetNestedJTokenValue(JToken token, string dotPath, JToken value) + private static void SetNestedJsonNodeValue(JsonNode node, string dotPath, JsonNode value) { var remaining = dotPath.AsSpan(); @@ -2034,7 +2188,7 @@ private static void SetNestedJTokenValue(JToken token, string dotPath, JToken va if (dotIndex < 0) { - if (token is JObject leaf) + if (node is JsonObject leaf) leaf[remaining.ToString()] = value; return; @@ -2042,40 +2196,41 @@ private static void SetNestedJTokenValue(JToken token, string dotPath, JToken va var segment = remaining[..dotIndex].ToString(); - if (token is not JObject current) + if (node is not JsonObject current) return; var next = current[segment]; - if (next is not JObject) + if (next is not JsonObject) { - next = new JObject(); + next = new JsonObject(); current[segment] = next; } - token = next; + node = next; remaining = remaining[(dotIndex + 1)..]; } } - private static Dictionary ToDictionary(JObject obj) + private static Dictionary ToDictionary(JsonObject obj) { var dict = new Dictionary(obj.Count); - foreach (var property in obj.Properties()) - dict[property.Name] = ToValue(property.Value); + foreach (var property in obj) + dict[property.Key] = ToValue(property.Value); return dict; } - private static object ToValue(JToken token) + private static object ToValue(JsonNode node) { - return token switch + return node switch { - JObject nested => ToDictionary(nested), - JArray array => array.Select(ToValue).ToList(), - JValue value => value.Value, - _ => token.ToObject() + JsonObject nested => ToDictionary(nested), + JsonArray array => array.Select(ToValue).ToList(), + JsonValue val => val.Deserialize(), + null => null, + _ => node.Deserialize() }; } } diff --git a/src/Foundatio.Repositories.Elasticsearch/Repositories/IParentChildDocument.cs b/src/Foundatio.Repositories.Elasticsearch/Repositories/IParentChildDocument.cs index 48a826ab..348cbcd9 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Repositories/IParentChildDocument.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Repositories/IParentChildDocument.cs @@ -1,5 +1,5 @@ -using Foundatio.Repositories.Models; -using Nest; +using Elastic.Clients.Elasticsearch; +using Foundatio.Repositories.Models; namespace Foundatio.Repositories.Elasticsearch; diff --git a/src/Foundatio.Repositories.Elasticsearch/Repositories/MigrationStateRepository.cs b/src/Foundatio.Repositories.Elasticsearch/Repositories/MigrationStateRepository.cs index 43e8e14b..3d33fb4a 100644 --- a/src/Foundatio.Repositories.Elasticsearch/Repositories/MigrationStateRepository.cs +++ b/src/Foundatio.Repositories.Elasticsearch/Repositories/MigrationStateRepository.cs @@ -1,6 +1,7 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Migrations; -using Nest; namespace Foundatio.Repositories.Elasticsearch; @@ -22,21 +23,22 @@ public MigrationIndex(IElasticConfiguration configuration, string name = "migrat _replicas = replicas; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p - .Keyword(f => f.Name(e => e.Id)) - .Number(f => f.Name(e => e.Version).Type(NumberType.Integer)) - .Date(f => f.Name(e => e.StartedUtc)) - .Date(f => f.Name(e => e.CompletedUtc)) + .Keyword(f => f.Id) + .IntegerNumber(f => f.Version) + .Date(f => f.StartedUtc) + .Date(f => f.CompletedUtc) ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx).Settings(s => s + base.ConfigureIndex(idx); + idx.Settings(s => s .NumberOfShards(1) .NumberOfReplicas(_replicas)); } diff --git a/src/Foundatio.Repositories.Elasticsearch/Utility/FieldValueHelper.cs b/src/Foundatio.Repositories.Elasticsearch/Utility/FieldValueHelper.cs new file mode 100644 index 00000000..a950f635 --- /dev/null +++ b/src/Foundatio.Repositories.Elasticsearch/Utility/FieldValueHelper.cs @@ -0,0 +1,31 @@ +using System; +using Elastic.Clients.Elasticsearch; + +namespace Foundatio.Repositories.Elasticsearch.Utility; + +public static class FieldValueHelper +{ + public static FieldValue ToFieldValue(object value) + { + return value switch + { + null => FieldValue.Null, + string s => FieldValue.String(s), + bool b => FieldValue.Boolean(b), + long l => FieldValue.Long(l), + int i => FieldValue.Long(i), + short s16 => FieldValue.Long(s16), + byte b8 => FieldValue.Long(b8), + sbyte sb => FieldValue.Long(sb), + uint ui => FieldValue.Long(ui), + ulong ul => ul <= (ulong)long.MaxValue ? FieldValue.Long((long)ul) : FieldValue.Double((double)ul), + ushort us => FieldValue.Long(us), + double d => FieldValue.Double(d), + float f => FieldValue.Double(f), + decimal m => FieldValue.Double((double)m), + DateTime dt => FieldValue.String(dt.ToString("o")), + DateTimeOffset dto => FieldValue.String(dto.ToString("o")), + _ => FieldValue.String(value.ToString()) + }; + } +} diff --git a/src/Foundatio.Repositories/Extensions/StringExtensions.cs b/src/Foundatio.Repositories/Extensions/StringExtensions.cs index 37e0a23f..a9f96503 100644 --- a/src/Foundatio.Repositories/Extensions/StringExtensions.cs +++ b/src/Foundatio.Repositories/Extensions/StringExtensions.cs @@ -1,31 +1,9 @@ using System; -using System.Text; namespace Foundatio.Repositories.Extensions; public static class StringExtensions { - public static string ToJTokenPath(this string path) - { - if (path.StartsWith("$")) - return path; - - var sb = new StringBuilder(); - string[] parts = path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < parts.Length; i++) - { - if (parts[i].IsNumeric()) - sb.Append("[").Append(parts[i]).Append("]"); - else - sb.Append(parts[i]); - - if (i < parts.Length - 1 && !parts[i + 1].IsNumeric()) - sb.Append("."); - } - - return sb.ToString(); - } - public static bool IsNumeric(this string value) { if (String.IsNullOrEmpty(value)) diff --git a/src/Foundatio.Repositories/JsonPatch/AbstractPatcher.cs b/src/Foundatio.Repositories/JsonPatch/AbstractPatcher.cs index 786f8377..e2cfc20e 100644 --- a/src/Foundatio.Repositories/JsonPatch/AbstractPatcher.cs +++ b/src/Foundatio.Repositories/JsonPatch/AbstractPatcher.cs @@ -1,4 +1,4 @@ -namespace Foundatio.Repositories.Utility; +namespace Foundatio.Repositories.Utility; public abstract class AbstractPatcher where TDoc : class { @@ -12,18 +12,27 @@ public virtual void Patch(ref TDoc target, PatchDocument document) public virtual TDoc ApplyOperation(Operation operation, TDoc target) { - if (operation is AddOperation) - Add((AddOperation)operation, target); - else if (operation is CopyOperation) - Copy((CopyOperation)operation, target); - else if (operation is MoveOperation) - Move((MoveOperation)operation, target); - else if (operation is RemoveOperation) - Remove((RemoveOperation)operation, target); - else if (operation is ReplaceOperation) - target = Replace((ReplaceOperation)operation, target) ?? target; - else if (operation is TestOperation) - Test((TestOperation)operation, target); + switch (operation) + { + case AddOperation add: + Add(add, target); + break; + case CopyOperation copy: + Copy(copy, target); + break; + case MoveOperation move: + Move(move, target); + break; + case RemoveOperation remove: + Remove(remove, target); + break; + case ReplaceOperation replace: + target = Replace(replace, target) ?? target; + break; + case TestOperation test: + Test(test, target); + break; + } return target; } diff --git a/src/Foundatio.Repositories/JsonPatch/AddOperation.cs b/src/Foundatio.Repositories/JsonPatch/AddOperation.cs index 584f9069..70b7c34c 100644 --- a/src/Foundatio.Repositories/JsonPatch/AddOperation.cs +++ b/src/Foundatio.Repositories/JsonPatch/AddOperation.cs @@ -1,13 +1,13 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; public class AddOperation : Operation { - public JToken Value { get; set; } + public JsonNode Value { get; set; } - public override void Write(JsonWriter writer) + public override void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); @@ -18,9 +18,9 @@ public override void Write(JsonWriter writer) writer.WriteEndObject(); } - public override void Read(JObject jOperation) + public override void Read(JsonObject jOperation) { - Path = jOperation.Value("path"); - Value = jOperation.GetValue("value"); + Path = jOperation["path"]?.GetValue(); + Value = jOperation["value"]?.DeepClone(); } } diff --git a/src/Foundatio.Repositories/JsonPatch/CopyOperation.cs b/src/Foundatio.Repositories/JsonPatch/CopyOperation.cs index 66d03b80..9b928eb4 100644 --- a/src/Foundatio.Repositories/JsonPatch/CopyOperation.cs +++ b/src/Foundatio.Repositories/JsonPatch/CopyOperation.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; @@ -7,7 +7,7 @@ public class CopyOperation : Operation { public string FromPath { get; set; } - public override void Write(JsonWriter writer) + public override void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); @@ -18,9 +18,9 @@ public override void Write(JsonWriter writer) writer.WriteEndObject(); } - public override void Read(JObject jOperation) + public override void Read(JsonObject jOperation) { - Path = jOperation.Value("path"); - FromPath = jOperation.Value("from"); + Path = jOperation["path"]?.GetValue(); + FromPath = jOperation["from"]?.GetValue(); } } diff --git a/src/Foundatio.Repositories/JsonPatch/JsonDiffer.cs b/src/Foundatio.Repositories/JsonPatch/JsonDiffer.cs index a63ad5cc..5a2f8464 100644 --- a/src/Foundatio.Repositories/JsonPatch/JsonDiffer.cs +++ b/src/Foundatio.Repositories/JsonPatch/JsonDiffer.cs @@ -1,30 +1,42 @@ using System; using System.Collections.Generic; using System.Linq; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; +/// +/// Computes the difference between two JSON documents and generates a JSON Patch document. +/// Converted from Newtonsoft.Json (JToken) to System.Text.Json (JsonNode) to align with +/// Elastic.Clients.Elasticsearch which exclusively uses System.Text.Json for serialization. +/// public class JsonDiffer { internal static string Extend(string path, string extension) { - // TODO: JSON property name needs escaping for path ?? - return path + "/" + extension; + return $"{path}/{EscapeJsonPointer(extension)}"; } - private static Operation Build(string op, string path, string key, JToken value) + private static string EscapeJsonPointer(string value) { - if (String.IsNullOrEmpty(key)) - return Operation.Parse("{ 'op' : '" + op + "' , path: '" + path + "', value: " + - (value == null ? "null" : value.ToString(Formatting.None)) + "}"); + return value.Replace("~", "~0").Replace("/", "~1"); + } + + private static Operation Build(string op, string path, string key, JsonNode value) + { + string fullPath = String.IsNullOrEmpty(key) ? path : Extend(path, key); + var jOperation = new JsonObject + { + ["op"] = op, + ["path"] = fullPath + }; + if (!String.Equals(op, "remove", StringComparison.Ordinal)) + jOperation["value"] = value?.DeepClone(); - return Operation.Parse("{ op : '" + op + "' , path : '" + Extend(path, key) + "' , value : " + - (value == null ? "null" : value.ToString(Formatting.None)) + "}"); + return Operation.Build(jOperation); } - internal static Operation Add(string path, string key, JToken value) + internal static Operation Add(string path, string key, JsonNode value) { return Build("add", path, key, value); } @@ -34,21 +46,21 @@ internal static Operation Remove(string path, string key) return Build("remove", path, key, null); } - internal static Operation Replace(string path, string key, JToken value) + internal static Operation Replace(string path, string key, JsonNode value) { return Build("replace", path, key, value); } - internal static IEnumerable CalculatePatch(JToken left, JToken right, bool useIdToDetermineEquality, + internal static IEnumerable CalculatePatch(JsonNode left, JsonNode right, bool useIdToDetermineEquality, string path = "") { - if (left.Type != right.Type) + if (GetNodeType(left) != GetNodeType(right)) { yield return JsonDiffer.Replace(path, "", right); yield break; } - if (left.Type == JTokenType.Array) + if (left is JsonArray) { Operation prev = null; foreach (var operation in ProcessArray(left, right, path, useIdToDetermineEquality)) @@ -69,27 +81,27 @@ internal static IEnumerable CalculatePatch(JToken left, JToken right, if (prev != null) yield return prev; } - else if (left.Type == JTokenType.Object) + else if (left is JsonObject leftObj && right is JsonObject rightObj) { - var lprops = ((IDictionary)left).OrderBy(p => p.Key); - var rprops = ((IDictionary)right).OrderBy(p => p.Key); + var lprops = leftObj.OrderBy(p => p.Key).ToList(); + var rprops = rightObj.OrderBy(p => p.Key).ToList(); - foreach (var removed in lprops.Except(rprops, MatchesKey.Instance)) + foreach (var removed in lprops.Where(l => !rprops.Any(r => r.Key == l.Key))) { yield return JsonDiffer.Remove(path, removed.Key); } - foreach (var added in rprops.Except(lprops, MatchesKey.Instance)) + foreach (var added in rprops.Where(r => !lprops.Any(l => l.Key == r.Key))) { yield return JsonDiffer.Add(path, added.Key, added.Value); } var matchedKeys = lprops.Select(x => x.Key).Intersect(rprops.Select(y => y.Key)); - var zipped = matchedKeys.Select(k => new { key = k, left = left[k], right = right[k] }); + var zipped = matchedKeys.Select(k => new { key = k, left = leftObj[k], right = rightObj[k] }); foreach (var match in zipped) { - string newPath = path + "/" + match.key; + string newPath = Extend(path, match.key); foreach (var patch in CalculatePatch(match.left, match.right, useIdToDetermineEquality, newPath)) yield return patch; } @@ -97,34 +109,49 @@ internal static IEnumerable CalculatePatch(JToken left, JToken right, } else { - // Two values, same type, not JObject so no properties + // Two values, same type, not JsonObject so no properties - if (left.ToString() == right.ToString()) + if (JsonNodeToString(left) == JsonNodeToString(right)) yield break; else yield return JsonDiffer.Replace(path, "", right); } } - private static IEnumerable ProcessArray(JToken left, JToken right, string path, + private static string GetNodeType(JsonNode node) + { + return node switch + { + null => "null", + JsonObject => "object", + JsonArray => "array", + JsonValue v => v.GetValueKind().ToString(), + _ => "unknown" + }; + } + + private static string JsonNodeToString(JsonNode node) + { + return node?.ToJsonString() ?? "null"; + } + + private static IEnumerable ProcessArray(JsonNode left, JsonNode right, string path, bool useIdPropertyToDetermineEquality) { - var comparer = new CustomCheckEqualityComparer(useIdPropertyToDetermineEquality, new JTokenEqualityComparer()); + var comparer = new CustomCheckEqualityComparer(useIdPropertyToDetermineEquality); int commonHead = 0; int commonTail = 0; - var array1 = left.ToArray(); + var array1 = (left as JsonArray)?.ToArray() ?? Array.Empty(); int len1 = array1.Length; - var array2 = right.ToArray(); + var array2 = (right as JsonArray)?.ToArray() ?? Array.Empty(); int len2 = array2.Length; - // if (len1 == 0 && len2 ==0 ) yield break; while (commonHead < len1 && commonHead < len2) { if (comparer.Equals(array1[commonHead], array2[commonHead]) == false) break; - //diff and yield objects here - foreach (var operation in CalculatePatch(array1[commonHead], array2[commonHead], useIdPropertyToDetermineEquality, path + "/" + commonHead)) + foreach (var operation in CalculatePatch(array1[commonHead], array2[commonHead], useIdPropertyToDetermineEquality, $"{path}/{commonHead}")) { yield return operation; } @@ -139,7 +166,7 @@ private static IEnumerable ProcessArray(JToken left, JToken right, st int index1 = len1 - 1 - commonTail; int index2 = len2 - 1 - commonTail; - foreach (var operation in CalculatePatch(array1[index1], array2[index2], useIdPropertyToDetermineEquality, path + "/" + index1)) + foreach (var operation in CalculatePatch(array1[index1], array2[index2], useIdPropertyToDetermineEquality, $"{path}/{index1}")) { yield return operation; } @@ -151,7 +178,7 @@ private static IEnumerable ProcessArray(JToken left, JToken right, st yield return new ReplaceOperation { Path = path, - Value = new JArray(array2) + Value = new JsonArray(array2.Select(n => n?.DeepClone()).ToArray()) }; yield break; } @@ -162,92 +189,71 @@ private static IEnumerable ProcessArray(JToken left, JToken right, st { yield return new RemoveOperation { - Path = path + "/" + commonHead + Path = $"{path}/{commonHead}" }; } for (int i = 0; i < rightMiddle.Length; i++) { yield return new AddOperation { - Value = rightMiddle[i], - Path = path + "/" + (i + commonHead) + Value = rightMiddle[i]?.DeepClone(), + Path = $"{path}/{i + commonHead}" }; } } - private class MatchesKey : IEqualityComparer> - { - public static MatchesKey Instance = new(); - - public bool Equals(KeyValuePair x, KeyValuePair y) - { - return x.Key.Equals(y.Key); - } - - public int GetHashCode(KeyValuePair obj) - { - return obj.Key.GetHashCode(); - } - } - - public PatchDocument Diff(JToken @from, JToken to, bool useIdPropertyToDetermineEquality) + public PatchDocument Diff(JsonNode @from, JsonNode to, bool useIdPropertyToDetermineEquality) { return new PatchDocument(CalculatePatch(@from, to, useIdPropertyToDetermineEquality).ToArray()); } } -internal class CustomCheckEqualityComparer : IEqualityComparer +internal class CustomCheckEqualityComparer : IEqualityComparer { private readonly bool _enableIdCheck; - private readonly IEqualityComparer _inner; - public CustomCheckEqualityComparer(bool enableIdCheck, IEqualityComparer inner) + public CustomCheckEqualityComparer(bool enableIdCheck) { _enableIdCheck = enableIdCheck; - _inner = inner; } - public bool Equals(JToken x, JToken y) + public bool Equals(JsonNode x, JsonNode y) { - if (!_enableIdCheck || x.Type != JTokenType.Object || y.Type != JTokenType.Object) - return _inner.Equals(x, y); - - var xIdToken = x["id"]; - var yIdToken = y["id"]; + if (!_enableIdCheck || x is not JsonObject xObj || y is not JsonObject yObj) + return JsonNode.DeepEquals(x, y); - string xId = xIdToken?.Value(); - string yId = yIdToken?.Value(); + string xId = xObj["id"]?.GetValue(); + string yId = yObj["id"]?.GetValue(); if (xId != null && xId == yId) { return true; } - return _inner.Equals(x, y); + return JsonNode.DeepEquals(x, y); } - public int GetHashCode(JToken obj) + public int GetHashCode(JsonNode obj) { - if (!_enableIdCheck || obj.Type != JTokenType.Object) - return _inner.GetHashCode(obj); + if (obj is null) + return 0; - var xIdToken = obj["id"]; - string xId = xIdToken != null && xIdToken.HasValues ? xIdToken.Value() : null; + if (!_enableIdCheck || obj is not JsonObject xObj) + return obj.ToJsonString().GetHashCode(); + + string xId = xObj["id"]?.GetValue(); if (xId != null) - return xId.GetHashCode() + _inner.GetHashCode(obj); + return xId.GetHashCode() + obj.ToJsonString().GetHashCode(); - return _inner.GetHashCode(obj); + return obj.ToJsonString().GetHashCode(); } - public static bool HaveEqualIds(JToken x, JToken y) + public static bool HaveEqualIds(JsonNode x, JsonNode y) { - if (x.Type != JTokenType.Object || y.Type != JTokenType.Object) + if (x is not JsonObject xObj || y is not JsonObject yObj) return false; - var xIdToken = x["id"]; - var yIdToken = y["id"]; - - string xId = xIdToken?.Value(); - string yId = yIdToken?.Value(); + string xId = xObj["id"]?.GetValue(); + string yId = yObj["id"]?.GetValue(); return xId != null && xId == yId; } diff --git a/src/Foundatio.Repositories/JsonPatch/JsonPatcher.cs b/src/Foundatio.Repositories/JsonPatch/JsonPatcher.cs index f1857b8d..2d27149b 100644 --- a/src/Foundatio.Repositories/JsonPatch/JsonPatcher.cs +++ b/src/Foundatio.Repositories/JsonPatch/JsonPatcher.cs @@ -1,184 +1,519 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; using Foundatio.Repositories.Extensions; -using Newtonsoft.Json.Linq; namespace Foundatio.Repositories.Utility; -public class JsonPatcher : AbstractPatcher +/// +/// Applies JSON Patch operations to a JSON document. +/// Converted from Newtonsoft.Json (JToken) to System.Text.Json (JsonNode) to align with +/// Elastic.Clients.Elasticsearch which exclusively uses System.Text.Json for serialization. +/// +public class JsonPatcher : AbstractPatcher { - protected override JToken Replace(ReplaceOperation operation, JToken target) + protected override JsonNode Replace(ReplaceOperation operation, JsonNode target) { var tokens = target.SelectPatchTokens(operation.Path).ToList(); if (tokens.Count == 0) { string[] parts = operation.Path.Split('/'); string parentPath = String.Join("/", parts.Select((p, i) => i < parts.Length - 1 ? p : String.Empty).Where(p => p.Length > 0)); - string propertyName = parts.LastOrDefault(); + string propertyName = JsonNodeExtensions.UnescapeJsonPointer(parts.LastOrDefault() ?? String.Empty); - if (target.SelectOrCreatePatchToken(parentPath) is not JObject parent) + if (target.SelectOrCreatePatchToken(parentPath) is not JsonObject parent) return target; - parent[propertyName] = operation.Value; + parent[propertyName] = operation.Value?.DeepClone(); return target; } foreach (var token in tokens) { - if (token.Parent != null) - token.Replace(operation.Value); + var parent = token.Parent; + if (parent is JsonObject parentObj) + { + var propName = parentObj.FirstOrDefault(p => ReferenceEquals(p.Value, token)).Key; + if (propName != null) + parentObj[propName] = operation.Value?.DeepClone(); + } + else if (parent is JsonArray parentArr) + { + var index = parentArr.ToList().IndexOf(token); + if (index >= 0) + parentArr[index] = operation.Value?.DeepClone(); + } else // root object - return operation.Value; + { + return operation.Value?.DeepClone(); + } } return target; } - protected override void Add(AddOperation operation, JToken target) + protected override void Add(AddOperation operation, JsonNode target) { string[] parts = operation.Path.Split('/'); string parentPath = String.Join("/", parts.Select((p, i) => i < parts.Length - 1 ? p : String.Empty).Where(p => p.Length > 0)); - string propertyName = parts.LastOrDefault(); + string propertyName = JsonNodeExtensions.UnescapeJsonPointer(parts.LastOrDefault() ?? String.Empty); if (propertyName == "-") { - var array = target.SelectOrCreatePatchArrayToken(parentPath) as JArray; - array?.Add(operation.Value); + var array = target.SelectOrCreatePatchArrayToken(parentPath) as JsonArray; + array?.Add(operation.Value?.DeepClone()); } else if (propertyName.IsNumeric()) { - var array = target.SelectOrCreatePatchArrayToken(parentPath) as JArray; + var array = target.SelectOrCreatePatchArrayToken(parentPath) as JsonArray; if (Int32.TryParse(propertyName, out int index)) - array?.Insert(index, operation.Value); + array?.Insert(index, operation.Value?.DeepClone()); } else { - var parent = target.SelectOrCreatePatchToken(parentPath) as JObject; - var property = parent?.Property(propertyName); - if (property == null) - parent?.Add(propertyName, operation.Value); - else - property.Value = operation.Value; + var parent = target.SelectOrCreatePatchToken(parentPath) as JsonObject; + if (parent != null) + { + if (parent.ContainsKey(propertyName)) + parent[propertyName] = operation.Value?.DeepClone(); + else + parent.Add(propertyName, operation.Value?.DeepClone()); + } } } - protected override void Remove(RemoveOperation operation, JToken target) + protected override void Remove(RemoveOperation operation, JsonNode target) { - var tokens = target.SelectPatchTokens(operation.Path).ToList(); - if (tokens.Count == 0) - return; - - foreach (var token in tokens) + // Handle JSONPath expressions (e.g., $.books[?(@.author == 'X')]) + if (operation.Path.StartsWith("$.", StringComparison.Ordinal) || operation.Path.StartsWith("$[", StringComparison.Ordinal)) { - if (token.Parent is JProperty) + var tokens = target.SelectPatchTokens(operation.Path).ToList(); + foreach (var token in tokens) { - token.Parent.Remove(); - } - else - { - token.Remove(); + var tokenParent = token.Parent; + if (tokenParent is JsonArray arr) + { + for (int i = 0; i < arr.Count; i++) + { + if (ReferenceEquals(arr[i], token)) + { + arr.RemoveAt(i); + break; + } + } + } + else if (tokenParent is JsonObject tokenParentObj) + { + var key = tokenParentObj.FirstOrDefault(p => ReferenceEquals(p.Value, token)).Key; + if (key != null) + tokenParentObj.Remove(key); + } } + return; + } + + string[] parts = operation.Path.Split('/'); + if (parts.Length == 0) + return; + + string parentPath = String.Join("/", parts.Select((p, i) => i < parts.Length - 1 ? p : String.Empty).Where(p => p.Length > 0)); + string propertyName = JsonNodeExtensions.UnescapeJsonPointer(parts.LastOrDefault() ?? String.Empty); + + if (String.IsNullOrEmpty(propertyName)) + return; + + var parent = target.SelectPatchToken(parentPath); + if (parent is JsonObject parentObjPointer) + { + if (parentObjPointer.ContainsKey(propertyName)) + parentObjPointer.Remove(propertyName); + } + else if (parent is JsonArray parentArr) + { + if (int.TryParse(propertyName, out int index) && index >= 0 && index < parentArr.Count) + parentArr.RemoveAt(index); } } - protected override void Move(MoveOperation operation, JToken target) + protected override void Move(MoveOperation operation, JsonNode target) { - if (operation.Path.StartsWith(operation.FromPath)) + if (operation.Path == operation.FromPath || operation.Path.StartsWith(operation.FromPath + "/")) throw new ArgumentException("To path cannot be below from path"); var token = target.SelectPatchToken(operation.FromPath); Remove(new RemoveOperation { Path = operation.FromPath }, target); - Add(new AddOperation { Path = operation.Path, Value = token }, target); + Add(new AddOperation { Path = operation.Path, Value = token?.DeepClone() }, target); } - protected override void Test(TestOperation operation, JToken target) + protected override void Test(TestOperation operation, JsonNode target) { var existingValue = target.SelectPatchToken(operation.Path); - if (!existingValue.Equals(target)) + if (!JsonNode.DeepEquals(existingValue, operation.Value)) { - throw new InvalidOperationException("Value at " + operation.Path + " does not match."); + throw new InvalidOperationException($"Value at {operation.Path} does not match."); } } - protected override void Copy(CopyOperation operation, JToken target) + protected override void Copy(CopyOperation operation, JsonNode target) { - var token = target.SelectPatchToken(operation.FromPath); // Do I need to clone this? - Add(new AddOperation { Path = operation.Path, Value = token }, target); + var token = target.SelectPatchToken(operation.FromPath); + Add(new AddOperation { Path = operation.Path, Value = token?.DeepClone() }, target); } } -public static class JTokenExtensions +/// +/// Extension methods for JsonNode to support JSON Pointer paths used in JSON Patch. +/// +public static class JsonNodeExtensions { - public static JToken SelectPatchToken(this JToken token, string path) + private static readonly Regex _dotPropFilterRegex = new(@"^@\.(\w+)\s*==\s*'(.+)'$", RegexOptions.Compiled); + private static readonly Regex _directValueFilterRegex = new(@"^@\s*==\s*'(.+)'$", RegexOptions.Compiled); + + public static JsonNode SelectPatchToken(this JsonNode token, string path) { - return token.SelectToken(path.ToJTokenPath()); + return SelectToken(token, path.ToJsonPointerPath()); } - public static IEnumerable SelectPatchTokens(this JToken token, string path) + public static IEnumerable SelectPatchTokens(this JsonNode token, string path) { - return token.SelectTokens(path.ToJTokenPath()); + if (path.StartsWith("$.", StringComparison.Ordinal) || path.StartsWith("$[", StringComparison.Ordinal)) + return SelectJsonPathTokens(token, path); + + var result = SelectToken(token, path.ToJsonPointerPath()); + if (result != null) + return new[] { result }; + return Enumerable.Empty(); } - public static JToken SelectOrCreatePatchToken(this JToken token, string path) + /// + /// Evaluates a subset of JSONPath expressions against a JSON node. + /// Supports filter expressions like $.array[?(@.prop == 'value')] and $.array[?(@ == 'value')]. + /// + private static IEnumerable SelectJsonPathTokens(JsonNode root, string path) { - var result = token.SelectToken(path.ToJTokenPath()); - if (result != null) - return result; + // Strip leading $ + string remaining = path.StartsWith("$.", StringComparison.Ordinal) ? path[2..] : path[1..]; - string[] parts = path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Any(p => p.IsNumeric())) - return null; + // Split on dots, but respect brackets + var segments = SplitJsonPathSegments(remaining); + IEnumerable current = new[] { root }; - JToken current = token; - for (int i = 0; i < parts.Length; i++) + foreach (var segment in segments) { - string part = parts[i]; - var partToken = current.SelectPatchToken(part); - if (partToken == null) + var next = new List(); + foreach (var node in current) { - if (current is JObject partObject) - current = partObject[part] = new JObject(); + if (node is null) continue; + + // Segment like "books[?(@.author == 'X')]" or "books[?(@.author == 'X')].prop" + // or pure filter "[?(@.author == 'X')]" + int bracketStart = segment.IndexOf('['); + if (bracketStart >= 0) + { + // Navigate to named property first (if any) + JsonNode target2 = node; + if (bracketStart > 0) + { + string propName = segment[..bracketStart]; + if (node is JsonObject propObj && propObj.TryGetPropertyValue(propName, out var propVal) && propVal is not null) + target2 = propVal; + else + continue; + } + + // Extract the bracket expression + int bracketEnd = segment.LastIndexOf(']'); + if (bracketEnd < 0) continue; + string expr = segment[(bracketStart + 1)..bracketEnd]; + + // Filter expression: [?(...)] + if (expr.StartsWith("?(", StringComparison.Ordinal) && expr.EndsWith(')')) + { + string filter = expr[2..^1]; + if (target2 is JsonArray arr) + next.AddRange(arr.Where(item => item is not null && EvaluateJsonPathFilter(item, filter)).Select(item => item!)); + } + } + else + { + if (node is JsonObject obj && obj.TryGetPropertyValue(segment, out var value) && value is not null) + next.Add(value); + } + } + current = next; + } + + return current; + } + + private static bool EvaluateJsonPathFilter(JsonNode node, string filter) + { + var dotPropMatch = _dotPropFilterRegex.Match(filter); + if (dotPropMatch.Success) + { + string prop = dotPropMatch.Groups[1].Value; + string expected = dotPropMatch.Groups[2].Value; + if (node is JsonObject obj && obj.TryGetPropertyValue(prop, out var val)) + return val?.GetValue() == expected; + return false; + } + + var directMatch = _directValueFilterRegex.Match(filter); + if (directMatch.Success) + { + string expected = directMatch.Groups[1].Value; + if (node is JsonValue jsonVal) + { + try + { + return jsonVal.GetValue() == expected; + } + catch (InvalidOperationException) { return false; } + catch (FormatException) { return false; } + } + + return false; + } + + return false; + } + + private static List SplitJsonPathSegments(string path) + { + var segments = new List(); + int start = 0; + int depth = 0; + + for (int i = 0; i < path.Length; i++) + { + char c = path[i]; + if (c == '[') depth++; + else if (c == ']') depth--; + else if (c == '.' && depth == 0) + { + if (i > start) + segments.Add(path[start..i]); + start = i + 1; + } + } + + if (start < path.Length) + segments.Add(path[start..]); + + return segments; + } + + private static JsonNode SelectToken(JsonNode node, string[] pathParts) + { + JsonNode current = node; + foreach (var part in pathParts) + { + if (current == null) + return null; + + if (current is JsonObject obj) + { + if (!obj.TryGetPropertyValue(part, out var value)) + return null; + current = value; + } + else if (current is JsonArray arr) + { + if (!int.TryParse(part, out int index) || index < 0 || index >= arr.Count) + return null; + current = arr[index]; } else { - current = partToken; + return null; } } return current; } - public static JToken SelectOrCreatePatchArrayToken(this JToken token, string path) + public static JsonNode SelectOrCreatePatchToken(this JsonNode token, string path) { - var result = token.SelectToken(path.ToJTokenPath()); - if (result != null) - return result; + var pathParts = path.ToJsonPointerPath(); + if (pathParts.Length == 0) + return token; + + // Validate that the path can be created: numeric parts must resolve to existing array indices + JsonNode validationNode = token; + for (int i = 0; i < pathParts.Length; i++) + { + string part = pathParts[i]; + + if (validationNode is JsonObject validationObj) + { + if (validationObj.TryGetPropertyValue(part, out var partToken)) + validationNode = partToken; + else if (part.IsNumeric()) + return null; + else + validationNode = null; + } + else if (validationNode is JsonArray validationArr) + { + if (int.TryParse(part, out int index) && index >= 0 && index < validationArr.Count) + validationNode = validationArr[index]; + else + return null; + } + else if (validationNode == null) + { + if (part.IsNumeric()) + return null; + } + else + { + return null; + } + } + + // Create missing intermediate objects + JsonNode current = token; + for (int i = 0; i < pathParts.Length; i++) + { + string part = pathParts[i]; + + if (current is JsonObject currentObj) + { + if (currentObj.TryGetPropertyValue(part, out var partToken)) + { + current = partToken; + } + else + { + var newObj = new JsonObject(); + currentObj[part] = newObj; + current = newObj; + } + } + else if (current is JsonArray currentArr) + { + if (int.TryParse(part, out int index) && index >= 0 && index < currentArr.Count) + { + current = currentArr[index]; + } + else + { + return null; + } + } + else + { + return null; + } + } + + return current; + } - string[] parts = path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Any(p => p.IsNumeric())) - return null; + public static JsonNode SelectOrCreatePatchArrayToken(this JsonNode token, string path) + { + var pathParts = path.ToJsonPointerPath(); + if (pathParts.Length == 0) + return token; - JToken current = token; - for (int i = 0; i < parts.Length; i++) + // Validate that the path can be created: numeric parts must resolve to existing array indices + JsonNode validationNode = token; + for (int i = 0; i < pathParts.Length; i++) { - string part = parts[i]; - var partToken = current.SelectPatchToken(part); - if (partToken == null) + string part = pathParts[i]; + + if (validationNode is JsonObject validationObj) + { + if (validationObj.TryGetPropertyValue(part, out var partToken)) + validationNode = partToken; + else if (part.IsNumeric()) + return null; + else + validationNode = null; + } + else if (validationNode is JsonArray validationArr) { - if (current is JObject partObject) + if (int.TryParse(part, out int index) && index >= 0 && index < validationArr.Count) + validationNode = validationArr[index]; + else + return null; + } + else if (validationNode == null) + { + if (part.IsNumeric()) + return null; + } + else + { + return null; + } + } + + // Create missing intermediate objects (arrays for last segment) + JsonNode current = token; + for (int i = 0; i < pathParts.Length; i++) + { + string part = pathParts[i]; + bool isLastPart = i == pathParts.Length - 1; + + if (current is JsonObject currentObj) + { + if (currentObj.TryGetPropertyValue(part, out var partToken)) + { + current = partToken; + } + else + { + JsonNode newNode = isLastPart ? new JsonArray() : new JsonObject(); + currentObj[part] = newNode; + current = newNode; + } + } + else if (current is JsonArray currentArr) + { + if (int.TryParse(part, out int index) && index >= 0 && index < currentArr.Count) + { + current = currentArr[index]; + } + else { - bool isLastPart = i == parts.Length - 1; - current = partObject[part] = isLastPart ? new JArray() : new JObject(); + return null; } } else { - current = partToken; + return null; } } return current; } + + /// + /// Converts a JSON Patch path to an array of path segments, unescaping per RFC 6901. + /// + private static string[] ToJsonPointerPath(this string path) + { + if (String.IsNullOrEmpty(path)) + return Array.Empty(); + + if (path.StartsWith('$')) + throw new NotSupportedException($"JSONPath expressions are not supported in patch operations. Use JSON Pointer format (e.g., '/foo/bar') instead of JSONPath (e.g., '$.foo.bar'). Path: {path}"); + + return path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries) + .Select(UnescapeJsonPointer) + .ToArray(); + } + + /// + /// Unescapes a JSON Pointer reference token per RFC 6901 Section 4. + /// Order matters: ~1 -> / first, then ~0 -> ~. + /// + internal static string UnescapeJsonPointer(string token) + { + return token.Replace("~1", "/").Replace("~0", "~"); + } } diff --git a/src/Foundatio.Repositories/JsonPatch/MoveOperation.cs b/src/Foundatio.Repositories/JsonPatch/MoveOperation.cs index aff2d290..c3b190d8 100644 --- a/src/Foundatio.Repositories/JsonPatch/MoveOperation.cs +++ b/src/Foundatio.Repositories/JsonPatch/MoveOperation.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; @@ -7,7 +7,7 @@ public class MoveOperation : Operation { public string FromPath { get; set; } - public override void Write(JsonWriter writer) + public override void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); @@ -18,9 +18,9 @@ public override void Write(JsonWriter writer) writer.WriteEndObject(); } - public override void Read(JObject jOperation) + public override void Read(JsonObject jOperation) { - Path = jOperation.Value("path"); - FromPath = jOperation.Value("from"); + Path = jOperation["path"]?.GetValue(); + FromPath = jOperation["from"]?.GetValue(); } } diff --git a/src/Foundatio.Repositories/JsonPatch/Operation.cs b/src/Foundatio.Repositories/JsonPatch/Operation.cs index 581740ae..9e04da89 100644 --- a/src/Foundatio.Repositories/JsonPatch/Operation.cs +++ b/src/Foundatio.Repositories/JsonPatch/Operation.cs @@ -1,51 +1,56 @@ using System; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; +/// +/// Base class for JSON Patch operations (RFC 6902). +/// Converted from Newtonsoft.Json to System.Text.Json to align with Elastic.Clients.Elasticsearch +/// which exclusively uses System.Text.Json for serialization. +/// public abstract class Operation { public string Path { get; set; } - public abstract void Write(JsonWriter writer); + public abstract void Write(Utf8JsonWriter writer); - protected static void WriteOp(JsonWriter writer, string op) + protected static void WriteOp(Utf8JsonWriter writer, string op) { - writer.WritePropertyName("op"); - writer.WriteValue(op); + writer.WriteString("op", op); } - protected static void WritePath(JsonWriter writer, string path) + protected static void WritePath(Utf8JsonWriter writer, string path) { - writer.WritePropertyName("path"); - writer.WriteValue(path); + writer.WriteString("path", path); } - protected static void WriteFromPath(JsonWriter writer, string path) + protected static void WriteFromPath(Utf8JsonWriter writer, string path) { - writer.WritePropertyName("from"); - writer.WriteValue(path); + writer.WriteString("from", path); } - protected static void WriteValue(JsonWriter writer, JToken value) + protected static void WriteValue(Utf8JsonWriter writer, JsonNode value) { writer.WritePropertyName("value"); - value.WriteTo(writer); + if (value == null) + writer.WriteNullValue(); + else + value.WriteTo(writer); } - public abstract void Read(JObject jOperation); + public abstract void Read(JsonObject jOperation); public static Operation Parse(string json) { - return Build(JObject.Parse(json)); + return Build(JsonNode.Parse(json)?.AsObject()); } - public static Operation Build(JObject jOperation) + public static Operation Build(JsonObject jOperation) { ArgumentNullException.ThrowIfNull(jOperation); - var opName = (string)jOperation["op"]; + var opName = jOperation["op"]?.GetValue(); ArgumentException.ThrowIfNullOrWhiteSpace(opName, "op"); var op = PatchDocument.CreateOperation(opName) diff --git a/src/Foundatio.Repositories/JsonPatch/PatchDocument.cs b/src/Foundatio.Repositories/JsonPatch/PatchDocument.cs index 89d9772d..5866f085 100644 --- a/src/Foundatio.Repositories/JsonPatch/PatchDocument.cs +++ b/src/Foundatio.Repositories/JsonPatch/PatchDocument.cs @@ -1,12 +1,17 @@ -using System; using System.Collections.Generic; using System.IO; using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; namespace Foundatio.Repositories.Utility; +/// +/// Represents a JSON Patch document (RFC 6902). +/// Converted from Newtonsoft.Json to System.Text.Json to align with Elastic.Clients.Elasticsearch +/// which exclusively uses System.Text.Json for serialization. +/// [JsonConverter(typeof(PatchDocumentConverter))] public class PatchDocument { @@ -22,16 +27,28 @@ public PatchDocument(params Operation[] operations) public List Operations => _operations; - public void Add(string path, JToken value) + public void Add(string path, JsonNode value) { Operations.Add(new AddOperation { Path = path, Value = value }); } - public void Replace(string path, JToken value) + public void Add(string path, string value) => Add(path, JsonValue.Create(value)); + public void Add(string path, int value) => Add(path, JsonValue.Create(value)); + public void Add(string path, long value) => Add(path, JsonValue.Create(value)); + public void Add(string path, double value) => Add(path, JsonValue.Create(value)); + public void Add(string path, bool value) => Add(path, JsonValue.Create(value)); + + public void Replace(string path, JsonNode value) { Operations.Add(new ReplaceOperation { Path = path, Value = value }); } + public void Replace(string path, string value) => Replace(path, JsonValue.Create(value)); + public void Replace(string path, int value) => Replace(path, JsonValue.Create(value)); + public void Replace(string path, long value) => Replace(path, JsonValue.Create(value)); + public void Replace(string path, double value) => Replace(path, JsonValue.Create(value)); + public void Replace(string path, bool value) => Replace(path, JsonValue.Create(value)); + public void Remove(string path) { Operations.Add(new RemoveOperation { Path = path }); @@ -49,17 +66,17 @@ public static PatchDocument Load(Stream document) return Parse(reader.ReadToEnd()); } - public static PatchDocument Load(JArray document) + public static PatchDocument Load(JsonArray document) { var root = new PatchDocument(); if (document == null) return root; - foreach (var child in document.Children()) + foreach (var item in document) { - if (child is not JObject jOperation) - throw new ArgumentException($"Invalid patch operation: expected a JSON object but found {child.Type}"); + if (item is not JsonObject jOperation) + throw new JsonException($"Invalid patch operation: expected a JSON object but found {item?.GetValueKind().ToString() ?? "null"}"); var op = Operation.Build(jOperation); root.AddOperation(op); @@ -70,7 +87,9 @@ public static PatchDocument Load(JArray document) public static PatchDocument Parse(string jsondocument) { - var root = JToken.Parse(jsondocument) as JArray; + var node = JsonNode.Parse(jsondocument); + if (node is not JsonArray root) + throw new JsonException("Invalid JSON Patch document: expected a JSON array per RFC 6902"); return Load(root); } @@ -104,32 +123,29 @@ public MemoryStream ToStream() return stream; } - public void CopyToStream(Stream stream, Formatting formatting = Formatting.Indented) + public void CopyToStream(Stream stream, bool indented = true) { - var sw = new JsonTextWriter(new StreamWriter(stream)) - { - Formatting = formatting - }; + using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = indented }); - sw.WriteStartArray(); + writer.WriteStartArray(); foreach (var operation in Operations) - operation.Write(sw); + operation.Write(writer); - sw.WriteEndArray(); + writer.WriteEndArray(); - sw.Flush(); + writer.Flush(); } public override string ToString() { - return ToString(Formatting.Indented); + return ToString(indented: true); } - public string ToString(Formatting formatting) + public string ToString(bool indented) { using var ms = new MemoryStream(); - CopyToStream(ms, formatting); + CopyToStream(ms, indented); ms.Position = 0; using StreamReader reader = new StreamReader(ms, Encoding.UTF8); return reader.ReadToEnd(); diff --git a/src/Foundatio.Repositories/JsonPatch/PatchDocumentConverter.cs b/src/Foundatio.Repositories/JsonPatch/PatchDocumentConverter.cs index c2491f7a..55469746 100644 --- a/src/Foundatio.Repositories/JsonPatch/PatchDocumentConverter.cs +++ b/src/Foundatio.Repositories/JsonPatch/PatchDocumentConverter.cs @@ -1,43 +1,53 @@ using System; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; namespace Foundatio.Repositories.Utility; -public class PatchDocumentConverter : JsonConverter +/// +/// JSON converter for PatchDocument using System.Text.Json. +/// Converted from Newtonsoft.Json to align with Elastic.Clients.Elasticsearch +/// which exclusively uses System.Text.Json for serialization. +/// +public class PatchDocumentConverter : JsonConverter { - public override bool CanConvert(Type objectType) + public override PatchDocument Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - return true; - } + if (typeToConvert != typeof(PatchDocument)) + throw new ArgumentException("Object must be of type PatchDocument", nameof(typeToConvert)); - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) - { - if (objectType != typeof(PatchDocument)) - throw new ArgumentException("Object must be of type PatchDocument", nameof(objectType)); + if (reader.TokenType == JsonTokenType.Null) + return null; try { - if (reader.TokenType == JsonToken.Null) - return null; + var node = JsonNode.Parse(ref reader); + if (node is not JsonArray array) + throw new JsonException("Invalid patch document: expected JSON array"); - var patch = JArray.Load(reader); - return PatchDocument.Parse(patch.ToString()); + return PatchDocument.Load(array); + } + catch (JsonException) + { + throw; } catch (Exception ex) { - throw new ArgumentException("Invalid patch document: " + ex.Message, ex); + throw new JsonException("Invalid patch document: " + ex.Message, ex); } } - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + public override void Write(Utf8JsonWriter writer, PatchDocument value, JsonSerializerOptions options) { - if (!(value is PatchDocument)) + if (value == null) + { + writer.WriteNullValue(); return; + } - var jsonPatchDoc = (PatchDocument)value; writer.WriteStartArray(); - foreach (var op in jsonPatchDoc.Operations) + foreach (var op in value.Operations) op.Write(writer); writer.WriteEndArray(); } diff --git a/src/Foundatio.Repositories/JsonPatch/RemoveOperation.cs b/src/Foundatio.Repositories/JsonPatch/RemoveOperation.cs index 7ca96f11..ff2db24c 100644 --- a/src/Foundatio.Repositories/JsonPatch/RemoveOperation.cs +++ b/src/Foundatio.Repositories/JsonPatch/RemoveOperation.cs @@ -1,11 +1,11 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; public class RemoveOperation : Operation { - public override void Write(JsonWriter writer) + public override void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); @@ -15,8 +15,8 @@ public override void Write(JsonWriter writer) writer.WriteEndObject(); } - public override void Read(JObject jOperation) + public override void Read(JsonObject jOperation) { - Path = jOperation.Value("path"); + Path = jOperation["path"]?.GetValue(); } } diff --git a/src/Foundatio.Repositories/JsonPatch/ReplaceOperation.cs b/src/Foundatio.Repositories/JsonPatch/ReplaceOperation.cs index 1efa37b7..b6d01956 100644 --- a/src/Foundatio.Repositories/JsonPatch/ReplaceOperation.cs +++ b/src/Foundatio.Repositories/JsonPatch/ReplaceOperation.cs @@ -1,13 +1,13 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; public class ReplaceOperation : Operation { - public JToken Value { get; set; } + public JsonNode Value { get; set; } - public override void Write(JsonWriter writer) + public override void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); @@ -18,9 +18,9 @@ public override void Write(JsonWriter writer) writer.WriteEndObject(); } - public override void Read(JObject jOperation) + public override void Read(JsonObject jOperation) { - Path = jOperation.Value("path"); - Value = jOperation.GetValue("value"); + Path = jOperation["path"]?.GetValue(); + Value = jOperation["value"]?.DeepClone(); } } diff --git a/src/Foundatio.Repositories/JsonPatch/TestOperation.cs b/src/Foundatio.Repositories/JsonPatch/TestOperation.cs index ba7f54aa..96763fed 100644 --- a/src/Foundatio.Repositories/JsonPatch/TestOperation.cs +++ b/src/Foundatio.Repositories/JsonPatch/TestOperation.cs @@ -1,13 +1,13 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; namespace Foundatio.Repositories.Utility; public class TestOperation : Operation { - public JToken Value { get; set; } + public JsonNode Value { get; set; } - public override void Write(JsonWriter writer) + public override void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); @@ -18,9 +18,9 @@ public override void Write(JsonWriter writer) writer.WriteEndObject(); } - public override void Read(JObject jOperation) + public override void Read(JsonObject jOperation) { - Path = jOperation.Value("path"); - Value = jOperation.GetValue("value"); + Path = jOperation["path"]?.GetValue(); + Value = jOperation["value"]?.DeepClone(); } } diff --git a/src/Foundatio.Repositories/Migration/MigrationManager.cs b/src/Foundatio.Repositories/Migration/MigrationManager.cs index a32c83a1..67728aaa 100644 --- a/src/Foundatio.Repositories/Migration/MigrationManager.cs +++ b/src/Foundatio.Repositories/Migration/MigrationManager.cs @@ -267,7 +267,7 @@ private IEnumerable GetDerivedTypes(IList assemblies = catch (ReflectionTypeLoadException ex) { string loaderMessages = String.Join(", ", ex.LoaderExceptions.ToList().Select(le => le.Message)); - _logger.LogInformation("Unable to search types from assembly {Assembly} for plugins of type {PluginType}: {LoaderMessages}", assembly.FullName, typeof(TAction).Name, loaderMessages); + _logger.LogInformation(ex, "Unable to search types from assembly \"{Assembly}\" for plugins of type \"{PluginType}\": {LoaderMessages}", assembly.FullName, typeof(TAction).Name, loaderMessages); } } diff --git a/src/Foundatio.Repositories/Models/Aggregations/IAggregate.cs b/src/Foundatio.Repositories/Models/Aggregations/IAggregate.cs index f235c1b0..bebc1ea9 100644 --- a/src/Foundatio.Repositories/Models/Aggregations/IAggregate.cs +++ b/src/Foundatio.Repositories/Models/Aggregations/IAggregate.cs @@ -1,5 +1,5 @@ using System.Collections.Generic; -using Foundatio.Repositories.Utility; +using Foundatio.Repositories.Serialization; namespace Foundatio.Repositories.Models; diff --git a/src/Foundatio.Repositories/Models/Aggregations/IBucket.cs b/src/Foundatio.Repositories/Models/Aggregations/IBucket.cs index 072287a4..d53dc870 100644 --- a/src/Foundatio.Repositories/Models/Aggregations/IBucket.cs +++ b/src/Foundatio.Repositories/Models/Aggregations/IBucket.cs @@ -1,5 +1,5 @@ using System.Collections.Generic; -using Foundatio.Repositories.Utility; +using Foundatio.Repositories.Serialization; namespace Foundatio.Repositories.Models; diff --git a/src/Foundatio.Repositories/Models/Aggregations/KeyedBucket.cs b/src/Foundatio.Repositories/Models/Aggregations/KeyedBucket.cs index 0b06ff39..bb5965cb 100644 --- a/src/Foundatio.Repositories/Models/Aggregations/KeyedBucket.cs +++ b/src/Foundatio.Repositories/Models/Aggregations/KeyedBucket.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; using System.Diagnostics; -using Foundatio.Repositories.Utility; +using Foundatio.Repositories.Serialization; namespace Foundatio.Repositories.Models; @@ -16,7 +16,7 @@ public KeyedBucket(IReadOnlyDictionary aggregations) : base( { } - [System.Text.Json.Serialization.JsonConverter(typeof(ObjectToInferredTypesConverter))] + [System.Text.Json.Serialization.JsonConverter(typeof(InferredTypesConverterFactory))] public T Key { get; set; } public string KeyAsString { get; set; } public long? Total { get; set; } diff --git a/src/Foundatio.Repositories/Models/Aggregations/ObjectValueAggregate.cs b/src/Foundatio.Repositories/Models/Aggregations/ObjectValueAggregate.cs index 45ef05d7..cdf738ce 100644 --- a/src/Foundatio.Repositories/Models/Aggregations/ObjectValueAggregate.cs +++ b/src/Foundatio.Repositories/Models/Aggregations/ObjectValueAggregate.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; using Foundatio.Serializer; -using Newtonsoft.Json.Linq; namespace Foundatio.Repositories.Models; @@ -10,18 +11,19 @@ public class ObjectValueAggregate : MetricAggregateBase { public object Value { get; set; } - public T ValueAs(ITextSerializer serializer = null) + public T ValueAs(ITextSerializer serializer) { - if (serializer != null) - { - if (Value is string stringValue) - return serializer.Deserialize(stringValue); - else if (Value is JToken jTokenValue) - return serializer.Deserialize(jTokenValue.ToString()); - } + ArgumentNullException.ThrowIfNull(serializer); - return Value is JToken jToken - ? jToken.ToObject() - : (T)Convert.ChangeType(Value, typeof(T)); + if (Value is string stringValue) + return serializer.Deserialize(stringValue); + + if (Value is JsonNode jNode) + return serializer.Deserialize(jNode.ToJsonString()); + + if (Value is JsonElement jElement) + return serializer.Deserialize(jElement.GetRawText()); + + return (T)Convert.ChangeType(Value, typeof(T)); } } diff --git a/src/Foundatio.Repositories/Models/Aggregations/TopHitsAggregate.cs b/src/Foundatio.Repositories/Models/Aggregations/TopHitsAggregate.cs index 39e0d86f..5b957d21 100644 --- a/src/Foundatio.Repositories/Models/Aggregations/TopHitsAggregate.cs +++ b/src/Foundatio.Repositories/Models/Aggregations/TopHitsAggregate.cs @@ -1,22 +1,56 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; +using System.Text; +using Foundatio.Serializer; namespace Foundatio.Repositories.Models; public class TopHitsAggregate : MetricAggregateBase { - private readonly IList _hits; + private readonly IList _hits = []; public long Total { get; set; } public double? MaxScore { get; set; } + /// + /// Raw JSON sources for each hit, used for serialization/deserialization round-tripping (e.g., caching). + /// + public IReadOnlyList Hits { get; set; } + public TopHitsAggregate(IList hits) { - _hits = hits ?? new List(); + _hits = hits ?? []; } - public IReadOnlyCollection Documents() where T : class + public TopHitsAggregate() { } + + /// + /// Required when this aggregate was round-tripped through the cache (i.e., is populated). + /// Can be omitted when reading directly from an Elasticsearch response where _hits are populated from + /// the live response. + /// + public IReadOnlyCollection Documents(ITextSerializer serializer = null) where T : class { - return _hits.Select(h => h.As()).ToList(); + if (_hits.Count > 0) + return _hits.Select(h => h.As()).ToList(); + + if (Hits is { Count: > 0 }) + { + ArgumentNullException.ThrowIfNull(serializer); + + return Hits + .Select(json => + { + if (String.IsNullOrEmpty(json)) + return null; + var lazy = new LazyDocument(Encoding.UTF8.GetBytes(json), serializer); + return lazy.As(); + }) + .Where(d => d != null) + .ToList(); + } + + return new List(); } } diff --git a/src/Foundatio.Repositories/Models/LazyDocument.cs b/src/Foundatio.Repositories/Models/LazyDocument.cs index 0caef486..8ce17b4b 100644 --- a/src/Foundatio.Repositories/Models/LazyDocument.cs +++ b/src/Foundatio.Repositories/Models/LazyDocument.cs @@ -39,11 +39,13 @@ public class LazyDocument : ILazyDocument /// Initializes a new instance of the class. /// /// The raw document data. - /// The serializer to use for deserialization. Defaults to JSON. - public LazyDocument(byte[] data, ITextSerializer serializer = null) + /// The serializer to use for deserialization. + public LazyDocument(byte[] data, ITextSerializer serializer) { + ArgumentNullException.ThrowIfNull(serializer); + _data = data; - _serializer = serializer ?? new JsonNetSerializer(); + _serializer = serializer; } /// diff --git a/src/Foundatio.Repositories/Utility/AggregationsNewtonsoftJsonConverter.cs b/src/Foundatio.Repositories/Serialization/AggregationsNewtonsoftJsonConverter.cs similarity index 89% rename from src/Foundatio.Repositories/Utility/AggregationsNewtonsoftJsonConverter.cs rename to src/Foundatio.Repositories/Serialization/AggregationsNewtonsoftJsonConverter.cs index 29bbe1b4..c60238a0 100644 --- a/src/Foundatio.Repositories/Utility/AggregationsNewtonsoftJsonConverter.cs +++ b/src/Foundatio.Repositories/Serialization/AggregationsNewtonsoftJsonConverter.cs @@ -4,7 +4,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace Foundatio.Repositories.Utility; +namespace Foundatio.Repositories.Serialization; public class AggregationsNewtonsoftJsonConverter : JsonConverter { @@ -27,7 +27,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist "percentiles" => DeserializePercentiles(item, serializer), "sbucket" => DeserializeSingleBucket(item, serializer), "stats" => new StatsAggregate(), - // TopHitsAggregate cannot be round-tripped: it holds ILazyDocument references (raw ES doc bytes) that require a serializer instance to materialize. + "tophits" => new TopHitsAggregate(), "value" => new ValueAggregate(), "dvalue" => new ValueAggregate(), _ => null @@ -59,6 +59,6 @@ private static SingleBucketAggregate DeserializeSingleBucket(JObject item, JsonS public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { - throw new NotImplementedException(); + throw new NotSupportedException(); } } diff --git a/src/Foundatio.Repositories/Utility/AggregationsSystemTextJsonConverter.cs b/src/Foundatio.Repositories/Serialization/AggregationsSystemTextJsonConverter.cs similarity index 89% rename from src/Foundatio.Repositories/Utility/AggregationsSystemTextJsonConverter.cs rename to src/Foundatio.Repositories/Serialization/AggregationsSystemTextJsonConverter.cs index ebbf29d5..ddd099e0 100644 --- a/src/Foundatio.Repositories/Utility/AggregationsSystemTextJsonConverter.cs +++ b/src/Foundatio.Repositories/Serialization/AggregationsSystemTextJsonConverter.cs @@ -3,7 +3,7 @@ using System.Text.Json; using Foundatio.Repositories.Models; -namespace Foundatio.Repositories.Utility; +namespace Foundatio.Repositories.Serialization; public class AggregationsSystemTextJsonConverter : System.Text.Json.Serialization.JsonConverter { @@ -25,7 +25,7 @@ public override IAggregate Read(ref Utf8JsonReader reader, Type typeToConvert, J "percentiles" => DeserializePercentiles(element, options), "sbucket" => DeserializeSingleBucket(element, options), "stats" => element.Deserialize(options), - // TopHitsAggregate cannot be round-tripped: it holds ILazyDocument references (raw ES doc bytes) that require a serializer instance to materialize. + "tophits" => element.Deserialize(options), "value" => element.Deserialize(options), "dvalue" => element.Deserialize>(options), _ => null @@ -36,12 +36,7 @@ public override IAggregate Read(ref Utf8JsonReader reader, Type typeToConvert, J public override void Write(Utf8JsonWriter writer, IAggregate value, JsonSerializerOptions options) { - var serializerOptions = new JsonSerializerOptions(options) - { - Converters = { new DoubleSystemTextJsonConverter() } - }; - - JsonSerializer.Serialize(writer, value, value.GetType(), serializerOptions); + JsonSerializer.Serialize(writer, value, value.GetType(), options); } private static PercentilesAggregate DeserializePercentiles(JsonElement element, JsonSerializerOptions options) @@ -83,7 +78,6 @@ private static SingleBucketAggregate DeserializeSingleBucket(JsonElement element private static string GetTokenType(JsonElement element) { var dataPropertyElement = GetProperty(element, "Data"); - if (dataPropertyElement != null && dataPropertyElement.Value.TryGetProperty("@type", out var typeElement)) return typeElement.ToString(); diff --git a/src/Foundatio.Repositories/Utility/BucketsNewtonsoftJsonConverter.cs b/src/Foundatio.Repositories/Serialization/BucketsNewtonsoftJsonConverter.cs similarity index 91% rename from src/Foundatio.Repositories/Utility/BucketsNewtonsoftJsonConverter.cs rename to src/Foundatio.Repositories/Serialization/BucketsNewtonsoftJsonConverter.cs index f1293d1b..a449f739 100644 --- a/src/Foundatio.Repositories/Utility/BucketsNewtonsoftJsonConverter.cs +++ b/src/Foundatio.Repositories/Serialization/BucketsNewtonsoftJsonConverter.cs @@ -1,10 +1,10 @@ -using System; +using System; using System.Collections.Generic; using Foundatio.Repositories.Models; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace Foundatio.Repositories.Utility; +namespace Foundatio.Repositories.Serialization; public class BucketsNewtonsoftJsonConverter : JsonConverter { @@ -47,6 +47,9 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist case "double": value = new KeyedBucket(aggregations); break; + case "geohash": + value = new KeyedBucket(aggregations); + break; case "object": value = new KeyedBucket(aggregations); break; @@ -65,6 +68,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { - throw new NotImplementedException(); + throw new NotSupportedException(); } } diff --git a/src/Foundatio.Repositories/Utility/BucketsSystemTextJsonConverter.cs b/src/Foundatio.Repositories/Serialization/BucketsSystemTextJsonConverter.cs similarity index 92% rename from src/Foundatio.Repositories/Utility/BucketsSystemTextJsonConverter.cs rename to src/Foundatio.Repositories/Serialization/BucketsSystemTextJsonConverter.cs index bff15fac..272f94ba 100644 --- a/src/Foundatio.Repositories/Utility/BucketsSystemTextJsonConverter.cs +++ b/src/Foundatio.Repositories/Serialization/BucketsSystemTextJsonConverter.cs @@ -2,7 +2,7 @@ using System.Text.Json; using Foundatio.Repositories.Models; -namespace Foundatio.Repositories.Utility; +namespace Foundatio.Repositories.Serialization; public class BucketsSystemTextJsonConverter : System.Text.Json.Serialization.JsonConverter { @@ -52,6 +52,9 @@ public override IBucket Read(ref Utf8JsonReader reader, Type typeToConvert, Json case "double": value = element.Deserialize>(options); break; + case "geohash": + value = element.Deserialize>(options); + break; case "object": value = element.Deserialize>(options); break; @@ -74,7 +77,7 @@ public override void Write(Utf8JsonWriter writer, IBucket value, JsonSerializerO if (element.TryGetProperty(propertyName, out var dataElement)) return dataElement; - if (element.TryGetProperty(propertyName.ToLower(), out dataElement)) + if (element.TryGetProperty(propertyName.ToLowerInvariant(), out dataElement)) return dataElement; return null; diff --git a/src/Foundatio.Repositories/Serialization/DoubleSystemTextJsonConverter.cs b/src/Foundatio.Repositories/Serialization/DoubleSystemTextJsonConverter.cs new file mode 100644 index 00000000..bdeeed5c --- /dev/null +++ b/src/Foundatio.Repositories/Serialization/DoubleSystemTextJsonConverter.cs @@ -0,0 +1,44 @@ +using System; +using System.Globalization; +using System.Text.Json; + +namespace Foundatio.Repositories.Serialization; + +/// +/// Preserves decimal points on whole-number doubles during JSON serialization. +/// Without this, 1.0 round-trips as 1, losing the floating-point representation. +/// +/// +public class DoubleSystemTextJsonConverter : System.Text.Json.Serialization.JsonConverter +{ + public override double Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return reader.GetDouble(); + } + + public override bool CanConvert(Type type) + { + return typeof(double).IsAssignableFrom(type); + } + + public override void Write(Utf8JsonWriter writer, double value, JsonSerializerOptions options) + { + if (double.IsNaN(value) || double.IsInfinity(value)) + { + writer.WriteRawValue("0.0"); + return; + } + + if (value != Math.Truncate(value)) + { + writer.WriteNumberValue(value); + return; + } + + string text = value.ToString("R", CultureInfo.InvariantCulture); + if (!text.Contains('.') && !text.Contains('E') && !text.Contains('e')) + text += ".0"; + + writer.WriteRawValue(text); + } +} diff --git a/src/Foundatio.Repositories/Utility/ObjectToInferredTypesConverter.cs b/src/Foundatio.Repositories/Serialization/InferredTypesConverterFactory.cs similarity index 73% rename from src/Foundatio.Repositories/Utility/ObjectToInferredTypesConverter.cs rename to src/Foundatio.Repositories/Serialization/InferredTypesConverterFactory.cs index 8bd7745c..05220760 100644 --- a/src/Foundatio.Repositories/Utility/ObjectToInferredTypesConverter.cs +++ b/src/Foundatio.Repositories/Serialization/InferredTypesConverterFactory.cs @@ -1,20 +1,25 @@ -using System; +using System; using System.Text.Json; using System.Text.Json.Serialization; -namespace Foundatio.Repositories.Utility; +namespace Foundatio.Repositories.Serialization; -public class ObjectToInferredTypesConverter : JsonConverterFactory +/// +/// A that infers CLR types from JSON tokens for generic typed properties +/// (e.g., KeyedBucket<T>.Key). Applied via [JsonConverter] attribute on properties +/// where the declared type is generic and the actual JSON value should be deserialized to a natural CLR type. +/// +public class InferredTypesConverterFactory : JsonConverterFactory { public override bool CanConvert(Type typeToConvert) => true; public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) { - var converterType = typeof(ObjectToInferredTypesConverterInner<>).MakeGenericType(typeToConvert); + var converterType = typeof(InferredTypesConverter<>).MakeGenericType(typeToConvert); return (JsonConverter)Activator.CreateInstance(converterType); } - private class ObjectToInferredTypesConverterInner : JsonConverter + private class InferredTypesConverter : JsonConverter { public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -38,11 +43,8 @@ JsonTokenType.String when reader.TryGetDateTime(out DateTime datetime) => dateti try { - // Special case for JsonElement if (result is JsonElement element) - { return JsonSerializer.Deserialize(element.GetRawText(), options)!; - } return (T)Convert.ChangeType(result, typeof(T)); } diff --git a/src/Foundatio.Repositories/Serialization/JsonSerializerOptionsExtensions.cs b/src/Foundatio.Repositories/Serialization/JsonSerializerOptionsExtensions.cs new file mode 100644 index 00000000..197ca742 --- /dev/null +++ b/src/Foundatio.Repositories/Serialization/JsonSerializerOptionsExtensions.cs @@ -0,0 +1,35 @@ +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Foundatio.Repositories.Serialization; + +public static class JsonSerializerOptionsExtensions +{ + /// + /// Configures with the defaults required for + /// Foundatio.Repositories document serialization and round-tripping: + /// + /// set to true for case-insensitive property matching + /// with camelCase naming and integer fallback for enum values stored as strings in Elasticsearch + /// to preserve decimal points on whole-number doubles (workaround for dotnet/runtime#35195) + /// to deserialize -typed properties as CLR primitives instead of + /// + /// + /// The same instance for chaining. + public static JsonSerializerOptions ConfigureFoundatioRepositoryDefaults(this JsonSerializerOptions options) + { + options.PropertyNameCaseInsensitive = true; + + if (!options.Converters.Any(c => c is JsonStringEnumConverter)) + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: true)); + + if (!options.Converters.Any(c => c is DoubleSystemTextJsonConverter)) + options.Converters.Add(new DoubleSystemTextJsonConverter()); + + if (!options.Converters.Any(c => c is ObjectToInferredTypesConverter)) + options.Converters.Add(new ObjectToInferredTypesConverter()); + + return options; + } +} diff --git a/src/Foundatio.Repositories/Serialization/ObjectToInferredTypesConverter.cs b/src/Foundatio.Repositories/Serialization/ObjectToInferredTypesConverter.cs new file mode 100644 index 00000000..ba087c8b --- /dev/null +++ b/src/Foundatio.Repositories/Serialization/ObjectToInferredTypesConverter.cs @@ -0,0 +1,157 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Foundatio.Repositories.Serialization; + +/// +/// A System.Text.Json converter that deserializes object-typed properties +/// into appropriate .NET types instead of the default behavior. +/// +/// +/// +/// By default, System.Text.Json deserializes properties typed as object into , +/// which requires additional handling to extract values. This converter infers the actual type from the JSON +/// token and deserializes directly to native .NET types: +/// +/// +/// true/false → +/// Numbers → for integers, for floats +/// Strings with ISO 8601 datetime format (containing 'T') → or +/// Other strings → +/// null → null +/// Objects → with +/// Arrays → of +/// +/// +/// +public sealed class ObjectToInferredTypesConverter : JsonConverter +{ + public override object Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return reader.TokenType switch + { + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Number => ReadNumber(ref reader), + JsonTokenType.String => ReadString(ref reader), + JsonTokenType.Null => null, + JsonTokenType.StartObject => ReadObject(ref reader, options), + JsonTokenType.StartArray => ReadArray(ref reader, options), + _ => JsonDocument.ParseValue(ref reader).RootElement.Clone() + }; + } + + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + if (value is null) + { + writer.WriteNullValue(); + return; + } + + if (value is JsonElement element) + { + element.WriteTo(writer); + return; + } + + JsonSerializer.Serialize(writer, value, value.GetType(), options); + } + + /// + /// Reads a JSON number, preserving the original representation (integer vs floating-point). + /// Checks the raw JSON bytes so that 0.0 stays as instead of becoming 0L. + /// + private static object ReadNumber(ref Utf8JsonReader reader) + { + ReadOnlySpan rawValue = reader.HasValueSequence + ? reader.ValueSequence.ToArray() + : reader.ValueSpan; + + if (rawValue.Contains((byte)'.') || rawValue.Contains((byte)'e') || rawValue.Contains((byte)'E')) + return reader.GetDouble(); + + if (reader.TryGetInt64(out long l)) + return l; + + return reader.GetDouble(); + } + + private static object ReadString(ref Utf8JsonReader reader) + { + // Only attempt date parsing for strings that contain 'T', which is required + // by ISO 8601 datetime format (e.g. "2025-01-01T10:30:00Z"). Date-only strings + // like "2025-01-01" also satisfy TryGetDateTimeOffset but should remain as strings + // since they are typically identifiers, labels, or display values — not timestamps. + ReadOnlySpan raw = reader.HasValueSequence + ? reader.ValueSequence.ToArray() + : reader.ValueSpan; + + if (raw.Contains((byte)'T')) + { + if (reader.TryGetDateTimeOffset(out DateTimeOffset dateTimeOffset)) + return dateTimeOffset; + + if (reader.TryGetDateTime(out DateTime dt)) + return dt; + } + + return reader.GetString(); + } + + private Dictionary ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + return dictionary; + + if (reader.TokenType != JsonTokenType.PropertyName) + continue; + + string propertyName = reader.GetString() ?? string.Empty; + + if (!reader.Read()) + continue; + + dictionary[propertyName] = ReadValue(ref reader, options); + } + + return dictionary; + } + + private List ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + var list = new List(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + return list; + + list.Add(ReadValue(ref reader, options)); + } + + return list; + } + + private object ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + return reader.TokenType switch + { + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Number => ReadNumber(ref reader), + JsonTokenType.String => ReadString(ref reader), + JsonTokenType.Null => null, + JsonTokenType.StartObject => ReadObject(ref reader, options), + JsonTokenType.StartArray => ReadArray(ref reader, options), + _ => JsonDocument.ParseValue(ref reader).RootElement.Clone() + }; + } +} diff --git a/src/Foundatio.Repositories/Utility/DoubleSystemTextJsonConverter.cs b/src/Foundatio.Repositories/Utility/DoubleSystemTextJsonConverter.cs deleted file mode 100644 index 3595adb0..00000000 --- a/src/Foundatio.Repositories/Utility/DoubleSystemTextJsonConverter.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Text.Json; - -namespace Foundatio.Repositories.Utility; - -// NOTE: This fixes an issue where doubles were converted to integers (https://github.com/dotnet/runtime/issues/35195) -public class DoubleSystemTextJsonConverter : System.Text.Json.Serialization.JsonConverter -{ - public override double Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override bool CanConvert(Type type) - { - return typeof(double).IsAssignableFrom(type); - } - - public override void Write(Utf8JsonWriter writer, double value, JsonSerializerOptions options) - { - writer.WriteRawValue($"{value:0.0}"); - } -} diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/AggregationQueryTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/AggregationQueryTests.cs index d3356c63..be5c27be 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/AggregationQueryTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/AggregationQueryTests.cs @@ -3,13 +3,12 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.Core.Search; using Exceptionless.DateTimeExtensions; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Foundatio.Repositories.Elasticsearch.Tests.Utility; using Foundatio.Repositories.Models; -using Foundatio.Serializer; using Microsoft.Extensions.Time.Testing; -using Nest; using Newtonsoft.Json; using Xunit; @@ -106,6 +105,56 @@ await _employeeRepository.AddAsync(new Employee Assert.Equal(1, result.Aggregations.Cardinality("cardinality_twitter").Value); } + [Fact] + public async Task GetNestedAggregationsAsync_WithPeerReviews_ReturnsNestedAndFilteredBuckets() + { + // Arrange + var utcToday = new DateTimeOffset(DateTime.UtcNow.Year, 1, 1, 12, 0, 0, TimeSpan.FromHours(5)); + var employees = new List { + EmployeeGenerator.Generate(nextReview: utcToday.SubtractDays(2)), + EmployeeGenerator.Generate(nextReview: utcToday.SubtractDays(1)) + }; + employees[0].Id = "employee1"; + employees[1].Id = "employee2"; + employees[0].PeerReviews = new PeerReview[] { new PeerReview { ReviewerEmployeeId = employees[1].Id, Rating = 4 } }; + employees[1].PeerReviews = new PeerReview[] { new PeerReview { ReviewerEmployeeId = employees[0].Id, Rating = 5 } }; + + await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); + + // Act + var nestedAggQuery = await _client.SearchAsync(d => d.Indices("employees").Aggregations(a => a + .Add("nested_reviewRating", agg => agg + .Nested(h => h.Path("peerReviews")) + .Aggregations(a1 => a1.Add("terms_rating", t => t.Terms(t1 => t1.Field("peerReviews.rating")).Meta(m => m.Add("@field_type", "integer"))))) + ), cancellationToken: TestCancellationToken); + + // Assert + var result = nestedAggQuery.Aggregations.ToAggregations(_serializer); + Assert.Single(result); + Assert.Equal(2, ((Foundatio.Repositories.Models.BucketAggregate)((Foundatio.Repositories.Models.SingleBucketAggregate)result["nested_reviewRating"]).Aggregations["terms_rating"]).Items.Count); + + // Act (with filter) + var nestedAggQueryWithFilter = await _client.SearchAsync(d => d.Indices("employees").Aggregations(a => a + .Add("nested_reviewRating", agg => agg + .Nested(h => h.Path("peerReviews")) + .Aggregations(a1 => a1 + .Add($"user_{employees[0].Id}", f => f + .Filter(q => q.Term(t => t.Field("peerReviews.reviewerEmployeeId").Value(employees[0].Id))) + .Aggregations(a2 => a2.Add("terms_rating", t => t.Terms(t1 => t1.Field("peerReviews.rating")).Meta(m => m.Add("@field_type", "integer"))))) + ))), cancellationToken: TestCancellationToken); + + // Assert (with filter) + result = nestedAggQueryWithFilter.Aggregations.ToAggregations(_serializer); + Assert.Single(result); + + var nestedAgg = (Foundatio.Repositories.Models.SingleBucketAggregate)result["nested_reviewRating"]; + var filteredAgg = (Foundatio.Repositories.Models.SingleBucketAggregate)nestedAgg.Aggregations[$"user_{employees[0].Id}"]; + Assert.NotNull(filteredAgg); + Assert.Single(filteredAgg.Aggregations.Terms("terms_rating").Buckets); + Assert.Equal("5", filteredAgg.Aggregations.Terms("terms_rating").Buckets.First().Key); + Assert.Equal(1, filteredAgg.Aggregations.Terms("terms_rating").Buckets.First().Total); + } + [Fact] public async Task GetAliasedNumberAggregationThatCausesMappingAsync() { @@ -304,7 +353,7 @@ await _employeeRepository.AddAsync(new List { var dateHistogramAgg = result.Aggregations.DateHistogram("date_nextReview"); Assert.Equal(3, dateHistogramAgg.Buckets.Count); - var oldestDate = DateTime.SpecifyKind(today.UtcDateTime.Date.SubtractDays(2).SubtractHours(1), DateTimeKind.Unspecified); + var oldestDate = DateTime.SpecifyKind(today.UtcDateTime.SubtractDays(2).AddHours(1).Date.SubtractHours(1), DateTimeKind.Unspecified); foreach (var bucket in dateHistogramAgg.Buckets) { AssertEqual(oldestDate, bucket.Date); @@ -355,17 +404,6 @@ public async Task GetTermAggregationsAsync() Assert.Equal(10, roundTripped.Aggregations.Terms("terms_age").Buckets.Count); Assert.Equal(1, roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19).Total); - // Test with all serializers - foreach (var serializer in SerializerTestHelper.GetTextSerializers()) - { - json = serializer.SerializeToString(result); - roundTripped = serializer.Deserialize(json); - Assert.Equal(10, roundTripped.Total); - Assert.Single(roundTripped.Aggregations); - Assert.Equal(10, roundTripped.Aggregations.Terms("terms_age").Buckets.Count); - Assert.Equal(1, roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19).Total); - } - result = await _employeeRepository.CountAsync(q => q.AggregationsExpression("terms:(age~2 @missing:0 terms:(years~2 @missing:0))")); Assert.Equal(10, result.Total); Assert.Single(result.Aggregations); @@ -375,19 +413,17 @@ public async Task GetTermAggregationsAsync() Assert.Single(bucket.Aggregations); Assert.Single(bucket.Aggregations.Terms("terms_years").Buckets); - // Test nested aggregations with all serializers - foreach (var serializer in SerializerTestHelper.GetTextSerializers()) - { - json = serializer.SerializeToString(result); - roundTripped = serializer.Deserialize(json); - Assert.Equal(10, roundTripped.Total); - Assert.Single(roundTripped.Aggregations); - Assert.Equal(2, roundTripped.Aggregations.Terms("terms_age").Buckets.Count); - bucket = roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19); - Assert.Equal(1, bucket.Total); - Assert.Single(bucket.Aggregations); - Assert.Single(bucket.Aggregations.Terms("terms_years").Buckets); - } + json = JsonConvert.SerializeObject(result, Formatting.Indented); + roundTripped = JsonConvert.DeserializeObject(json); + string roundTrippedJson = JsonConvert.SerializeObject(roundTripped, Formatting.Indented); + Assert.Equal(json, roundTrippedJson); + Assert.Equal(10, roundTripped.Total); + Assert.Single(roundTripped.Aggregations); + Assert.Equal(2, roundTripped.Aggregations.Terms("terms_age").Buckets.Count); + bucket = roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19); + Assert.Equal(1, bucket.Total); + Assert.Single(bucket.Aggregations); + Assert.Single(bucket.Aggregations.Terms("terms_years").Buckets); } [Fact] @@ -427,23 +463,6 @@ await _employeeRepository.AddAsync(new List { Assert.Equal(1, bucket.Total); oldestDate = oldestDate.AddDays(1); } - - // Test with all serializers - foreach (var serializer in SerializerTestHelper.GetTextSerializers()) - { - json = serializer.SerializeToString(result); - roundTripped = serializer.Deserialize(json); - - dateHistogramAgg = roundTripped.Aggregations.DateHistogram("date_nextReview"); - Assert.Equal(3, dateHistogramAgg.Buckets.Count); - oldestDate = DateTime.SpecifyKind(utcToday.UtcDateTime.Date.SubtractDays(2).SubtractHours(1), DateTimeKind.Unspecified); - foreach (var bucket in dateHistogramAgg.Buckets) - { - AssertEqual(oldestDate, bucket.Date); - Assert.Equal(1, bucket.Total); - oldestDate = oldestDate.AddDays(1); - } - } } [Fact] @@ -469,16 +488,6 @@ await _employeeRepository.AddAsync(new List { dateTermsAgg = roundTripped.Aggregations.Min("min_nextReview"); Assert.Equal(utcToday.SubtractDays(2), dateTermsAgg.Value); - - // Test with all serializers - foreach (var serializer in SerializerTestHelper.GetTextSerializers()) - { - json = serializer.SerializeToString(result); - roundTripped = serializer.Deserialize(json); - - dateTermsAgg = roundTripped.Aggregations.Min("min_nextReview"); - Assert.Equal(utcToday.SubtractDays(2), dateTermsAgg.Value); - } } [Fact] @@ -509,60 +518,61 @@ public async Task GetTermAggregationsWithTopHitsAsync() bucket = roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19); Assert.Equal(1, bucket.Total); - // Test with all serializers - foreach (var serializer in SerializerTestHelper.GetTextSerializers()) - { - json = serializer.SerializeToString(result); - roundTripped = serializer.Deserialize(json); - Assert.Equal(10, roundTripped.Total); - Assert.Single(roundTripped.Aggregations); - Assert.Equal(10, roundTripped.Aggregations.Terms("terms_age").Buckets.Count); - bucket = roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19); - Assert.Equal(1, bucket.Total); - } + string systemTextJson = System.Text.Json.JsonSerializer.Serialize(result); + Assert.True(System.Text.Json.Nodes.JsonNode.DeepEquals( + System.Text.Json.Nodes.JsonNode.Parse(json), + System.Text.Json.Nodes.JsonNode.Parse(systemTextJson)), + "Newtonsoft and System.Text.Json serialization should produce semantically equivalent JSON"); + roundTripped = System.Text.Json.JsonSerializer.Deserialize(systemTextJson); + Assert.Equal(10, roundTripped.Total); + Assert.Single(roundTripped.Aggregations); + Assert.Equal(10, roundTripped.Aggregations.Terms("terms_age").Buckets.Count); + bucket = roundTripped.Aggregations.Terms("terms_age").Buckets.First(f => f.Key == 19); + Assert.Equal(1, bucket.Total); - // TopHitsAggregate cannot survive JSON round-tripping because it holds ILazyDocument - // references (raw ES document bytes) that are lost during serialization. If caching of - // tophits results is needed, the converter would need to serialize the raw documents - // as JSON arrays and reconstruct LazyDocument wrappers on deserialization. + tophits = bucket.Aggregations.TopHits(); + Assert.NotNull(tophits); + employees = tophits.Documents(_serializer); + Assert.Single(employees); + Assert.Equal(19, employees.First().Age); + Assert.Equal(1, employees.First().YearsEmployed); } [Fact] public void CanDeserializeHit() { - /* language = json */ - const string json = """ - { - "_index" : "employees", - "_type" : "_doc", - "_id" : "53cc5800d3e0d1fed81452fd", - "_score" : 0.0, - "_source" : { - "id" : "53cc5800d3e0d1fed81452fd", - "companyId" : "62d982efd3e0d1fed81452f3", - "companyName" : null, - "unmappedCompanyName" : null, - "name" : null, - "emailAddress" : null, - "unmappedEmailAddress" : null, - "age" : 45, - "unmappedAge" : 45, - "location" : "20,20", - "yearsEmployed" : 8, - "lastReview" : "0001-01-01T00:00:00", - "nextReview" : "0001-01-01T00:00:00+00:00", - "createdUtc" : "2014-07-21T00:00:00Z", - "updatedUtc" : "2022-07-21T16:46:39.6914481Z", - "version" : null, - "isDeleted" : false, - "peerReviews" : null, - "phoneNumbers" : [ ], - "data" : { } - } - } - """; - - var employeeHit = _configuration.Client.ConnectionSettings.RequestResponseSerializer.Deserialize>(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))); + string json = @" + { + ""_index"" : ""employees"", + ""_type"" : ""_doc"", + ""_id"" : ""53cc5800d3e0d1fed81452fd"", + ""_score"" : 0.0, + ""_source"" : { + ""id"" : ""53cc5800d3e0d1fed81452fd"", + ""companyId"" : ""62d982efd3e0d1fed81452f3"", + ""companyName"" : null, + ""unmappedCompanyName"" : null, + ""name"" : null, + ""emailAddress"" : null, + ""unmappedEmailAddress"" : null, + ""age"" : 45, + ""unmappedAge"" : 45, + ""location"" : ""20,20"", + ""yearsEmployed"" : 8, + ""lastReview"" : ""0001-01-01T00:00:00"", + ""nextReview"" : ""0001-01-01T00:00:00+00:00"", + ""createdUtc"" : ""2014-07-21T00:00:00Z"", + ""updatedUtc"" : ""2022-07-21T16:46:39.6914481Z"", + ""version"" : null, + ""isDeleted"" : false, + ""peerReviews"" : null, + ""phoneNumbers"" : [ ], + ""data"" : { } + } + }"; + + using var hitStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json)); + var employeeHit = _configuration.Client.ElasticsearchClientSettings.RequestResponseSerializer.Deserialize>(hitStream); Assert.Equal("employees", employeeHit.Index); Assert.Equal("62d982efd3e0d1fed81452f3", employeeHit.Source.CompanyId); } @@ -592,8 +602,7 @@ public async Task CanGetDupesAsync() private Task CreateDataAsync() { var utcToday = DateTime.UtcNow.Date; - return _employeeRepository.AddAsync(new List - { + return _employeeRepository.AddAsync(new List { EmployeeGenerator.Generate(age: 19, yearsEmployed: 1, location: "10,10", createdUtc: utcToday.SubtractYears(1), updatedUtc: utcToday.SubtractYears(1)), EmployeeGenerator.Generate(age: 22, yearsEmployed: 2, location: "10,10", createdUtc: utcToday.SubtractYears(2), updatedUtc: utcToday.SubtractYears(2)), EmployeeGenerator.Generate(age: 25, yearsEmployed: 3, location: "10,10", createdUtc: utcToday.SubtractYears(3), updatedUtc: utcToday.SubtractYears(3)), diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/ElasticRepositoryTestBase.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/ElasticRepositoryTestBase.cs index 33c1997f..63a05cb8 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/ElasticRepositoryTestBase.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/ElasticRepositoryTestBase.cs @@ -2,16 +2,17 @@ using System.Diagnostics; using System.Threading; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Messaging; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Queues; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; +using Foundatio.Serializer; using Foundatio.Utility; using Foundatio.Xunit; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using IAsyncLifetime = Xunit.IAsyncLifetime; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -22,9 +23,10 @@ public abstract class ElasticRepositoryTestBase : TestWithLoggingBase, IAsyncLif { protected readonly MyAppElasticConfiguration _configuration; protected readonly InMemoryCacheClient _cache; - protected readonly IElasticClient _client; + protected readonly ElasticsearchClient _client; protected readonly IQueue _workItemQueue; protected readonly InMemoryMessageBus _messageBus; + protected readonly ITextSerializer _serializer; public ElasticRepositoryTestBase(ITestOutputHelper output) : base(output) { @@ -35,6 +37,7 @@ public ElasticRepositoryTestBase(ITestOutputHelper output) : base(output) _workItemQueue = new InMemoryQueue(new InMemoryQueueOptions { LoggerFactory = Log }); _configuration = new MyAppElasticConfiguration(_workItemQueue, _cache, _messageBus, Log); _client = _configuration.Client; + _serializer = _configuration.Serializer; } private static bool _elasticsearchReady; @@ -56,7 +59,7 @@ protected virtual async Task RemoveDataAsync(bool configureIndexes = true) await _workItemQueue.DeleteQueueAsync(); await _configuration.DeleteIndexesAsync(); - await _client.Indices.DeleteAsync(Indices.Parse("employee*")); + await DeleteWildcardIndicesAsync("employee*"); if (configureIndexes) await _configuration.ConfigureIndexesAsync(null, false); @@ -70,5 +73,17 @@ protected virtual async Task RemoveDataAsync(bool configureIndexes = true) Log.DefaultLogLevel = minimumLevel; } + protected async Task DeleteWildcardIndicesAsync(string pattern) + { + // Use GetAsync to resolve wildcards to actual index names to avoid issues with action.destructive_requires_name=true. + // Note: ResolveIndexAsync sends a body in ES 9.x client which ES rejects; use GetAsync instead. + var getResponse = await _client.Indices.GetAsync(Indices.Parse(pattern), d => d.IgnoreUnavailable()); + if (getResponse.IsValidResponse && getResponse.Indices != null && getResponse.Indices.Count > 0) + { + var indexNames = string.Join(",", getResponse.Indices.Keys); + await _client.Indices.DeleteAsync(Indices.Parse(indexNames), i => i.IgnoreUnavailable()); + } + } + public virtual ValueTask DisposeAsync() => ValueTask.CompletedTask; } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/ElasticsearchExtensions.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/ElasticsearchExtensions.cs index d9999ead..79b8973b 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/ElasticsearchExtensions.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/ElasticsearchExtensions.cs @@ -1,30 +1,85 @@ -using System.Threading.Tasks; -using Nest; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Xunit; namespace Foundatio.Repositories.Elasticsearch.Tests; public static class ElasticsearchExtensions { - public static async Task AssertSingleIndexAlias(this IElasticClient client, string indexName, string aliasName) + public static async Task AssertSingleIndexAlias(this ElasticsearchClient client, string indexName, string aliasName) { - var aliasResponse = await client.Indices.GetAliasAsync(aliasName, a => a.IgnoreUnavailable()); - Assert.True(aliasResponse.IsValid); - Assert.Contains(indexName, aliasResponse.Indices); - Assert.Single(aliasResponse.Indices); - var aliasedIndex = aliasResponse.Indices[indexName]; + var aliasResponse = await client.Indices.GetAliasAsync((Indices)aliasName, a => a.IgnoreUnavailable()); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Contains(indexName, indices.Keys); + Assert.Single(indices); + var aliasedIndex = indices[indexName]; Assert.NotNull(aliasedIndex); - Assert.Contains(aliasName, aliasedIndex.Aliases); + Assert.Contains(aliasName, aliasedIndex.Aliases.Keys); Assert.Single(aliasedIndex.Aliases); } - public static async Task GetAliasIndexCount(this IElasticClient client, string aliasName) + public static async Task GetAliasIndexCount(this ElasticsearchClient client, string aliasName) { - var response = await client.Indices.GetAliasAsync(aliasName, a => a.IgnoreUnavailable()); - if (!response.IsValid && response.ServerError?.Status == 404) - return 0; + var response = await client.Indices.GetAliasAsync((Indices)aliasName, a => a.IgnoreUnavailable()); - Assert.True(response.IsValid); - return response.Indices.Count; + if (!response.IsValidResponse) + { + if (response.ApiCallDetails is { HttpStatusCode: 404 }) + return 0; + + throw new InvalidOperationException($"Failed to get alias '{aliasName}': {response.ElasticsearchServerError?.Error?.Reason ?? "Unknown error"}"); + } + +#if ELASTICSEARCH9 + return response.Aliases.Count; +#else + return response.Values.Count; +#endif + } + + public static async Task> GetIndicesPointingToAliasAsync(this ElasticsearchClient client, string aliasName) + { + var response = await client.Indices.GetAliasAsync((Indices)aliasName, a => a.IgnoreUnavailable()); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails is { HttpStatusCode: 404 }) + return []; + + throw new InvalidOperationException($"Failed to get alias '{aliasName}': {response.ElasticsearchServerError?.Error?.Reason ?? "Unknown error"}"); + } + +#if ELASTICSEARCH9 + return response.Aliases.Keys.ToList(); +#else + return response.Values.Keys.ToList(); +#endif + } + + public static IReadOnlyCollection GetIndicesPointingToAlias(this ElasticsearchClient client, string aliasName) + { + var response = client.Indices.GetAlias((Indices)aliasName, a => a.IgnoreUnavailable()); + + if (!response.IsValidResponse) + { + if (response.ApiCallDetails is { HttpStatusCode: 404 }) + return []; + + throw new InvalidOperationException($"Failed to get alias '{aliasName}': {response.ElasticsearchServerError?.Error?.Reason ?? "Unknown error"}"); + } + +#if ELASTICSEARCH9 + return response.Aliases.Keys.ToList(); +#else + return response.Values.Keys.ToList(); +#endif } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/TimeSpanExtensionsTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/TimeSpanExtensionsTests.cs new file mode 100644 index 00000000..18eaecf3 --- /dev/null +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Extensions/TimeSpanExtensionsTests.cs @@ -0,0 +1,115 @@ +using System; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Xunit; + +namespace Foundatio.Repositories.Elasticsearch.Tests.Extensions; + +public class TimeSpanExtensionsTests +{ + [Theory] + [InlineData(500, "500ms")] + [InlineData(1, "1ms")] + [InlineData(0, "0ms")] + [InlineData(999, "999ms")] + public void ToElasticDuration_SubSecond_ReturnsMilliseconds(int ms, string expected) + { + // Arrange + var timeSpan = TimeSpan.FromMilliseconds(ms); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(1, "1s")] + [InlineData(30, "30s")] + [InlineData(59, "59s")] + public void ToElasticDuration_WholeSeconds_ReturnsSeconds(int seconds, string expected) + { + // Arrange + var timeSpan = TimeSpan.FromSeconds(seconds); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(1, "1m")] + [InlineData(5, "5m")] + [InlineData(59, "59m")] + public void ToElasticDuration_WholeMinutes_ReturnsMinutes(int minutes, string expected) + { + // Arrange + var timeSpan = TimeSpan.FromMinutes(minutes); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(1, "1h")] + [InlineData(12, "12h")] + [InlineData(23, "23h")] + public void ToElasticDuration_WholeHours_ReturnsHours(int hours, string expected) + { + // Arrange + var timeSpan = TimeSpan.FromHours(hours); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(1, "1d")] + [InlineData(7, "7d")] + [InlineData(30, "30d")] + public void ToElasticDuration_WholeDays_ReturnsDays(int days, string expected) + { + // Arrange + var timeSpan = TimeSpan.FromDays(days); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal(expected, result); + } + + [Fact] + public void ToElasticDuration_MixedSecondsAndMilliseconds_ReturnsMilliseconds() + { + // Arrange + var timeSpan = TimeSpan.FromMilliseconds(1500); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal("1500ms", result); + } + + [Fact] + public void ToElasticDuration_MixedMinutesAndSeconds_ReturnsSeconds() + { + // Arrange + var timeSpan = TimeSpan.FromSeconds(90); + + // Act + var result = timeSpan.ToElasticDuration(); + + // Assert + Assert.Equal("90s", result); + } +} diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Foundatio.Repositories.Elasticsearch.Tests.csproj b/tests/Foundatio.Repositories.Elasticsearch.Tests/Foundatio.Repositories.Elasticsearch.Tests.csproj index 232662cd..6002743b 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Foundatio.Repositories.Elasticsearch.Tests.csproj +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Foundatio.Repositories.Elasticsearch.Tests.csproj @@ -6,7 +6,6 @@ - diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/IndexTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/IndexTests.cs index 32b9d027..cc6f7e98 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/IndexTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/IndexTests.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.DateTimeExtensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; @@ -12,7 +14,6 @@ using Foundatio.Repositories.Utility; using Foundatio.Utility; using Microsoft.Extensions.Time.Testing; -using Nest; using Xunit; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -50,17 +51,22 @@ public async Task CanCreateDailyAliasesAsync(DateTime utcNow) Assert.NotNull(employee.Id); Assert.Equal(1, await index.GetCurrentVersionAsync()); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -86,17 +92,22 @@ public async Task CanCreateMonthlyAliasesAsync(DateTime utcNow) Assert.NotNull(employee.Id); Assert.Equal(1, await index.GetCurrentVersionAsync()); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -121,9 +132,9 @@ public async Task GetByDateBasedIndexAsync() var indexes = await _client.GetIndicesPointingToAliasAsync(_configuration.DailyLogEvents.Name); Assert.Empty(indexes); - var alias = await _client.Indices.GetAliasAsync(_configuration.DailyLogEvents.Name, ct: TestCancellationToken); + var alias = await _client.Indices.GetAliasAsync((Indices)_configuration.DailyLogEvents.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(alias); - Assert.False(alias.IsValid); + Assert.False(alias.IsValidResponse); var utcNow = DateTime.UtcNow; ILogEventRepository repository = new DailyLogEventRepository(_configuration); @@ -135,10 +146,15 @@ public async Task GetByDateBasedIndexAsync() Assert.NotNull(logEvent); Assert.NotNull(logEvent.Id); - alias = await _client.Indices.GetAliasAsync(_configuration.DailyLogEvents.Name, ct: TestCancellationToken); + alias = await _client.Indices.GetAliasAsync((Indices)_configuration.DailyLogEvents.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(alias); - Assert.True(alias.IsValid); - Assert.Equal(2, alias.Indices.Count); + Assert.True(alias.IsValidResponse); +#if ELASTICSEARCH9 + var aliasIndices = alias.Aliases; +#else + var aliasIndices = alias.Values; +#endif + Assert.Equal(2, aliasIndices.Count); indexes = await _client.GetIndicesPointingToAliasAsync(_configuration.DailyLogEvents.Name); Assert.Equal(2, indexes.Count); @@ -163,12 +179,12 @@ public async Task MaintainWillCreateAliasOnVersionedIndexAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); // delete all aliases @@ -176,19 +192,34 @@ public async Task MaintainWillCreateAliasOnVersionedIndexAsync() await DeleteAliasesAsync(version1Index.VersionedName); await DeleteAliasesAsync(version2Index.VersionedName); - await _client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); - var aliasesResponse = await _client.Indices.GetAliasAsync($"{version1Index.VersionedName},{version2Index.VersionedName}", ct: TestCancellationToken); - Assert.Empty(aliasesResponse.Indices.Values.SelectMany(i => i.Aliases)); + await _client.Indices.RefreshAsync(Elastic.Clients.Elasticsearch.Indices.All, cancellationToken: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)$"{version1Index.VersionedName},{version2Index.VersionedName}", cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Empty(indices.Values.SelectMany(i => i.Aliases)); // Indexes exist but no alias so the oldest index version will be used. Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); await version1Index.MaintainAsync(); - aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.VersionedName, ct: TestCancellationToken); - Assert.Single(aliasesResponse.Indices.Single().Value.Aliases); - aliasesResponse = await _client.Indices.GetAliasAsync(version2Index.VersionedName, ct: TestCancellationToken); - Assert.Empty(aliasesResponse.Indices.Single().Value.Aliases); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)version1Index.VersionedName, cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Single(indices.Single().Value.Aliases); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.VersionedName, cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Empty(indices.Single().Value.Aliases); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); @@ -212,7 +243,7 @@ public async Task MaintainWillCreateAliasesOnTimeSeriesIndexAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); await version1Index.EnsureIndexAsync(utcNow.UtcDateTime); - Assert.True((await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow.UtcDateTime), ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow.UtcDateTime), cancellationToken: TestCancellationToken)).Exists); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); // delete all aliases @@ -222,26 +253,41 @@ public async Task MaintainWillCreateAliasesOnTimeSeriesIndexAsync() await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); await version2Index.EnsureIndexAsync(utcNow.UtcDateTime); - Assert.True((await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow.UtcDateTime), ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow.UtcDateTime), cancellationToken: TestCancellationToken)).Exists); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); // delete all aliases await _configuration.Cache.RemoveAllAsync(); await DeleteAliasesAsync(version2Index.GetVersionedIndex(utcNow.UtcDateTime)); - await _client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); - var aliasesResponse = await _client.Indices.GetAliasAsync($"{version1Index.GetVersionedIndex(utcNow.UtcDateTime)},{version2Index.GetVersionedIndex(utcNow.UtcDateTime)}", ct: TestCancellationToken); - Assert.Empty(aliasesResponse.Indices.Values.SelectMany(i => i.Aliases)); + await _client.Indices.RefreshAsync(Elastic.Clients.Elasticsearch.Indices.All, cancellationToken: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)$"{version1Index.GetVersionedIndex(utcNow.UtcDateTime)},{version2Index.GetVersionedIndex(utcNow.UtcDateTime)}", cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Empty(indices.Values.SelectMany(i => i.Aliases)); // Indexes exist but no alias so the oldest index version will be used. Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); await version1Index.MaintainAsync(); - aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetVersionedIndex(utcNow.UtcDateTime), ct: TestCancellationToken); - Assert.Equal(version1Index.Aliases.Count + 1, aliasesResponse.Indices.Single().Value.Aliases.Count); - aliasesResponse = await _client.Indices.GetAliasAsync(version2Index.GetVersionedIndex(utcNow.UtcDateTime), ct: TestCancellationToken); - Assert.Empty(aliasesResponse.Indices.Single().Value.Aliases); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)version1Index.GetVersionedIndex(utcNow.UtcDateTime), cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Equal(version1Index.Aliases.Count + 1, indices.Single().Value.Aliases.Count); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.GetVersionedIndex(utcNow.UtcDateTime), cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Empty(indices.Single().Value.Aliases); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); @@ -249,11 +295,16 @@ public async Task MaintainWillCreateAliasesOnTimeSeriesIndexAsync() private async Task DeleteAliasesAsync(string index) { - var aliasesResponse = await _client.Indices.GetAliasAsync(index, ct: TestCancellationToken); - var aliases = aliasesResponse.Indices.Single(a => a.Key == index).Value.Aliases.Select(s => s.Key).ToList(); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index, cancellationToken: TestCancellationToken); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + var aliases = indices.Single(a => a.Key == index).Value.Aliases.Select(s => s.Key).ToList(); foreach (string alias in aliases) { - await _client.Indices.DeleteAliasAsync(new DeleteAliasRequest(index, alias), TestCancellationToken); + await _client.Indices.DeleteAliasAsync(new Elastic.Clients.Elasticsearch.IndexManagement.DeleteAliasRequest(index, alias), cancellationToken: TestCancellationToken); } } @@ -275,45 +326,55 @@ public async Task MaintainDailyIndexesAsync() await index.MaintainAsync(); Assert.Equal(1, await index.GetCurrentVersionAsync()); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, timeProvider.GetUtcNow().UtcDateTime, employee.CreatedUtc), String.Join(", ", aliases)); timeProvider.Advance(TimeSpan.FromDays(6)); index.MaxIndexAge = TimeSpan.FromDays(10); await index.MaintainAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, timeProvider.GetUtcNow().UtcDateTime, employee.CreatedUtc), String.Join(", ", aliases)); timeProvider.Advance(TimeSpan.FromDays(9)); await index.MaintainAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); - aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.False(aliasesResponse.IsValid); + Assert.False(aliasesResponse.IsValidResponse); } [Fact] @@ -340,17 +401,22 @@ public async Task MaintainMonthlyIndexesAsync() Assert.NotNull(employee.Id); Assert.Equal(1, await index.GetCurrentVersionAsync()); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow.UtcDateTime, employee.CreatedUtc), String.Join(", ", aliases)); @@ -366,17 +432,22 @@ public async Task MaintainMonthlyIndexesAsync() Assert.NotNull(employee.Id); Assert.Equal(1, await index.GetCurrentVersionAsync()); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow.UtcDateTime, employee.CreatedUtc), String.Join(", ", aliases)); @@ -394,17 +465,17 @@ public async Task MaintainOnlyOldIndexesAsync() }; await index.EnsureIndexAsync(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); index.MaxIndexAge = _configuration.TimeProvider.GetUtcNow().UtcDateTime.EndOfMonth() - _configuration.TimeProvider.GetUtcNow().UtcDateTime.StartOfMonth(); await index.MaintainAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } @@ -414,15 +485,15 @@ public async Task CanCreateAndDeleteIndex() var index = new EmployeeIndex(_configuration); await index.ConfigureAsync(); - var existsResponse = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); await index.DeleteAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } @@ -432,58 +503,77 @@ public async Task CanChangeIndexSettings() var index1 = new VersionedEmployeeIndex(_configuration, 1, i => i .Settings(s => s .NumberOfReplicas(0) - .Setting("index.mapping.total_fields.limit", 2000) - .Analysis(a => a.Analyzers(a1 => a1.Custom("custom1", c => c.Filters("uppercase").Tokenizer("whitespace")))) + .AddOtherSetting("index.mapping.total_fields.limit", 2000) + .Analysis(a => a.Analyzers(a1 => a1.Custom("custom1", c => c.Filter("uppercase").Tokenizer("whitespace")))) )); await index1.DeleteAsync(); await index1.ConfigureAsync(); - var settings = await _client.Indices.GetSettingsAsync(index1.VersionedName, ct: TestCancellationToken); - Assert.Equal(0, settings.Indices[index1.VersionedName].Settings.NumberOfReplicas); - Assert.NotNull(settings.Indices[index1.VersionedName].Settings.Analysis.Analyzers["custom1"]); + var settings = await _client.Indices.GetSettingsAsync((Indices)index1.VersionedName, cancellationToken: TestCancellationToken); + var indexSettings = settings.Settings[index1.VersionedName].Settings; + // NumberOfReplicas is Union - need to extract the actual value + var replicas = indexSettings.Index?.NumberOfReplicas ?? indexSettings.NumberOfReplicas; + Assert.NotNull(replicas); + // Match returns the value from either side of the union + var replicaCount = replicas.Match(i => i, s => int.Parse(s)); + Assert.Equal(0, replicaCount); + Assert.NotNull(indexSettings.Index); + Assert.NotNull(indexSettings.Index.Analysis); + Assert.NotNull(indexSettings.Index.Analysis.Analyzers["custom1"]); var index2 = new VersionedEmployeeIndex(_configuration, 1, i => i.Settings(s => s .NumberOfReplicas(1) - .Setting("index.mapping.total_fields.limit", 3000) - .Analysis(a => a.Analyzers(a1 => a1.Custom("custom1", c => c.Filters("uppercase").Tokenizer("whitespace")).Custom("custom2", c => c.Filters("uppercase").Tokenizer("whitespace")))) + .AddOtherSetting("index.mapping.total_fields.limit", 3000) + .Analysis(a => a.Analyzers(a1 => a1.Custom("custom1", c => c.Filter("uppercase").Tokenizer("whitespace")).Custom("custom2", c => c.Filter("uppercase").Tokenizer("whitespace")))) )); await index2.ConfigureAsync(); - settings = await _client.Indices.GetSettingsAsync(index1.VersionedName, ct: TestCancellationToken); - Assert.Equal(1, settings.Indices[index1.VersionedName].Settings.NumberOfReplicas); - Assert.NotNull(settings.Indices[index1.VersionedName].Settings.Analysis.Analyzers["custom1"]); + settings = await _client.Indices.GetSettingsAsync((Indices)index1.VersionedName, cancellationToken: TestCancellationToken); + indexSettings = settings.Settings[index1.VersionedName].Settings; + replicas = indexSettings.Index?.NumberOfReplicas ?? indexSettings.NumberOfReplicas; + Assert.NotNull(replicas); + replicaCount = replicas.Match(i => i, s => int.Parse(s)); + Assert.Equal(1, replicaCount); + Assert.NotNull(indexSettings.Index); + Assert.NotNull(indexSettings.Index.Analysis); + Assert.NotNull(indexSettings.Index.Analysis.Analyzers["custom1"]); + Assert.NotNull(indexSettings.Index.Analysis.Analyzers["custom2"]); } [Fact] public async Task CanAddIndexMappings() { - var index1 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Keyword(k => k.Name(n => n.EmailAddress)))); + var index1 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Keyword(e => e.EmailAddress))); await index1.DeleteAsync(); await index1.ConfigureAsync(); - var fieldMapping = await _client.Indices.GetFieldMappingAsync("emailAddress", d => d.Index(index1.VersionedName), TestCancellationToken); - Assert.NotNull(fieldMapping.Indices[index1.VersionedName].Mappings["emailAddress"]); + var fieldMapping = await _client.Indices.GetFieldMappingAsync(new Field("emailAddress"), d => d.Indices(index1.VersionedName), cancellationToken: TestCancellationToken); + Assert.True(fieldMapping.IsValidResponse); + Assert.True(fieldMapping.FieldMappings.TryGetValue(index1.VersionedName, out var indexMapping)); + Assert.True(indexMapping.Mappings.ContainsKey("emailAddress")); - var index2 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Keyword(k => k.Name(n => n.EmailAddress)).Number(k => k.Name(n => n.Age)))); + var index2 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Keyword(e => e.EmailAddress).IntegerNumber(e => e.Age))); await index2.ConfigureAsync(); - fieldMapping = await _client.Indices.GetFieldMappingAsync("age", d => d.Index(index2.VersionedName), TestCancellationToken); - Assert.NotNull(fieldMapping.Indices[index2.VersionedName].Mappings["age"]); + fieldMapping = await _client.Indices.GetFieldMappingAsync(new Field("age"), d => d.Indices(index2.VersionedName), cancellationToken: TestCancellationToken); + Assert.True(fieldMapping.IsValidResponse); + Assert.True(fieldMapping.FieldMappings.TryGetValue(index2.VersionedName, out indexMapping)); + Assert.True(indexMapping.Mappings.ContainsKey("age")); } [Fact] public async Task WillWarnWhenAttemptingToChangeFieldMappingType() { - var index1 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Keyword(k => k.Name(n => n.EmailAddress)))); + var index1 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Keyword(e => e.EmailAddress))); await index1.DeleteAsync(); await index1.ConfigureAsync(); - var existsResponse = await _client.Indices.ExistsAsync(index1.VersionedName, ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index1.VersionedName, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var index2 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.Number(k => k.Name(n => n.EmailAddress)))); + var index2 = new VersionedEmployeeIndex(_configuration, 1, null, m => m.Properties(p => p.IntegerNumber(e => e.EmailAddress))); await index2.ConfigureAsync(); Assert.Contains(Log.LogEntries, l => l.LogLevel == LogLevel.Error && l.Message.Contains("requires a new index version")); @@ -496,17 +586,17 @@ public async Task CanCreateAndDeleteVersionedIndex() await index.DeleteAsync(); await index.ConfigureAsync(); - var existsResponse = await _client.Indices.ExistsAsync(index.VersionedName, ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.VersionedName, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); await _client.AssertSingleIndexAlias(index.VersionedName, index.Name); await index.DeleteAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.VersionedName, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.VersionedName, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); Assert.Equal(0, await _client.GetAliasIndexCount(index.Name)); @@ -527,26 +617,26 @@ public async Task CanCreateAndDeleteDailyIndex() await index.EnsureIndexAsync(todayDate); await index.EnsureIndexAsync(yesterdayDate); - var existsResponse = await _client.Indices.ExistsAsync(todayIndex, ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(todayIndex, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); await index.DeleteAsync(); - existsResponse = await _client.Indices.ExistsAsync(todayIndex, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(todayIndex, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); - existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(yesterdayIndex, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } @@ -561,18 +651,18 @@ public async Task MaintainOnlyOldIndexesWithNoExistingAliasesAsync() }; await index.EnsureIndexAsync(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); index.MaxIndexAge = _configuration.TimeProvider.GetUtcNow().UtcDateTime.EndOfMonth() - _configuration.TimeProvider.GetUtcNow().UtcDateTime.StartOfMonth(); await DeleteAliasesAsync(index.GetVersionedIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12))); await index.MaintainAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } @@ -588,18 +678,18 @@ public async Task MaintainOnlyOldIndexesWithPartialAliasesAsync() await index.EnsureIndexAsync(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(11)); await index.EnsureIndexAsync(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); index.MaxIndexAge = _configuration.TimeProvider.GetUtcNow().UtcDateTime.EndOfMonth() - _configuration.TimeProvider.GetUtcNow().UtcDateTime.StartOfMonth(); await DeleteAliasesAsync(index.GetVersionedIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12))); await index.MaintainAsync(); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(_configuration.TimeProvider.GetUtcNow().UtcDateTime.SubtractMonths(12)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } @@ -624,16 +714,21 @@ public async Task DailyAliasMaxAgeAsync(DateTime utcNow) Assert.NotNull(employee); Assert.NotNull(employee.Id); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -641,16 +736,21 @@ public async Task DailyAliasMaxAgeAsync(DateTime utcNow) Assert.NotNull(employee); Assert.NotNull(employee.Id); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -658,16 +758,21 @@ public async Task DailyAliasMaxAgeAsync(DateTime utcNow) Assert.NotNull(employee); Assert.NotNull(employee.Id); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); } @@ -692,16 +797,21 @@ public async Task MonthlyAliasMaxAgeAsync(DateTime utcNow) Assert.NotNull(employee); Assert.NotNull(employee.Id); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasesResponse.Aliases; +#else + var indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + var aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -709,16 +819,21 @@ public async Task MonthlyAliasMaxAgeAsync(DateTime utcNow) Assert.NotNull(employee); Assert.NotNull(employee.Id); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -726,16 +841,21 @@ public async Task MonthlyAliasMaxAgeAsync(DateTime utcNow) Assert.NotNull(employee); Assert.NotNull(employee.Id); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - aliasesResponse = await _client.Indices.GetAliasAsync(index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Single(aliasesResponse.Indices); - aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasesResponse.Aliases; +#else + indices = aliasesResponse.Values; +#endif + Assert.Single(indices); + aliases = indices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeMonthlyAliases(index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); } @@ -756,21 +876,21 @@ public async Task DailyIndexMaxAgeAsync(DateTime utcNow) await index.ConfigureAsync(); await index.EnsureIndexAsync(utcNow); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); await index.EnsureIndexAsync(utcNow.SubtractDays(1)); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow.SubtractDays(1)), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow.SubtractDays(1)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); await Assert.ThrowsAsync(async () => await index.EnsureIndexAsync(utcNow.SubtractDays(2))); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow.SubtractDays(2)), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow.SubtractDays(2)), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } @@ -790,24 +910,24 @@ public async Task MonthlyIndexMaxAgeAsync(DateTime utcNow) await index.ConfigureAsync(); await index.EnsureIndexAsync(utcNow); - var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); await index.EnsureIndexAsync(utcNow.Subtract(index.MaxIndexAge.GetValueOrDefault())); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow.Subtract(index.MaxIndexAge.GetValueOrDefault())), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(utcNow.Subtract(index.MaxIndexAge.GetValueOrDefault())), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); var endOfTwoMonthsAgo = utcNow.SubtractMonths(2).EndOfMonth(); if (utcNow - endOfTwoMonthsAgo >= index.MaxIndexAge.GetValueOrDefault()) { await Assert.ThrowsAsync(async () => await index.EnsureIndexAsync(endOfTwoMonthsAgo)); - existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(endOfTwoMonthsAgo), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(index.GetIndex(endOfTwoMonthsAgo), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); } } @@ -872,12 +992,12 @@ public async Task Index_MaintainThenIndexing_ShouldCreateIndexWhenNeeded() // Verify the correct versioned index was created string expectedVersionedIndex = index.GetVersionedIndex(utcNow.UtcDateTime); - var indexExists = await _client.Indices.ExistsAsync(expectedVersionedIndex, ct: TestCancellationToken); + var indexExists = await _client.Indices.ExistsAsync(expectedVersionedIndex, cancellationToken: TestCancellationToken); Assert.True(indexExists.Exists); // Verify the alias exists string expectedAlias = index.GetIndex(utcNow.UtcDateTime); - var aliasExists = await _client.Indices.AliasExistsAsync(expectedAlias, ct: TestCancellationToken); + var aliasExists = await _client.Indices.ExistsAliasAsync(Names.Parse(expectedAlias), cancellationToken: TestCancellationToken); Assert.True(aliasExists.Exists); } @@ -915,11 +1035,11 @@ public async Task Index_ParallelOperations_ShouldNotInterfereWithEachOther() string expectedVersionedIndex = "monthly-employees-v2-2025.06"; string expectedAlias = "monthly-employees-2025.06"; - var indexExistsResponse = await _client.Indices.ExistsAsync(expectedVersionedIndex, ct: TestCancellationToken); + var indexExistsResponse = await _client.Indices.ExistsAsync(expectedVersionedIndex, cancellationToken: TestCancellationToken); Assert.True(indexExistsResponse.Exists, $"Versioned index {expectedVersionedIndex} should exist"); - var aliasResponse = await _client.Indices.GetAliasAsync(expectedAlias, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid, $"Alias {expectedAlias} should exist"); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)expectedAlias, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse, $"Alias {expectedAlias} should exist"); } [Fact] @@ -960,41 +1080,47 @@ public async Task EnsuredDates_AddingManyDates_CouldLeakMemory() var baseDate = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); // Act + Employee lastEmployee = null; for (int i = 0; i < UNIQUE_DATES; i++) { var testDate = baseDate.AddDays(i); var employee = EmployeeGenerator.Generate(createdUtc: testDate); - await repository.AddAsync(employee); + lastEmployee = await repository.AddAsync(employee); } - // This test verifies that adding many dates doesn't throw exceptions, - // but it highlights the fact that _ensuredDates will grow unbounded. - // A proper fix would implement a cleanup mechanism. + // Assert + Assert.NotNull(lastEmployee); + Assert.NotNull(lastEmployee!.Id); + var retrieved = await repository.GetByIdAsync(lastEmployee.Id); + Assert.NotNull(retrieved); + Assert.Equal(lastEmployee.Id, retrieved.Id); } - [Fact(Skip = "Shows an issue where we cannot recover if an index exists with an alias name")] - public async Task UpdateAliasesAsync_CreateAliasFailure_ShouldHandleGracefully() + [Fact] + public async Task UpdateAliasesAsync_CreateAliasFailure_ShouldThrow() { // Arrange var index = new DailyEmployeeIndex(_configuration, 2); await index.DeleteAsync(); - await using AsyncDisposableAction _ = new(() => index.DeleteAsync()); - - // Create a scenario that causes alias creation to fail string indexName = index.GetIndex(DateTime.UtcNow); - // First create a conflicting index without the alias + await using AsyncDisposableAction _ = new(async () => + { + await _client.Indices.DeleteAsync(indexName); + await index.DeleteAsync(); + }); + await _client.Indices.CreateAsync(indexName, d => d - .Map(m => m.AutoMap()) - .Settings(s => s.NumberOfReplicas(0)), TestCancellationToken); + .Mappings(m => m.Properties(p => p.Keyword("id"))) + .Settings(s => s.NumberOfReplicas(0)), cancellationToken: TestCancellationToken); - // Act var repository = new EmployeeRepository(index); var employee = EmployeeGenerator.Generate(createdUtc: DateTime.UtcNow); - // This should handle the conflict gracefully in the future - await repository.AddAsync(employee); + // Act & Assert: exception type depends on ES transport layer error handling + var ex = await Assert.ThrowsAnyAsync(() => repository.AddAsync(employee)); + Assert.Contains("alias", ex.Message, StringComparison.OrdinalIgnoreCase); } [Fact] @@ -1070,7 +1196,7 @@ public void GetIndexes_ThreeMonthPeriod_ShouldReturnEmptyForDailyIndex() string[] indexes = index.GetIndexes(startDate, endDate); // Assert - Assert.Empty(indexes); // Should return empty for periods >= 3 months + Assert.Empty(indexes); } [Fact] @@ -1103,9 +1229,9 @@ public async Task PatchAsync_WhenActionPatchAndSingleIndexMissing_CreatesIndex() await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); + response = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1124,9 +1250,9 @@ public async Task PatchAsync_WhenPartialPatchAndSingleIndexMissing_CreatesIndex( await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); + response = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1147,9 +1273,9 @@ public async Task PatchAsync_WhenJsonPatchAndSingleIndexMissing_CreatesIndex() await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); + response = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1168,9 +1294,9 @@ public async Task PatchAsync_WhenScriptPatchAndSingleIndexMissing_CreatesIndex() await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); + response = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1188,7 +1314,7 @@ public async Task PatchAllAsync_WhenActionPatchAndSingleIndexMissing_CreatesInde await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1206,7 +1332,7 @@ public async Task PatchAllAsync_WhenPartialPatchAndSingleIndexMissing_CreatesInd await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1226,7 +1352,7 @@ public async Task PatchAllAsync_WhenJsonPatchAndSingleIndexMissing_CreatesIndex( await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1244,7 +1370,7 @@ public async Task PatchAllAsync_WhenScriptPatchAndSingleIndexMissing_CreatesInde await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1263,10 +1389,10 @@ public async Task PatchAsync_WhenActionPatchAndMonthlyIndexMissing_CreatesIndex( await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); - Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); - Assert.True(response.Exists); + var aliasResponse = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.Exists); + var indexResponse = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); + Assert.True(indexResponse.Exists); } [Fact] @@ -1284,10 +1410,10 @@ public async Task PatchAsync_WhenPartialPatchAndMonthlyIndexMissing_CreatesIndex await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); - Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); - Assert.True(response.Exists); + var aliasResponse = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.Exists); + var indexResponse = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); + Assert.True(indexResponse.Exists); } [Fact] @@ -1307,10 +1433,10 @@ public async Task PatchAsync_WhenJsonPatchAndMonthlyIndexMissing_CreatesIndex() await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); - Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); - Assert.True(response.Exists); + var aliasResponse = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.Exists); + var indexResponse = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); + Assert.True(indexResponse.Exists); } [Fact] @@ -1328,10 +1454,10 @@ public async Task PatchAsync_WhenScriptPatchAndMonthlyIndexMissing_CreatesIndex( await Assert.ThrowsAsync(async () => await repository.PatchAsync(id, patch)); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); - Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id), ct: TestCancellationToken); - Assert.True(response.Exists); + var aliasResponse = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.Exists); + var indexResponse = await _client.Indices.ExistsAsync(index.GetIndex(id), cancellationToken: TestCancellationToken); + Assert.True(indexResponse.Exists); } [Fact] @@ -1348,7 +1474,7 @@ public async Task PatchAllAsync_WhenActionPatchAndMonthlyIndexMissing_DoesNotCre await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); Assert.False(response.Exists); } @@ -1366,7 +1492,7 @@ public async Task PatchAllAsync_WhenPartialPatchAndMonthlyIndexMissing_DoesNotCr await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); Assert.False(response.Exists); } @@ -1386,7 +1512,7 @@ public async Task PatchAllAsync_WhenJsonPatchAndMonthlyIndexMissing_DoesNotCreat await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); Assert.False(response.Exists); } @@ -1404,7 +1530,7 @@ public async Task PatchAllAsync_WhenScriptPatchAndMonthlyIndexMissing_DoesNotCre await repository.PatchAllAsync(q => q, patch); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); Assert.False(response.Exists); } @@ -1424,11 +1550,11 @@ public async Task PatchAllAsync_ByQuery_CreatesAllRelevantDailyIndices() await repository.PatchAllAsync(q => q.Id(id1, id2), new ActionPatch(e => e.Name = "Patched")); // Assert - var response = await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken); + var response = await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken); Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id1), ct: TestCancellationToken); + response = await _client.Indices.ExistsAsync(index.GetIndex(id1), cancellationToken: TestCancellationToken); Assert.True(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id2), ct: TestCancellationToken); + response = await _client.Indices.ExistsAsync(index.GetIndex(id2), cancellationToken: TestCancellationToken); Assert.True(response.Exists); } @@ -1448,11 +1574,61 @@ public async Task PatchAllAsync_ByQueryAcrossMultipleDays_DoesNotCreateAllReleva await repository.PatchAllAsync(q => q.Id(id1, id2), new ActionPatch(e => e.Name = "Patched")); // Assert - var response = await _client.Indices.AliasExistsAsync(index.Name, ct: TestCancellationToken); - Assert.False(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id1), ct: TestCancellationToken); - Assert.False(response.Exists); - response = await _client.Indices.ExistsAsync(index.GetIndex(id2), ct: TestCancellationToken); - Assert.False(response.Exists); + var aliasResponse = await _client.Indices.ExistsAliasAsync(index.Name, cancellationToken: TestCancellationToken); + Assert.False(aliasResponse.Exists); + var indexResponse1 = await _client.Indices.ExistsAsync(index.GetIndex(id1), cancellationToken: TestCancellationToken); + Assert.False(indexResponse1.Exists); + var indexResponse2 = await _client.Indices.ExistsAsync(index.GetIndex(id2), cancellationToken: TestCancellationToken); + Assert.False(indexResponse2.Exists); + } + + [Fact] + public async Task CreateIndexAsync_WithInvalidSettings_DoesNotCacheEnsured() + { + // Arrange + var badIndex = new VersionedEmployeeIndex(_configuration, 1, + createIndex: d => d + .Settings(s => s + .NumberOfReplicas(0) + .NumberOfShards(1) + .Analysis(a => a.Analyzers(an => an.Custom("broken_analyzer", c => c.Tokenizer("nonexistent_tokenizer"))))) + .Mappings(map => map + .Dynamic(DynamicMapping.False) + .Properties(p => p.Keyword(e => e.CompanyName))), + createMappings: _ => { }); + await badIndex.DeleteAsync(); + await using AsyncDisposableAction cleanup = new(() => badIndex.DeleteAsync()); + + // Act + await Assert.ThrowsAsync(() => badIndex.ConfigureAsync()); + + // Assert - index should not exist and a subsequent valid configure should succeed + var existsResponse = await _client.Indices.ExistsAsync(badIndex.VersionedName, cancellationToken: TestCancellationToken); + Assert.False(existsResponse.Exists); + + var goodIndex = new VersionedEmployeeIndex(_configuration, 1); + await using AsyncDisposableAction goodCleanup = new(() => goodIndex.DeleteAsync()); + await goodIndex.ConfigureAsync(); + + existsResponse = await _client.Indices.ExistsAsync(goodIndex.VersionedName, cancellationToken: TestCancellationToken); + Assert.True(existsResponse.Exists); + } + + [Fact] + public async Task GetIndexListAsync_WithExistingIndex_ReturnsIndexNames() + { + // Arrange + var index = new VersionedEmployeeIndex(_configuration, 1); + await index.DeleteAsync(); + await using AsyncDisposableAction _ = new(() => index.DeleteAsync()); + await index.ConfigureAsync(); + + // Act + var utility = new ElasticUtility(_client, _logger); + var indexes = await utility.GetIndexListAsync(); + + // Assert + Assert.NotNull(indexes); + Assert.Contains(indexes, i => i.Contains("employees")); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/MigrationTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/MigrationTests.cs index f3f1e609..79647d0e 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/MigrationTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/MigrationTests.cs @@ -69,7 +69,7 @@ public async Task WillSetVersionToLatestIfNoMigrationsRun() Assert.False(migrationStatus.NeedsMigration); Assert.Equal(3, migrationStatus.CurrentVersion); - await _client.Indices.RefreshAsync(ct: TestCancellationToken); + await _client.Indices.RefreshAsync(cancellationToken: TestCancellationToken); var migrations = await _migrationStateRepository.GetAllAsync(); Assert.Single(migrations.Documents); @@ -103,7 +103,7 @@ await _migrationStateRepository.AddAsync(new MigrationState var result = await _migrationManager.RunMigrationsAsync(TestCancellationToken); Assert.Equal(MigrationResult.Success, result); - await _client.Indices.RefreshAsync(ct: TestCancellationToken); + await _client.Indices.RefreshAsync(cancellationToken: TestCancellationToken); var migrations = await _migrationStateRepository.GetAllAsync(); Assert.Equal(2, migrations.Documents.Count); @@ -137,7 +137,7 @@ await _migrationStateRepository.AddAsync(new MigrationState var result = await _migrationManager.RunMigrationsAsync(TestCancellationToken); Assert.Equal(MigrationResult.Success, result); - await _client.Indices.RefreshAsync(ct: TestCancellationToken); + await _client.Indices.RefreshAsync(cancellationToken: TestCancellationToken); var migrations = await _migrationStateRepository.GetAllAsync(); Assert.Equal(2, migrations.Documents.Count); @@ -204,7 +204,7 @@ await _migrationStateRepository.AddAsync(new MigrationState var failingMigration = _serviceProvider.GetRequiredService(); Assert.Equal(1, failingMigration.Attempts); - await _client.Indices.RefreshAsync(ct: TestCancellationToken); + await _client.Indices.RefreshAsync(cancellationToken: TestCancellationToken); var migrations = await _migrationStateRepository.GetAllAsync(); Assert.Equal(2, migrations.Documents.Count); @@ -241,7 +241,7 @@ await _migrationStateRepository.AddAsync(new MigrationState var failingMigration = _serviceProvider.GetRequiredService(); Assert.Equal(3, failingMigration.Attempts); - await _client.Indices.RefreshAsync(ct: TestCancellationToken); + await _client.Indices.RefreshAsync(cancellationToken: TestCancellationToken); var migrations = await _migrationStateRepository.GetAllAsync(); Assert.Equal(2, migrations.Documents.Count); @@ -327,10 +327,8 @@ public FailingMigration() public int Attempts { get; set; } -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public override async Task RunAsync(MigrationContext context) + public override Task RunAsync(MigrationContext context) { -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously Attempts++; throw new ApplicationException("Boom"); } @@ -346,13 +344,13 @@ public FailingResumableMigration() public int Attempts { get; set; } -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public override async Task RunAsync(MigrationContext context) + public override Task RunAsync(MigrationContext context) { -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously Attempts++; if (Attempts <= 3) throw new ApplicationException("Boom"); + + return Task.CompletedTask; } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/NestedFieldTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/NestedFieldTests.cs index 29443945..48dabf89 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/NestedFieldTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/NestedFieldTests.cs @@ -89,8 +89,7 @@ public async Task CountAsync_WithNestedPeerReviewAggregation_ReturnsAggregationD Assert.Equal(3, result.Total); Assert.Single(result.Aggregations); - var nestedPeerReviewsAgg = result.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(nestedPeerReviewsAgg); + var nestedPeerReviewsAgg = Assert.IsType(result.Aggregations["nested_peerReviews"]); Assert.NotEmpty(nestedPeerReviewsAgg.Aggregations); } @@ -145,39 +144,38 @@ public async Task SearchAsync_WithNestedAggregations_ReturnsCorrectBuckets() await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); // Act - var nestedAggQuery = _client.Search(d => d.Index("employees").Aggregations(a => a - .Nested("nested_reviewRating", h => h.Path("peerReviews") - .Aggregations(a1 => a1.Terms("terms_rating", t => t.Field("peerReviews.rating").Meta(m => m.Add("@field_type", "integer"))))) - )); + var nestedAggQuery = await _client.SearchAsync(d => d.Indices("employees").Aggregations(a => a + .Add("nested_reviewRating", agg => agg + .Nested(h => h.Path("peerReviews")) + .Aggregations(a1 => a1.Add("terms_rating", t => t.Terms(t1 => t1.Field("peerReviews.rating")).Meta(m => m.Add("@field_type", "integer"))))) + ), cancellationToken: TestCancellationToken); // Assert - var result = nestedAggQuery.Aggregations.ToAggregations(); + var result = nestedAggQuery.Aggregations.ToAggregations(_serializer); Assert.Single(result); - var nestedReviewRatingAgg = result["nested_reviewRating"] as SingleBucketAggregate; - Assert.NotNull(nestedReviewRatingAgg); + var nestedReviewRatingAgg = Assert.IsType(result["nested_reviewRating"]); - var termsRatingAgg = nestedReviewRatingAgg.Aggregations["terms_rating"] as BucketAggregate; - Assert.NotNull(termsRatingAgg); + var termsRatingAgg = Assert.IsType(nestedReviewRatingAgg.Aggregations["terms_rating"]); Assert.Equal(2, termsRatingAgg.Items.Count); // Act - Test nested aggregation with filter - var nestedAggQueryWithFilter = _client.Search(d => d.Index("employees").Aggregations(a => a - .Nested("nested_reviewRating", h => h.Path("peerReviews") + var nestedAggQueryWithFilter = await _client.SearchAsync(d => d.Indices("employees").Aggregations(a => a + .Add("nested_reviewRating", agg => agg + .Nested(h => h.Path("peerReviews")) .Aggregations(a1 => a1 - .Filter("user_" + employees[0].Id, f => f.Filter(q => q.Term(t => t.Field("peerReviews.reviewerEmployeeId").Value(employees[0].Id))) - .Aggregations(a2 => a2.Terms("terms_rating", t => t.Field("peerReviews.rating").Meta(m => m.Add("@field_type", "integer"))))) - )))); + .Add($"user_{employees[0].Id}", f => f + .Filter(q => q.Term(t => t.Field("peerReviews.reviewerEmployeeId").Value(employees[0].Id))) + .Aggregations(a2 => a2.Add("terms_rating", t => t.Terms(t1 => t1.Field("peerReviews.rating")).Meta(m => m.Add("@field_type", "integer"))))) + ))), cancellationToken: TestCancellationToken); // Assert - Verify filtered aggregation - result = nestedAggQueryWithFilter.Aggregations.ToAggregations(); + result = nestedAggQueryWithFilter.Aggregations.ToAggregations(_serializer); Assert.Single(result); - var nestedReviewRatingFilteredAgg = result["nested_reviewRating"] as SingleBucketAggregate; - Assert.NotNull(nestedReviewRatingFilteredAgg); + var nestedReviewRatingFilteredAgg = Assert.IsType(result["nested_reviewRating"]); - var userFilteredAgg = nestedReviewRatingFilteredAgg.Aggregations["user_" + employees[0].Id] as SingleBucketAggregate; - Assert.NotNull(userFilteredAgg); + var userFilteredAgg = Assert.IsType(nestedReviewRatingFilteredAgg.Aggregations[$"user_{employees[0].Id}"]); Assert.Single(userFilteredAgg.Aggregations.Terms("terms_rating").Buckets); Assert.Equal("5", userFilteredAgg.Aggregations.Terms("terms_rating").Buckets.First().Key); Assert.Equal(1, userFilteredAgg.Aggregations.Terms("terms_rating").Buckets.First().Total); @@ -217,8 +215,7 @@ public async Task CountAsync_WithNestedLuceneBasedAggregations_ReturnsCorrectMet Assert.Equal(3, result.Total); Assert.Single(result.Aggregations); - var nestedPeerReviewsAgg = result.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(nestedPeerReviewsAgg); + var nestedPeerReviewsAgg = Assert.IsType(result.Aggregations["nested_peerReviews"]); var reviewerTermsAgg = nestedPeerReviewsAgg.Aggregations.Terms("terms_peerReviews.reviewerEmployeeId"); Assert.Equal(3, reviewerTermsAgg.Buckets.Count); @@ -265,8 +262,7 @@ public async Task CountAsync_WithNestedAggregationsIncludeFiltering_ReturnsFilte Assert.Equal(3, resultWithInclude.Total); Assert.Single(resultWithInclude.Aggregations); - var nestedPeerReviewsAggWithInclude = resultWithInclude.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(nestedPeerReviewsAggWithInclude); + var nestedPeerReviewsAggWithInclude = Assert.IsType(resultWithInclude.Aggregations["nested_peerReviews"]); var reviewerTermsAggWithInclude = nestedPeerReviewsAggWithInclude.Aggregations.Terms("terms_peerReviews.reviewerEmployeeId"); Assert.Equal(2, reviewerTermsAggWithInclude.Buckets.Count); // Only employee1 and employee2 should be included @@ -315,8 +311,7 @@ public async Task CountAsync_WithNestedAggregationsExcludeFiltering_ReturnsFilte Assert.Equal(3, resultWithExclude.Total); Assert.Single(resultWithExclude.Aggregations); - var nestedPeerReviewsAggWithExclude = resultWithExclude.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(nestedPeerReviewsAggWithExclude); + var nestedPeerReviewsAggWithExclude = Assert.IsType(resultWithExclude.Aggregations["nested_peerReviews"]); var reviewerTermsAggWithExclude = nestedPeerReviewsAggWithExclude.Aggregations.Terms("terms_peerReviews.reviewerEmployeeId"); Assert.Equal(2, reviewerTermsAggWithExclude.Buckets.Count); // employee3 should be excluded @@ -363,8 +358,7 @@ public async Task CountAsync_WithNestedAggregationsSerialization_CanRoundtripBot Assert.Equal(3, result.Total); Assert.Single(result.Aggregations); - var nestedPeerReviewsAgg = result.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(nestedPeerReviewsAgg); + var nestedPeerReviewsAgg = Assert.IsType(result.Aggregations["nested_peerReviews"]); var ratingTermsAgg = nestedPeerReviewsAgg.Aggregations.Terms("terms_peerReviews.rating"); Assert.Equal(4, ratingTermsAgg.Buckets.Count); @@ -377,8 +371,7 @@ public async Task CountAsync_WithNestedAggregationsSerialization_CanRoundtripBot Assert.Equal(3, roundTripped.Total); Assert.Single(roundTripped.Aggregations); - var roundTrippedNestedAgg = roundTripped.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(roundTrippedNestedAgg); + var roundTrippedNestedAgg = Assert.IsType(roundTripped.Aggregations["nested_peerReviews"]); var roundTrippedRatingTermsAgg = roundTrippedNestedAgg.Aggregations.Terms("terms_peerReviews.rating"); Assert.NotNull(roundTrippedRatingTermsAgg); @@ -388,13 +381,15 @@ public async Task CountAsync_WithNestedAggregationsSerialization_CanRoundtripBot // Test System.Text.Json serialization string systemTextJson = System.Text.Json.JsonSerializer.Serialize(result); - Assert.Equal(json, systemTextJson); + Assert.True(System.Text.Json.Nodes.JsonNode.DeepEquals( + System.Text.Json.Nodes.JsonNode.Parse(json), + System.Text.Json.Nodes.JsonNode.Parse(systemTextJson)), + "Newtonsoft and System.Text.Json serialization should produce semantically equivalent JSON"); roundTripped = System.Text.Json.JsonSerializer.Deserialize(systemTextJson); Assert.Equal(3, roundTripped.Total); Assert.Single(roundTripped.Aggregations); - roundTrippedNestedAgg = roundTripped.Aggregations["nested_peerReviews"] as SingleBucketAggregate; - Assert.NotNull(roundTrippedNestedAgg); + roundTrippedNestedAgg = Assert.IsType(roundTripped.Aggregations["nested_peerReviews"]); roundTrippedRatingTermsAgg = roundTrippedNestedAgg.Aggregations.Terms("terms_peerReviews.rating"); Assert.Equal(4, roundTrippedRatingTermsAgg.Buckets.Count); diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/ParentChildTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/ParentChildTests.cs index 30e745cc..a047fb60 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/ParentChildTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/ParentChildTests.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Foundatio.Repositories.Elasticsearch.Tests.Repositories; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; +using Microsoft.Extensions.Logging; using Xunit; namespace Foundatio.Repositories.Elasticsearch.Tests; @@ -99,25 +100,25 @@ public async Task DeletedParentWillFilterChild() parent.IsDeleted = false; await _parentRepository.SaveAsync(parent, o => o.ImmediateConsistency()); - Assert.Equal(1, await _childRepository.GetAllAsync()); + Assert.Equal(1, await _childRepository.CountAsync()); } [Fact] public async Task CanQueryByParent() { var parent = ParentGenerator.Default; - parent = await _parentRepository.AddAsync(parent); + parent = await _parentRepository.AddAsync(parent, o => o.ImmediateConsistency()); Assert.NotNull(parent); Assert.NotNull(parent.Id); - await _parentRepository.AddAsync(ParentGenerator.Generate()); + await _parentRepository.AddAsync(ParentGenerator.Generate(), o => o.ImmediateConsistency()); var child = ChildGenerator.Default; child = await _childRepository.AddAsync(child, o => o.ImmediateConsistency()); Assert.NotNull(child); Assert.NotNull(child.Id); - var childResults = await _childRepository.FindAsync(q => q.ParentQuery(p => p.Id(parent.Id))); + var childResults = await _childRepository.FindAsync(q => q.ParentQuery(p => p.Id(parent.Id)), o => o.QueryLogLevel(LogLevel.Warning)); Assert.Equal(1, childResults.Total); } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/PipelineTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/PipelineTests.cs index 2cac26bb..70452d75 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/PipelineTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/PipelineTests.cs @@ -1,160 +1,216 @@ -// using System; -// using System.Collections.Generic; -// using System.Linq; -// using System.Threading.Tasks; -// using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; -// using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Types; -// using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -// using Foundatio.Repositories.JsonPatch; -// using Foundatio.Repositories.Models; -// using Foundatio.Repositories.Utility; -// using Foundatio.Utility; -// using Xunit; -// using Xunit; - -// namespace Foundatio.Repositories.Elasticsearch.Tests { -// public sealed class PipelineTests : ElasticRepositoryTestBase { -// private readonly IEmployeeRepository _employeeRepository; - -// public PipelineTests(ITestOutputHelper output) : base(output) { -// // configure type so pipeline is created. -// var employeeType = new EmployeeTypeWithWithPipeline(new EmployeeIndex(_configuration)); -// employeeType.ConfigureAsync().GetAwaiter().GetResult(); - -// _employeeRepository = new EmployeeRepository(employeeType); -// RemoveDataAsync().GetAwaiter().GetResult(); -// } - -// [Fact] -// public async Task AddAsync() { -// var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Generate(name: " BLAKE ")); -// Assert.NotNull(employee?.Id); - -// var result = await _employeeRepository.GetByIdAsync(employee.Id); -// Assert.Equal("blake", result.Name); -// } - -// [Fact] -// public async Task AddCollectionAsync() { -// var employees = new List { -// EmployeeGenerator.Generate(name: " BLAKE "), -// EmployeeGenerator.Generate(name: "\tBLAKE ") -// }; -// await _employeeRepository.AddAsync(employees); - -// var result = await _employeeRepository.GetByIdsAsync(new Ids(employees.Select(e => e.Id))); -// Assert.Equal(2, result.Count); -// Assert.True(result.All(e => String.Equals(e.Name, "blake"))); -// } - -// [Fact] -// public async Task SaveCollectionAsync() { -// var employee1 = EmployeeGenerator.Generate(id: ObjectId.GenerateNewId().ToString()); -// var employee2 = EmployeeGenerator.Generate(id: ObjectId.GenerateNewId().ToString()); -// await _employeeRepository.AddAsync(new List { employee1, employee2 }); - -// employee1.Name = " BLAKE "; -// employee2.Name = "\tBLAKE "; -// await _employeeRepository.SaveAsync(new List { employee1, employee2 }); -// var result = await _employeeRepository.GetByIdsAsync(new List { employee1.Id, employee2.Id }); -// Assert.Equal(2, result.Count); -// Assert.True(result.All(e => String.Equals(e.Name, "blake"))); -// } - -// [Fact] -// public async Task JsonPatchAsync() { -// var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default); -// var patch = new PatchDocument(new ReplaceOperation { Path = "name", Value = "Patched" }); -// await _employeeRepository.PatchAsync(employee.Id, new Models.JsonPatch(patch)); - -// employee = await _employeeRepository.GetByIdAsync(employee.Id); -// Assert.Equal(EmployeeGenerator.Default.Age, employee.Age); -// Assert.Equal("patched", employee.Name); -// Assert.Equal(2, employee.Version); -// } - -// [Fact] -// public async Task JsonPatchAllAsync() { -// var utcNow = SystemClock.UtcNow; -// var employees = new List { -// EmployeeGenerator.Generate(ObjectId.GenerateNewId(utcNow.AddDays(-1)).ToString(), createdUtc: utcNow.AddDays(-1), companyId: "1", yearsEmployed: 0), -// EmployeeGenerator.Generate(createdUtc: utcNow, companyId: "1", yearsEmployed: 0), -// EmployeeGenerator.Generate(createdUtc: utcNow, companyId: "2", yearsEmployed: 0), -// }; - -// await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); - -// var patch = new PatchDocument(new ReplaceOperation { Path = "name", Value = "Patched" }); -// await _employeeRepository.PatchAsync(employees.Select(l => l.Id).ToArray(), new Models.JsonPatch(patch), o => o.ImmediateConsistency()); - -// var results = await _employeeRepository.GetAllByCompanyAsync("1"); -// Assert.Equal(2, results.Documents.Count); -// foreach (var document in results.Documents) { -// Assert.Equal("1", document.CompanyId); -// Assert.Equal("patched", document.Name); -// } -// } - -// [Fact (Skip = "Not yet supported: https://github.com/elastic/elasticsearch/issues/17895")] -// public async Task PartialPatchAsync() { -// var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default); -// await _employeeRepository.PatchAsync(employee.Id, new PartialPatch(new { name = "Patched" })); - -// employee = await _employeeRepository.GetByIdAsync(employee.Id); -// Assert.Equal(EmployeeGenerator.Default.Age, employee.Age); -// Assert.Equal("patched", employee.Name); -// Assert.Equal(2, employee.Version); -// } - -// [Fact(Skip = "Not yet supported: https://github.com/elastic/elasticsearch/issues/17895")] -// public async Task PartialPatchAllAsync() { -// var utcNow = SystemClock.UtcNow; -// var employees = new List { -// EmployeeGenerator.Generate(ObjectId.GenerateNewId(utcNow.AddDays(-1)).ToString(), createdUtc: utcNow.AddDays(-1), companyId: "1", yearsEmployed: 0), -// EmployeeGenerator.Generate(createdUtc: utcNow, companyId: "1", yearsEmployed: 0), -// EmployeeGenerator.Generate(createdUtc: utcNow, companyId: "2", yearsEmployed: 0), -// }; - -// await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); -// await _employeeRepository.PatchAsync(employees.Select(l => l.Id).ToArray(), new PartialPatch(new { name = "Patched" }), o => o.ImmediateConsistency()); - -// var results = await _employeeRepository.GetAllByCompanyAsync("1"); -// Assert.Equal(2, results.Documents.Count); -// foreach (var document in results.Documents) { -// Assert.Equal("1", document.CompanyId); -// Assert.Equal("patched", document.Name); -// } -// } - -// [Fact(Skip = "Not yet supported: https://github.com/elastic/elasticsearch/issues/17895")] -// public async Task ScriptPatchAsync() { -// var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default); -// await _employeeRepository.PatchAsync(employee.Id, new ScriptPatch("ctx._source.name = 'Patched';")); - -// employee = await _employeeRepository.GetByIdAsync(employee.Id); -// Assert.Equal(EmployeeGenerator.Default.Age, employee.Age); -// Assert.Equal("patched", employee.Name); -// Assert.Equal(2, employee.Version); -// } - -// [Fact(Skip = "Not yet supported: https://github.com/elastic/elasticsearch/issues/17895")] -// public async Task ScriptPatchAllAsync() { -// var utcNow = SystemClock.UtcNow; -// var employees = new List { -// EmployeeGenerator.Generate(ObjectId.GenerateNewId(utcNow.AddDays(-1)).ToString(), createdUtc: utcNow.AddDays(-1), companyId: "1", yearsEmployed: 0), -// EmployeeGenerator.Generate(createdUtc: utcNow, companyId: "1", yearsEmployed: 0), -// EmployeeGenerator.Generate(createdUtc: utcNow, companyId: "2", yearsEmployed: 0), -// }; - -// await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); -// await _employeeRepository.PatchAsync(employees.Select(l => l.Id).ToArray(), new ScriptPatch("ctx._source.name = 'Patched';"), o => o.ImmediateConsistency()); - -// var results = await _employeeRepository.GetAllByCompanyAsync("1"); -// Assert.Equal(2, results.Documents.Count); -// foreach (var document in results.Documents) { -// Assert.Equal("1", document.CompanyId); -// Assert.Equal("patched", document.Name); -// } -// } -// } -// } +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Utility; +using Xunit; + +namespace Foundatio.Repositories.Elasticsearch.Tests; + +public sealed class PipelineTests : ElasticRepositoryTestBase +{ + private const string PipelineId = "employee-lowercase-name"; + private readonly EmployeeWithPipelineRepository _employeeRepository; + + public PipelineTests(ITestOutputHelper output) : base(output) + { + _employeeRepository = new EmployeeWithPipelineRepository(_configuration.Employees, PipelineId); + } + + public override async ValueTask InitializeAsync() + { + await base.InitializeAsync(); + + var response = await _client.Ingest.PutPipelineAsync(PipelineId, p => p + .Description("Lowercases the name field for pipeline tests") + .Processors(pr => pr.Lowercase(l => l.Field("name")))); + Assert.True(response.IsValidResponse, $"Failed to create pipeline: {response.ElasticsearchServerError?.Error?.Reason}"); + + await RemoveDataAsync(); + } + + [Fact] + public async Task AddAsync_WithLowercasePipeline_LowercasesName() + { + // Arrange + var employee = EmployeeGenerator.Generate(name: " BLAKE "); + + // Act + employee = await _employeeRepository.AddAsync(employee, o => o.ImmediateConsistency()); + + // Assert + Assert.NotNull(employee); + Assert.NotNull(employee.Id); + var result = await _employeeRepository.GetByIdAsync(employee.Id); + Assert.NotNull(result); + Assert.Equal(" blake ", result.Name); + } + + [Fact] + public async Task AddCollectionAsync_WithLowercasePipeline_LowercasesNames() + { + // Arrange + var employees = new List + { + EmployeeGenerator.Generate(name: " BLAKE "), + EmployeeGenerator.Generate(name: "\tBLAKE ") + }; + + // Act + await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); + + // Assert + var result = await _employeeRepository.GetByIdsAsync(new Ids(employees.Select(e => e.Id))); + Assert.Equal(2, result.Count); + Assert.Contains(result, e => String.Equals(e.Name, " blake ")); + Assert.Contains(result, e => String.Equals(e.Name, "\tblake ")); + } + + [Fact] + public async Task SaveCollectionAsync_WithLowercasePipeline_LowercasesNames() + { + // Arrange + var employee1 = EmployeeGenerator.Generate(id: ObjectId.GenerateNewId().ToString(), name: "Original1"); + var employee2 = EmployeeGenerator.Generate(id: ObjectId.GenerateNewId().ToString(), name: "Original2"); + await _employeeRepository.AddAsync(new List { employee1, employee2 }, o => o.ImmediateConsistency()); + + // Act + employee1.Name = " BLAKE "; + employee2.Name = "\tBLAKE "; + await _employeeRepository.SaveAsync(new List { employee1, employee2 }, o => o.ImmediateConsistency()); + + // Assert + var result = await _employeeRepository.GetByIdsAsync(new List { employee1.Id, employee2.Id }); + Assert.Equal(2, result.Count); + Assert.Contains(result, e => String.Equals(e.Name, " blake ")); + Assert.Contains(result, e => String.Equals(e.Name, "\tblake ")); + } + + [Fact] + public async Task JsonPatchAsync_WithLowercasePipeline_LowercasesName() + { + // Arrange + var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); + + // Act + var patch = new PatchDocument(new ReplaceOperation { Path = "name", Value = JsonValue.Create("Patched") }); + await _employeeRepository.PatchAsync(employee.Id, new JsonPatch(patch), o => o.ImmediateConsistency()); + + // Assert + employee = await _employeeRepository.GetByIdAsync(employee.Id); + Assert.NotNull(employee); + Assert.Equal(EmployeeGenerator.Default.Age, employee.Age); + Assert.Equal("patched", employee.Name); + } + + [Fact] + public async Task JsonPatchAllAsync_WithLowercasePipeline_LowercasesNames() + { + // Arrange + var employees = new List + { + EmployeeGenerator.Generate(companyId: "1", name: "employee1"), + EmployeeGenerator.Generate(companyId: "1", name: "employee2"), + }; + await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); + + // Act + var patch = new PatchDocument(new ReplaceOperation { Path = "name", Value = JsonValue.Create("Patched") }); + await _employeeRepository.PatchAsync(employees.Select(e => e.Id).ToArray(), new JsonPatch(patch), o => o.ImmediateConsistency()); + + // Assert + var results = await _employeeRepository.GetByIdsAsync(employees.Select(e => e.Id).ToList()); + Assert.Equal(2, results.Count); + Assert.All(results, e => Assert.Equal("patched", e.Name)); + } + + [Fact] + public async Task PartialPatchAsync_WithLowercasePipeline_LowercasesName() + { + // Arrange + var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); + + // Act + await _employeeRepository.PatchAsync(employee.Id, new PartialPatch(new { name = "Patched" }), o => o.ImmediateConsistency()); + + // Assert + employee = await _employeeRepository.GetByIdAsync(employee.Id); + Assert.NotNull(employee); + Assert.Equal(EmployeeGenerator.Default.Age, employee.Age); + Assert.Equal("patched", employee.Name); + } + + [Fact] + public async Task PartialPatchAllAsync_WithLowercasePipeline_LowercasesNames() + { + // Arrange + var employees = new List + { + EmployeeGenerator.Generate(companyId: "1", name: "employee1"), + EmployeeGenerator.Generate(companyId: "1", name: "employee2"), + }; + await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); + + // Act + await _employeeRepository.PatchAsync(employees.Select(e => e.Id).ToArray(), new PartialPatch(new { name = "Patched" }), o => o.ImmediateConsistency()); + + // Assert + var results = await _employeeRepository.GetByIdsAsync(employees.Select(e => e.Id).ToList()); + Assert.Equal(2, results.Count); + Assert.All(results, e => Assert.Equal("patched", e.Name)); + } + + [Fact] + public async Task ScriptPatchAsync_WithLowercasePipeline_LowercasesName() + { + // Arrange + var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); + + // Act + await _employeeRepository.PatchAsync(employee.Id, new ScriptPatch("ctx._source.name = 'Patched';"), o => o.ImmediateConsistency()); + + // Assert + employee = await _employeeRepository.GetByIdAsync(employee.Id); + Assert.NotNull(employee); + Assert.Equal(EmployeeGenerator.Default.Age, employee.Age); + Assert.Equal("patched", employee.Name); + } + + [Fact] + public async Task ScriptPatchAllAsync_WithLowercasePipeline_LowercasesNames() + { + // Arrange + var employees = new List + { + EmployeeGenerator.Generate(companyId: "1", name: "employee1"), + EmployeeGenerator.Generate(companyId: "1", name: "employee2"), + }; + await _employeeRepository.AddAsync(employees, o => o.ImmediateConsistency()); + + // Act + await _employeeRepository.PatchAsync(employees.Select(e => e.Id).ToArray(), new ScriptPatch("ctx._source.name = 'Patched';"), o => o.ImmediateConsistency()); + + // Assert + var results = await _employeeRepository.GetByIdsAsync(employees.Select(e => e.Id).ToList()); + Assert.Equal(2, results.Count); + Assert.All(results, e => Assert.Equal("patched", e.Name)); + } + + public override async ValueTask DisposeAsync() + { + await _client.Ingest.DeletePipelineAsync(PipelineId); + await base.DisposeAsync(); + } +} + +internal class EmployeeWithPipelineRepository : ElasticRepositoryBase +{ + public EmployeeWithPipelineRepository(IIndex index, string pipelineId) : base(index) + { + DefaultPipeline = pipelineId; + } +} diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryBuilderTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryBuilderTests.cs index 4b421848..8112e2dd 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryBuilderTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryBuilderTests.cs @@ -1,10 +1,10 @@ using System.Linq; using System.Threading.Tasks; +using Foundatio.Parsers; using Foundatio.Parsers.ElasticQueries.Visitors; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Xunit; -using Nest; using Xunit; namespace Foundatio.Repositories.Elasticsearch.Tests; @@ -16,21 +16,61 @@ public RuntimeFieldsQueryBuilderTests(ITestOutputHelper output) : base(output) } [Fact] - public async Task BuildAsync_MultipleFields() + public async Task BuildAsync_WithRuntimeFields_TransfersFieldsToContext() { + // Arrange + var queryBuilder = new AddRuntimeFieldsToContextQueryBuilder(); + var query = new RepositoryQuery() + .RuntimeField("field_one", ElasticRuntimeFieldType.Keyword) + .RuntimeField(new ElasticRuntimeField { Name = "field_two", FieldType = ElasticRuntimeFieldType.Long, Script = "emit(doc['age'].value)" }); + var ctx = new QueryBuilderContext(query, new CommandOptions()); + var ctxElastic = (IElasticQueryVisitorContext)ctx; + + Assert.Empty(ctxElastic.RuntimeFields); + + // Act + await queryBuilder.BuildAsync(ctx); + + // Assert + Assert.Equal(2, ctxElastic.RuntimeFields.Count); + Assert.Equal("field_one", ctxElastic.RuntimeFields.ElementAt(0).Name); + Assert.Equal(ElasticRuntimeFieldType.Keyword, ctxElastic.RuntimeFields.ElementAt(0).FieldType); + Assert.Equal("field_two", ctxElastic.RuntimeFields.ElementAt(1).Name); + Assert.Equal(ElasticRuntimeFieldType.Long, ctxElastic.RuntimeFields.ElementAt(1).FieldType); + Assert.Equal("emit(doc['age'].value)", ctxElastic.RuntimeFields.ElementAt(1).Script); + } + + [Fact] + public async Task BuildAsync_WithContextFields_ConsumesFields() + { + // Arrange + var queryBuilder = new RuntimeFieldsQueryBuilder(); + var query = new RepositoryQuery(); + var ctx = new QueryBuilderContext(query, new CommandOptions()); + var ctxElastic = (IElasticQueryVisitorContext)ctx; + ctxElastic.RuntimeFields.Add(new ElasticRuntimeField { Name = "field_one", FieldType = ElasticRuntimeFieldType.Keyword }); + ctxElastic.RuntimeFields.Add(new ElasticRuntimeField { Name = "field_two", FieldType = ElasticRuntimeFieldType.Long, Script = "emit(doc['age'].value)" }); + + // Act + await queryBuilder.BuildAsync(ctx); + + // Assert + Assert.Equal(2, ctxElastic.RuntimeFields.Count); + } + + [Fact] + public async Task BuildAsync_WithEmptyFields_DoesNotMutateSearch() + { + // Arrange var queryBuilder = new RuntimeFieldsQueryBuilder(); var query = new RepositoryQuery(); - string runtimeField1 = "One", runtimeField2 = "Two"; var ctx = new QueryBuilderContext(query, new CommandOptions()); - var ctxElastic = ctx as IElasticQueryVisitorContext; - ctxElastic.RuntimeFields.Add(new Parsers.ElasticRuntimeField() { Name = runtimeField1 }); - ctxElastic.RuntimeFields.Add(new Parsers.ElasticRuntimeField() { Name = runtimeField2 }); + var ctxElastic = (IElasticQueryVisitorContext)ctx; + // Act await queryBuilder.BuildAsync(ctx); - ISearchRequest request = ctx.Search; - Assert.Equal(2, request.RuntimeFields.Count); - Assert.Equal(runtimeField1, request.RuntimeFields.First().Key); - Assert.Equal(runtimeField2, request.RuntimeFields.Last().Key); + // Assert + Assert.Empty(ctxElastic.RuntimeFields); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryTests.cs index be0ffbe3..4902ce85 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryTests.cs @@ -174,11 +174,12 @@ public async Task GetByMissingFieldAsync() [Fact] public async Task GetByCompanyWithIncludedFields() { - var log = await _dailyRepository.AddAsync(LogEventGenerator.Generate(companyId: "1234567890", message: "test"), o => o.ImmediateConsistency()); + Log.SetLogLevel(LogLevel.Warning); + var log = await _dailyRepository.AddAsync(LogEventGenerator.Generate(companyId: "1234567890", message: "test"), o => o.ImmediateConsistency().QueryLogLevel(LogLevel.Warning)); Assert.NotNull(log); Assert.NotNull(log.Id); - var results = await _dailyRepository.FindAsync(q => q.Company(log.CompanyId)); + var results = await _dailyRepository.FindAsync(q => q.Company(log.CompanyId), o => o.QueryLogLevel(LogLevel.Warning)); Assert.Single(results.Documents); Assert.Equal(log, results.Documents.First()); diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryableRepositoryTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryableRepositoryTests.cs index e70e9365..766ffddd 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryableRepositoryTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/QueryableRepositoryTests.cs @@ -149,9 +149,4 @@ public async Task SearchByQueryWithTimeSeriesAsync() searchResults = await _dailyRepository.FindAsync(q => q.Id(yesterdayLog.Id)); Assert.Equal(1, searchResults.Total); } - - //[Fact] - //public async Task GetAggregations() { - // throw new NotImplementedException(); - //} } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/ReadOnlyRepositoryTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/ReadOnlyRepositoryTests.cs index c43ef7e8..42108cdd 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/ReadOnlyRepositoryTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/ReadOnlyRepositoryTests.cs @@ -372,11 +372,12 @@ public async Task GetByIdWithTimeSeriesAsync() Assert.Equal(nowLog, await _dailyRepository.GetByIdAsync(nowLog.Id)); } - [Fact(Skip = "We need to look into how we want to handle this.")] + [Fact] public async Task GetByIdWithOutOfSyncIndexAsync() { var utcNow = DateTime.UtcNow; - var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId().ToString(), createdUtc: utcNow.AddDays(-1))); + var yesterday = utcNow.AddDays(-1); + var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId(yesterday).ToString(), createdUtc: yesterday)); Assert.NotNull(yesterdayLog); Assert.NotNull(yesterdayLog.Id); @@ -517,11 +518,12 @@ public async Task GetByIdsWithTimeSeriesAsync() Assert.Equal(2, results.Count); } - [Fact(Skip = "We need to look into how we want to handle this.")] + [Fact] public async Task GetByIdsWithOutOfSyncIndexAsync() { var utcNow = DateTime.UtcNow; - var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId().ToString(), createdUtc: utcNow.AddDays(-1))); + var yesterday = utcNow.AddDays(-1); + var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId(yesterday).ToString(), createdUtc: yesterday)); Assert.NotNull(yesterdayLog); Assert.NotNull(yesterdayLog.Id); @@ -600,13 +602,13 @@ public async Task GetAllWithSnapshotPagingAsync() { var identity1 = await _identityRepository.AddAsync(IdentityGenerator.Default, o => o.ImmediateConsistency()); Assert.NotNull(identity1); - Assert.NotNull(identity1.Id); var identity2 = await _identityRepository.AddAsync(IdentityGenerator.Generate(), o => o.ImmediateConsistency()); Assert.NotNull(identity2); - Assert.NotNull(identity2.Id); - await _client.ClearScrollAsync(ct: TestCancellationToken); + var allIds = new HashSet { identity1.Id, identity2.Id }; + + await _client.ClearScrollAsync(cancellationToken: TestCancellationToken); long baselineScrollCount = await GetCurrentScrollCountAsync(); var results = await _identityRepository.GetAllAsync(o => o.PageLimit(1).SnapshotPagingLifetime(TimeSpan.FromMinutes(10))); @@ -614,7 +616,8 @@ public async Task GetAllWithSnapshotPagingAsync() Assert.Single(results.Documents); Assert.Equal(1, results.Page); Assert.True(results.HasMore); - Assert.Equal(identity1.Id, results.Documents.First().Id); + Assert.Contains(results.Documents.First().Id, allIds); + var firstPageId = results.Documents.First().Id; Assert.Equal(2, results.Total); long currentScrollCount = await GetCurrentScrollCountAsync(); Assert.Equal(baselineScrollCount + 1, currentScrollCount); @@ -623,7 +626,8 @@ public async Task GetAllWithSnapshotPagingAsync() Assert.Single(results.Documents); Assert.Equal(2, results.Page); Assert.Equal(2, results.Total); - Assert.Equal(identity2.Id, results.Documents.First().Id); + Assert.Contains(results.Documents.First().Id, allIds); + Assert.NotEqual(firstPageId, results.Documents.First().Id); // Ensure we got a different document // returns true even though there are no more results because we don't know if there are more or not for scrolls until we try to get the next page Assert.True(results.HasMore); var secondDoc = results.Documents.First(); @@ -853,7 +857,7 @@ public async Task FindWithResolvedRuntimeFieldsAsync() Assert.NotNull(employee2); Assert.NotNull(employee2.Id); - var results = await _employeeRepository.FindAsync(q => q.FilterExpression("unmappedcompanyname:" + employee1.CompanyName), o => o.RuntimeFieldResolver(f => String.Equals(f, "unmappedCompanyName", StringComparison.OrdinalIgnoreCase) ? Task.FromResult(new ElasticRuntimeField { Name = "unmappedCompanyName", FieldType = ElasticRuntimeFieldType.Keyword }) : Task.FromResult(null))); + var results = await _employeeRepository.FindAsync(q => q.FilterExpression($"unmappedcompanyname:{employee1.CompanyName}"), o => o.RuntimeFieldResolver(f => String.Equals(f, "unmappedCompanyName", StringComparison.OrdinalIgnoreCase) ? Task.FromResult(new ElasticRuntimeField { Name = "unmappedCompanyName", FieldType = ElasticRuntimeFieldType.Keyword }) : Task.FromResult(null))); Assert.NotNull(results); Assert.Single(results.Documents); } @@ -869,15 +873,15 @@ public async Task CanUseOptInRuntimeFieldResolving() Assert.NotNull(employee2); Assert.NotNull(employee2.Id); - var results = await _employeeRepository.FindAsync(q => q.FilterExpression("unmappedemailaddress:" + employee1.UnmappedEmailAddress)); + var results = await _employeeRepository.FindAsync(q => q.FilterExpression($"unmappedemailaddress:{employee1.UnmappedEmailAddress}")); Assert.NotNull(results); Assert.Empty(results.Documents); - results = await _employeeRepository.FindAsync(q => q.FilterExpression("unmappedemailaddress:" + employee1.UnmappedEmailAddress), o => o.EnableRuntimeFieldResolver()); + results = await _employeeRepository.FindAsync(q => q.FilterExpression($"unmappedemailaddress:{employee1.UnmappedEmailAddress}"), o => o.EnableRuntimeFieldResolver()); Assert.NotNull(results); Assert.Single(results.Documents); - results = await _employeeRepository.FindAsync(q => q.FilterExpression("unmappedemailaddress:" + employee1.UnmappedEmailAddress), o => o.EnableRuntimeFieldResolver(false)); + results = await _employeeRepository.FindAsync(q => q.FilterExpression($"unmappedemailaddress:{employee1.UnmappedEmailAddress}"), o => o.EnableRuntimeFieldResolver(false)); Assert.NotNull(results); Assert.Empty(results.Documents); } @@ -931,7 +935,7 @@ public async Task FindWithSearchAfterPagingAsync() Assert.NotEmpty(results.GetSearchAfterToken()); // try search before - results = await _employeeRepository.FindAsync(q => q.SortDescending(d => d.Name), o => o.PageLimit(1).SearchBeforeToken(searchBeforeToken)); + results = await _employeeRepository.FindAsync(q => q.SortDescending(d => d.Name), o => o.PageLimit(1).SearchBeforeToken(searchBeforeToken, _serializer)); Assert.NotNull(results); Assert.Single(results.Documents); Assert.Equal(1, results.Page); @@ -999,19 +1003,14 @@ public async Task GetAllWithSearchAfterPagingWithCustomSortAsync() Assert.Equal(2, results.Page); Assert.False(results.HasMore); Assert.Equal(2, results.Total); - - // var secondPageResults = await _identityRepository.GetAllAsync(o => o.PageNumber(2).PageLimit(1)); - // Assert.Equal(secondDoc, secondPageResults.Documents.First()); } [Fact] public async Task GetAllAsync_WithNoSortAndPaging_ReturnsAllDocumentsWithoutDuplicates() { - // Arrange var identities = IdentityGenerator.GenerateIdentities(100); await _identityRepository.AddAsync(identities, o => o.ImmediateConsistency()); - // Act var results = await _identityRepository.GetAllAsync(o => o.PageLimit(10)); var viewedIds = new HashSet(); int pagedRecords = 0; @@ -1021,7 +1020,6 @@ public async Task GetAllAsync_WithNoSortAndPaging_ReturnsAllDocumentsWithoutDupl pagedRecords += results.Documents.Count; } while (await results.NextPageAsync()); - // Assert Assert.Equal(100, pagedRecords); Assert.Equal(100, viewedIds.Count); Assert.True(identities.All(e => viewedIds.Contains(e.Id))); @@ -1030,15 +1028,12 @@ public async Task GetAllAsync_WithNoSortAndPaging_ReturnsAllDocumentsWithoutDupl [Fact] public async Task GetAllAsync_WithNoSort_ReturnsDocumentsSortedByIdAscending() { - // Arrange var identities = IdentityGenerator.GenerateIdentities(100); await _identityRepository.AddAsync(identities, o => o.ImmediateConsistency()); - // Act var results = await _identityRepository.GetAllAsync(o => o.PageLimit(100)); var ids = results.Documents.Select(d => d.Id).ToList(); - // Assert Assert.Equal(100, ids.Count); Assert.Equal(ids.OrderBy(id => id).ToList(), ids); } @@ -1046,11 +1041,9 @@ public async Task GetAllAsync_WithNoSort_ReturnsDocumentsSortedByIdAscending() [Fact] public async Task FindAsync_WithNoSortAndPaging_ReturnsAllDocumentsWithoutDuplicates() { - // Arrange var identities = IdentityGenerator.GenerateIdentities(100); await _identityRepository.AddAsync(identities, o => o.ImmediateConsistency()); - // Act var results = await _identityRepository.FindAsync(q => q, o => o.PageLimit(10)); var viewedIds = new HashSet(); int pagedRecords = 0; @@ -1060,7 +1053,6 @@ public async Task FindAsync_WithNoSortAndPaging_ReturnsAllDocumentsWithoutDuplic pagedRecords += results.Documents.Count; } while (await results.NextPageAsync()); - // Assert Assert.Equal(100, pagedRecords); Assert.Equal(100, viewedIds.Count); Assert.True(identities.All(e => viewedIds.Contains(e.Id))); @@ -1114,8 +1106,7 @@ public async Task GetAllWithAliasedDateRangeAsync() Assert.Single(results.Aggregations); Assert.True(results.Aggregations.ContainsKey("date_next")); - var aggregation = results.Aggregations["date_next"] as BucketAggregate; - Assert.NotNull(aggregation); + var aggregation = Assert.IsType(results.Aggregations["date_next"]); Assert.InRange(aggregation.Items.Count, 120, 121); Assert.Equal(0, aggregation.Total); } @@ -1305,7 +1296,7 @@ public async Task CanSearchAfterAndBeforeWithMultipleSorts(string secondarySort) do { page++; - var employees = await _employeeRepository.FindAsync(q => q.Sort(e => e.Name).Sort(e => e.CompanyName).SortDescending(secondarySort), o => o.SearchAfterToken(searchAfterToken).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); + var employees = await _employeeRepository.FindAsync(q => q.Sort(e => e.Name).Sort(e => e.CompanyName).SortDescending(secondarySort), o => o.SearchAfterToken(searchAfterToken, _serializer).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); searchBeforeToken = employees.GetSearchBeforeToken(); searchAfterToken = employees.GetSearchAfterToken(); if (page == 1) @@ -1345,7 +1336,7 @@ public async Task CanSearchAfterAndBeforeWithMultipleSorts(string secondarySort) do { page--; - var employees = await _employeeRepository.FindAsync(q => q.Sort(e => e.Name).Sort(e => e.CompanyName).SortDescending(e => e.Age), o => o.SearchBeforeToken(searchBeforeToken).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); + var employees = await _employeeRepository.FindAsync(q => q.Sort(e => e.Name).Sort(e => e.CompanyName).SortDescending(e => e.Age), o => o.SearchBeforeToken(searchBeforeToken, _serializer).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); searchBeforeToken = employees.GetSearchBeforeToken(); searchAfterToken = employees.GetSearchAfterToken(); if (page == 1) @@ -1383,7 +1374,7 @@ public async Task CanSearchAfterAndBeforeWithSortExpression() do { page++; - var employees = await _employeeRepository.FindAsync(q => q.SortExpression("name companyname -age"), o => o.SearchAfterToken(searchAfterToken).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); + var employees = await _employeeRepository.FindAsync(q => q.SortExpression("name companyname -age"), o => o.SearchAfterToken(searchAfterToken, _serializer).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); searchBeforeToken = employees.GetSearchBeforeToken(); searchAfterToken = employees.GetSearchAfterToken(); if (page == 1) @@ -1423,7 +1414,7 @@ public async Task CanSearchAfterAndBeforeWithSortExpression() do { page--; - var employees = await _employeeRepository.FindAsync(q => q.SortExpression("name companyname -age"), o => o.SearchBeforeToken(searchBeforeToken).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); + var employees = await _employeeRepository.FindAsync(q => q.SortExpression("name companyname -age"), o => o.SearchBeforeToken(searchBeforeToken, _serializer).PageLimit(pageSize).QueryLogLevel(LogLevel.Information)); searchBeforeToken = employees.GetSearchBeforeToken(); searchAfterToken = employees.GetSearchAfterToken(); if (page == 1) diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/ReindexTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/ReindexTests.cs index 37815612..18b4e9c7 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/ReindexTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/ReindexTests.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.AsyncEx; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; @@ -12,7 +14,6 @@ using Foundatio.Repositories.Utility; using Foundatio.Utility; using Microsoft.Extensions.Logging; -using Nest; using Xunit; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -31,7 +32,7 @@ public override async ValueTask InitializeAsync() await RemoveDataAsync(false); } - [Fact(Skip = "This will only work if the mapping is manually updated.")] + [Fact] public async Task CanReindexSameIndexAsync() { var index = new EmployeeIndex(_configuration); @@ -39,37 +40,30 @@ public async Task CanReindexSameIndexAsync() await using AsyncDisposableAction _ = new(() => index.DeleteAsync()); await index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(index.Name, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(index.Name, cancellationToken: TestCancellationToken)).Exists); IEmployeeRepository repository = new EmployeeRepository(_configuration); var employee = await repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); Assert.NotNull(employee); Assert.NotNull(employee.Id); - var countResponse = await _client.CountAsync(ct: TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); - var mappingResponse = await _client.Indices.GetMappingAsync(ct: TestCancellationToken); - _logger.LogRequest(mappingResponse); - Assert.True(mappingResponse.IsValid); - Assert.NotNull(mappingResponse.GetMappingFor(index.Name)); - + // ES does not support reindexing into the same index -- verify data is preserved after the failed reindex attempt var newIndex = new EmployeeIndexWithYearsEmployed(_configuration); await newIndex.ReindexAsync(); - countResponse = await _client.CountAsync(ct: TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); - string version1Mappings = ToJson(mappingResponse.GetMappingFor()); - mappingResponse = await _client.Indices.GetMappingAsync(ct: TestCancellationToken); - _logger.LogRequest(mappingResponse); - Assert.True(mappingResponse.IsValid); - Assert.NotNull(mappingResponse.GetMappingFor()); - Assert.NotEqual(version1Mappings, ToJson(mappingResponse.GetMappingFor())); + var result = await repository.GetByIdAsync(employee.Id); + Assert.NotNull(result); + Assert.Equal(employee.Id, result.Id); } [Fact] @@ -85,20 +79,20 @@ public async Task CanResumeReindexAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); IEmployeeRepository version1Repository = new EmployeeRepository(_configuration); await version1Repository.AddAsync(EmployeeGenerator.GenerateEmployees(numberOfEmployeesToCreate), o => o.ImmediateConsistency()); - var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name), TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(version1Index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(numberOfEmployeesToCreate, countResponse.Count); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); // Throw error before second repass. await Assert.ThrowsAsync(async () => await version2Index.ReindexAsync((progress, message) => @@ -116,20 +110,25 @@ await Assert.ThrowsAsync(async () => await version2Index.R await version1Repository.AddAsync(EmployeeGenerator.Generate(ObjectId.GenerateNewId(DateTime.UtcNow.AddMinutes(1)).ToString()), o => o.ImmediateConsistency()); await version2Index.ReindexAsync(); - var aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version2Index.VersionedName, indices.First().Key); Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(numberOfEmployeesToCreate + 1, countResponse.Count); - Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); } [Fact] @@ -143,58 +142,66 @@ public async Task CanHandleReindexFailureAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); IEmployeeRepository version1Repository = new EmployeeRepository(_configuration); await version1Repository.AddAsync(EmployeeGenerator.Generate(), o => o.ImmediateConsistency()); - var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name), TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(version1Index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); //Create invalid mappings - var response = await _client.Indices.CreateAsync(version2Index.VersionedName, d => d.Map(map => map - .Dynamic(false) + var response = await _client.Indices.CreateAsync(version2Index.VersionedName, d => d.Mappings(map => map + .Dynamic(DynamicMapping.False) .Properties(p => p - .Number(f => f.Name(e => e.Id)) - )), TestCancellationToken); + .IntegerNumber(e => e.Id) + )), cancellationToken: TestCancellationToken); _logger.LogRequest(response); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await version2Index.ReindexAsync(); - await version2Index.Configuration.Client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); - - var aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.True(aliasResponse.Indices.ContainsKey(version1Index.VersionedName)); - - var indexResponse = await _client.Cat.IndicesAsync(d => d.Index(Indices.Index("employees-*")), TestCancellationToken); - Assert.NotNull(indexResponse.Records.FirstOrDefault(r => r.Index == version1Index.VersionedName)); - Assert.NotNull(indexResponse.Records.FirstOrDefault(r => r.Index == version2Index.VersionedName)); - Assert.NotNull(indexResponse.Records.FirstOrDefault(r => r.Index == $"{version2Index.VersionedName}-error")); + await version2Index.Configuration.Client.Indices.RefreshAsync(Indices.All, cancellationToken: TestCancellationToken); + + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.True(indices.ContainsKey(version1Index.VersionedName)); + + // Verify indices exist + var index1Exists = await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken); + Assert.True(index1Exists.Exists); + var index2Exists = await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken); + Assert.True(index2Exists.Exists); + var errorIndexExists = await _client.Indices.ExistsAsync($"{version2Index.VersionedName}-error", cancellationToken: TestCancellationToken); + Assert.True(errorIndexExists.Exists); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); - countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version1Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(0, countResponse.Count); - countResponse = await _client.CountAsync(d => d.Index($"{version2Index.VersionedName}-error"), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices($"{version2Index.VersionedName}-error"), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); } @@ -209,79 +216,94 @@ public async Task CanReindexVersionedIndexAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); var indexes = _client.GetIndicesPointingToAlias(version1Index.Name); Assert.Single(indexes); - var aliasResponse = await _client.Indices.GetAliasAsync(version1Index.Name, ct: TestCancellationToken); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version1Index.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(aliasResponse); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version1Index.VersionedName, indices.First().Key); IEmployeeRepository version1Repository = new EmployeeRepository(_configuration); var employee = await version1Repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); Assert.NotNull(employee); Assert.NotNull(employee.Id); - var countResponse = await _client.CountAsync(d => d.Index(version1Index.Name), TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(version1Index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); // Make sure we can write to the index still. Should go to the old index until after the reindex is complete. IEmployeeRepository version2Repository = new EmployeeRepository(_configuration); await version2Repository.AddAsync(EmployeeGenerator.Generate(), o => o.ImmediateConsistency()); - countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version1Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(2, countResponse.Count); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(0, countResponse.Count); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); // alias should still point to the old version until reindex - aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key); + aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasResponse.Aliases; +#else + indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version1Index.VersionedName, indices.First().Key); await version2Index.ReindexAsync(); - aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key); + aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasResponse.Aliases; +#else + indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version2Index.VersionedName, indices.First().Key); Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(2, countResponse.Count); - Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); employee = await version2Repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); Assert.NotNull(employee); Assert.NotNull(employee.Id); - countResponse = await _client.CountAsync(d => d.Index(version2Index.Name), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(3, countResponse.Count); } @@ -308,27 +330,27 @@ public async Task CanReindexVersionedIndexWithCorrectMappingsAsync() await version2Index.ReindexAsync(); - var existsResponse = await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - var mappingResponse = await _client.Indices.GetMappingAsync(m => m.Index(version1Index.VersionedName), TestCancellationToken); + var mappingResponse = await _client.Indices.GetMappingAsync(m => m.Indices(version1Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(mappingResponse); - Assert.True(mappingResponse.IsValid); - var mappingsV1 = mappingResponse.Indices[version1Index.VersionedName]; + Assert.True(mappingResponse.IsValidResponse); + var mappingsV1 = mappingResponse.Mappings[version1Index.VersionedName]; Assert.NotNull(mappingsV1); - existsResponse = await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); string version1Mappings = ToJson(mappingsV1); - mappingResponse = await _client.Indices.GetMappingAsync(m => m.Index(version2Index.VersionedName), TestCancellationToken); + mappingResponse = await _client.Indices.GetMappingAsync(m => m.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(mappingResponse); - Assert.True(mappingResponse.IsValid); - var mappingsV2 = mappingResponse.Indices[version2Index.VersionedName]; + Assert.True(mappingResponse.IsValidResponse); + var mappingsV2 = mappingResponse.Mappings[version2Index.VersionedName]; Assert.NotNull(mappingsV2); string version2Mappings = ToJson(mappingsV2); Assert.Equal(version1Mappings, version2Mappings); @@ -415,10 +437,15 @@ public async Task HandleFailureInReindexScriptAsync() await version22Index.ConfigureAsync(); await version22Index.ReindexAsync(); - var aliasResponse = await _client.Indices.GetAliasAsync(version1Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version1Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version1Index.VersionedName, indices.First().Key); } [Fact] @@ -432,7 +459,7 @@ public async Task CanReindexVersionedIndexWithDataInBothIndexesAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); IEmployeeRepository version1Repository = new EmployeeRepository(_configuration); var employee = await version1Repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); @@ -441,56 +468,66 @@ public async Task CanReindexVersionedIndexWithDataInBothIndexesAsync() await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); // swap the alias so we write to v1 and v2 and try to reindex. - await _client.Indices.BulkAliasAsync(x => x - .Remove(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName)) - .Add(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName)), TestCancellationToken); + await _client.Indices.UpdateAliasesAsync(x => x.Actions( + a => a.Remove(r => r.Alias(version1Index.Name).Index(version1Index.VersionedName)), + a => a.Add(ad => ad.Alias(version2Index.Name).Index(version2Index.VersionedName))), cancellationToken: TestCancellationToken); IEmployeeRepository version2Repository = new EmployeeRepository(_configuration); await version2Repository.AddAsync(EmployeeGenerator.Generate(), o => o.ImmediateConsistency()); - var countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName), TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(version1Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(1, countResponse.Count); // swap back the alias - await _client.Indices.BulkAliasAsync(x => x - .Remove(a => a.Alias(version2Index.Name).Index(version2Index.VersionedName)) - .Add(a => a.Alias(version1Index.Name).Index(version1Index.VersionedName)), TestCancellationToken); + await _client.Indices.UpdateAliasesAsync(x => x.Actions( + a => a.Remove(r => r.Alias(version2Index.Name).Index(version2Index.VersionedName)), + a => a.Add(ad => ad.Alias(version1Index.Name).Index(version1Index.VersionedName))), cancellationToken: TestCancellationToken); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); // alias should still point to the old version until reindex - var aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version1Index.VersionedName, indices.First().Key); await version2Index.ReindexAsync(); - aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key); + aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasResponse.Aliases; +#else + indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version2Index.VersionedName, indices.First().Key); Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); - await _client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + await _client.Indices.RefreshAsync(Indices.All, cancellationToken: TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(2, countResponse.Count); - Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); } [Fact] @@ -504,7 +541,7 @@ public async Task CanReindexVersionedIndexWithUpdatedDocsAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); IEmployeeRepository repository = new EmployeeRepository(_configuration); var employee = await repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); @@ -513,29 +550,36 @@ public async Task CanReindexVersionedIndexWithUpdatedDocsAsync() await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); // alias should still point to the old version until reindex - var aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(aliasResponse); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version1Index.VersionedName, indices.First().Key); var countdown = new AsyncCountdownEvent(1); var reindexTask = version2Index.ReindexAsync(async (progress, message) => { _logger.LogInformation("Reindex Progress {Progress}%: {Message}", progress, message); - if (progress == 91) + // Signal after any progress is made (reindex has started processing) + if (progress > 0 && countdown.CurrentCount > 0) { countdown.Signal(); - await Task.Delay(1000); + await Task.Delay(1000, TestCancellationToken); } }); - // Wait until the first reindex pass is done. - await countdown.WaitAsync(TestCancellationToken); + // Wait until the first reindex pass is done (with timeout to prevent hang). + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await countdown.WaitAsync(cts.Token); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await repository.AddAsync(EmployeeGenerator.Generate(createdUtc: DateTime.UtcNow)); employee.Name = "Updated"; @@ -543,23 +587,30 @@ public async Task CanReindexVersionedIndexWithUpdatedDocsAsync() // Resume after everythings been indexed. await reindexTask; - aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key); + aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + indices = aliasResponse.Aliases; +#else + indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version2Index.VersionedName, indices.First().Key); Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); - await _client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); - var countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + await _client.Indices.RefreshAsync(Indices.All, cancellationToken: TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid); + Assert.True(countResponse.IsValidResponse); Assert.Equal(2, countResponse.Count); var result = await repository.GetByIdAsync(employee.Id); + Assert.NotNull(result); + employee.Version = result.Version; // SeqNo/PrimaryTerm is not preserved across reindex Assert.Equal(ToJson(employee), ToJson(result)); - Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); } [Fact] @@ -573,7 +624,7 @@ public async Task CanReindexVersionedIndexWithDeletedDocsAsync() await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); await version1Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); IEmployeeRepository repository = new EmployeeRepository(_configuration); var employee = await repository.AddAsync(EmployeeGenerator.Default, o => o.ImmediateConsistency()); @@ -582,55 +633,70 @@ public async Task CanReindexVersionedIndexWithDeletedDocsAsync() await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); await version2Index.ConfigureAsync(); - Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, ct: TestCancellationToken)).Exists); + Assert.True((await _client.Indices.ExistsAsync(version2Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); Assert.Equal(1, await version2Index.GetCurrentVersionAsync()); // alias should still point to the old version until reindex - var aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); + var aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(aliasResponse); - Assert.True(aliasResponse.IsValid); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version1Index.VersionedName, aliasResponse.Indices.First().Key); + Assert.True(aliasResponse.IsValidResponse); +#if ELASTICSEARCH9 + var indices = aliasResponse.Aliases; +#else + var indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version1Index.VersionedName, indices.First().Key); var countdown = new AsyncCountdownEvent(1); var reindexTask = version2Index.ReindexAsync(async (progress, message) => { _logger.LogInformation("Reindex Progress {Progress}%: {Message}", progress, message); - if (progress == 91) + // Signal after any progress is made (reindex has started processing) + if (progress > 0 && countdown.CurrentCount > 0) { countdown.Signal(); - await Task.Delay(1000); + await Task.Delay(1000, TestCancellationToken); } }); - // Wait until the first reindex pass is done. - await countdown.WaitAsync(TestCancellationToken); + // Wait until the first reindex pass is done (with timeout to prevent hang). + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await countdown.WaitAsync(cts.Token); Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); await repository.RemoveAllAsync(o => o.ImmediateConsistency()); // Resume after everythings been indexed. await reindexTask; - aliasResponse = await _client.Indices.GetAliasAsync(version2Index.Name, ct: TestCancellationToken); + aliasResponse = await _client.Indices.GetAliasAsync((Indices)version2Index.Name, cancellationToken: TestCancellationToken); _logger.LogRequest(aliasResponse); - Assert.True(aliasResponse.IsValid, aliasResponse.GetErrorMessage()); - Assert.Single(aliasResponse.Indices); - Assert.Equal(version2Index.VersionedName, aliasResponse.Indices.First().Key); + Assert.True(aliasResponse.IsValidResponse, aliasResponse.GetErrorMessage()); +#if ELASTICSEARCH9 + indices = aliasResponse.Aliases; +#else + indices = aliasResponse.Values; +#endif + Assert.Single(indices); + Assert.Equal(version2Index.VersionedName, indices.First().Key); Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); - var countResponse = await _client.CountAsync(d => d.Index(version1Index.VersionedName), TestCancellationToken); + var countResponse = await _client.CountAsync(d => d.Indices(version1Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.ApiCall.HttpStatusCode == 404, countResponse.GetErrorMessage()); + Assert.Equal(404, countResponse.ApiCallDetails.HttpStatusCode); Assert.Equal(0, countResponse.Count); - countResponse = await _client.CountAsync(d => d.Index(version2Index.VersionedName), TestCancellationToken); + countResponse = await _client.CountAsync(d => d.Indices(version2Index.VersionedName), cancellationToken: TestCancellationToken); _logger.LogRequest(countResponse); - Assert.True(countResponse.IsValid, countResponse.GetErrorMessage()); + Assert.True(countResponse.IsValidResponse, countResponse.GetErrorMessage()); Assert.Equal(1, countResponse.Count); - Assert.Equal(employee, await repository.GetByIdAsync(employee.Id)); - Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, ct: TestCancellationToken)).Exists); + var reindexedEmployee = await repository.GetByIdAsync(employee.Id); + Assert.NotNull(reindexedEmployee); + employee.Version = reindexedEmployee.Version; // SeqNo/PrimaryTerm is not preserved across reindex + Assert.Equal(employee, reindexedEmployee); + Assert.False((await _client.Indices.ExistsAsync(version1Index.VersionedName, cancellationToken: TestCancellationToken)).Exists); } [Fact] @@ -653,19 +719,19 @@ public async Task CanReindexTimeSeriesIndexAsync() Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); - var aliasCountResponse = await _client.CountAsync(d => d.Index(version1Index.Name), TestCancellationToken); + var aliasCountResponse = await _client.CountAsync(d => d.Indices(version1Index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasCountResponse); - Assert.True(aliasCountResponse.IsValid); + Assert.True(aliasCountResponse.IsValidResponse); Assert.Equal(1, aliasCountResponse.Count); - var indexCountResponse = await _client.CountAsync(d => d.Index(version1Index.GetIndex(utcNow)), TestCancellationToken); + var indexCountResponse = await _client.CountAsync(d => d.Indices(version1Index.GetIndex(utcNow)), cancellationToken: TestCancellationToken); _logger.LogRequest(indexCountResponse); - Assert.True(indexCountResponse.IsValid); + Assert.True(indexCountResponse.IsValidResponse); Assert.Equal(1, indexCountResponse.Count); - indexCountResponse = await _client.CountAsync(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1)), TestCancellationToken); + indexCountResponse = await _client.CountAsync(d => d.Indices(version1Index.GetVersionedIndex(utcNow, 1)), cancellationToken: TestCancellationToken); _logger.LogRequest(indexCountResponse); - Assert.True(indexCountResponse.IsValid); + Assert.True(indexCountResponse.IsValidResponse); Assert.Equal(1, indexCountResponse.Count); await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); @@ -676,28 +742,33 @@ public async Task CanReindexTimeSeriesIndexAsync() // Make sure we write to the old index. await version2Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency()); - aliasCountResponse = await _client.CountAsync(d => d.Index(version1Index.Name), TestCancellationToken); + aliasCountResponse = await _client.CountAsync(d => d.Indices(version1Index.Name), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasCountResponse); - Assert.True(aliasCountResponse.IsValid); + Assert.True(aliasCountResponse.IsValidResponse); Assert.Equal(2, aliasCountResponse.Count); - indexCountResponse = await _client.CountAsync(d => d.Index(version1Index.GetVersionedIndex(utcNow, 1)), TestCancellationToken); + indexCountResponse = await _client.CountAsync(d => d.Indices(version1Index.GetVersionedIndex(utcNow, 1)), cancellationToken: TestCancellationToken); _logger.LogRequest(indexCountResponse); - Assert.True(indexCountResponse.IsValid); + Assert.True(indexCountResponse.IsValidResponse); Assert.Equal(2, indexCountResponse.Count); - var existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); // alias should still point to the old version until reindex - var aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + var aliasesResponse = await _client.Indices.GetAliasAsync((Indices)version1Index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 1), aliasesResponse.Indices.Single().Key); - - var aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse); +#if ELASTICSEARCH9 + var aliasIndices = aliasesResponse.Aliases; +#else + var aliasIndices = aliasesResponse.Values; +#endif + Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 1), aliasIndices.Single().Key); + + var aliases = aliasIndices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); @@ -706,26 +777,67 @@ public async Task CanReindexTimeSeriesIndexAsync() Assert.Equal(2, await version1Index.GetCurrentVersionAsync()); Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); - aliasesResponse = await _client.Indices.GetAliasAsync(version1Index.GetIndex(employee.CreatedUtc), ct: TestCancellationToken); + aliasesResponse = await _client.Indices.GetAliasAsync((Indices)version1Index.GetIndex(employee.CreatedUtc), cancellationToken: TestCancellationToken); _logger.LogRequest(aliasesResponse); - Assert.True(aliasesResponse.IsValid); - Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 2), aliasesResponse.Indices.Single().Key); - - aliases = aliasesResponse.Indices.Values.Single().Aliases.Select(s => s.Key).ToList(); + Assert.True(aliasesResponse.IsValidResponse, aliasesResponse.GetErrorMessage()); +#if ELASTICSEARCH9 + aliasIndices = aliasesResponse.Aliases; +#else + aliasIndices = aliasesResponse.Values; +#endif + Assert.Equal(version1Index.GetVersionedIndex(employee.CreatedUtc, 2), aliasIndices.Single().Key); + + aliases = aliasIndices.Values.Single().Aliases.Select(s => s.Key).ToList(); aliases.Sort(); Assert.Equal(GetExpectedEmployeeDailyAliases(version1Index, utcNow, employee.CreatedUtc), String.Join(", ", aliases)); - existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.False(existsResponse.Exists); - existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2), ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(version2Index.GetVersionedIndex(utcNow, 2), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); } + [Fact] + public async Task ReindexAsync_DailyIndexWithReindexScript_ExecutesScript() + { + // Arrange + var version1Index = new DailyEmployeeIndexWithReindexScripts(_configuration, 1); + await version1Index.DeleteAsync(); + + var version2Index = new DailyEmployeeIndexWithReindexScripts(_configuration, 2) { DiscardIndexesOnReindex = false }; + await version2Index.DeleteAsync(); + + await using AsyncDisposableAction _ = new(() => version1Index.DeleteAsync()); + await version1Index.ConfigureAsync(); + IEmployeeRepository version1Repository = new EmployeeRepository(version1Index); + + var utcNow = DateTime.UtcNow; + var employee = await version1Repository.AddAsync(EmployeeGenerator.Generate(createdUtc: utcNow), o => o.ImmediateConsistency()); + Assert.NotNull(employee); + Assert.NotNull(employee.Id); + Assert.NotEqual("daily-scripted", employee.CompanyName); + Assert.Equal(1, await version1Index.GetCurrentVersionAsync()); + + await using AsyncDisposableAction version2Scope = new(() => version2Index.DeleteAsync()); + await version2Index.ConfigureAsync(); + + // Act + await version2Index.ReindexAsync(); + + // Assert + Assert.Equal(2, await version2Index.GetCurrentVersionAsync()); + + IEmployeeRepository version2Repository = new EmployeeRepository(version2Index); + var result = await version2Repository.GetByIdAsync(employee.Id); + Assert.NotNull(result); + Assert.Equal("daily-scripted", result.CompanyName); + } + [Fact] public async Task CanReindexTimeSeriesIndexWithCorrectMappingsAsync() { @@ -749,29 +861,29 @@ public async Task CanReindexTimeSeriesIndexWithCorrectMappingsAsync() await version2Index.ReindexAsync(); - var existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1), ct: TestCancellationToken); + var existsResponse = await _client.Indices.ExistsAsync(version1Index.GetVersionedIndex(utcNow, 1), cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); string indexV1 = version1Index.GetVersionedIndex(utcNow, 1); - var mappingResponse = await _client.Indices.GetMappingAsync(m => m.Index(indexV1), TestCancellationToken); + var mappingResponse = await _client.Indices.GetMappingAsync(m => m.Indices(indexV1), cancellationToken: TestCancellationToken); _logger.LogRequest(mappingResponse); - Assert.True(mappingResponse.IsValid); - var mappingsV1 = mappingResponse.Indices[indexV1]; + Assert.True(mappingResponse.IsValidResponse); + var mappingsV1 = mappingResponse.Mappings[indexV1]; Assert.NotNull(mappingsV1); string version1Mappings = ToJson(mappingsV1); string indexV2 = version2Index.GetVersionedIndex(utcNow, 2); - existsResponse = await _client.Indices.ExistsAsync(indexV2, ct: TestCancellationToken); + existsResponse = await _client.Indices.ExistsAsync(indexV2, cancellationToken: TestCancellationToken); _logger.LogRequest(existsResponse); - Assert.True(existsResponse.ApiCall.Success); + Assert.True(existsResponse.ApiCallDetails.HasSuccessfulStatusCode); Assert.True(existsResponse.Exists); - mappingResponse = await _client.Indices.GetMappingAsync(m => m.Index(indexV2), TestCancellationToken); + mappingResponse = await _client.Indices.GetMappingAsync(m => m.Indices(indexV2), cancellationToken: TestCancellationToken); _logger.LogRequest(mappingResponse); - Assert.True(mappingResponse.IsValid); - var mappingsV2 = mappingResponse.Indices[indexV2]; + Assert.True(mappingResponse.IsValidResponse); + var mappingsV2 = mappingResponse.Mappings[indexV2]; Assert.NotNull(mappingsV2); string version2Mappings = ToJson(mappingsV2); Assert.Equal(version1Mappings, version2Mappings); @@ -839,10 +951,9 @@ public async Task RenameFieldScript_WithTopLevelField_RenamesFieldDuringReindex( await version2Index.ReindexAsync(); // Assert - var response = await _client.GetAsync>( - new DocumentPath>(employee.Id).Index(version2Index.VersionedName), - ct: TestCancellationToken); - Assert.True(response.IsValid); + var request = new GetRequest(version2Index.VersionedName, employee.Id); + var response = await _client.GetAsync>(request, cancellationToken: TestCancellationToken); + Assert.True(response.IsValidResponse); Assert.True(response.Source.TryGetValue("companyNameRenamed", out var companyNameRenamed)); Assert.Equal("TestCompany", companyNameRenamed?.ToString()); Assert.False(response.Source.ContainsKey("companyName")); @@ -874,10 +985,9 @@ public async Task RenameFieldScript_WithNestedField_RenamesNestedFieldDuringRein await version2Index.ReindexAsync(); // Assert - var response = await _client.GetAsync>( - new DocumentPath>(employee.Id).Index(version2Index.VersionedName), - ct: TestCancellationToken); - Assert.True(response.IsValid); + var request = new GetRequest(version2Index.VersionedName, employee.Id); + var response = await _client.GetAsync>(request, cancellationToken: TestCancellationToken); + Assert.True(response.IsValidResponse); Assert.True(response.Source.TryGetValue("data", out var data)); string json = ToJson(data); @@ -910,10 +1020,9 @@ public async Task RemoveFieldScript_WithTopLevelField_RemovesFieldDuringReindex( await version2Index.ReindexAsync(); // Assert - var response = await _client.GetAsync>( - new DocumentPath>(employee.Id).Index(version2Index.VersionedName), - ct: TestCancellationToken); - Assert.True(response.IsValid); + var request = new GetRequest(version2Index.VersionedName, employee.Id); + var response = await _client.GetAsync>(request, cancellationToken: TestCancellationToken); + Assert.True(response.IsValidResponse); Assert.False(response.Source.ContainsKey("companyName")); } @@ -944,10 +1053,9 @@ public async Task RemoveFieldScript_WithNestedField_RemovesNestedFieldDuringRein await version2Index.ReindexAsync(); // Assert - var response = await _client.GetAsync>( - new DocumentPath>(employee.Id).Index(version2Index.VersionedName), - ct: TestCancellationToken); - Assert.True(response.IsValid); + var request = new GetRequest(version2Index.VersionedName, employee.Id); + var response = await _client.GetAsync>(request, cancellationToken: TestCancellationToken); + Assert.True(response.IsValidResponse); Assert.True(response.Source.TryGetValue("data", out var data)); string json = ToJson(data); @@ -971,6 +1079,10 @@ private static string GetExpectedEmployeeDailyAliases(IIndex index, DateTime utc private string ToJson(object data) { - return _client.SourceSerializer.SerializeToString(data); + using var stream = new System.IO.MemoryStream(); + _client.ElasticsearchClientSettings.SourceSerializer.Serialize(data, stream); + stream.Position = 0; + using var reader = new System.IO.StreamReader(stream); + return reader.ReadToEnd(); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ChildRepository.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ChildRepository.cs index c6def0c1..4cab9369 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ChildRepository.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ChildRepository.cs @@ -1,10 +1,10 @@ using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories; @@ -21,7 +21,7 @@ public ChildRepository(MyAppElasticConfiguration elasticConfiguration) : base(el private Task OnDocumentsChanging(object sender, DocumentsChangeEventArgs args) { - foreach (var doc in args.Documents.Select(d => d.Value).Cast()) + foreach (var doc in args.Documents.Select(d => d.Value)) doc.Discriminator = JoinField.Link(doc.ParentId); return Task.CompletedTask; diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/ElasticsearchJsonNetSerializer.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/ElasticsearchJsonNetSerializer.cs deleted file mode 100644 index d3837e83..00000000 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/ElasticsearchJsonNetSerializer.cs +++ /dev/null @@ -1,24 +0,0 @@ -using Elasticsearch.Net; -using Nest; -using Nest.JsonNetSerializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; - -public class ElasticsearchJsonNetSerializer : ConnectionSettingsAwareSerializerBase -{ - public ElasticsearchJsonNetSerializer(IElasticsearchSerializer builtinSerializer, IConnectionSettingsValues connectionSettings) - : base(builtinSerializer, connectionSettings) { } - - protected override JsonSerializerSettings CreateJsonSerializerSettings() => - new JsonSerializerSettings - { - NullValueHandling = NullValueHandling.Include - }; - - protected override void ModifyContractResolver(ConnectionSettingsAwareContractResolver resolver) - { - resolver.NamingStrategy = new CamelCaseNamingStrategy(); - } -} diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyFileAccessHistoryIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyFileAccessHistoryIndex.cs index 81394a7d..1c08db8a 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyFileAccessHistoryIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyFileAccessHistoryIndex.cs @@ -1,6 +1,6 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -10,8 +10,8 @@ public DailyFileAccessHistoryIndex(IElasticConfiguration configuration) : base(c { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyLogEventIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyLogEventIndex.cs index 4f0f49b5..6f6540a3 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyLogEventIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/DailyLogEventIndex.cs @@ -1,9 +1,11 @@ using System; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Queries; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -16,15 +18,14 @@ public DailyLogEventIndex(IElasticConfiguration configuration) : base(configurat AddAlias($"{Name}-last30days", TimeSpan.FromDays(30)); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return base.ConfigureIndexMapping(map) - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Date(f => f.Name(e => e.Date)) - .Date(f => f.Name(e => e.CreatedUtc)) + .SetupDefaults() + .Keyword(e => e.CompanyId) + .Date(e => e.Date) ); } @@ -33,8 +34,8 @@ protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) builder.Register(); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeIndex.cs index bbaaec78..75d29258 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeIndex.cs @@ -1,9 +1,10 @@ using System; -using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers; using Foundatio.Parsers.ElasticQueries; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Elasticsearch.Extensions; @@ -12,8 +13,6 @@ using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Queries; using Foundatio.Serializer; using Microsoft.Extensions.Logging.Abstractions; -using Nest; - namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; public sealed class EmployeeIndex : Index @@ -26,48 +25,47 @@ public EmployeeIndex(IElasticConfiguration configuration) : base(configuration, AddCustomFieldType(new CalculatedIntegerFieldType(new ScriptService(new SystemTextJsonSerializer(), NullLogger.Instance))); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Setting("index.mapping.ignore_malformed", "true") + base.ConfigureIndex(idx.Settings(s => s + .AddOtherSetting("index.mapping.ignore_malformed", "true") .NumberOfReplicas(0) .NumberOfShards(1) .Analysis(a => a.AddSortNormalizer()))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Text(f => f.Name("_all")) - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.EmailAddress)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.EmploymentType)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields().CopyTo(c => c.Field("_all"))) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .FieldAlias(a => a.Name("aliasedage").Path(f => f.Age)) - .Scalar(f => f.DecimalAge, f => f.Name(e => e.DecimalAge)) - .Scalar(f => f.NextReview, f => f.Name(e => e.NextReview)) - .FieldAlias(a => a.Name("next").Path(f => f.NextReview)) - .GeoPoint(f => f.Name(e => e.Location)) - .FieldAlias(a => a.Name("phone").Path(f => f.PhoneNumbers.First().Number)) - .Object(f => f - .Name(u => u.PhoneNumbers.First()).Properties(mp => mp - .Text(fu => fu.Name(m => m.Number).CopyTo(c => c.Field("_all"))))) - .FieldAlias(a => a.Name("twitter").Path("data.@user_meta.twitter_id")) - .FieldAlias(a => a.Name("followers").Path("data.@user_meta.twitter_followers")) - .Object>(f => f.Name(e => e.Data).Properties(p1 => p1 - .Object(f2 => f2.Name("@user_meta").Properties(p2 => p2 - .Keyword(f3 => f3.Name("twitter_id").CopyTo(c => c.Field("_all"))) - .Number(f3 => f3.Name("twitter_followers")) - )))) - .Nested(f => f.Name(e => e.PeerReviews).Properties(p1 => p1 - .Keyword(f2 => f2.Name(p2 => p2.ReviewerEmployeeId)) - .Scalar(p3 => p3.Rating, f2 => f2.Name(p3 => p3.Rating)))) + .Text("_all") + .Keyword(e => e.EmailAddress) + .Keyword(e => e.CompanyId) + .Keyword(e => e.EmploymentType) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordAndSortFields().CopyTo("_all")) + .IntegerNumber(e => e.Age) + .FieldAlias("aliasedage", a => a.Path(e => e.Age)) + .DoubleNumber(e => e.DecimalAge) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + .GeoPoint(e => e.Location) + .FieldAlias("phone", a => a.Path("phoneNumbers.number")) + .Object(e => e.PhoneNumbers, mp => mp + .Properties(p => p.Text("number", t => t.CopyTo("_all")))) + .FieldAlias("twitter", a => a.Path("data.@user_meta.twitter_id")) + .FieldAlias("followers", a => a.Path("data.@user_meta.twitter_followers")) + .Object(e => e.Data, p1 => p1 + .Properties(p => p.Object("@user_meta", p2 => p2 + .Properties(p3 => p3 + .Keyword("twitter_id", f3 => f3.CopyTo("_all")) + .LongNumber("twitter_followers"))))) + .Nested(e => e.PeerReviews, p1 => p1 + .Properties(p => p + .Keyword("reviewerEmployeeId") + .IntegerNumber("rating"))) ); } @@ -110,38 +108,37 @@ public sealed class EmployeeIndexWithYearsEmployed : Index { public EmployeeIndexWithYearsEmployed(IElasticConfiguration configuration) : base(configuration, "employees") { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return base.ConfigureIndexMapping(map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .Scalar(f => f.YearsEmployed, f => f.Name(e => e.YearsEmployed)) - .Date(f => f.Name(e => e.LastReview)) - .Scalar(f => f.NextReview, f => f.Name(e => e.NextReview)) - .FieldAlias(a => a.Name("next").Path(f2 => f2.NextReview)) - )); + .Keyword(e => e.CompanyId) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordField()) + .IntegerNumber(e => e.Age) + .IntegerNumber(e => e.YearsEmployed) + .Date(e => e.LastReview) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + ); } } public sealed class VersionedEmployeeIndex : VersionedIndex { - private readonly Func _createIndex; - private readonly Func, TypeMappingDescriptor> _createMappings; + private readonly Action _createIndex; + private readonly Action> _createMappings; public VersionedEmployeeIndex(IElasticConfiguration configuration, int version, - Func createIndex = null, - Func, TypeMappingDescriptor> createMappings = null) : base(configuration, "employees", version) + Action createIndex = null, + Action> createMappings = null) : base(configuration, "employees", version) { _createIndex = createIndex; _createMappings = createMappings; @@ -150,20 +147,26 @@ public VersionedEmployeeIndex(IElasticConfiguration configuration, int version, AddReindexScript(22, "ctx._source.FAIL = 'should not work"); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { if (_createIndex != null) - return _createIndex(idx); + { + _createIndex(idx); + return; + } - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { if (_createMappings != null) - return _createMappings(map); + { + _createMappings(map); + return; + } - return base.ConfigureIndexMapping(map); + base.ConfigureIndexMapping(map); } } @@ -176,26 +179,63 @@ public DailyEmployeeIndex(IElasticConfiguration configuration, int version) : ba AddAlias($"{Name}-last30days", TimeSpan.FromDays(30)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return base.ConfigureIndexMapping(map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .Date(f => f.Name(e => e.LastReview)) - .Scalar(f => f.NextReview, f => f.Name(e => e.NextReview)) - .FieldAlias(a => a.Name("next").Path(f2 => f2.NextReview)) - )); + .Keyword(e => e.CompanyId) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordField()) + .IntegerNumber(e => e.Age) + .Date(e => e.LastReview) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + ); + } + + protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) + { + builder.Register(); + builder.Register(); + } +} + +public sealed class DailyEmployeeIndexWithReindexScripts : DailyIndex +{ + public DailyEmployeeIndexWithReindexScripts(IElasticConfiguration configuration, int version) : base(configuration, "daily-employees", version) + { + AddAlias($"{Name}-today", TimeSpan.FromDays(1)); + AddAlias($"{Name}-last7days", TimeSpan.FromDays(7)); + AddAlias($"{Name}-last30days", TimeSpan.FromDays(30)); + AddReindexScript(2, "ctx._source.companyName = 'daily-scripted';"); + } + + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) + { + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + } + + public override void ConfigureIndexMapping(TypeMappingDescriptor map) + { + map + .Dynamic(DynamicMapping.False) + .Properties(p => p + .SetupDefaults() + .Keyword(e => e.CompanyId) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordField()) + .IntegerNumber(e => e.Age) + .Date(e => e.LastReview) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + ); } protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) @@ -210,69 +250,73 @@ public sealed class DailyEmployeeIndexWithWrongEmployeeType : DailyIndex +public sealed class MonthlyEmployeeIndex : MonthlyIndex { - public DailyEmployeeIndexWithReindexScript(IElasticConfiguration configuration, int version) : base(configuration, "daily-employees", version) + public MonthlyEmployeeIndex(IElasticConfiguration configuration, int version) : base(configuration, "monthly-employees", version) { AddAlias($"{Name}-today", TimeSpan.FromDays(1)); AddAlias($"{Name}-last7days", TimeSpan.FromDays(7)); AddAlias($"{Name}-last30days", TimeSpan.FromDays(30)); - AddReindexScript(2, "ctx._source.companyName = 'daily-reindex-script';"); + AddAlias($"{Name}-last60days", TimeSpan.FromDays(60)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return base.ConfigureIndexMapping(map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .Date(f => f.Name(e => e.LastReview)) - .Scalar(f => f.NextReview, f => f.Name(e => e.NextReview)) - .FieldAlias(a => a.Name("next").Path(f2 => f2.NextReview)) - )); + .Keyword(e => e.CompanyId) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordField()) + .IntegerNumber(e => e.Age) + .Date(e => e.LastReview) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + ); + } + + protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) + { + builder.Register(); + builder.Register(); } } -public sealed class MonthlyEmployeeIndex : MonthlyIndex +public sealed class DailyEmployeeIndexWithReindexScript : DailyIndex { - public MonthlyEmployeeIndex(IElasticConfiguration configuration, int version) : base(configuration, "monthly-employees", version) + public DailyEmployeeIndexWithReindexScript(IElasticConfiguration configuration, int version) : base(configuration, "daily-employees", version) { AddAlias($"{Name}-today", TimeSpan.FromDays(1)); AddAlias($"{Name}-last7days", TimeSpan.FromDays(7)); AddAlias($"{Name}-last30days", TimeSpan.FromDays(30)); - AddAlias($"{Name}-last60days", TimeSpan.FromDays(60)); + AddReindexScript(2, "ctx._source.companyName = 'daily-reindex-script';"); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return base.ConfigureIndexMapping(map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .Date(f => f.Name(e => e.LastReview)) - .Scalar(f => f.NextReview, f => f.Name(e => e.NextReview)) - .FieldAlias(a => a.Name("next").Path(f => f.NextReview)) - )); + .Keyword(e => e.CompanyId) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordField()) + .IntegerNumber(e => e.Age) + .Date(e => e.LastReview) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + ); } protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) @@ -290,9 +334,9 @@ public VersionedEmployeeIndexWithFieldRename(IElasticConfiguration configuration RenameFieldScript(2, "companyName", "companyNameRenamed"); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } @@ -304,9 +348,9 @@ public VersionedEmployeeIndexWithNestedFieldRename(IElasticConfiguration configu RenameFieldScript(2, "data.oldField", "data.newField"); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } @@ -318,9 +362,9 @@ public VersionedEmployeeIndexWithFieldRemove(IElasticConfiguration configuration RemoveFieldScript(2, "companyName"); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } @@ -332,8 +376,8 @@ public VersionedEmployeeIndexWithNestedFieldRemove(IElasticConfiguration configu RemoveFieldScript(2, "data.oldField"); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithCustomFieldsIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithCustomFieldsIndex.cs index 2b024009..53d83f4e 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithCustomFieldsIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithCustomFieldsIndex.cs @@ -1,9 +1,10 @@ using System; -using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Parsers; using Foundatio.Parsers.ElasticQueries; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Elasticsearch.Extensions; @@ -12,7 +13,6 @@ using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Queries; using Foundatio.Serializer; using Microsoft.Extensions.Logging.Abstractions; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -26,46 +26,45 @@ public EmployeeWithCustomFieldsIndex(IElasticConfiguration configuration) : base AddCustomFieldType(new CalculatedIntegerFieldType(new ScriptService(new SystemTextJsonSerializer(), NullLogger.Instance))); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Setting("index.mapping.ignore_malformed", "true") + base.ConfigureIndex(idx.Settings(s => s + .AddOtherSetting("index.mapping.ignore_malformed", "true") .NumberOfReplicas(0) .NumberOfShards(1) .Analysis(a => a.AddSortNormalizer()))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Text(f => f.Name("_all")) - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.EmailAddress)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Text(f => f.Name(e => e.Name).AddKeywordAndSortFields().CopyTo(c => c.Field("_all"))) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .FieldAlias(a => a.Name("aliasedage").Path(f => f.Age)) - .Scalar(f => f.NextReview, f => f.Name(e => e.NextReview)) - .FieldAlias(a => a.Name("next").Path(f => f.NextReview)) - .GeoPoint(f => f.Name(e => e.Location)) - .FieldAlias(a => a.Name("phone").Path(f => f.PhoneNumbers.First().Number)) - .Object(f => f - .Name(u => u.PhoneNumbers.First()).Properties(mp => mp - .Text(fu => fu.Name(m => m.Number).CopyTo(c => c.Field("_all"))))) - .FieldAlias(a => a.Name("twitter").Path("data.@user_meta.twitter_id")) - .FieldAlias(a => a.Name("followers").Path("data.@user_meta.twitter_followers")) - .Object>(f => f.Name(e => e.Data).Properties(p1 => p1 - .Object(f2 => f2.Name("@user_meta").Properties(p2 => p2 - .Keyword(f3 => f3.Name("twitter_id").CopyTo(c => c.Field("_all"))) - .Number(f3 => f3.Name("twitter_followers")) - )))) - .Nested(f => f.Name(e => e.PeerReviews).Properties(p1 => p1 - .Keyword(f2 => f2.Name(p2 => p2.ReviewerEmployeeId)) - .Scalar(p3 => p3.Rating, f2 => f2.Name(p3 => p3.Rating)))) + .Text("_all") + .Keyword(e => e.EmailAddress) + .Keyword(e => e.CompanyId) + .Keyword(e => e.CompanyName) + .Text(e => e.Name, t => t.AddKeywordAndSortFields().CopyTo("_all")) + .IntegerNumber(e => e.Age) + .FieldAlias("aliasedage", a => a.Path(e => e.Age)) + .Date(e => e.NextReview) + .FieldAlias("next", a => a.Path(e => e.NextReview)) + .GeoPoint(e => e.Location) + .FieldAlias("phone", a => a.Path("phoneNumbers.number")) + .Object(e => e.PhoneNumbers, mp => mp + .Properties(p => p.Text("number", t => t.CopyTo("_all")))) + .FieldAlias("twitter", a => a.Path("data.@user_meta.twitter_id")) + .FieldAlias("followers", a => a.Path("data.@user_meta.twitter_followers")) + .Object(e => e.Data, p1 => p1 + .Properties(p => p.Object("@user_meta", p2 => p2 + .Properties(p3 => p3 + .Keyword("twitter_id", f3 => f3.CopyTo("_all")) + .LongNumber("twitter_followers"))))) + .Nested(e => e.PeerReviews, p1 => p1 + .Properties(p => p + .Keyword("reviewerEmployeeId") + .IntegerNumber("rating"))) ); } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithDateMetaDataIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithDateMetaDataIndex.cs index b6005493..4a8c4904 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithDateMetaDataIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/EmployeeWithDateMetaDataIndex.cs @@ -1,34 +1,37 @@ +using System.Text.Json; +using Elastic.Clients.Elasticsearch.Mapping; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; public sealed class EmployeeWithDateMetaDataIndex : Index { + private static string CamelCase(string name) => JsonNamingPolicy.CamelCase.ConvertName(name); + public EmployeeWithDateMetaDataIndex(IElasticConfiguration configuration) : base(configuration, "employees-metadata") { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(Elastic.Clients.Elasticsearch.IndexManagement.CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Text(f => f.Name(e => e.Name)) - .Scalar(f => f.Age, f => f.Name(e => e.Age)) - .Keyword(f => f.Name(e => e.CompanyName)) - .Keyword(f => f.Name(e => e.CompanyId)) - .Object(o => o.Name(e => e.MetaData).Properties(mp => mp - .Date(d => d.Name(m => m.DateCreatedUtc)) - .Date(d => d.Name(m => m.DateUpdatedUtc)) - )) + .Text(e => e.Name) + .IntegerNumber(e => e.Age) + .Keyword(e => e.CompanyName) + .Keyword(e => e.CompanyId) + .Object(e => e.MetaData, mp => mp + .Properties(p2 => p2 + .Date(CamelCase(nameof(DateMetaData.DateCreatedUtc))) + .Date(CamelCase(nameof(DateMetaData.DateUpdatedUtc))) + )) ); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/IdentityIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/IdentityIndex.cs index 388d97e5..19e0387f 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/IdentityIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/IdentityIndex.cs @@ -1,6 +1,8 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Repositories.Elasticsearch.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -8,17 +10,17 @@ public sealed class IdentityIndex : Index { public IdentityIndex(IElasticConfiguration configuration) : base(configuration, "identity") { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p - .Keyword(f => f.Name(e => e.Id)) + .SetupDefaults() ); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyFileAccessHistoryIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyFileAccessHistoryIndex.cs index c636a05f..aee5c9f5 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyFileAccessHistoryIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyFileAccessHistoryIndex.cs @@ -1,6 +1,6 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -10,8 +10,8 @@ public MonthlyFileAccessHistoryIndex(IElasticConfiguration configuration) : base { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyLogEventIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyLogEventIndex.cs index d5f9e50f..faefa200 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyLogEventIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/MonthlyLogEventIndex.cs @@ -1,7 +1,7 @@ using System; +using Elastic.Clients.Elasticsearch.IndexManagement; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -13,8 +13,8 @@ public MonthlyLogEventIndex(IElasticConfiguration configuration) : base(configur AddAlias($"{Name}-last3months", TimeSpan.FromDays(100)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); + base.ConfigureIndex(idx.Settings(s => s.NumberOfReplicas(0).NumberOfShards(1))); } } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/ParentChildIndex.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/ParentChildIndex.cs index 309c48fa..1bdd947d 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/ParentChildIndex.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/Indexes/ParentChildIndex.cs @@ -1,7 +1,6 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; @@ -9,19 +8,15 @@ public sealed class ParentChildIndex : VersionedIndex { public ParentChildIndex(IElasticConfiguration configuration) : base(configuration, "parentchild", 1) { } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx + base.ConfigureIndex(idx .Settings(s => s.NumberOfReplicas(0).NumberOfShards(1)) - .Map(m => m - //.RoutingField(r => r.Required()) - .AutoMap() - .AutoMap() + .Mappings(m => m .Properties(p => p .SetupDefaults() - .Join(j => j - .Name(n => n.Discriminator) - .Relations(r => r.Join()) + .Join(d => d.Discriminator, j => j + .Relations(r => r.Add("parent", new[] { "child" })) ) ))); } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/MyAppElasticConfiguration.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/MyAppElasticConfiguration.cs index 484f5c0a..d2a25abd 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/MyAppElasticConfiguration.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Configuration/MyAppElasticConfiguration.cs @@ -1,9 +1,7 @@ using System; -using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Net.NetworkInformation; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Messaging; @@ -12,14 +10,13 @@ using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration.Indexes; using Microsoft.Extensions.Logging; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; public class MyAppElasticConfiguration : ElasticConfiguration { public MyAppElasticConfiguration(IQueue workItemQueue, ICacheClient cacheClient, IMessageBus messageBus, ILoggerFactory loggerFactory) - : base(workItemQueue, cacheClient, messageBus, null, null, loggerFactory) + : base(workItemQueue, cacheClient, messageBus, loggerFactory: loggerFactory) { AddIndex(Identities = new IdentityIndex(this)); AddIndex(Employees = new EmployeeIndex(this)); @@ -34,57 +31,24 @@ public MyAppElasticConfiguration(IQueue workItemQueue, ICacheClien CustomFields = AddCustomFieldIndex(replicas: 0); } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { - string connectionString = null; - bool fiddlerIsRunning = Process.GetProcessesByName("fiddler").Length > 0; + string connectionString = Environment.GetEnvironmentVariable("ELASTICSEARCH_URL"); + bool fiddlerIsRunning = String.Equals(Environment.GetEnvironmentVariable("USE_FIDDLER_PROXY"), "true", StringComparison.OrdinalIgnoreCase); - var servers = new List(); if (!String.IsNullOrEmpty(connectionString)) { - servers.AddRange( - connectionString.Split(',') - .Select(url => new Uri(fiddlerIsRunning ? url.Replace("localhost", "ipv4.fiddler") : url))); + var servers = connectionString.Split(',') + .Select(url => new Uri(fiddlerIsRunning ? url.Replace("localhost", "ipv4.fiddler") : url)) + .ToList(); + return new StaticNodePool(servers); } - else - { - servers.Add(new Uri($"http://{(fiddlerIsRunning ? "ipv4.fiddler" : "elastic.localtest.me")}:9200")); - if (IsPortOpen(9201)) - servers.Add(new Uri($"http://{(fiddlerIsRunning ? "ipv4.fiddler" : "localhost")}:9201")); - if (IsPortOpen(9202)) - servers.Add(new Uri($"http://{(fiddlerIsRunning ? "ipv4.fiddler" : "localhost")}:9202")); - } - - return new StaticConnectionPool(servers); - } - - private static bool IsPortOpen(int port) - { - var ipGlobalProperties = IPGlobalProperties.GetIPGlobalProperties(); - var tcpConnInfoArray = ipGlobalProperties.GetActiveTcpListeners(); - - foreach (var endpoint in tcpConnInfoArray) - { - if (endpoint.Port == port) - return true; - } - - return false; - } - - protected override IElasticClient CreateElasticClient() - { - //var settings = new ConnectionSettings(CreateConnectionPool() ?? new SingleNodeConnectionPool(new Uri("http://localhost:9200")), sourceSerializer: (serializer, values) => new ElasticsearchJsonNetSerializer(serializer, values)); - var settings = new ConnectionSettings(CreateConnectionPool() ?? new SingleNodeConnectionPool(new Uri("http://localhost:9200"))); - settings.EnableApiVersioningHeader(); - ConfigureSettings(settings); - foreach (var index in Indexes) - index.ConfigureSettings(settings); - return new ElasticClient(settings); + var host = fiddlerIsRunning ? "ipv4.fiddler" : "elastic.localtest.me"; + return new SingleNodePool(new Uri($"http://{host}:9200")); } - protected override void ConfigureSettings(ConnectionSettings settings) + protected override void ConfigureSettings(ElasticsearchClientSettings settings) { // only do this in test and dev mode settings.DisableDirectStreaming().PrettyJson(); diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeRepository.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeRepository.cs index c6e829c7..0e605407 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeRepository.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeRepository.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests; @@ -117,12 +117,12 @@ public Task GetCountByCompanyAsync(string companyId) public Task GetNumberOfEmployeesWithMissingCompanyName(string company) { - return CountAsync(q => q.Company(company).ElasticFilter(!Query.Exists(f => f.Field(e => e.CompanyName)))); + return CountAsync(q => q.Company(company).ElasticFilter(new BoolQuery { MustNot = [new ExistsQuery { Field = "companyName" }] })); } public Task GetNumberOfEmployeesWithMissingName(string company) { - return CountAsync(q => q.Company(company).ElasticFilter(!Query.Exists(f => f.Field(e => e.Name)))); + return CountAsync(q => q.Company(company).ElasticFilter(new BoolQuery { MustNot = [new ExistsQuery { Field = "name" }] })); } /// diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeWithCustomFieldsRepository.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeWithCustomFieldsRepository.cs index 6b3f2faf..bcee7b78 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeWithCustomFieldsRepository.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/EmployeeWithCustomFieldsRepository.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests; @@ -104,12 +104,12 @@ public Task GetCountByCompanyAsync(string companyId) public Task GetNumberOfEmployeesWithMissingCompanyName(string company) { - return CountAsync(q => q.Company(company).ElasticFilter(!Query.Exists(f => f.Field(e => e.CompanyName)))); + return CountAsync(q => q.Company(company).ElasticFilter(new BoolQuery { MustNot = [new ExistsQuery { Field = "companyName" }] })); } public Task GetNumberOfEmployeesWithMissingName(string company) { - return CountAsync(q => q.Company(company).ElasticFilter(!Query.Exists(f => f.Field(e => e.Name)))); + return CountAsync(q => q.Company(company).ElasticFilter(new BoolQuery { MustNot = [new ExistsQuery { Field = "name" }] })); } /// diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Child.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Child.cs index 6033a219..306c4444 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Child.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Child.cs @@ -1,7 +1,8 @@ using System; +using System.Text.Json.Serialization; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Models; using Foundatio.Repositories.Utility; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; @@ -9,7 +10,10 @@ public class Child : IParentChildDocument, IHaveDates, ISupportSoftDeletes { public string Id { get; set; } public string ParentId { get; set; } - JoinField IParentChildDocument.Discriminator { get; set; } + + [JsonPropertyName("discriminator")] + public JoinField Discriminator { get; set; } + public string ChildProperty { get; set; } public DateTime CreatedUtc { get; set; } public DateTime UpdatedUtc { get; set; } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Employee.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Employee.cs index 3f72a508..537661fa 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Employee.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Employee.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using Exceptionless; using Exceptionless.DateTimeExtensions; @@ -113,22 +113,6 @@ public class EmployeeWithCustomFields : Employee, IHaveCustomFields { public IDictionary Idx { get; set; } = new Dictionary(); - //IDictionary IHaveVirtualCustomFields.GetCustomFields() { - // return Data; - //} - - //object IHaveVirtualCustomFields.GetCustomField(string name) { - // return Data[name]; - //} - - //void IHaveVirtualCustomFields.SetCustomField(string name, object value) { - // Data[name] = value; - //} - - //void IHaveVirtualCustomFields.RemoveCustomField(string name) { - // Data.Remove(name); - //} - string IHaveCustomFields.GetTenantKey() { return CompanyId; diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Parent.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Parent.cs index ac5fa30a..43da8655 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Parent.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Models/Parent.cs @@ -1,7 +1,8 @@ using System; +using System.Text.Json.Serialization; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Models; using Foundatio.Repositories.Utility; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; @@ -9,7 +10,10 @@ public class Parent : IParentChildDocument, IHaveDates, ISupportSoftDeletes { public string Id { get; set; } string IParentChildDocument.ParentId { get; set; } - JoinField IParentChildDocument.Discriminator { get; set; } + + [JsonPropertyName("discriminator")] + public JoinField Discriminator { get; set; } + public string ParentProperty { get; set; } public DateTime CreatedUtc { get; set; } public DateTime UpdatedUtc { get; set; } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ParentRepository.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ParentRepository.cs index c5e0dfd3..56a27dcc 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ParentRepository.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/ParentRepository.cs @@ -1,9 +1,9 @@ using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Configuration; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Models; -using Nest; namespace Foundatio.Repositories.Elasticsearch.Tests.Repositories; @@ -19,7 +19,7 @@ public ParentRepository(MyAppElasticConfiguration elasticConfiguration) : base(e private Task OnDocumentsChanging(object sender, DocumentsChangeEventArgs args) { - foreach (var doc in args.Documents.Select(d => d.Value).Cast()) + foreach (var doc in args.Documents.Select(d => d.Value)) doc.Discriminator = JoinField.Root(); return Task.CompletedTask; diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/AgeQuery.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/AgeQuery.cs index bbb42586..6ecb232f 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/AgeQuery.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/AgeQuery.cs @@ -1,10 +1,11 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -49,9 +50,9 @@ public class AgeQueryBuilder : IElasticQueryBuilder return Task.CompletedTask; if (ages.Count == 1) - ctx.Filter &= Query.Term(f => f.Age, ages.First()); + ctx.Filter &= new TermQuery { Field = Infer.Field(f => f.Age), Value = ages.First() }; else - ctx.Filter &= Query.Terms(d => d.Field(f => f.Age).Terms(ages)); + ctx.Filter &= new TermsQuery { Field = Infer.Field(f => f.Age), Terms = new TermsQueryField(ages.Select(a => FieldValue.Long(a)).ToArray()) }; return Task.CompletedTask; } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/CompanyQuery.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/CompanyQuery.cs index 682f920b..34f17e4e 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/CompanyQuery.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/CompanyQuery.cs @@ -1,10 +1,11 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -41,9 +42,9 @@ public class CompanyQueryBuilder : IElasticQueryBuilder return Task.CompletedTask; if (companyIds.Count == 1) - ctx.Filter &= Query.Term(f => f.CompanyId, companyIds.Single()); + ctx.Filter &= new TermQuery { Field = Infer.Field(f => f.CompanyId), Value = companyIds.Single() }; else - ctx.Filter &= Query.Terms(d => d.Field(f => f.CompanyId).Terms(companyIds)); + ctx.Filter &= new TermsQuery { Field = Infer.Field(f => f.CompanyId), Terms = new TermsQueryField(companyIds.Select(c => FieldValue.String(c)).ToArray()) }; return Task.CompletedTask; } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/EmailAddressQuery.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/EmailAddressQuery.cs index 19389dd6..2b560f8b 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/EmailAddressQuery.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/Repositories/Queries/EmailAddressQuery.cs @@ -1,9 +1,10 @@ using System; using System.Threading.Tasks; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Foundatio.Repositories { @@ -39,7 +40,7 @@ public class EmailAddressQueryBuilder : IElasticQueryBuilder if (String.IsNullOrEmpty(emailAddress)) return Task.CompletedTask; - ctx.Filter &= Query.Term(f => f.EmailAddress, emailAddress); + ctx.Filter &= new TermQuery { Field = Infer.Field(f => f.EmailAddress), Value = emailAddress }; return Task.CompletedTask; } diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/RepositoryTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/RepositoryTests.cs index 9ce21568..5e90bd13 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/RepositoryTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/RepositoryTests.cs @@ -1198,19 +1198,19 @@ public async Task ConcurrentJsonPatchAsync() var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.Cache(false)); string employeeId = employee.Id; - _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (i, ct) => + _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (iteration, ct) => { var e = await _employeeRepository.GetByIdAsync(employeeId, o => o.Cache(false)); resetEvent.Set(); - e.CompanyName = "Company " + i; + e.CompanyName = $"Company {iteration}"; try { await _employeeRepository.SaveAsync(e, o => o.Cache(false)); - _logger.LogInformation("Set company {Iteration}", i); + _logger.LogInformation("Set company {Iteration}", iteration); } catch (VersionConflictDocumentException) { - _logger.LogInformation("Got version conflict {Iteration}", i); + _logger.LogInformation("Got version conflict {Iteration}", iteration); } }); @@ -1253,19 +1253,19 @@ public async Task ConcurrentJsonPatchAllAsync() var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.Cache(false)); string employeeId = employee.Id; - _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (i, ct) => + _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (iteration, ct) => { var e = await _employeeRepository.GetByIdAsync(employeeId, o => o.Cache(false)); resetEvent.Set(); - e.CompanyName = "Company " + i; + e.CompanyName = $"Company {iteration}"; try { await _employeeRepository.SaveAsync(e, o => o.Cache(false)); - _logger.LogInformation("Set company {Iteration}", i); + _logger.LogInformation("Set company {Iteration}", iteration); } catch (VersionConflictDocumentException) { - _logger.LogInformation("Got version conflict {Iteration}", i); + _logger.LogInformation("Got version conflict {Iteration}", iteration); } }); @@ -1312,6 +1312,28 @@ public async Task PartialPatchAsync() Assert.Equal("1:1", employee.Version); } + /// + /// Documents known limitation: PartialPatch cannot set fields to null because the ES client's + /// SourceSerializer uses JsonIgnoreCondition.WhenWritingNull (elastic/elasticsearch-net#8763). + /// Use ScriptPatch or JsonPatch to set fields to null instead. + /// + [Fact] + public async Task PartialPatchAsync_WithNullField_RetainsOriginalValue() + { + // Arrange + var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Generate(companyName: "OriginalCompany")); + Assert.Equal("OriginalCompany", employee.CompanyName); + + // Act + await _employeeRepository.PatchAsync(employee.Id, new PartialPatch(new { companyName = (string)null })); + + // Assert + employee = await _employeeRepository.GetByIdAsync(employee.Id); + // TODO: This should be null once elastic/elasticsearch-net#8763 is fixed. + // Currently the null value is silently dropped, so the field retains its original value. + Assert.Equal("OriginalCompany", employee.CompanyName); + } + [Fact] public async Task ScriptPatchAsync() { @@ -1332,19 +1354,19 @@ public async Task ConcurrentScriptPatchAsync() var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.Cache(false)); string employeeId = employee.Id; - _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (i, ct) => + _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (iteration, ct) => { var e = await _employeeRepository.GetByIdAsync(employeeId, o => o.Cache(false)); resetEvent.Set(); - e.CompanyName = "Company " + i; + e.CompanyName = $"Company {iteration}"; try { await _employeeRepository.SaveAsync(e, o => o.Cache(false)); - _logger.LogInformation("Set company {Iteration}", i); + _logger.LogInformation("Set company {Iteration}", iteration); } catch (VersionConflictDocumentException) { - _logger.LogInformation("Got version conflict {Iteration}", i); + _logger.LogInformation("Got version conflict {Iteration}", iteration); } }); @@ -1409,19 +1431,19 @@ public async Task ConcurrentActionPatchAsync() var employee = await _employeeRepository.AddAsync(EmployeeGenerator.Default, o => o.Cache(false)); string employeeId = employee.Id; - _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (i, ct) => + _ = Parallel.ForEachAsync(Enumerable.Range(1, 100), new ParallelOptions { MaxDegreeOfParallelism = 2, CancellationToken = cts.Token }, async (iteration, ct) => { var e = await _employeeRepository.GetByIdAsync(employeeId, o => o.Cache(false)); resetEvent.Set(); - e.CompanyName = "Company " + i; + e.CompanyName = $"Company {iteration}"; try { await _employeeRepository.SaveAsync(e, o => o.Cache(false)); - _logger.LogInformation("Set company {Iteration}", i); + _logger.LogInformation("Set company {Iteration}", iteration); } catch (VersionConflictDocumentException) { - _logger.LogInformation("Got version conflict {Iteration}", i); + _logger.LogInformation("Got version conflict {Iteration}", iteration); } }); @@ -1538,7 +1560,7 @@ public async Task PatchAllBulkAsync() await _dailyRepository.AddAsync(LogEventGenerator.GenerateLogs(BATCH_SIZE)); added += BATCH_SIZE; } while (added < COUNT); - await _client.Indices.RefreshAsync(_configuration.DailyLogEvents.Name, ct: TestCancellationToken); + await _client.Indices.RefreshAsync(_configuration.DailyLogEvents.Name, cancellationToken: TestCancellationToken); Log.SetLogLevel(LogLevel.Trace); Assert.Equal(COUNT, await _dailyRepository.IncrementValueAsync(Array.Empty())); @@ -1556,7 +1578,7 @@ public async Task PatchAllBulkConcurrentlyAsync() await _dailyRepository.AddAsync(LogEventGenerator.GenerateLogs(BATCH_SIZE)); added += BATCH_SIZE; } while (added < COUNT); - await _client.Indices.RefreshAsync(_configuration.DailyLogEvents.Name, ct: TestCancellationToken); + await _client.Indices.RefreshAsync(_configuration.DailyLogEvents.Name, cancellationToken: TestCancellationToken); Log.SetLogLevel(LogLevel.Trace); var tasks = Enumerable.Range(1, 6).Select(async i => @@ -1619,11 +1641,12 @@ public async Task RemoveWithTimeSeriesAsync() Assert.Equal(0, await _dailyRepository.CountAsync()); } - [Fact(Skip = "We need to look into how we want to handle this.")] + [Fact] public async Task RemoveWithOutOfSyncIndexAsync() { var utcNow = DateTime.UtcNow; - var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId().ToString(), createdUtc: utcNow.AddDays(-1)), o => o.ImmediateConsistency()); + var yesterday = utcNow.AddDays(-1); + var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId(yesterday).ToString(), createdUtc: yesterday), o => o.ImmediateConsistency()); Assert.NotNull(yesterdayLog); Assert.NotNull(yesterdayLog.Id); @@ -1631,7 +1654,7 @@ public async Task RemoveWithOutOfSyncIndexAsync() await _dailyRepository.RemoveAsync(yesterdayLog, o => o.ImmediateConsistency()); - Assert.Equal(1, await _dailyRepository.CountAsync()); + Assert.Equal(0, await _dailyRepository.CountAsync()); } [Fact] @@ -1739,11 +1762,12 @@ public async Task RemoveCollectionWithCachingAsync() Assert.Equal(0, await _identityRepository.CountAsync()); } - [Fact(Skip = "We need to look into how we want to handle this.")] + [Fact] public async Task RemoveCollectionWithOutOfSyncIndexAsync() { var utcNow = DateTime.UtcNow; - var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId().ToString(), createdUtc: utcNow.AddDays(-1)), o => o.ImmediateConsistency()); + var yesterday = utcNow.AddDays(-1); + var yesterdayLog = await _dailyRepository.AddAsync(LogEventGenerator.Generate(ObjectId.GenerateNewId(yesterday).ToString(), createdUtc: yesterday), o => o.ImmediateConsistency()); Assert.NotNull(yesterdayLog); Assert.NotNull(yesterdayLog.Id); @@ -1751,7 +1775,7 @@ public async Task RemoveCollectionWithOutOfSyncIndexAsync() await _dailyRepository.RemoveAsync(new List { yesterdayLog }, o => o.ImmediateConsistency()); - Assert.Equal(1, await _dailyRepository.CountAsync()); + Assert.Equal(0, await _dailyRepository.CountAsync()); } [Fact] diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/Utility/SerializerTestHelper.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/Utility/SerializerTestHelper.cs deleted file mode 100644 index d0b5579c..00000000 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/Utility/SerializerTestHelper.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Foundatio.Serializer; - -namespace Foundatio.Repositories.Elasticsearch.Tests.Utility; - -public static class SerializerTestHelper -{ - public static ITextSerializer[] GetTextSerializers() => - [ - new SystemTextJsonSerializer(), - new JsonNetSerializer() - ]; -} diff --git a/tests/Foundatio.Repositories.Elasticsearch.Tests/VersionedTests.cs b/tests/Foundatio.Repositories.Elasticsearch.Tests/VersionedTests.cs index 73e516fc..9cd46277 100644 --- a/tests/Foundatio.Repositories.Elasticsearch.Tests/VersionedTests.cs +++ b/tests/Foundatio.Repositories.Elasticsearch.Tests/VersionedTests.cs @@ -3,14 +3,13 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Elasticsearch.Tests.Repositories.Models; using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Utility; -using Nest; using Xunit; namespace Foundatio.Repositories.Elasticsearch.Tests; @@ -121,13 +120,13 @@ public async Task SaveAsync() var request = new UpdateRequest(_configuration.Employees.Name, employee.Id) { - Script = new InlineScript("ctx._source.version = '112:2'"), + Script = new Script { Source = "ctx._source.version = '112:2'" }, Refresh = Refresh.True }; var response = await _client.UpdateAsync(request, TestCancellationToken); _logger.LogRequest(response); - Assert.True(response.IsValid); + Assert.True(response.IsValidResponse); employee = await _employeeRepository.GetByIdAsync(employee.Id); Assert.Equal("1:2", employee.Version); @@ -306,7 +305,7 @@ public async Task CanUseSnapshotPagingAsync() var employees = EmployeeGenerator.GenerateEmployees(NUMBER_OF_EMPLOYEES, companyId: "1"); await _employeeRepository.AddAsync(employees); - await _client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); + await _client.Indices.RefreshAsync(Indices.All, cancellationToken: TestCancellationToken); Assert.Equal(NUMBER_OF_EMPLOYEES, await _employeeRepository.CountAsync()); @@ -342,7 +341,7 @@ public async Task CanUseSnapshotWithScrollIdAsync() var employees = EmployeeGenerator.GenerateEmployees(NUMBER_OF_EMPLOYEES, companyId: "1"); await _employeeRepository.AddAsync(employees); - await _client.Indices.RefreshAsync(Indices.All, ct: TestCancellationToken); + await _client.Indices.RefreshAsync(Indices.All, cancellationToken: TestCancellationToken); Assert.Equal(NUMBER_OF_EMPLOYEES, await _employeeRepository.CountAsync()); diff --git a/tests/Foundatio.Repositories.Tests/JsonPatch/JsonPatchTests.cs b/tests/Foundatio.Repositories.Tests/JsonPatch/JsonPatchTests.cs index 89b3e71a..c96497ba 100644 --- a/tests/Foundatio.Repositories.Tests/JsonPatch/JsonPatchTests.cs +++ b/tests/Foundatio.Repositories.Tests/JsonPatch/JsonPatchTests.cs @@ -1,12 +1,17 @@ -using System; +using System; using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; using Foundatio.Repositories.Utility; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Xunit; namespace Foundatio.Repositories.Tests.JsonPatch; +/// +/// Tests for JSON Patch (RFC 6902) operations. +/// Converted from Newtonsoft.Json (JToken) to System.Text.Json (JsonNode) to align with +/// Elastic.Clients.Elasticsearch which exclusively uses System.Text.Json for serialization. +/// public class JsonPatchTests { [Fact] @@ -17,12 +22,12 @@ public void Add_an_array_element() var patchDocument = new PatchDocument(); string pointer = "/books/-"; - patchDocument.AddOperation(new AddOperation { Path = pointer, Value = new JObject(new[] { new JProperty("author", "James Brown") }) }); + patchDocument.AddOperation(new AddOperation { Path = pointer, Value = JsonNode.Parse(@"{ ""author"": ""James Brown"" }") }); var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); - var list = sample["books"] as JArray; + var list = sample["books"] as JsonArray; Assert.Equal(3, list.Count); } @@ -35,79 +40,30 @@ public void Add_an_array_element_to_non_existent_property() var patchDocument = new PatchDocument(); string pointer = "/someobject/somearray/-"; - patchDocument.AddOperation(new AddOperation { Path = pointer, Value = new JObject(new[] { new JProperty("author", "James Brown") }) }); + patchDocument.AddOperation(new AddOperation { Path = pointer, Value = JsonNode.Parse(@"{ ""author"": ""James Brown"" }") }); var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); - var list = sample["someobject"]["somearray"] as JArray; + var list = sample["someobject"]["somearray"] as JsonArray; Assert.Single(list); } [Fact] - public void Remove_array_item_by_matching() - { - var sample = JToken.Parse(@"{ - 'books': [ - { - 'title' : 'The Great Gatsby', - 'author' : 'F. Scott Fitzgerald' - }, - { - 'title' : 'The Grapes of Wrath', - 'author' : 'John Steinbeck' - }, - { - 'title' : 'Some Other Title', - 'author' : 'John Steinbeck' - } - ] -}"); - - var patchDocument = new PatchDocument(); - string pointer = "$.books[?(@.author == 'John Steinbeck')]"; - - patchDocument.AddOperation(new RemoveOperation { Path = pointer }); - - new JsonPatcher().Patch(ref sample, patchDocument); - - var list = sample["books"] as JArray; - - Assert.Single(list); - } - - [Fact] - public void Remove_array_item_by_value() - { - var sample = JToken.Parse("{ 'tags': [ 'tag1', 'tag2', 'tag3' ] }"); - - var patchDocument = new PatchDocument(); - string pointer = "$.tags[?(@ == 'tag2')]"; - - patchDocument.AddOperation(new RemoveOperation { Path = pointer }); - - new JsonPatcher().Patch(ref sample, patchDocument); - - var list = sample["tags"] as JArray; - - Assert.Equal(2, list.Count); - } - - [Fact] - public void Add_an_existing_member_property() // Why isn't this replace? + public void Add_an_existing_member_property() { var sample = GetSample2(); var patchDocument = new PatchDocument(); string pointer = "/books/0/title"; - patchDocument.AddOperation(new AddOperation { Path = pointer, Value = "Little Red Riding Hood" }); + patchDocument.AddOperation(new AddOperation { Path = pointer, Value = JsonValue.Create("Little Red Riding Hood") }); var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); - string result = sample.SelectPatchToken(pointer).Value(); + string result = sample.SelectPatchToken(pointer)?.GetValue(); Assert.Equal("Little Red Riding Hood", result); } @@ -119,12 +75,12 @@ public void Add_an_non_existing_member_property() var patchDocument = new PatchDocument(); string pointer = "/books/0/SBN"; - patchDocument.AddOperation(new AddOperation { Path = pointer, Value = "213324234343" }); + patchDocument.AddOperation(new AddOperation { Path = pointer, Value = JsonValue.Create("213324234343") }); var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); - string result = sample.SelectPatchToken(pointer).Value(); + string result = sample.SelectPatchToken(pointer)?.GetValue(); Assert.Equal("213324234343", result); } @@ -143,7 +99,7 @@ public void Copy_array_element() patcher.Patch(ref sample, patchDocument); var result = sample.SelectPatchToken("/books/2"); - Assert.IsType(result); + Assert.IsType(result); } [Fact] @@ -155,14 +111,14 @@ public void Copy_property() string frompointer = "/books/0/ISBN"; string topointer = "/books/1/ISBN"; - patchDocument.AddOperation(new AddOperation { Path = frompointer, Value = new JValue("21123123") }); + patchDocument.AddOperation(new AddOperation { Path = frompointer, Value = JsonValue.Create("21123123") }); patchDocument.AddOperation(new CopyOperation { FromPath = frompointer, Path = topointer }); var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); var result = sample.SelectPatchToken("/books/1/ISBN"); - Assert.Equal("21123123", result); + Assert.Equal("21123123", result?.GetValue()); } [Fact] @@ -179,7 +135,7 @@ public void Move_property() var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); - string result = sample.SelectPatchToken(topointer).Value(); + string result = sample.SelectPatchToken(topointer)?.GetValue(); Assert.Equal("F. Scott Fitzgerald", result); } @@ -198,33 +154,33 @@ public void Move_array_element() patcher.Patch(ref sample, patchDocument); var result = sample.SelectPatchToken(topointer); - Assert.IsType(result); + Assert.IsType(result); } [Fact] public void CreateEmptyPatch() { var sample = GetSample2(); - string sampletext = sample.ToString(); + string sampletext = sample.ToJsonString(); var patchDocument = new PatchDocument(); new JsonPatcher().Patch(ref sample, patchDocument); - Assert.Equal(sampletext, sample.ToString()); + Assert.Equal(sampletext, sample.ToJsonString()); } [Fact] public void TestExample1() { - var targetDoc = JToken.Parse("{ 'foo': 'bar'}"); + var targetDoc = JsonNode.Parse(@"{ ""foo"": ""bar""}"); var patchDoc = PatchDocument.Parse(@"[ - { 'op': 'add', 'path': '/baz', 'value': 'qux' } + { ""op"": ""add"", ""path"": ""/baz"", ""value"": ""qux"" } ]"); new JsonPatcher().Patch(ref targetDoc, patchDoc); - Assert.True(JToken.DeepEquals(JToken.Parse(@"{ - 'foo': 'bar', - 'baz': 'qux' + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(@"{ + ""foo"": ""bar"", + ""baz"": ""qux"" }"), targetDoc)); } @@ -232,25 +188,26 @@ public void TestExample1() public void SerializePatchDocument() { var patchDoc = new PatchDocument( - new TestOperation { Path = "/a/b/c", Value = new JValue("foo") }, + new TestOperation { Path = "/a/b/c", Value = JsonValue.Create("foo") }, new RemoveOperation { Path = "/a/b/c" }, - new AddOperation { Path = "/a/b/c", Value = new JArray(new JValue("foo"), new JValue("bar")) }, - new ReplaceOperation { Path = "/a/b/c", Value = new JValue(42) }, + new AddOperation { Path = "/a/b/c", Value = new JsonArray(JsonValue.Create("foo"), JsonValue.Create("bar")) }, + new ReplaceOperation { Path = "/a/b/c", Value = JsonValue.Create(42) }, new MoveOperation { FromPath = "/a/b/c", Path = "/a/b/d" }, new CopyOperation { FromPath = "/a/b/d", Path = "/a/b/e" }); - string json = JsonConvert.SerializeObject(patchDoc); - var roundTripped = JsonConvert.DeserializeObject(json); - string roundTrippedJson = JsonConvert.SerializeObject(roundTripped); + string json = JsonSerializer.Serialize(patchDoc); + var roundTripped = JsonSerializer.Deserialize(json); + string roundTrippedJson = JsonSerializer.Serialize(roundTripped); Assert.Equal(json, roundTrippedJson); - var outputstream = patchDoc.ToStream(); - string output = new StreamReader(outputstream).ReadToEnd(); + using var outputstream = patchDoc.ToStream(); + using var streamReader = new StreamReader(outputstream); + string output = streamReader.ReadToEnd(); - var jOutput = JToken.Parse(output); + var jOutput = JsonNode.Parse(output); Assert.Equal(@"[{""op"":""test"",""path"":""/a/b/c"",""value"":""foo""},{""op"":""remove"",""path"":""/a/b/c""},{""op"":""add"",""path"":""/a/b/c"",""value"":[""foo"",""bar""]},{""op"":""replace"",""path"":""/a/b/c"",""value"":42},{""op"":""move"",""path"":""/a/b/d"",""from"":""/a/b/c""},{""op"":""copy"",""path"":""/a/b/e"",""from"":""/a/b/d""}]", - jOutput.ToString(Formatting.None)); + jOutput.ToJsonString()); } [Fact] @@ -287,14 +244,14 @@ public void Remove_an_array_element() [Fact] public void Remove_an_array_element_with_numbered_custom_fields() { - var sample = JToken.Parse(@"{ - 'data': { - '2017PropertyOne' : '2017 property one value', - '2017PropertyTwo' : '2017 property two value', - '2017Properties' : ['First value from 2017','Second value from 2017'], - '2018PropertyOne' : '2018 property value', - '2018PropertyTwo' : '2018 property two value', - '2018Properties' : ['First value from 2018','Second value from 2018'] + var sample = JsonNode.Parse(@"{ + ""data"": { + ""2017PropertyOne"" : ""2017 property one value"", + ""2017PropertyTwo"" : ""2017 property two value"", + ""2017Properties"" : [""First value from 2017"",""Second value from 2017""], + ""2018PropertyOne"" : ""2018 property value"", + ""2018PropertyTwo"" : ""2018 property two value"", + ""2018Properties"" : [""First value from 2018"",""Second value from 2018""] } }"); @@ -319,59 +276,59 @@ public void Replace_a_property_value_with_a_new_value() var patchDocument = new PatchDocument(); string pointer = "/books/0/author"; - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = "Bob Brown" }); + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonValue.Create("Bob Brown") }); new JsonPatcher().Patch(ref sample, patchDocument); - Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer).Value()); + Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer)?.GetValue()); } [Fact] public void Replace_non_existant_property() { - var sample = JToken.Parse(@"{ ""data"": {} }"); + var sample = JsonNode.Parse(@"{ ""data"": {} }"); var patchDocument = new PatchDocument(); string pointer = "/data/author"; - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = "Bob Brown" }); + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonValue.Create("Bob Brown") }); new JsonPatcher().Patch(ref sample, patchDocument); - Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer).Value()); + Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer)?.GetValue()); - sample = JToken.Parse("{}"); + sample = JsonNode.Parse("{}"); patchDocument = new PatchDocument(); pointer = "/data/author"; - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = "Bob Brown" }); + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonValue.Create("Bob Brown") }); new JsonPatcher().Patch(ref sample, patchDocument); - Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer).Value()); + Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer)?.GetValue()); - sample = JToken.Parse("{}"); + sample = JsonNode.Parse("{}"); patchDocument = new PatchDocument(); pointer = "/"; - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = "Bob Brown" }); + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonValue.Create("Bob Brown") }); new JsonPatcher().Patch(ref sample, patchDocument); - Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer).Value()); + Assert.Equal("Bob Brown", sample.SelectPatchToken(pointer)?.GetValue()); - sample = JToken.Parse("{}"); + sample = JsonNode.Parse("{}"); patchDocument = new PatchDocument(); pointer = "/hey/now/0/you"; - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = "Bob Brown" }); + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonValue.Create("Bob Brown") }); new JsonPatcher().Patch(ref sample, patchDocument); - Assert.Equal("{}", sample.ToString(Formatting.None)); + Assert.Equal("{}", sample.ToJsonString()); } [Fact] @@ -382,46 +339,12 @@ public void Replace_a_property_value_with_an_object() var patchDocument = new PatchDocument(); string pointer = "/books/0/author"; - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = new JObject(new[] { new JProperty("hello", "world") }) }); + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonNode.Parse(@"{ ""hello"": ""world"" }") }); new JsonPatcher().Patch(ref sample, patchDocument); string newPointer = "/books/0/author/hello"; - Assert.Equal("world", sample.SelectPatchToken(newPointer).Value()); - } - - [Fact] - public void Replace_multiple_property_values_with_jsonpath() - { - var sample = JToken.Parse(@"{ - 'books': [ - { - 'title' : 'The Great Gatsby', - 'author' : 'F. Scott Fitzgerald' - }, - { - 'title' : 'The Grapes of Wrath', - 'author' : 'John Steinbeck' - }, - { - 'title' : 'Some Other Title', - 'author' : 'John Steinbeck' - } - ] -}"); - - var patchDocument = new PatchDocument(); - string pointer = "$.books[?(@.author == 'John Steinbeck')].author"; - - patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = "Eric" }); - - new JsonPatcher().Patch(ref sample, patchDocument); - - string newPointer = "/books/1/author"; - Assert.Equal("Eric", sample.SelectPatchToken(newPointer).Value()); - - newPointer = "/books/2/author"; - Assert.Equal("Eric", sample.SelectPatchToken(newPointer).Value()); + Assert.Equal("world", sample.SelectPatchToken(newPointer)?.GetValue()); } [Fact] @@ -429,12 +352,12 @@ public void SyncValuesWithRemovesAndReplaces() { const string operations = "[{\"op\":\"remove\",\"path\":\"/data/Address/full_address\"},{\"op\":\"remove\",\"path\":\"/data/Address/longitude\"},{\"op\":\"remove\",\"path\":\"/data/Address/latitude\"},{\"op\":\"remove\",\"path\":\"/data/Address/geo_locality\"},{\"op\":\"remove\",\"path\":\"/data/Address/geo_level2\"},{\"op\":\"remove\",\"path\":\"/data/Address/geo_level1\"},{\"op\":\"remove\",\"path\":\"/data/Address/geo_country\"},{\"op\":\"remove\",\"path\":\"/data/Address/normalized_geo_hash\"},{\"op\":\"remove\",\"path\":\"/data/Address/geo_hash\"},{\"op\":\"remove\",\"path\":\"/data/Address/geo\"},{\"op\":\"replace\",\"path\":\"/data/Address/country\",\"value\":\"US\"},{\"op\":\"replace\",\"path\":\"/data/Address/postal_code\",\"value\":\"54173\"},{\"op\":\"replace\",\"path\":\"/data/Address/state\",\"value\":\"Wi\"},{\"op\":\"replace\",\"path\":\"/data/Address/city\",\"value\":\"Suamico\"},{\"op\":\"remove\",\"path\":\"/data/Address/address2\"},{\"op\":\"replace\",\"path\":\"/data/Address/address1\",\"value\":\"100 Main Street\"}]"; - var patchDocument = JsonConvert.DeserializeObject(operations); - var token = JToken.Parse("{ \"data\": { \"Address\": { \"address1\": null, \"address2\": null, \"city\": \"e\", \"state\": null, \"postal_code\": null, \"country\": null, \"geo\": null, \"geo_hash\": null, \"normalized_geo_hash\": null, \"geo_country\": null, \"geo_level1\": null, \"geo_level2\": null, \"geo_locality\": null, \"latitude\": null, \"longitude\": null, \"full_address\": null } } }"); + var patchDocument = JsonSerializer.Deserialize(operations); + var token = JsonNode.Parse("{ \"data\": { \"Address\": { \"address1\": null, \"address2\": null, \"city\": \"e\", \"state\": null, \"postal_code\": null, \"country\": null, \"geo\": null, \"geo_hash\": null, \"normalized_geo_hash\": null, \"geo_country\": null, \"geo_level1\": null, \"geo_level2\": null, \"geo_locality\": null, \"latitude\": null, \"longitude\": null, \"full_address\": null } } }"); new JsonPatcher().Patch(ref token, patchDocument); - Assert.Equal("{\"data\":{\"Address\":{\"address1\":\"100 Main Street\",\"city\":\"Suamico\",\"state\":\"Wi\",\"postal_code\":\"54173\",\"country\":\"US\"}}}", token.ToString(Formatting.None)); + Assert.Equal("{\"data\":{\"Address\":{\"address1\":\"100 Main Street\",\"city\":\"Suamico\",\"state\":\"Wi\",\"postal_code\":\"54173\",\"country\":\"US\"}}}", token.ToJsonString()); } [Fact] @@ -445,7 +368,7 @@ public void Test_a_value() var patchDocument = new PatchDocument(); string pointer = "/books/0/author"; - patchDocument.AddOperation(new TestOperation { Path = pointer, Value = new JValue("Billy Burton") }); + patchDocument.AddOperation(new TestOperation { Path = pointer, Value = JsonValue.Create("Billy Burton") }); Assert.Throws(() => { @@ -457,32 +380,132 @@ public void Test_a_value() [Fact] public void Can_replace_existing_boolean() { - var sample = JToken.FromObject(new MyConfigClass { RequiresConfiguration = true }); + var sample = JsonSerializer.SerializeToNode(new MyConfigClass { RequiresConfiguration = true }); var patchDocument = new PatchDocument(); - patchDocument.AddOperation(new ReplaceOperation { Path = "/RequiresConfiguration", Value = new JValue(false) }); + patchDocument.AddOperation(new ReplaceOperation { Path = "/RequiresConfiguration", Value = JsonValue.Create(false) }); var patcher = new JsonPatcher(); patcher.Patch(ref sample, patchDocument); - Assert.False(sample.ToObject().RequiresConfiguration); + Assert.False(sample.Deserialize().RequiresConfiguration); } - public static JToken GetSample2() + public static JsonNode GetSample2() { - return JToken.Parse(@"{ - 'books': [ + return JsonNode.Parse(@"{ + ""books"": [ { - 'title' : 'The Great Gatsby', - 'author' : 'F. Scott Fitzgerald' + ""title"" : ""The Great Gatsby"", + ""author"" : ""F. Scott Fitzgerald"" }, { - 'title' : 'The Grapes of Wrath', - 'author' : 'John Steinbeck' + ""title"" : ""The Grapes of Wrath"", + ""author"" : ""John Steinbeck"" } ] }"); } + + [Fact] + public void Remove_WithJsonPathFilter_RemovesMatchingArrayItems() + { + // Arrange + var sample = JsonNode.Parse(@"{ + ""books"": [ + { + ""title"" : ""The Great Gatsby"", + ""author"" : ""F. Scott Fitzgerald"" + }, + { + ""title"" : ""The Grapes of Wrath"", + ""author"" : ""John Steinbeck"" + }, + { + ""title"" : ""Some Other Title"", + ""author"" : ""John Steinbeck"" + } + ] +}"); + + var patchDocument = new PatchDocument(); + string pointer = "$.books[?(@.author == 'John Steinbeck')]"; + patchDocument.AddOperation(new RemoveOperation { Path = pointer }); + + // Act + new JsonPatcher().Patch(ref sample, patchDocument); + + // Assert + var list = sample["books"] as System.Text.Json.Nodes.JsonArray; + Assert.Single(list); + } + + [Fact] + public void Remove_WithJsonPathValueFilter_RemovesMatchingItem() + { + // Arrange + var sample = JsonNode.Parse(@"{ ""tags"": [ ""tag1"", ""tag2"", ""tag3"" ] }"); + + var patchDocument = new PatchDocument(); + string pointer = "$.tags[?(@ == 'tag2')]"; + patchDocument.AddOperation(new RemoveOperation { Path = pointer }); + + // Act + new JsonPatcher().Patch(ref sample, patchDocument); + + // Assert + var list = sample["tags"] as System.Text.Json.Nodes.JsonArray; + Assert.NotNull(list); + Assert.Equal(2, list.Count); + } + + [Fact] + public void Replace_WithJsonPathFilter_ReplacesMatchingProperties() + { + // Arrange + var sample = JsonNode.Parse(@"{ + ""books"": [ + { + ""title"" : ""The Great Gatsby"", + ""author"" : ""F. Scott Fitzgerald"" + }, + { + ""title"" : ""The Grapes of Wrath"", + ""author"" : ""John Steinbeck"" + }, + { + ""title"" : ""Some Other Title"", + ""author"" : ""John Steinbeck"" + } + ] +}"); + + var patchDocument = new PatchDocument(); + string pointer = "$.books[?(@.author == 'John Steinbeck')].author"; + patchDocument.AddOperation(new ReplaceOperation { Path = pointer, Value = JsonValue.Create("Eric") }); + + // Act + new JsonPatcher().Patch(ref sample, patchDocument); + + // Assert + string newPointer = "/books/1/author"; + Assert.Equal("Eric", sample.SelectPatchToken(newPointer)?.GetValue()); + + newPointer = "/books/2/author"; + Assert.Equal("Eric", sample.SelectPatchToken(newPointer)?.GetValue()); + } + + [Fact] + public void Parse_WithNonArrayJson_ThrowsJsonException() + { + Assert.Throws(() => PatchDocument.Parse(@"{ ""op"": ""add"" }")); + } + + [Fact] + public void Parse_WithJsonPrimitive_ThrowsJsonException() + { + Assert.Throws(() => PatchDocument.Parse(@"""hello""")); + } } public class MyConfigClass diff --git a/tests/Foundatio.Repositories.Tests/Serialization/DoubleSystemTextJsonConverterTests.cs b/tests/Foundatio.Repositories.Tests/Serialization/DoubleSystemTextJsonConverterTests.cs new file mode 100644 index 00000000..d3527c0f --- /dev/null +++ b/tests/Foundatio.Repositories.Tests/Serialization/DoubleSystemTextJsonConverterTests.cs @@ -0,0 +1,192 @@ +using System.Linq; +using System.Text.Json; +using Foundatio.Serializer; +using Xunit; + +namespace Foundatio.Repositories.Tests.Serialization; + +public class DoubleSystemTextJsonConverterTests +{ + private static readonly ITextSerializer _serializer = + SerializerTestHelper.GetTextSerializers().OfType().First(); + + [Fact] + public void Write_WithWholeDouble_PreservesDecimalPoint() + { + // Arrange + double value = 1.0; + + // Act + string json = _serializer.SerializeToString(value); + + // Assert + Assert.Equal("1.0", json); + } + + [Fact] + public void Write_WithZero_PreservesDecimalPoint() + { + // Arrange + double value = 0.0; + + // Act + string json = _serializer.SerializeToString(value); + + // Assert + Assert.Equal("0.0", json); + } + + [Fact] + public void Write_WithFractionalDouble_PreservesFullPrecision() + { + // Arrange + double value = 3.14; + + // Act + string json = _serializer.SerializeToString(value); + + // Assert + Assert.Equal("3.14", json); + } + + [Fact] + public void Read_WithWholeNumber_ReturnsDouble() + { + // Arrange + string json = "1"; + + // Act + double result = _serializer.Deserialize(json); + + // Assert + Assert.Equal(1.0, result); + } + + [Fact] + public void Read_WithDecimalNumber_ReturnsExactDouble() + { + // Arrange + string json = "3.14"; + + // Act + double result = _serializer.Deserialize(json); + + // Assert + Assert.Equal(3.14, result); + } + + [Fact] + public void RoundTrip_WithWholeDouble_PreservesDecimalRepresentation() + { + // Arrange + double original = 42.0; + + // Act + string json = _serializer.SerializeToString(original); + double roundTripped = _serializer.Deserialize(json); + + // Assert + Assert.Contains(".", json); + Assert.Equal(original, roundTripped); + } + + [Fact] + public void RoundTrip_WithNegativeDouble_PreservesValue() + { + // Arrange + double original = -99.5; + + // Act + string json = _serializer.SerializeToString(original); + double roundTripped = _serializer.Deserialize(json); + + // Assert + Assert.Equal(original, roundTripped); + } + + [Fact] + public void Write_WithDoubleInObject_PreservesDecimalPoint() + { + // Arrange + var obj = new { Value = 1.0 }; + + // Act + string json = _serializer.SerializeToString(obj); + + // Assert + Assert.Contains("1.0", json); + } + + [Fact] + public void Write_WithNaN_WritesZero() + { + string json = _serializer.SerializeToString(double.NaN); + Assert.Equal("0.0", json); + } + + [Fact] + public void Write_WithPositiveInfinity_WritesZero() + { + string json = _serializer.SerializeToString(double.PositiveInfinity); + Assert.Equal("0.0", json); + } + + [Fact] + public void Write_WithNegativeInfinity_WritesZero() + { + string json = _serializer.SerializeToString(double.NegativeInfinity); + Assert.Equal("0.0", json); + } + + [Fact] + public void Write_WithMaxValue_ProducesValidJson() + { + string json = _serializer.SerializeToString(double.MaxValue); + double roundTripped = _serializer.Deserialize(json); + Assert.Equal(double.MaxValue, roundTripped); + } + + [Fact] + public void Write_WithMinValue_ProducesValidJson() + { + string json = _serializer.SerializeToString(double.MinValue); + double roundTripped = _serializer.Deserialize(json); + Assert.Equal(double.MinValue, roundTripped); + } + + [Fact] + public void Write_WithEpsilon_ProducesValidJson() + { + string json = _serializer.SerializeToString(double.Epsilon); + double roundTripped = _serializer.Deserialize(json); + Assert.Equal(double.Epsilon, roundTripped); + } + + [Fact] + public void Write_WithLargePrecisionDouble_PreservesValue() + { + double value = 123456789.123456; + string json = _serializer.SerializeToString(value); + double roundTripped = _serializer.Deserialize(json); + Assert.Equal(value, roundTripped); + } + + [Fact] + public void Write_WithNegativeWholeDouble_PreservesDecimalPoint() + { + string json = _serializer.SerializeToString(-5.0); + Assert.Equal("-5.0", json); + } + + [Theory] + [InlineData(0.1)] + [InlineData(0.001)] + [InlineData(999999.999999)] + [InlineData(-0.5)] + public void RoundTrip_WithVariousFractions_PreservesValue(double value) + { + string json = _serializer.SerializeToString(value); + double roundTripped = _serializer.Deserialize(json); + Assert.Equal(value, roundTripped); + } +} diff --git a/tests/Foundatio.Repositories.Tests/Serialization/JsonSerializerOptionsExtensionsTests.cs b/tests/Foundatio.Repositories.Tests/Serialization/JsonSerializerOptionsExtensionsTests.cs new file mode 100644 index 00000000..cb426149 --- /dev/null +++ b/tests/Foundatio.Repositories.Tests/Serialization/JsonSerializerOptionsExtensionsTests.cs @@ -0,0 +1,131 @@ +using System.Collections.Generic; +using System.Text.Json; +using Foundatio.Repositories.Serialization; +using Foundatio.Serializer; +using Xunit; + +namespace Foundatio.Repositories.Tests.Serialization; + +public class JsonSerializerOptionsExtensionsTests +{ + [Fact] + public void ConfigureDefaults_WithNewOptions_SetsPropertyNameCaseInsensitive() + { + // Arrange + var options = new JsonSerializerOptions(); + + // Act + options.ConfigureFoundatioRepositoryDefaults(); + + // Assert + Assert.True(options.PropertyNameCaseInsensitive); + } + + [Fact] + public void ConfigureDefaults_WithNewOptions_RegistersDoubleConverter() + { + // Arrange + var options = new JsonSerializerOptions(); + + // Act + options.ConfigureFoundatioRepositoryDefaults(); + + // Assert + Assert.Contains(options.Converters, c => c is DoubleSystemTextJsonConverter); + } + + [Fact] + public void ConfigureDefaults_WithNewOptions_RegistersObjectConverter() + { + // Arrange + var options = new JsonSerializerOptions(); + + // Act + options.ConfigureFoundatioRepositoryDefaults(); + + // Assert + Assert.Contains(options.Converters, c => c is ObjectToInferredTypesConverter); + } + + [Fact] + public void ConfigureDefaults_WithEnumValue_SerializesAsCamelCase() + { + // Arrange + var serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + + // Act + string json = serializer.SerializeToString(TestEnum.SomeValue); + + // Assert + Assert.Equal("\"someValue\"", json); + } + + [Fact] + public void ConfigureDefaults_WithCamelCaseEnumString_DeserializesToEnum() + { + // Arrange + var serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + + // Act + var result = serializer.Deserialize("\"someValue\""); + + // Assert + Assert.Equal(TestEnum.SomeValue, result); + } + + [Fact] + public void ConfigureDefaults_WithIntegerEnumValue_DeserializesToEnum() + { + // Arrange + var serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + + // Act + var result = serializer.Deserialize("1"); + + // Assert + Assert.Equal(TestEnum.SomeValue, result); + } + + [Fact] + public void ConfigureDefaults_WithMixedTypeObject_DeserializesObjectValuesAsClrTypes() + { + // Arrange + var serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + string json = "{\"name\":\"test\",\"count\":42,\"active\":true,\"score\":1.5}"; + + // Act + var result = serializer.Deserialize>(json); + + // Assert + Assert.IsType(result["name"]); + Assert.IsType(result["count"]); + Assert.IsType(result["active"]); + Assert.IsType(result["score"]); + } + + [Fact] + public void ConfigureDefaults_WithWholeDouble_PreservesDecimalPointInRoundTrip() + { + // Arrange + var serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + var obj = new { value = 1.0 }; + + // Act + string json = serializer.SerializeToString(obj); + + // Assert + Assert.Contains("1.0", json); + } + + private enum TestEnum + { + Default = 0, + SomeValue = 1, + AnotherValue = 2 + } +} diff --git a/tests/Foundatio.Repositories.Tests/Serialization/Models/BucketSerializationTests.cs b/tests/Foundatio.Repositories.Tests/Serialization/Models/BucketSerializationTests.cs index 66b0a0e6..f25de82c 100644 --- a/tests/Foundatio.Repositories.Tests/Serialization/Models/BucketSerializationTests.cs +++ b/tests/Foundatio.Repositories.Tests/Serialization/Models/BucketSerializationTests.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using Foundatio.Repositories.Models; -using Foundatio.Repositories.Tests.Utility; using Foundatio.Serializer; using Xunit; diff --git a/tests/Foundatio.Repositories.Tests/Serialization/Models/FindResultsSerializationTests.cs b/tests/Foundatio.Repositories.Tests/Serialization/Models/FindResultsSerializationTests.cs index b12ac2ed..802d6f97 100644 --- a/tests/Foundatio.Repositories.Tests/Serialization/Models/FindResultsSerializationTests.cs +++ b/tests/Foundatio.Repositories.Tests/Serialization/Models/FindResultsSerializationTests.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using Foundatio.Repositories.Models; -using Foundatio.Repositories.Tests.Utility; using Foundatio.Serializer; using Xunit; diff --git a/tests/Foundatio.Repositories.Tests/Serialization/ObjectToInferredTypesConverterTests.cs b/tests/Foundatio.Repositories.Tests/Serialization/ObjectToInferredTypesConverterTests.cs new file mode 100644 index 00000000..6c0ed725 --- /dev/null +++ b/tests/Foundatio.Repositories.Tests/Serialization/ObjectToInferredTypesConverterTests.cs @@ -0,0 +1,317 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using Foundatio.Repositories.Serialization; +using Foundatio.Serializer; +using Xunit; + +namespace Foundatio.Repositories.Tests.Serialization; + +public class ObjectToInferredTypesConverterTests +{ + private static readonly ITextSerializer _serializer = new SystemTextJsonSerializer( + new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()); + + [Fact] + public void Read_WithInteger_ReturnsLong() + { + // Arrange + string json = "42"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(42L, result); + } + + [Fact] + public void Read_WithNegativeInteger_ReturnsLong() + { + // Arrange + string json = "-100"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(-100L, result); + } + + [Fact] + public void Read_WithZeroInteger_ReturnsLong() + { + // Arrange + string json = "0"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(0L, result); + } + + [Fact] + public void Read_WithFloatingPoint_ReturnsDouble() + { + // Arrange + string json = "42.5"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(42.5, result); + } + + [Fact] + public void Read_WithZeroPointZero_ReturnsDouble() + { + // Arrange + string json = "0.0"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(0.0, result); + } + + [Fact] + public void Read_WithScientificNotation_ReturnsDouble() + { + // Arrange + string json = "1.5e3"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(1500.0, result); + } + + [Fact] + public void Read_WithString_ReturnsString() + { + // Arrange + string json = "\"hello\""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal("hello", result); + } + + [Fact] + public void Read_WithTrue_ReturnsBoolTrue() + { + // Arrange + string json = "true"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.True((bool)result); + } + + [Fact] + public void Read_WithFalse_ReturnsBoolFalse() + { + // Arrange + string json = "false"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.False((bool)result); + } + + [Fact] + public void Read_WithNull_ReturnsNull() + { + // Arrange + string json = "null"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.Null(result); + } + + [Fact] + public void Read_WithIso8601Datetime_ReturnsDateTimeOffset() + { + // Arrange + string json = "\"2026-03-03T12:00:00Z\""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.IsType(result); + Assert.Equal(new DateTimeOffset(2026, 3, 3, 12, 0, 0, TimeSpan.Zero), result); + } + + [Theory] + [InlineData("\"2026-03-03\"")] // date-only ISO 8601 + [InlineData("\"12/31/2025\"")] // US date format + [InlineData("\"January 1, 2025\"")] // long form date + public void Read_WithDateOnlyOrNonIsoString_ReturnsString(string json) + { + // Act + var result = _serializer.Deserialize(json); + + // Assert — date-only strings must NOT be silently promoted to DateTimeOffset + Assert.IsType(result); + } + + [Theory] + [InlineData("\"2026-03-03T12:00:00Z\"")] + [InlineData("\"2026-03-03T10:30:00+05:00\"")] + [InlineData("\"2026-03-03T00:00:00\"")] + public void Read_WithIso8601DatetimeWithTimeComponent_ReturnsDateTimeOffset(string json) + { + // Act + var result = _serializer.Deserialize(json); + + // Assert — only strings with 'T' (time component) are treated as dates + Assert.IsType(result); + } + + [Fact] + public void Read_WithJsonObject_ReturnsCaseInsensitiveDictionary() + { + // Arrange + string json = "{\"Name\":\"test\",\"Count\":42}"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + var dict = Assert.IsType>(result); + Assert.Equal("test", dict["name"]); + Assert.Equal("test", dict["NAME"]); + Assert.Equal(42L, dict["count"]); + } + + [Fact] + public void Read_WithJsonArray_ReturnsList() + { + // Arrange + string json = "[1,2,3]"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + var list = Assert.IsType>(result); + Assert.Equal(3, list.Count); + Assert.Equal(1L, list[0]); + Assert.Equal(2L, list[1]); + Assert.Equal(3L, list[2]); + } + + [Fact] + public void Read_WithMixedArray_PreservesElementTypes() + { + // Arrange + string json = "[\"hello\",42,true,null,1.5]"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + var list = Assert.IsType>(result); + Assert.Equal(5, list.Count); + Assert.IsType(list[0]); + Assert.IsType(list[1]); + Assert.IsType(list[2]); + Assert.Null(list[3]); + Assert.IsType(list[4]); + } + + [Fact] + public void Read_WithNestedObject_PreservesStructure() + { + // Arrange + string json = "{\"outer\":{\"inner\":\"value\"}}"; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + var dict = Assert.IsType>(result); + var inner = Assert.IsType>(dict["outer"]); + Assert.Equal("value", inner["inner"]); + } + + [Fact] + public void RoundTrip_WithObjectArray_PreservesTypesAndValues() + { + // Arrange + var original = new object[] { "test", 42L, true, 1.5 }; + + // Act + string json = _serializer.SerializeToString(original); + var result = _serializer.Deserialize(json); + + // Assert + Assert.Equal(4, result.Length); + Assert.Equal("test", result[0]); + Assert.Equal(42L, result[1]); + Assert.Equal(true, result[2]); + Assert.Equal(1.5, result[3]); + } + + [Fact] + public void Write_WithNull_WritesNullLiteral() + { + // Arrange + object value = null; + + // Act + string json = _serializer.SerializeToString(value); + + // Assert + Assert.Equal("null", json); + } + + [Fact] + public void Write_WithLong_WritesNumber() + { + // Arrange + object value = 42L; + + // Act + string json = _serializer.SerializeToString(value); + + // Assert + Assert.Equal("42", json); + } + + [Fact] + public void Write_WithString_WritesQuotedString() + { + // Arrange + object value = "hello"; + + // Act + string json = _serializer.SerializeToString(value); + + // Assert + Assert.Equal("\"hello\"", json); + } +} diff --git a/tests/Foundatio.Repositories.Tests/Serialization/SerializerTestHelper.cs b/tests/Foundatio.Repositories.Tests/Serialization/SerializerTestHelper.cs new file mode 100644 index 00000000..d69b41ae --- /dev/null +++ b/tests/Foundatio.Repositories.Tests/Serialization/SerializerTestHelper.cs @@ -0,0 +1,14 @@ +using System.Text.Json; +using Foundatio.Repositories.Serialization; +using Foundatio.Serializer; + +namespace Foundatio.Repositories.Tests.Serialization; + +public static class SerializerTestHelper +{ + public static ITextSerializer[] GetTextSerializers() => + [ + new SystemTextJsonSerializer(new JsonSerializerOptions().ConfigureFoundatioRepositoryDefaults()), + new JsonNetSerializer() + ]; +} diff --git a/tests/Foundatio.Repositories.Tests/Utility/SerializerTestHelper.cs b/tests/Foundatio.Repositories.Tests/Utility/SerializerTestHelper.cs deleted file mode 100644 index 9a59d0e4..00000000 --- a/tests/Foundatio.Repositories.Tests/Utility/SerializerTestHelper.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Foundatio.Serializer; - -namespace Foundatio.Repositories.Tests.Utility; - -public static class SerializerTestHelper -{ - public static ITextSerializer[] GetTextSerializers() => - [ - new SystemTextJsonSerializer(), - new JsonNetSerializer() - ]; -}