diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/HypertableOperationGenerator.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/HypertableOperationGenerator.cs new file mode 100644 index 0000000..031bc0f --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/HypertableOperationGenerator.cs @@ -0,0 +1,129 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators +{ + public class HypertableOperationGenerator + { + public static void Generate(CreateHypertableOperation operation, IndentedStringBuilder builder) + { + List statements = + [ + $"SELECT create_hypertable('\"\"{operation.TableName}\"\"', '{operation.TimeColumnName}');" + ]; + + // ChunkTimeInterval + if (!string.IsNullOrEmpty(operation.ChunkTimeInterval)) + { + // Check if the interval is a plain number (e.g., for microseconds). + if (long.TryParse(operation.ChunkTimeInterval, out _)) + { + // If it's a number, don't wrap it in quotes. + statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', {operation.ChunkTimeInterval}::bigint);"); + } + else + { + // If it's a string like '7 days', wrap it in quotes. + statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', INTERVAL '{operation.ChunkTimeInterval}');"); + } + } + + // EnableCompression + if (operation.EnableCompression || operation.ChunkSkipColumns?.Count > 0) + { + bool enableCompression = operation.EnableCompression || (operation.ChunkSkipColumns != null && operation.ChunkSkipColumns.Count > 0); + statements.Add($"ALTER TABLE \"\"{operation.TableName}\"\" SET (timescaledb.compress = {enableCompression.ToString().ToLower()});"); + } + + // ChunkSkipColumns + if (operation.ChunkSkipColumns != null && operation.ChunkSkipColumns.Count > 0) + { + statements.Add("SET timescaledb.enable_chunk_skipping = 'ON';"); + + foreach (string column in operation.ChunkSkipColumns) + { + statements.Add($"SELECT enable_chunk_skipping('\"\"{operation.TableName}\"\"', '{column}');"); + } + } + + // AdditionalDimensions + if (operation.AdditionalDimensions != null && operation.AdditionalDimensions.Count > 0) + { + foreach (Dimension dimension in operation.AdditionalDimensions) + { + if (dimension.Type == EDimensionType.Range) + { + statements.Add($"SELECT add_dimension('\"\"{operation.TableName}\"\"', by_range('{dimension.ColumnName}', INTERVAL '{dimension.Interval}'));"); + } + else if (dimension.Type == EDimensionType.Hash) + { + statements.Add($"SELECT add_dimension('\"\"{operation.TableName}\"\"', by_hash('{dimension.ColumnName}', {dimension.NumberOfPartitions}));"); + } + } + } + + MigrationBuilderSqlHelper.BuildQueryString(statements, builder); + } + + public static void Generate(AlterHypertableOperation operation, IndentedStringBuilder builder) + { + List statements = []; + + // Check for ChunkTimeInterval change + if (operation.ChunkTimeInterval != operation.OldChunkTimeInterval) + { + if (operation.ChunkTimeInterval != operation.OldChunkTimeInterval) + { + // Check if the interval is a plain number (e.g., for microseconds). + if (long.TryParse(operation.ChunkTimeInterval, out _)) + { + // If it's a number, don't wrap it in quotes. + statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', {operation.ChunkTimeInterval}::bigint);"); + } + else + { + // If it's a string like '7 days', wrap it in quotes. + statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', INTERVAL '{operation.ChunkTimeInterval}');"); + } + } + } + + // Check for EnableCompression change + bool newCompressionState = operation.EnableCompression || (operation.ChunkSkipColumns != null && operation.ChunkSkipColumns.Any()); + bool oldCompressionState = operation.OldEnableCompression || (operation.OldChunkSkipColumns != null && operation.OldChunkSkipColumns.Any()); + + if (newCompressionState != oldCompressionState) + { + string compressionValue = newCompressionState.ToString().ToLower(); + statements.Add($"ALTER TABLE \"\"{operation.TableName}\"\" SET (timescaledb.compress = {compressionValue});"); + } + + // Handle ChunkSkipColumns + IReadOnlyList newColumns = operation.ChunkSkipColumns ?? []; + IReadOnlyList oldColumns = operation.OldChunkSkipColumns ?? []; + List addedColumns = [.. newColumns.Except(oldColumns)]; + + if (addedColumns.Count != 0) + { + statements.Add("SET timescaledb.enable_chunk_skipping = 'ON';"); + + foreach (string column in addedColumns) + { + statements.Add($"SELECT enable_chunk_skipping('\"\"{operation.TableName}\"\"', '{column}');"); + } + } + + List removedColumns = [.. oldColumns.Except(newColumns)]; + if (removedColumns.Count != 0) + { + foreach (string column in removedColumns) + { + statements.Add($"SELECT disable_chunk_skipping('\"\"{operation.TableName}\"\"', '{column}');"); + } + } + + MigrationBuilderSqlHelper.BuildQueryString(statements, builder); + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/MigrationBuilderSqlHelper.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/MigrationBuilderSqlHelper.cs new file mode 100644 index 0000000..f0fd7fd --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/MigrationBuilderSqlHelper.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators +{ + public static class MigrationBuilderSqlHelper + { + public static void BuildQueryString(List statements, IndentedStringBuilder builder) + { + if (statements.Count > 0) + { + builder.AppendLine(".Sql(@\""); + using (builder.Indent()) + { + foreach (string statement in statements) + { + builder.AppendLine(statement); + } + } + builder.Append("\")"); + } + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/ReorderPolicyOperationGenerator.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/ReorderPolicyOperationGenerator.cs new file mode 100644 index 0000000..21c3d31 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/Generators/ReorderPolicyOperationGenerator.cs @@ -0,0 +1,145 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; +using Microsoft.EntityFrameworkCore.Infrastructure; +using System.Globalization; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators +{ + public class ReorderPolicyOperationGenerator + { + public static void Generate(AddReorderPolicyOperation operation, IndentedStringBuilder builder) + { + List statements = + [ + BuildAddReorderPolicySql(operation.TableName, operation.IndexName, operation.InitialStart) + ]; + + List alterJobClauses = BuildAlterJobClauses(operation); + if (alterJobClauses.Count != 0) + { + statements.Add(BuildAlterJobSql(operation.TableName, alterJobClauses)); + } + + MigrationBuilderSqlHelper.BuildQueryString(statements, builder); + } + + public static void Generate(AlterReorderPolicyOperation operation, IndentedStringBuilder builder) + { + List statements = []; + bool needsRecreation = operation.IndexName != operation.OldIndexName || operation.InitialStart != operation.OldInitialStart; + + if (needsRecreation) + { + statements.Add($"SELECT remove_reorder_policy('\"\"{operation.TableName}\"\"', if_exists => true);"); + statements.Add(BuildAddReorderPolicySql(operation.TableName, operation.IndexName, operation.InitialStart)); + + // Create a temporary "add" operation representing the final desired state to ensure existing settings are reapplied. + AddReorderPolicyOperation finalStateOperation = new() + { + TableName = operation.TableName, + IndexName = operation.IndexName, + InitialStart = operation.InitialStart, + ScheduleInterval = operation.ScheduleInterval, + MaxRuntime = operation.MaxRuntime, + MaxRetries = operation.MaxRetries, + RetryPeriod = operation.RetryPeriod + }; + + List finalStateClauses = BuildAlterJobClauses(finalStateOperation); + if (finalStateClauses.Count != 0) + { + statements.Add(BuildAlterJobSql(operation.TableName, finalStateClauses)); + } + } + else + { + List changedClauses = BuildAlterJobClauses(operation); + if (changedClauses.Count != 0) + { + statements.Add(BuildAlterJobSql(operation.TableName, changedClauses)); + } + } + + MigrationBuilderSqlHelper.BuildQueryString(statements, builder); + } + + public static void Generate(DropReorderPolicyOperation operation, IndentedStringBuilder builder) + { + List statements = + [ + $"SELECT remove_reorder_policy('\"\"{operation.TableName}\"\"', if_exists => true);" + ]; + MigrationBuilderSqlHelper.BuildQueryString(statements, builder); + } + + private static List BuildAlterJobClauses(AddReorderPolicyOperation operation) + { + List clauses = []; + if (!string.IsNullOrWhiteSpace(operation.ScheduleInterval) && operation.ScheduleInterval != DefaultValues.ReorderPolicyScheduleInterval) + clauses.Add($"schedule_interval => INTERVAL '{operation.ScheduleInterval}'"); + + if (!string.IsNullOrWhiteSpace(operation.MaxRuntime) && operation.MaxRuntime != DefaultValues.ReorderPolicyMaxRuntime) + clauses.Add($"max_runtime => INTERVAL '{operation.MaxRuntime}'"); + + if (operation.MaxRetries != null && operation.MaxRetries != DefaultValues.ReorderPolicyMaxRetries) + clauses.Add($"max_retries => {operation.MaxRetries}"); + + if (!string.IsNullOrWhiteSpace(operation.RetryPeriod) && operation.RetryPeriod != DefaultValues.ReorderPolicyRetryPeriod) + clauses.Add($"retry_period => INTERVAL '{operation.RetryPeriod}'"); + + return clauses; + } + + private static List BuildAlterJobClauses(AlterReorderPolicyOperation operation) + { + List clauses = []; + + if (!string.IsNullOrWhiteSpace(operation.ScheduleInterval) && operation.ScheduleInterval != operation.OldScheduleInterval) + clauses.Add($"schedule_interval => INTERVAL '{operation.ScheduleInterval}'"); + + if (!string.IsNullOrWhiteSpace(operation.MaxRuntime) && operation.MaxRuntime != operation.OldMaxRuntime) + { + string maxRuntimeValue = string.IsNullOrWhiteSpace(operation.MaxRuntime) ? "NULL" : $"INTERVAL '{operation.MaxRuntime}'"; + clauses.Add($"max_runtime => {maxRuntimeValue}"); + } + + if (operation.MaxRetries != null && operation.MaxRetries != operation.OldMaxRetries) + clauses.Add($"max_retries => {operation.MaxRetries}"); + + if (!string.IsNullOrWhiteSpace(operation.RetryPeriod) && operation.RetryPeriod != operation.OldRetryPeriod) + clauses.Add($"retry_period => INTERVAL '{operation.RetryPeriod}'"); + + return clauses; + } + + private static string BuildAlterJobSql(string tableName, IEnumerable clauses) + { + return $@" + SELECT alter_job(job_id, {string.Join(", ", clauses)}) + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_reorder' AND hypertable_name = '{tableName}';".Trim(); + } + + private static string BuildAddReorderPolicySql(string tableName, string indexName, DateTime? initialStart) + { + string baseSql = $"SELECT add_reorder_policy('\"\"{tableName}\"\"', '{indexName}'"; + + List optionalArgs = []; + + // Add optional arguments if they are provided + if (initialStart.HasValue) + { + // Use ISO 8601 format for timestamps to avoid ambiguity + string timestamp = initialStart.Value.ToUniversalTime().ToString("o", CultureInfo.InvariantCulture); + optionalArgs.Add($"initial_start => '{timestamp}'"); + } + + if (optionalArgs.Count > 0) + { + baseSql += $", {string.Join(", ", optionalArgs)}"; + } + + baseSql += ");"; + return baseSql; + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpMigrationOperationGenerator.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpMigrationOperationGenerator.cs index a4615ce..97f2d8b 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpMigrationOperationGenerator.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpMigrationOperationGenerator.cs @@ -1,4 +1,4 @@ -using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators; using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations.Design; @@ -16,148 +16,27 @@ protected override void Generate(MigrationOperation operation, IndentedStringBui switch (operation) { case CreateHypertableOperation create: - Generate(create, builder); + HypertableOperationGenerator.Generate(create, builder); break; case AlterHypertableOperation alter: - Generate(alter, builder); + HypertableOperationGenerator.Generate(alter, builder); break; - default: - base.Generate(operation, builder); + case AddReorderPolicyOperation addReorder: + ReorderPolicyOperationGenerator.Generate(addReorder, builder); + break; + case AlterReorderPolicyOperation alterReorder: + ReorderPolicyOperationGenerator.Generate(alterReorder, builder); + break; + case DropReorderPolicyOperation dropReorder: + ReorderPolicyOperationGenerator.Generate(dropReorder, builder); break; - } - } - - private static void Generate(CreateHypertableOperation operation, IndentedStringBuilder builder) - { - List statements = - [ - $"SELECT create_hypertable('\"\"{operation.TableName}\"\"', '{operation.TimeColumnName}');" - ]; - - // ChunkTimeInterval - if (!string.IsNullOrEmpty(operation.ChunkTimeInterval)) - { - // Check if the interval is a plain number (e.g., for microseconds). - if (long.TryParse(operation.ChunkTimeInterval, out _)) - { - // If it's a number, don't wrap it in quotes. - statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', {operation.ChunkTimeInterval}::bigint);"); - } - else - { - // If it's a string like '7 days', wrap it in quotes. - statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', INTERVAL '{operation.ChunkTimeInterval}');"); - } - } - - // EnableCompression - if (operation.EnableCompression) - { - statements.Add($"ALTER TABLE \"\"{operation.TableName}\"\" SET (timescaledb.compress = true);"); - } - - // ChunkSkipColumns - if (operation.ChunkSkipColumns != null && operation.ChunkSkipColumns.Count > 0) - { - statements.Add("SET timescaledb.enable_chunk_skipping = 'ON';"); - - foreach (string column in operation.ChunkSkipColumns) - { - statements.Add($"SELECT enable_chunk_skipping('\"\"{operation.TableName}\"\"', '{column}');"); - } - } - - // AdditionalDimensions - if (operation.AdditionalDimensions != null && operation.AdditionalDimensions.Count > 0) - { - foreach (Dimension dimension in operation.AdditionalDimensions) - { - if (dimension.Type == EDimensionType.Range) - { - statements.Add($"SELECT add_dimension('\"\"{operation.TableName}\"\"', by_range('{dimension.ColumnName}', INTERVAL '{dimension.Interval}'));"); - } - else if (dimension.Type == EDimensionType.Hash) - { - statements.Add($"SELECT add_dimension('\"\"{operation.TableName}\"\"', by_hash('{dimension.ColumnName}', {dimension.NumberOfPartitions}));"); - } - } - } - - BuildQueryString(statements, builder); - } - - private static void Generate(AlterHypertableOperation operation, IndentedStringBuilder builder) - { - List statements = []; - - // Check for ChunkTimeInterval change - if (operation.ChunkTimeInterval != operation.OldChunkTimeInterval) - { - if (operation.ChunkTimeInterval != operation.OldChunkTimeInterval) - { - // Check if the interval is a plain number (e.g., for microseconds). - if (long.TryParse(operation.ChunkTimeInterval, out _)) - { - // If it's a number, don't wrap it in quotes. - statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', {operation.ChunkTimeInterval}::bigint);"); - } - else - { - // If it's a string like '7 days', wrap it in quotes. - statements.Add($"SELECT set_chunk_time_interval('\"\"{operation.TableName}\"\"', INTERVAL '{operation.ChunkTimeInterval}');"); - } - } - } - - // Check for EnableCompression change - if (operation.EnableCompression != operation.OldEnableCompression) - { - string compressionValue = operation.EnableCompression.ToString().ToLower(); - statements.Add($"ALTER TABLE \"\"{operation.TableName}\"\" SET (timescaledb.compress = {compressionValue});"); - } - - // Handle ChunkSkipColumns - IReadOnlyList newColumns = operation.ChunkSkipColumns ?? []; - IReadOnlyList oldColumns = operation.OldChunkSkipColumns ?? []; - List addedColumns = [.. newColumns.Except(oldColumns)]; - - if (addedColumns.Count != 0) - { - statements.Add("SET timescaledb.enable_chunk_skipping = 'ON';"); - - foreach (string column in addedColumns) - { - statements.Add($"SELECT enable_chunk_skipping('\"\"{operation.TableName}\"\"', '{column}');"); - } - } - - List removedColumns = [.. oldColumns.Except(newColumns)]; - if (removedColumns.Count != 0) - { - foreach (string column in removedColumns) - { - statements.Add($"SELECT disable_chunk_skipping('\"\"{operation.TableName}\"\"', '{column}');"); - } - } - - BuildQueryString(statements, builder); - } - private static void BuildQueryString(List statements, IndentedStringBuilder builder) - { - if (statements.Count > 0) - { - builder.AppendLine(".Sql(@\""); - using (builder.Indent()) - { - foreach (string statement in statements) - { - builder.AppendLine(statement); - } - } - builder.Append("\")"); + default: + base.Generate(operation, builder); + break; } } + } } \ No newline at end of file diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpModelGenerator.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpModelGenerator.cs index d8109e6..408897c 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpModelGenerator.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/TimescaleCSharpModelGenerator.cs @@ -1,10 +1,12 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Scaffolding; using Microsoft.EntityFrameworkCore.Scaffolding.Metadata; using Npgsql.EntityFrameworkCore.PostgreSQL.Scaffolding.Internal; +using System.Data; using System.Data.Common; using System.Text.Json; @@ -22,15 +24,30 @@ private sealed record HypertableInfo( List AdditionalDimensions ); + private sealed record ReorderPolicyInfo( + string IndexName, + DateTime? InitialStart, + string? ScheduleInterval, + string? MaxRuntime, + int? MaxRetries, + string? RetryPeriod + ); + public override DatabaseModel Create(DbConnection connection, DatabaseModelFactoryOptions options) { DatabaseModel databaseModel = base.Create(connection, options); Dictionary<(string, string), HypertableInfo> hypertables = GetHypertables(connection); + Dictionary<(string, string), ReorderPolicyInfo> reorderPolicies = GetReorderPolicies(connection); // Annotate the tables in the model foreach (DatabaseTable table in databaseModel.Tables) { - if (table?.Schema != null && hypertables.TryGetValue((table.Schema, table.Name), out HypertableInfo? info)) + if (table?.Schema == null) continue; + + (string Schema, string Name) tableKey = (table.Schema, table.Name); + + // Annotations for Hypertables + if (hypertables.TryGetValue(tableKey, out HypertableInfo? info)) { table[HypertableAnnotations.IsHypertable] = true; table[HypertableAnnotations.HypertableTimeColumn] = info.TimeColumnName; @@ -47,6 +64,39 @@ public override DatabaseModel Create(DbConnection connection, DatabaseModelFacto table[HypertableAnnotations.AdditionalDimensions] = JsonSerializer.Serialize(info.AdditionalDimensions); } } + + // Annotate for Reorder Policies + if (reorderPolicies.TryGetValue(tableKey, out ReorderPolicyInfo? policyInfo)) + { + table[ReorderPolicyAnnotations.HasReorderPolicy] = true; + table[ReorderPolicyAnnotations.IndexName] = policyInfo.IndexName; + + if (policyInfo.InitialStart.HasValue) + { + table[ReorderPolicyAnnotations.InitialStart] = policyInfo.InitialStart.Value; + } + + // Set annotations only if they differ from TimescaleDB defaults + if (policyInfo.ScheduleInterval != DefaultValues.ReorderPolicyScheduleInterval) + { + table[ReorderPolicyAnnotations.ScheduleInterval] = policyInfo.ScheduleInterval; + } + + if (policyInfo.MaxRuntime != DefaultValues.ReorderPolicyMaxRuntime) + { + table[ReorderPolicyAnnotations.MaxRuntime] = policyInfo.MaxRuntime; + } + + if (policyInfo.MaxRetries != DefaultValues.ReorderPolicyMaxRetries) + { + table[ReorderPolicyAnnotations.MaxRetries] = policyInfo.MaxRetries; + } + + if (policyInfo.RetryPeriod != DefaultValues.ReorderPolicyRetryPeriod) + { + table[ReorderPolicyAnnotations.RetryPeriod] = policyInfo.RetryPeriod; + } + } } return databaseModel; @@ -177,6 +227,69 @@ FROM _timescaledb_catalog.chunk_column_stats AS ccs } } } + + private static Dictionary<(string, string), ReorderPolicyInfo> GetReorderPolicies(DbConnection connection) + { + bool wasOpen = connection.State == ConnectionState.Open; + if (!wasOpen) + { + connection.Open(); + } + + try + { + Dictionary<(string, string), ReorderPolicyInfo> reorderPolicies = []; + using (DbCommand command = connection.CreateCommand()) + { + command.CommandText = @" + SELECT + j.hypertable_schema, + j.hypertable_name, + j.config ->> 'index_name' AS index_name, + j.initial_start, + j.schedule_interval::text, + j.max_runtime::text, + j.max_retries, + j.retry_period::text + FROM timescaledb_information.jobs AS j + WHERE j.proc_name = 'policy_reorder';"; + + using DbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + string schema = reader.GetString(0); + string name = reader.GetString(1); + string indexName = reader.GetString(2); + DateTime? initialStart = reader.IsDBNull(3) ? null : reader.GetDateTime(3); + + string? scheduleInterval = reader.IsDBNull(4) ? null : reader.GetString(4); + string? maxRuntime = reader.IsDBNull(5) ? null : reader.GetString(5); + int? maxRetries = reader.IsDBNull(6) ? null : reader.GetInt32(6); + string? retryPeriod = reader.IsDBNull(7) ? null : reader.GetString(7); + + if (!string.IsNullOrEmpty(indexName)) + { + reorderPolicies[(schema, name)] = new ReorderPolicyInfo( + indexName, + initialStart, + scheduleInterval, + maxRuntime, + maxRetries, + retryPeriod + ); + } + } + } + return reorderPolicies; + } + finally + { + if (!wasOpen) + { + connection.Close(); + } + } + } } #pragma warning restore EF1001 } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/README.md b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/README.md index 3409517..a2e7a99 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/README.md +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/README.md @@ -24,6 +24,7 @@ dotnet ef dbcontext scaffold --schema public --context-dir . --context MyTimescaleDbContext + --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst ``` This command will: diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs index 607bbcf..d645ce3 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs @@ -1,4 +1,5 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy; using CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -13,6 +14,7 @@ public void Configure(EntityTypeBuilder builder) builder.HasNoKey() .IsHypertable(x => x.Timestamp) .WithChunkTimeInterval("1 day"); + builder.WithReorderPolicy("Trades_Timestamp_idx", DateTime.Parse("2025-09-23T09:15:19.3905112Z"), "2 days", "10 minutes", null, "1 minute"); } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs index 9a5a949..f7d983c 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs @@ -12,7 +12,7 @@ public void Configure(EntityTypeBuilder builder) builder.ToTable("TradesWithId"); builder.HasKey(x => new { x.Id, x.Timestamp }); builder.IsHypertable(x => x.Timestamp) - .WithChunkTimeInterval("1 day"); + .WithChunkTimeInterval("2 day"); } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/DeviceReading.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/DeviceReading.cs index fbc1d1e..12bb12b 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/DeviceReading.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/DeviceReading.cs @@ -1,9 +1,11 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy; using Microsoft.EntityFrameworkCore; namespace CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models { [Hypertable(nameof(Time), ChunkSkipColumns = new[] { "Time" }, ChunkTimeInterval = "1 day")] + [ReorderPolicy("DeviceReadings_Time_idx", InitialStart = "2025-09-23T09:15:19.3905112Z", ScheduleInterval = "1 day", MaxRuntime = "00:00:00", RetryPeriod = "00:05:00")] [PrimaryKey(nameof(Id), nameof(Time))] public class DeviceReading { diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs index 4c2708b..363d111 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs @@ -1,6 +1,4 @@ -using System.ComponentModel.DataAnnotations; - -namespace CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models { /// /// Represents a single trade event with a standard primary key. @@ -12,7 +10,6 @@ public class TradeWithId /// The unique identifier for the trade record. /// This is the primary key. /// - [Key] public long Id { get; set; } /// diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example/README.md b/CmdScale.EntityFrameworkCore.TimescaleDB.Example/README.md index 9d4993b..ed3990b 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example/README.md +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example/README.md @@ -8,19 +8,24 @@ This project demonstrates how to use the **Code-First** approach with [Timescale Use the following commands to manage your EF Core migrations and database updates. -### 📌 Add a New Migration +### 📌 Add a new migration ```bash dotnet ef migrations add --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess --startup-project CmdScale.EntityFrameworkCore.TimescaleDB.Example ``` -### ✅ Apply Migrations to the Database +### ✅ Apply migrations to the database ```bash dotnet ef database update --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess --startup-project CmdScale.EntityFrameworkCore.TimescaleDB.Example ``` -### 🧹 Reset All Migrations (Rollback to Initial State) +### ❌ Remove last migration (if not applied to the database, yet) +``` + dotnet ef migrations remove --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess --startup-project CmdScale.EntityFrameworkCore.TimescaleDB.Example +``` + +### 🧹 Reset all migrations (rollback to initial state) ```bash dotnet ef database update 0 --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess --startup-project CmdScale.EntityFrameworkCore.TimescaleDB.Example diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example/TimescaleDBDesignTimeService.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example/TimescaleDBDesignTimeService.cs index 3ea8ae9..148960a 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example/TimescaleDBDesignTimeService.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example/TimescaleDBDesignTimeService.cs @@ -15,8 +15,8 @@ public void ConfigureDesignTimeServices(IServiceCollection services) // NOTE: When using project references, instead of package referneces, you need to uncomment these lines to find inject the correct ICSharpMigrationOperationGenerator // The reason for this is, because the CmdScale.EntityFrameworkCore.TimescaleDB.Design project only copies the required assembly-attribute when being packaged. - //services.AddSingleton() - // .AddSingleton(); + services.AddSingleton() + .AddSingleton(); } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Generators/HypertableOperationGeneratorTests.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Generators/HypertableOperationGeneratorTests.cs new file mode 100644 index 0000000..3371d91 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Generators/HypertableOperationGeneratorTests.cs @@ -0,0 +1,197 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators; +using CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests.Utils; +using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests.Generators +{ + public class HypertableOperationGeneratorTests + { + /// + /// A helper to run the generator and capture its string output. + /// + private static string GetGeneratedCode(dynamic operation) + { + IndentedStringBuilder builder = new(); + HypertableOperationGenerator.Generate(operation, builder); + return builder.ToString(); + } + + // --- Tests for CreateHypertableOperation --- + + [Fact] + public void Generate_Create_with_minimal_details_generates_correct_sql() + { + // Arrange + CreateHypertableOperation operation = new() + { + TableName = "MinimalTable", + TimeColumnName = "Timestamp" + }; + + string expected = @".Sql(@"" + SELECT create_hypertable('""""MinimalTable""""', 'Timestamp'); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Create_with_all_options_generates_comprehensive_sql() + { + // Arrange + CreateHypertableOperation operation = new() + { + TableName = "FullTable", + TimeColumnName = "EventTime", + ChunkTimeInterval = "1 day", + EnableCompression = true, + ChunkSkipColumns = ["DeviceId"], + AdditionalDimensions = + [ + Dimension.CreateHash("LocationId", 4) + ] + }; + + string expected = @".Sql(@"" + SELECT create_hypertable('""""FullTable""""', 'EventTime'); + SELECT set_chunk_time_interval('""""FullTable""""', INTERVAL '1 day'); + ALTER TABLE """"FullTable"""" SET (timescaledb.compress = true); + SET timescaledb.enable_chunk_skipping = 'ON'; + SELECT enable_chunk_skipping('""""FullTable""""', 'DeviceId'); + SELECT add_dimension('""""FullTable""""', by_hash('LocationId', 4)); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Alter_WhenAddingChunkSkippingToUncompressedTable_ShouldAlsoEnableCompression() + { + // Arrange + AlterHypertableOperation operation = new() + { + TableName = "Metrics", + OldEnableCompression = false, + OldChunkSkipColumns = [], + EnableCompression = false, + ChunkSkipColumns = ["device_id"] + }; + + string expected = @".Sql(@"" + ALTER TABLE """"Metrics"""" SET (timescaledb.compress = true); + SET timescaledb.enable_chunk_skipping = 'ON'; + SELECT enable_chunk_skipping('""""Metrics""""', 'device_id'); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + // --- Tests for AlterHypertableOperation --- + + [Fact] + public void Generate_Alter_when_changing_compression_generates_correct_sql() + { + // Arrange + AlterHypertableOperation operation = new() + { + TableName = "SensorData", + EnableCompression = true, + OldEnableCompression = false + }; + + string expected = @".Sql(@"" + ALTER TABLE """"SensorData"""" SET (timescaledb.compress = true); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Alter_when_adding_and_removing_skip_columns_generates_correct_sql() + { + // Arrange + AlterHypertableOperation operation = new() + { + TableName = "Metrics", + ChunkSkipColumns = ["host", "service"], + OldChunkSkipColumns = ["host", "region"] + }; + + string expected = @".Sql(@"" + SET timescaledb.enable_chunk_skipping = 'ON'; + SELECT enable_chunk_skipping('""""Metrics""""', 'service'); + SELECT disable_chunk_skipping('""""Metrics""""', 'region'); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Alter_when_no_properties_change_generates_no_sql() + { + // Arrange + AlterHypertableOperation operation = new() + { + TableName = "NoChangeTable", + EnableCompression = true, + OldEnableCompression = true, + ChunkTimeInterval = "7 days", + OldChunkTimeInterval = "7 days" + }; + + string expected = ""; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(expected, result); + } + + [Fact] + public void Generate_Alter_WhenRemovingLastChunkSkipColumn_ShouldDisableCompression_IfNotExplicitlyEnabled() + { + // Arrange + AlterHypertableOperation operation = new() + { + TableName = "Logs", + OldEnableCompression = false, + OldChunkSkipColumns = ["trace_id"], + EnableCompression = false, + ChunkSkipColumns = [] + }; + string expected = @".Sql(@"" + ALTER TABLE """"Logs"""" SET (timescaledb.compress = false); + SELECT disable_chunk_skipping('""""Logs""""', 'trace_id'); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Generators/ReorderPolicyOperationGeneratorTests.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Generators/ReorderPolicyOperationGeneratorTests.cs new file mode 100644 index 0000000..890e883 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Generators/ReorderPolicyOperationGeneratorTests.cs @@ -0,0 +1,182 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators; +using CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests.Utils; +using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests.Generators +{ + public class ReorderPolicyOperationGeneratorTests + { + /// + /// A helper to run the generator and capture its string output. + /// + private static string GetGeneratedCode(dynamic operation) + { + IndentedStringBuilder builder = new(); + ReorderPolicyOperationGenerator.Generate(operation, builder); + return builder.ToString(); + } + + [Fact] + public void Generate_Add_with_minimal_details_creates_only_add_policy_sql() + { + // Arrange + AddReorderPolicyOperation operation = new() + { + TableName = "TestTable", + IndexName = "IX_TestTable_Time" + }; + + string expected = @".Sql(@"" + SELECT add_reorder_policy('""""TestTable""""', 'IX_TestTable_Time'); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Add_with_non_default_schedule_creates_add_and_alter_sql() + { + // Arrange + DateTime testDate = new(2025, 10, 20, 12, 30, 0, DateTimeKind.Utc); + AddReorderPolicyOperation operation = new() + { + TableName = "TestTable", + IndexName = "IX_TestTable_Time", + InitialStart = testDate, + ScheduleInterval = "2 days", + MaxRuntime = "1 hour", + MaxRetries = 5, + RetryPeriod = "10 minutes" + }; + + string expected = @".Sql(@"" + SELECT add_reorder_policy('""""TestTable""""', 'IX_TestTable_Time', initial_start => '2025-10-20T12:30:00.0000000Z'); + SELECT alter_job(job_id, schedule_interval => INTERVAL '2 days', max_runtime => INTERVAL '1 hour', max_retries => 5, retry_period => INTERVAL '10 minutes') + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_reorder' AND hypertable_name = 'TestTable'; + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + // --- Tests for DropReorderPolicyOperation --- + + [Fact] + public void Generate_Drop_creates_correct_remove_policy_sql() + { + // Arrange + DropReorderPolicyOperation operation = new() { TableName = "TestTable" }; + + string expected = @".Sql(@"" + SELECT remove_reorder_policy('""""TestTable""""', if_exists => true); + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + // --- Tests for AlterReorderPolicyOperation --- + + [Fact] + public void Generate_Alter_when_only_job_settings_change_creates_only_alter_job_sql() + { + // Arrange + AlterReorderPolicyOperation operation = new() + { + TableName = "TestTable", + // Fundamental properties are the same + IndexName = "IX_TestTable_Time", + OldIndexName = "IX_TestTable_Time", + InitialStart = null, + OldInitialStart = null, + // Job properties have changed + ScheduleInterval = "2 days", + OldScheduleInterval = "1 day" + }; + + string expected = @".Sql(@"" + SELECT alter_job(job_id, schedule_interval => INTERVAL '2 days') + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_reorder' AND hypertable_name = 'TestTable'; + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Alter_when_fundamental_property_changes_creates_drop_and_add_sql() + { + // Arrange + AlterReorderPolicyOperation operation = new() + { + TableName = "TestTable", + IndexName = "IX_New_Name", + OldIndexName = "IX_Old_Name", + ScheduleInterval = "2 days", + OldScheduleInterval = "2 days" + }; + + string expected = @".Sql(@"" + SELECT remove_reorder_policy('""""TestTable""""', if_exists => true); + SELECT add_reorder_policy('""""TestTable""""', 'IX_New_Name'); + SELECT alter_job(job_id, schedule_interval => INTERVAL '2 days') + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_reorder' AND hypertable_name = 'TestTable'; + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + + [Fact] + public void Generate_Alter_when_both_fundamental_and_job_settings_change_creates_full_sequence() + { + // Arrange + AlterReorderPolicyOperation operation = new() + { + TableName = "TestTable", + IndexName = "IX_New_Name", + OldIndexName = "IX_Old_Name", + ScheduleInterval = "2 days", + OldScheduleInterval = "1 day", + MaxRetries = 5, + OldMaxRetries = -1, + RetryPeriod = "10 minutes", + OldRetryPeriod = "10 minutes" + }; + + string expected = @".Sql(@"" + SELECT remove_reorder_policy('""""TestTable""""', if_exists => true); + SELECT add_reorder_policy('""""TestTable""""', 'IX_New_Name'); + SELECT alter_job(job_id, schedule_interval => INTERVAL '2 days', max_retries => 5, retry_period => INTERVAL '10 minutes') + FROM timescaledb_information.jobs + WHERE proc_name = 'policy_reorder' AND hypertable_name = 'TestTable'; + "")"; + + // Act + string result = GetGeneratedCode(operation); + + // Assert + Assert.Equal(SqlHelper.NormalizeSql(expected), SqlHelper.NormalizeSql(result)); + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Utils/SqlHelper.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Utils/SqlHelper.cs new file mode 100644 index 0000000..6a29aed --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests/Utils/SqlHelper.cs @@ -0,0 +1,20 @@ +namespace CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests.Utils +{ + internal static class SqlHelper + { + /// + /// Normalizes a multi-line SQL string for comparison by trimming each line + /// and removing empty lines, making the comparison insensitive to indentation. + /// + public static string NormalizeSql(string sql) + { + // Split into lines, trim each line, and filter out empty ones + IEnumerable lines = sql.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries) + .Select(line => line.Trim()) + .Where(line => !string.IsNullOrWhiteSpace(line)); + + // Join back with a consistent newline character + return string.Join("\n", lines); + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableConvention.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableConvention.cs index 128ad77..b3305a6 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableConvention.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableConvention.cs @@ -49,16 +49,6 @@ public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilde entityTypeBuilder.HasAnnotation(HypertableAnnotations.ChunkSkipColumns, string.Join(",", attribute.ChunkSkipColumns ?? [])); entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true); } - - bool compressionAnnotationValue = attribute.EnableCompression; - if (hasChunkSkipping) - { - entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true); - } - else - { - entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, compressionAnnotationValue); - } } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableTypeBuilder.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableTypeBuilder.cs index f8f83f9..fa5cf7a 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableTypeBuilder.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/Hypertable/HypertableTypeBuilder.cs @@ -105,9 +105,6 @@ public static EntityTypeBuilder WithChunkSkipping( this EntityTypeBuilder entityTypeBuilder, params Expression>[] chunkSkipColumns) where TEntity : class { - // You can't use chunk skipping without compression enabled - entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true); - string[] columnNames = [.. chunkSkipColumns.Select(GetPropertyName)]; entityTypeBuilder.HasAnnotation(HypertableAnnotations.ChunkSkipColumns, string.Join(",", columnNames)); return entityTypeBuilder; diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyAnnotations.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyAnnotations.cs new file mode 100644 index 0000000..e489b74 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyAnnotations.cs @@ -0,0 +1,17 @@ +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy +{ + /// + /// Contains constants for annotations used by the TimescaleDB provider extension. + /// + public static class ReorderPolicyAnnotations + { + public const string HasReorderPolicy = "TimescaleDB:HasReorderPolicy"; + public const string IndexName = "TimescaleDB:ReorderPolicy:IndexName"; + public const string InitialStart = "TimescaleDB:ReorderPolicy:InitialStart"; + + public const string ScheduleInterval = "TimescaleDB:ReorderPolicy:ScheduleInterval"; + public const string MaxRuntime = "TimescaleDB:ReorderPolicy:MaxRuntime"; + public const string MaxRetries = "TimescaleDB:ReorderPolicy:MaxRetries"; + public const string RetryPeriod = "TimescaleDB:ReorderPolicy:RetryPeriod"; + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyAttribute.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyAttribute.cs new file mode 100644 index 0000000..1920bd7 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyAttribute.cs @@ -0,0 +1,67 @@ +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] + public sealed class ReorderPolicyAttribute : Attribute + { + /// + /// Gets the name of the existing index that the reorder policy will use to sort the data. + /// + /// + /// "IX_Readings_DeviceId_Time" + /// + public string IndexName { get; set; } = string.Empty; + + /// + /// Gets or sets the first time the policy job is scheduled to run. + /// Can be specified as a UTC date-time string in ISO 8601 format. + /// If not set, the first run is scheduled based on the schedule_interval. + /// + /// + /// "2025-10-01T03:00:00Z" + /// + public string? InitialStart { get; set; } + + /// + /// Gets or sets the interval at which the reorder policy job runs. + /// If not set, it defaults to '1 day'. + /// + /// + /// "2 days" + /// + public string? ScheduleInterval { get; set; } + + /// + /// Gets or sets the maximum amount of time the job is allowed to run before being stopped. + /// If not set, there is no time limit. + /// + /// + /// "1 hour" + /// + public string? MaxRuntime { get; set; } + + /// + /// Gets or sets the number of times the job is retried if it fails. + /// If not set, it defaults to -1 (retry indefinitely). + /// + public int MaxRetries { get; set; } = -1; + + /// + /// Gets or sets the amount of time the scheduler waits between retries of a failed job. + /// If not set, it defaults to '00:05:00'. + /// + /// + /// "30 minutes" + /// + public string? RetryPeriod { get; set; } + + public ReorderPolicyAttribute(string indexName) + { + if (string.IsNullOrWhiteSpace(indexName)) + { + throw new ArgumentException("IndexName must be provided.", nameof(indexName)); + } + + IndexName = indexName; + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyConvention.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyConvention.cs new file mode 100644 index 0000000..377e3da --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyConvention.cs @@ -0,0 +1,56 @@ +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Metadata.Conventions; +using System.Reflection; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy +{ + /// + /// A convention that configures the reorder policy for a hypertable based on the presence of + /// the [ReorderPolicy] attribute. + /// + public class ReorderPolicyConvention : IEntityTypeAddedConvention + { + /// + /// Called when an entity type is added to the model. + /// + /// The builder for the entity type. + /// Additional information available during convention execution. + public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilder, IConventionContext context) + { + IConventionEntityType entityType = entityTypeBuilder.Metadata; + ReorderPolicyAttribute? attribute = entityType.ClrType?.GetCustomAttribute(); + + if (attribute != null) + { + // Apply the annotations that the Fluent API would have applied. + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.HasReorderPolicy, true); + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.IndexName, attribute.IndexName); + + if (!string.IsNullOrWhiteSpace(attribute.InitialStart)) + { + if (DateTime.TryParse(attribute.InitialStart, out DateTime parsedDateTimeOffset)) + { + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.ScheduleInterval, parsedDateTimeOffset); + } + else + { + throw new InvalidOperationException($"InitialStart '{attribute.InitialStart}' is not a valid DateTime format. Please use a valid DateTime string."); + } + } + + if (!string.IsNullOrWhiteSpace(attribute.ScheduleInterval)) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.ScheduleInterval, attribute.ScheduleInterval); + + if (!string.IsNullOrWhiteSpace(attribute.MaxRuntime)) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.MaxRuntime, attribute.MaxRuntime); + + if (attribute.MaxRetries > -1) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.MaxRetries, attribute.MaxRetries); + + if (!string.IsNullOrWhiteSpace(attribute.RetryPeriod)) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.RetryPeriod, attribute.RetryPeriod); + } + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyTypeBulder.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyTypeBulder.cs new file mode 100644 index 0000000..9d37e83 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Configuration/ReorderPolicy/ReorderPolicyTypeBulder.cs @@ -0,0 +1,66 @@ +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy +{ + /// + /// Provides extension methods for configuring TimescaleDB hypertables reorder policies using the EF Core Fluent API. + /// + public static class ReorderPolicyTypeBuilder + { + /// + /// Configures a TimescaleDB reorder policy for the entity using a fluent API. + /// + /// + /// A reorder policy physically reorders data on disk according to a specified index to improve query performance. + /// This method applies the necessary annotations to the entity's metadata, which are then used by the custom + /// migrations infrastructure to generate the appropriate add_reorder_policy() and alter_job() SQL commands. + /// + /// + /// + /// modelBuilder.Entity<DeviceReading>() + /// .WithReorderPolicy( + /// indexName: "IX_DeviceReadings_DeviceId_Time", + /// scheduleInterval: "2 days", + /// maxRetries: 5); + /// + /// + /// The type of the entity being configured. + /// The builder for the entity type being configured. + /// The name of the existing index that the reorder policy will use to sort the data. + /// The first time the policy job is scheduled to run. If null, it's based on the schedule interval. + /// The interval at which the reorder policy job runs. Defaults to '1 day' if not specified. + /// The maximum amount of time the job is allowed to run. If null, there is no time limit. + /// The number of times the job is retried if it fails. Defaults to -1 (retry indefinitely) if not specified. + /// The amount of time the scheduler waits between retries. Defaults to '1 hour' if not specified. + /// The same builder instance so that multiple calls can be chained. + public static EntityTypeBuilder WithReorderPolicy( + this EntityTypeBuilder entityTypeBuilder, + string indexName, + DateTime? initialStart = null, + string? scheduleInterval = null, + string? maxRuntime = null, + int? maxRetries = null, + string? retryPeriod = null) where TEntity : class + { + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.HasReorderPolicy, true); + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.IndexName, indexName); + + if (initialStart.HasValue) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.InitialStart, initialStart); + + if (!string.IsNullOrWhiteSpace(scheduleInterval)) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.ScheduleInterval, scheduleInterval); + + if (!string.IsNullOrWhiteSpace(maxRuntime)) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.MaxRuntime, maxRuntime); + + if (maxRetries.HasValue) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.MaxRetries, maxRetries.Value); + + if (!string.IsNullOrWhiteSpace(retryPeriod)) + entityTypeBuilder.HasAnnotation(ReorderPolicyAnnotations.RetryPeriod, retryPeriod); + + return entityTypeBuilder; + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/DefaultValues.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/DefaultValues.cs index 271cceb..8b6d3e4 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/DefaultValues.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/DefaultValues.cs @@ -7,5 +7,9 @@ public static class DefaultValues { public const string ChunkTimeInterval = "7 days"; public const long ChunkTimeIntervalLong = 604_800_000_000L; + public const string ReorderPolicyScheduleInterval = "1 day"; + public const int ReorderPolicyMaxRetries = -1; + public const string ReorderPolicyMaxRuntime = "00:00:00"; + public const string ReorderPolicyRetryPeriod = "00:05:00"; } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AddReorderPolicyOperation.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AddReorderPolicyOperation.cs new file mode 100644 index 0000000..3fd646f --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AddReorderPolicyOperation.cs @@ -0,0 +1,15 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Operations +{ + public class AddReorderPolicyOperation : MigrationOperation + { + public string TableName { get; set; } = string.Empty; + public string IndexName { get; set; } = string.Empty; + public DateTime? InitialStart { get; set; } + public string? ScheduleInterval { get; set; } + public string? MaxRuntime { get; set; } + public int? MaxRetries { get; set; } + public string? RetryPeriod { get; set; } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterHypertableOperation.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterHypertableOperation.cs index 94f489a..2af4570 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterHypertableOperation.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterHypertableOperation.cs @@ -5,16 +5,16 @@ namespace CmdScale.EntityFrameworkCore.TimescaleDB.Operations public class AlterHypertableOperation : MigrationOperation { public string TableName { get; set; } = string.Empty; - public string ChunkTimeInterval { get; set; } = DefaultValues.ChunkTimeInterval; - public bool EnableCompression { get; set; } = false; + public string ChunkTimeInterval { get; set; } = string.Empty; + public bool EnableCompression { get; set; } - // For chunk skipping, you need to anebl it with SET timescaledb.enable_chunk_skipping = 'on' + // For chunk skipping, you need to enable it with SET timescaledb.enable_chunk_skipping = 'on' // Only timestamp-like and Integer-like columns are supported for chunk skipping // Cannot be reverted once enabled - public IReadOnlyList? ChunkSkipColumns { get; set; } = null; + public IReadOnlyList? ChunkSkipColumns { get; set; } - public string OldChunkTimeInterval { get; set; } = DefaultValues.ChunkTimeInterval; - public bool OldEnableCompression { get; set; } = false; - public IReadOnlyList? OldChunkSkipColumns { get; set; } = null; + public string OldChunkTimeInterval { get; set; } = string.Empty; + public bool OldEnableCompression { get; set; } + public IReadOnlyList? OldChunkSkipColumns { get; set; } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterReorderPolicyOperation.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterReorderPolicyOperation.cs new file mode 100644 index 0000000..436649b --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/AlterReorderPolicyOperation.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Operations +{ + public class AlterReorderPolicyOperation : MigrationOperation + { + public string TableName { get; set; } = string.Empty; + + public string IndexName { get; set; } = string.Empty; + public DateTime? InitialStart { get; set; } + public string? ScheduleInterval { get; set; } + public string? MaxRuntime { get; set; } + public int? MaxRetries { get; set; } + public string? RetryPeriod { get; set; } + + public string OldIndexName { get; set; } = string.Empty; + public DateTime? OldInitialStart { get; set; } + public string? OldScheduleInterval { get; set; } + public string? OldMaxRuntime { get; set; } + public int? OldMaxRetries { get; set; } + public string? OldRetryPeriod { get; set; } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/CreateHypertableOperation.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/CreateHypertableOperation.cs index bbe49a0..330b910 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/CreateHypertableOperation.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/CreateHypertableOperation.cs @@ -7,9 +7,9 @@ public class CreateHypertableOperation : MigrationOperation { public string TableName { get; set; } = string.Empty; public string TimeColumnName { get; set; } = string.Empty; - public string ChunkTimeInterval { get; set; } = DefaultValues.ChunkTimeInterval; - public bool EnableCompression { get; set; } = false; - public IReadOnlyList? ChunkSkipColumns { get; set; } = null; - public IReadOnlyList? AdditionalDimensions { get; set; } = null; + public string ChunkTimeInterval { get; set; } = string.Empty; + public bool EnableCompression { get; set; } + public IReadOnlyList? ChunkSkipColumns { get; set; } + public IReadOnlyList? AdditionalDimensions { get; set; } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/DropReorderPolicyOperation.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/DropReorderPolicyOperation.cs new file mode 100644 index 0000000..e17a7a6 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Operations/DropReorderPolicyOperation.cs @@ -0,0 +1,9 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Operations +{ + public class DropReorderPolicyOperation : MigrationOperation + { + public string TableName { get; set; } = string.Empty; + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbContextOptionsBuilderExtensions.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbContextOptionsBuilderExtensions.cs index c71246e..491f322 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbContextOptionsBuilderExtensions.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbContextOptionsBuilderExtensions.cs @@ -1,4 +1,5 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata.Conventions; @@ -72,6 +73,7 @@ public class HypertableConventionSetPlugin : IConventionSetPlugin public ConventionSet ModifyConventions(ConventionSet conventionSet) { conventionSet.EntityTypeAddedConventions.Add(new HypertableConvention()); + conventionSet.EntityTypeAddedConventions.Add(new ReorderPolicyConvention()); return conventionSet; } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleMigrationsModelDiffer.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleMigrationsModelDiffer.cs index 6c76beb..d686b88 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleMigrationsModelDiffer.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleMigrationsModelDiffer.cs @@ -1,5 +1,6 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ReorderPolicy; using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -30,6 +31,8 @@ public override IReadOnlyList GetDifferences(IRelationalMode { // Get the standard migration operations (CreateTable, AddColumn, etc.) from the base MigrationsModelDiffer. List operations = [.. base.GetDifferences(source, target)]; + + // Hypertable diffs List targetHypertables = [.. GetHypertables(target)]; List sourceHypertables = [.. GetHypertables(source)]; @@ -80,6 +83,57 @@ op is CreateTableOperation createTable && operations.Add(alterOperation); } + // Reorder diffs + List sourcePolicies = [.. GetReorderPolicies(source)]; + List targetPolicies = [.. GetReorderPolicies(target)]; + + // Identiy new reorder policies + IEnumerable newReorderPolicies = targetPolicies.Where(t => !sourcePolicies.Any(s => s.TableName == t.TableName)); + operations.AddRange(newReorderPolicies); + + // Identify updated reorder policies + var updatedReorderPolicies = targetPolicies + .Join( + sourcePolicies, + targetPolicy => targetPolicy.TableName, + sourcePolicy => sourcePolicy.TableName, + (targetPolicy, sourcePolicy) => new { Target = targetPolicy, Source = sourcePolicy } + ) + .Where(x => + x.Target.IndexName != x.Source.IndexName || + x.Target.InitialStart != x.Source.InitialStart || + x.Target.ScheduleInterval != x.Source.ScheduleInterval || + x.Target.MaxRuntime != x.Source.MaxRuntime || + x.Target.MaxRetries != x.Source.MaxRetries || + x.Target.RetryPeriod != x.Source.RetryPeriod + ); + + foreach (var policy in updatedReorderPolicies) + { + operations.Add(new AlterReorderPolicyOperation + { + TableName = policy.Target.TableName, + IndexName = policy.Target.IndexName, + InitialStart = policy.Target.InitialStart, + ScheduleInterval = policy.Target.ScheduleInterval, + MaxRuntime = policy.Target.MaxRuntime, + MaxRetries = policy.Target.MaxRetries, + RetryPeriod = policy.Target.RetryPeriod, + + OldIndexName = policy.Source.IndexName, + OldInitialStart = policy.Source.InitialStart, + OldScheduleInterval = policy.Source.ScheduleInterval, + OldMaxRuntime = policy.Source.MaxRuntime, + OldMaxRetries = policy.Source.MaxRetries, + OldRetryPeriod = policy.Source.RetryPeriod + }); + } + + IEnumerable removedReorderPolicies = sourcePolicies + .Where(s => !targetPolicies.Any(t => t.TableName == s.TableName)) + .Select(p => new DropReorderPolicyOperation { TableName = p.TableName }); + operations.AddRange(removedReorderPolicies); + return operations; } @@ -118,7 +172,7 @@ private static IEnumerable GetHypertables(IRelational { TableName = entityType.GetTableName()!, TimeColumnName = timeColumnName, - ChunkTimeInterval = chunkTimeInterval, + ChunkTimeInterval = chunkTimeInterval ?? DefaultValues.ChunkTimeInterval, EnableCompression = enableCompression, ChunkSkipColumns = chunkSkipColumns, AdditionalDimensions = additionalDimensions @@ -127,6 +181,35 @@ private static IEnumerable GetHypertables(IRelational } } + private static IEnumerable GetReorderPolicies(IRelationalModel? relationalModel) + { + if (relationalModel == null) + { + yield break; + } + foreach (IEntityType entityType in relationalModel.Model.GetEntityTypes()) + { + // Retrieve the annotations set by the convention + bool hasReorderPolicy = entityType.FindAnnotation(ReorderPolicyAnnotations.HasReorderPolicy)?.Value as bool? ?? false; + string? indexName = entityType.FindAnnotation(ReorderPolicyAnnotations.IndexName)?.Value as string; + DateTime? initialStart = entityType.FindAnnotation(ReorderPolicyAnnotations.InitialStart)?.Value as DateTime?; + + if (hasReorderPolicy && !string.IsNullOrWhiteSpace(indexName) && !string.IsNullOrWhiteSpace(indexName)) + { + yield return new AddReorderPolicyOperation + { + TableName = entityType.GetTableName()!, + IndexName = indexName!, + InitialStart = initialStart, + ScheduleInterval = entityType.FindAnnotation(ReorderPolicyAnnotations.ScheduleInterval)?.Value as string ?? DefaultValues.ReorderPolicyScheduleInterval, + MaxRuntime = entityType.FindAnnotation(ReorderPolicyAnnotations.MaxRuntime)?.Value as string ?? DefaultValues.ReorderPolicyMaxRuntime, + MaxRetries = entityType.FindAnnotation(ReorderPolicyAnnotations.MaxRetries)?.Value as int? ?? DefaultValues.ReorderPolicyMaxRetries, + RetryPeriod = entityType.FindAnnotation(ReorderPolicyAnnotations.RetryPeriod)?.Value as string ?? DefaultValues.ReorderPolicyRetryPeriod + }; + } + } + } + // Helper method to compare two lists of chunk skip columns private static bool AreChunkSkipColumnsEqual(IReadOnlyList? list1, IReadOnlyList? list2) {