diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/BulkCopyToAsyncBenchmarks.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/BulkCopyToAsyncBenchmarks.cs deleted file mode 100644 index e843228..0000000 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/BulkCopyToAsyncBenchmarks.cs +++ /dev/null @@ -1,67 +0,0 @@ -using BenchmarkDotNet.Attributes; -using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; -using CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models; - -namespace CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks -{ - [Config(typeof(InProcessConfig))] - [MemoryDiagnoser] - [ThreadingDiagnoser] - public class BulkCopyToAsyncBenchmarks - { - [Params(50_000)] - public int NumberOfRecords; - - [Params(5000, 10_000, 20_000, 30_000, 50_000)] - public int MaxBatchSize; - - [Params(1, 2, 4, 8)] - public int NumberOfWorkers; - - private readonly string _connectionString = "Host=localhost;Database=cmdscale-ef-timescaledb;Username=timescale_admin;Password=R#!kro#GP43ra8Ae;Include Error Detail=True"; - private readonly List trades = []; - - [IterationSetup] - public void IterationSetup() - { - trades.Clear(); - - // --- Data Variety Setup --- - Random random = new(); - string[] tickers = ["AAPL", "GOOGL", "MSFT", "TSLA", "AMZN", "NVDA", "JPM", "V"]; - string[] exchanges = ["NASDAQ", "NYSE", "ARCA"]; - DateTime baseTimestamp = DateTime.UtcNow.AddMinutes(-30); - Dictionary basePrices = tickers.ToDictionary(t => t, t => (decimal)(100 + random.NextDouble() * 400)); - - // --- Data Generation Loop --- - for (int i = 0; i < NumberOfRecords; i++) - { - string currentTicker = tickers[random.Next(tickers.Length)]; - decimal priceJitter = (decimal)(random.NextDouble() * 2 - 1); - decimal currentPrice = basePrices[currentTicker] + priceJitter; - - trades.Add(new Trade - { - Timestamp = baseTimestamp.AddMicroseconds(i), - Ticker = currentTicker, - Price = Math.Round(currentPrice, 2), - Size = random.Next(1, 2500), - Exchange = exchanges[random.Next(exchanges.Length)], - }); - } - - Console.WriteLine($"Generated {trades.Count} records."); - } - - [Benchmark] - public async Task BulkCopyAsyncPerformance() - { - TimescaleCopyConfig config = new TimescaleCopyConfig() - .ToTable("Trades") - .WithWorkers(NumberOfWorkers) - .WithBatchSize(MaxBatchSize); - - await trades.BulkCopyToAsync(_connectionString, config); - } - } -} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks.csproj b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks.csproj index ac74926..503b9b0 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks.csproj +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks.csproj @@ -13,6 +13,8 @@ + + diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/README.md b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/README.md index 73c0a54..408cc6b 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/README.md +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/README.md @@ -7,59 +7,9 @@ This project uses **BenchmarkDotNet** to measure the performance of high-through ## Prerequisites - .NET 8 SDK or later -- Docker and Docker Compose +- Docker ---- - -## Step 1: Start a Clean Database - -The benchmarks require a running TimescaleDB instance. A `docker-compose.yml` file is provided in the project root to simplify this process. - -### 🔄 Stop and Reset (if needed) - -To ensure you start with a clean slate, run this command to stop any running containers and permanently delete all existing data. - -```bash -docker compose down -v -``` - -### ▶️ Start the Database - -Launch a new TimescaleDB instance in the background. - -```bash -docker compose up -d -``` - -### ✅ Verify Connection String - -Ensure the connection string in `BulkCopyToAsyncBenchmarks.cs` matches the settings in your `docker-compose.yml` file (the default should work). - ---- - -## Step 2: Apply Migrations - -Next, create the necessary tables in the new database using EF Core migrations. - -### ➕ Add a Migration - -Create a new migration. Run this from the root of the solution. - -```bash -dotnet ef migrations add --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess --startup-project CmdScale.EntityFrameworkCore.TimescaleDB.Example -``` - -### ⬆️ Update the Database - -Apply the migrations to create the hypertable schema. - -```bash -dotnet ef database update --project CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess --startup-project CmdScale.EntityFrameworkCore.TimescaleDB.Example -``` - ---- - -## Step 3: Run the Benchmarks +## Run the Benchmarks Once the database is set up, you can run the performance tests. diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsBenchmarkBase.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsBenchmarkBase.cs new file mode 100644 index 0000000..4a3d0d8 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsBenchmarkBase.cs @@ -0,0 +1,68 @@ +using BenchmarkDotNet.Attributes; +using CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess; +using Microsoft.EntityFrameworkCore; +using Testcontainers.PostgreSql; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks +{ + public abstract class WriteRecordsBenchmarkBase where T : class + { + public int NumberOfRecords; + public int MaxBatchSize; + public int NumberOfWorkers; + + private readonly PostgreSqlContainer _dbContainer = new PostgreSqlBuilder() + .WithImage("timescale/timescaledb:latest-pg17") + .WithDatabase("benchmark_tests_db") + .WithUsername("test_user") + .WithPassword("test_password") + .Build(); + + protected string ConnectionString = ""; + protected readonly List Trades = []; + protected TimescaleContext? Context; + + [GlobalSetup] + public async Task Setup() + { + await _dbContainer.StartAsync(); + ConnectionString = _dbContainer.GetConnectionString(); + + DbContextOptionsBuilder optionsBuilder = new(); + optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + Context = new TimescaleContext(optionsBuilder.Options); + + await Context.Database.MigrateAsync(); + Console.WriteLine("Migration applied successfully."); + } + + [GlobalCleanup] + public async Task GlobalCleanup() + { + await _dbContainer.DisposeAsync(); + } + + [IterationSetup] + public void IterationSetup() + { + Trades.Clear(); + var random = new Random(); + string[] tickers = ["AAPL", "GOOGL", "MSFT", "TSLA", "AMZN"]; + var baseTimestamp = DateTime.UtcNow.AddMinutes(-30); + + for (int i = 0; i < NumberOfRecords; i++) + { + var trade = CreateTradeInstance(i, baseTimestamp, tickers[random.Next(tickers.Length)], random); + Trades.Add(trade); + } + + // Truncate the table before each iteration for a clean slate + string tableName = GetTableName(); + string sql = $"TRUNCATE TABLE \"{tableName}\""; + Context!.Database.ExecuteSqlRaw(sql); + } + + protected abstract T CreateTradeInstance(int index, DateTime baseTimestamp, string ticker, Random random); + protected abstract string GetTableName(); + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsKeylessBenchmarks.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsKeylessBenchmarks.cs new file mode 100644 index 0000000..7a98a4a --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsKeylessBenchmarks.cs @@ -0,0 +1,110 @@ +using BenchmarkDotNet.Attributes; +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models; +using Npgsql; +using NpgsqlTypes; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks +{ + [Config(typeof(InProcessConfig))] + [MemoryDiagnoser] + [ThreadingDiagnoser] + public class WriteRecordsKeylessBenchmarks : WriteRecordsBenchmarkBase + { + [Params(100_000, 500_000)] + public new int NumberOfRecords; + + [Params(25_000, 50_000, 100_000)] + public new int MaxBatchSize; + + [Params(8)] + public new int NumberOfWorkers; + + private readonly List tasks = []; + private int totalRecords; + private int workerChunkSize; + + protected override Trade CreateTradeInstance(int index, DateTime baseTimestamp, string ticker, Random random) + { + return new Trade + { + Timestamp = baseTimestamp.AddMicroseconds(index), + Ticker = ticker, + Price = (decimal)(100 + random.NextDouble() * 400), + Size = random.Next(1, 100) + }; + } + + protected override string GetTableName() => "Trades"; + + [Benchmark] + public async Task BulkCopyAsync() + { + TimescaleCopyConfig config = new TimescaleCopyConfig() + .ToTable(GetTableName()) + .WithWorkers(NumberOfWorkers) + .WithBatchSize(MaxBatchSize); + + await Trades.BulkCopyAsync(ConnectionString, config); + } + + [Benchmark] + public async Task HardcodedBulkCopyAsync() + { + totalRecords = Trades.Count; + workerChunkSize = (int)Math.Ceiling((double)totalRecords / NumberOfWorkers); + + for (int i = 0; i < NumberOfWorkers; i++) + { + int startIndex = i * workerChunkSize; + int currentWorkerDataSize = Math.Min(workerChunkSize, totalRecords - startIndex); + + if (currentWorkerDataSize <= 0) + { + break; + } + + List workerData = [.. Trades.Skip(startIndex).Take(currentWorkerDataSize)]; + tasks.Add(Task.Run(async () => + { + // Open new connection to DB + using NpgsqlConnection connection = new(ConnectionString); + await connection.OpenAsync(); + + // Command to copy data in a binary format from a client-application + string copyCommand = "COPY \"Trades\" (\"Time\", \"Value\", \"SegmentId\", \"SignalId\") FROM STDIN (FORMAT BINARY)"; + + for (int j = 0; j < workerData.Count; j += MaxBatchSize) + { + List currentBatch = [.. workerData.Skip(j).Take(MaxBatchSize)]; + + // Start a binary import stream + await using NpgsqlBinaryImporter writer = connection.BeginBinaryImport(copyCommand); + foreach (Trade item in currentBatch) + { + // IMPORTANT: Columns must be inserted in the exact order + await writer.WriteAsync(item.Timestamp, NpgsqlDbType.TimestampTz); + await writer.WriteAsync(item.Ticker, NpgsqlDbType.Text); + await writer.WriteAsync(item.Price, NpgsqlDbType.Numeric); + await writer.WriteAsync(item.Size, NpgsqlDbType.Integer); + await writer.WriteAsync(item.Exchange, NpgsqlDbType.Text); + } + + await writer.CompleteAsync(); + } + })); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task BatchedBulkInsertOptimizedAsync() + { + foreach (Trade[] chunk in Trades.Chunk(MaxBatchSize)) + { + await Context!.BulkInsertOptimizedAsync(chunk.ToList()); + } + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsWithKeyBenchmarks.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsWithKeyBenchmarks.cs new file mode 100644 index 0000000..010cb3d --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks/WriteRecordsWithKeyBenchmarks.cs @@ -0,0 +1,120 @@ +using BenchmarkDotNet.Attributes; +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models; +using Npgsql; +using NpgsqlTypes; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Benchmarks +{ + [Config(typeof(InProcessConfig))] + [MemoryDiagnoser] + [ThreadingDiagnoser] + public class WriteRecordsWithKeyBenchmarks : WriteRecordsBenchmarkBase + { + [Params(1_000, 5_000, 10_000)] + public new int NumberOfRecords; + + [Params(500, 1_000, 5_000)] + public new int MaxBatchSize; + + [Params(1, 4, 8)] + public new int NumberOfWorkers; + + private readonly List tasks = []; + private int totalRecords; + private int workerChunkSize; + + protected override TradeWithId CreateTradeInstance(int index, DateTime baseTimestamp, string ticker, Random random) + { + return new TradeWithId + { + Timestamp = baseTimestamp.AddMicroseconds(index), + Ticker = ticker, + Price = (decimal)(100 + random.NextDouble() * 400), + Size = random.Next(1, 100) + }; + } + + protected override string GetTableName() => "TradesWithId"; + + [Benchmark] + public Task BulkCopyAsync() + { + TimescaleCopyConfig config = new TimescaleCopyConfig() + .ToTable(GetTableName()) + .WithWorkers(NumberOfWorkers) + .WithBatchSize(MaxBatchSize); + return Trades.BulkCopyAsync(ConnectionString, config); + } + + [Benchmark] + public async Task HardcodedBulkCopyAsync() + { + totalRecords = Trades.Count; + workerChunkSize = (int)Math.Ceiling((double)totalRecords / NumberOfWorkers); + + for (int i = 0; i < NumberOfWorkers; i++) + { + int startIndex = i * workerChunkSize; + int currentWorkerDataSize = Math.Min(workerChunkSize, totalRecords - startIndex); + + if (currentWorkerDataSize <= 0) + { + break; + } + + List workerData = [.. Trades.Skip(startIndex).Take(currentWorkerDataSize)]; + tasks.Add(Task.Run(async () => + { + // Open new connection to DB + using NpgsqlConnection connection = new(ConnectionString); + await connection.OpenAsync(); + + // Command to copy data in a binary format from a client-application + string copyCommand = "COPY \"TradesWithId\" (\"Time\", \"Value\", \"SegmentId\", \"SignalId\") FROM STDIN (FORMAT BINARY)"; + + for (int j = 0; j < workerData.Count; j += MaxBatchSize) + { + List currentBatch = [.. workerData.Skip(j).Take(MaxBatchSize)]; + + // Start a binary import stream + await using NpgsqlBinaryImporter writer = connection.BeginBinaryImport(copyCommand); + foreach (TradeWithId item in currentBatch) + { + // IMPORTANT: Columns must be inserted in the exact order + await writer.WriteAsync(item.Timestamp, NpgsqlDbType.TimestampTz); + await writer.WriteAsync(item.Ticker, NpgsqlDbType.Text); + await writer.WriteAsync(item.Price, NpgsqlDbType.Numeric); + await writer.WriteAsync(item.Size, NpgsqlDbType.Integer); + await writer.WriteAsync(item.Exchange, NpgsqlDbType.Text); + } + + await writer.CompleteAsync(); + } + })); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task BatchedSaveChangesAsync() + { + foreach (TradeWithId[] chunk in Trades.Chunk(MaxBatchSize)) + { + Context!.AddRange(chunk); + await Context!.SaveChangesAsync(); + Context.ChangeTracker.Clear(); + } + } + + [Benchmark] + public async Task BatchedBulkInsertOptimizedAsync() + { + foreach (TradeWithId[] chunk in Trades.Chunk(MaxBatchSize)) + { + await Context!.BulkInsertOptimizedAsync(chunk.ToList()); + } + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/CmdScale.EntityFrameworkCore.TimescaleDB.Design.csproj b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/CmdScale.EntityFrameworkCore.TimescaleDB.Design.csproj index 1274f9a..27bed89 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Design/CmdScale.EntityFrameworkCore.TimescaleDB.Design.csproj +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Design/CmdScale.EntityFrameworkCore.TimescaleDB.Design.csproj @@ -22,7 +22,6 @@ timescaledb;timescale;efcore;ef-core;entityframeworkcore;postgresql;postgres;time-series;timeseries;data;database;efcore-provider;provider;design;migrations;scaffolding;codegen;cli;tools - @@ -41,4 +40,7 @@ + + + \ No newline at end of file diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst.csproj b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst.csproj index 32f0f0e..e141375 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst.csproj +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.DbFirst.csproj @@ -1,13 +1,13 @@  + + + + net8.0 enable enable - - - - diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.csproj b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.csproj index 62a1544..10afeef 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.csproj +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.csproj @@ -1,13 +1,13 @@  + + + + net8.0 enable enable - - - - diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs index f588b12..607bbcf 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeConfiguration.cs @@ -9,6 +9,7 @@ public class TradeConfiguration : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.ToTable("Trades"); builder.HasNoKey() .IsHypertable(x => x.Timestamp) .WithChunkTimeInterval("1 day"); diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs new file mode 100644 index 0000000..9a5a949 --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Configurations/TradeWithIdConfiguration.cs @@ -0,0 +1,18 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Configurations +{ + public class TradeWithIdConfiguration : IEntityTypeConfiguration + { + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("TradesWithId"); + builder.HasKey(x => new { x.Id, x.Timestamp }); + builder.IsHypertable(x => x.Timestamp) + .WithChunkTimeInterval("1 day"); + } + } +} diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs new file mode 100644 index 0000000..4c2708b --- /dev/null +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Models/TradeWithId.cs @@ -0,0 +1,43 @@ +using System.ComponentModel.DataAnnotations; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models +{ + /// + /// Represents a single trade event with a standard primary key. + /// Used for benchmark comparison against keyless or composite-key entities. + /// + public class TradeWithId + { + /// + /// The unique identifier for the trade record. + /// This is the primary key. + /// + [Key] + public long Id { get; set; } + + /// + /// The precise UTC timestamp when the trade was executed. + /// + public DateTime Timestamp { get; set; } + + /// + /// The stock ticker symbol (e.g., "TSLA", "AAPL"). + /// + public string Ticker { get; set; } = string.Empty; + + /// + /// The price at which the trade was executed. + /// + public decimal Price { get; set; } + + /// + /// The number of shares traded. + /// + public int Size { get; set; } + + /// + /// The exchange where the trade occurred (e.g., "NASDAQ", "NYSE"). + /// + public string Exchange { get; set; } = string.Empty; + } +} \ No newline at end of file diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Repositories/TradeRepository.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Repositories/TradeRepository.cs index f6cca68..fb3f643 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Repositories/TradeRepository.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/Repositories/TradeRepository.cs @@ -20,7 +20,7 @@ public async Task IngestTradesAsync(List trades) .WithWorkers(8) .WithBatchSize(20_000); - await trades.BulkCopyToAsync(_connectionString, config); + await trades.BulkCopyAsync(_connectionString, config); } /// @@ -43,7 +43,7 @@ public async Task IngestTradesAsyncWithColumnMapping(List trades) .MapColumn("Size", t => t.Size, NpgsqlDbType.Integer) .MapColumn("Exchange", t => t.Exchange, NpgsqlDbType.Text); - await trades.BulkCopyToAsync(_connectionString, config); + await trades.BulkCopyAsync(_connectionString, config); } /// @@ -53,7 +53,7 @@ public async Task IngestTradesAsyncWithColumnMapping(List trades) /// public async Task IngestTradesAsyncWithDefaultConfig(List trades) { - await trades.BulkCopyToAsync(_connectionString); + await trades.BulkCopyAsync(_connectionString); } } } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/TimescaleContext.cs b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/TimescaleContext.cs index 49f66a2..89eb315 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/TimescaleContext.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess/TimescaleContext.cs @@ -10,6 +10,7 @@ public class TimescaleContext(DbContextOptions options) : DbCo public DbSet WeatherData { get; set; } public DbSet OrderStatusEvents { get; set; } public DbSet Trades { get; set; } + public DbSet TradesWithId { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB.Example/CmdScale.EntityFrameworkCore.TimescaleDB.Example.csproj b/CmdScale.EntityFrameworkCore.TimescaleDB.Example/CmdScale.EntityFrameworkCore.TimescaleDB.Example.csproj index d2dd4af..beec9d6 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB.Example/CmdScale.EntityFrameworkCore.TimescaleDB.Example.csproj +++ b/CmdScale.EntityFrameworkCore.TimescaleDB.Example/CmdScale.EntityFrameworkCore.TimescaleDB.Example.csproj @@ -8,12 +8,12 @@ - + diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/Abstractions/TimescaleCopyConfig.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/Abstractions/TimescaleCopyConfig.cs index 260a077..a815cc2 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/Abstractions/TimescaleCopyConfig.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/Abstractions/TimescaleCopyConfig.cs @@ -42,7 +42,7 @@ public class TimescaleCopyConfig /// The order of the properties in this dictionary is critical. It must precisely match the /// column order in the SQL COPY command (and in the database) to ensure a successful binary copy operation. /// - public Dictionary ColumnMappings { get; } = []; + public Dictionary Getter, NpgsqlDbType DbType)> ColumnMappings { get; } = []; public TimescaleCopyConfig() { @@ -57,8 +57,13 @@ public TimescaleCopyConfig() // Attempt to map the C# type to an NpgsqlDbType if (MapClrTypeToNpgsqlDbType(property.PropertyType, out NpgsqlDbType dbType)) { - // By default, the column name is the same as the property name - ColumnMappings[property.Name] = (property, dbType); + // Auto-discover properties and create compiled getters for them. + var parameter = Expression.Parameter(typeof(T), "x"); + var member = Expression.Property(parameter, property); + var conversion = Expression.Convert(member, typeof(object)); + var lambda = Expression.Lambda>(conversion, parameter); + + ColumnMappings[property.Name] = (lambda.Compile(), dbType); } } } @@ -108,26 +113,8 @@ public TimescaleCopyConfig WithBatchSize(int maxBatchSize) /// The same configuration instance for fluent chaining. public TimescaleCopyConfig MapColumn(string columnName, Expression> propertySelector, NpgsqlDbType dbType) { - MemberExpression memberExpression; - - // Check if the expression body is a direct member access or needs to be unwrapped from a convert operation. - if (propertySelector.Body is MemberExpression directMember) - { - memberExpression = directMember; - } - else if (propertySelector.Body is UnaryExpression unary && unary.Operand is MemberExpression indirectMember) - { - memberExpression = indirectMember; - } - else - { - throw new ArgumentException("Expression must be a property selector.", nameof(propertySelector)); - } - - MemberExpression memberExpr = memberExpression ?? (MemberExpression)((UnaryExpression)propertySelector.Body).Operand; - PropertyInfo propertyInfo = (PropertyInfo)memberExpr.Member; - - ColumnMappings[columnName] = (propertyInfo, dbType); + Func getter = propertySelector.Compile(); + ColumnMappings[columnName] = (getter, dbType); return this; } diff --git a/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbCopyExtensions.cs b/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbCopyExtensions.cs index 6392ee7..3324c2f 100644 --- a/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbCopyExtensions.cs +++ b/CmdScale.EntityFrameworkCore.TimescaleDB/TimescaleDbCopyExtensions.cs @@ -1,7 +1,5 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using Npgsql; -using NpgsqlTypes; -using System.Reflection; namespace CmdScale.EntityFrameworkCore.TimescaleDB { @@ -16,7 +14,7 @@ public static class TimescaleDbCopyExtensions /// The database connection string. /// A object that configures the bulk copy operation, including table name, column mappings, and parallelism. /// A that represents the asynchronous completion of the entire bulk copy operation. - public static async Task BulkCopyToAsync( + public static async Task BulkCopyAsync( this IEnumerable data, string connectionString, TimescaleCopyConfig? config = null) @@ -28,16 +26,15 @@ public static async Task BulkCopyToAsync( // Create parallel workers to ingest the data List tasks = []; - List dataList = [.. data]; - int totalRecords = dataList.Count; + int totalRecords = data.Count(); int workerChunkSize = (int)Math.Ceiling((double)totalRecords / config.NumberOfWorkers); for (int i = 0; i < config.NumberOfWorkers; i++) { int startIndex = i * workerChunkSize; - List workerData = [.. dataList.Skip(startIndex).Take(workerChunkSize)]; + IEnumerable workerData = [.. data.Skip(startIndex).Take(workerChunkSize)]; - if (workerData.Count == 0) + if (!workerData.Any()) { break; } @@ -47,7 +44,7 @@ public static async Task BulkCopyToAsync( using NpgsqlConnection connection = new(connectionString); await connection.OpenAsync(); - for (int j = 0; j < workerData.Count; j += config.MaxBatchSize) + for (int j = 0; j < workerData.Count(); j += config.MaxBatchSize) { IEnumerable batch = workerData.Skip(j).Take(config.MaxBatchSize); @@ -58,10 +55,10 @@ public static async Task BulkCopyToAsync( await writer.StartRowAsync(); // Write each configured column in the specified order - foreach (KeyValuePair mapping in config.ColumnMappings) + foreach (var (Getter, DbType) in config.ColumnMappings.Values) { - object? value = mapping.Value.PropertyInfo.GetValue(item); - await writer.WriteAsync(value, mapping.Value.DbType); + object? value = Getter(item); + await writer.WriteAsync(value, DbType); } } await writer.CompleteAsync(); diff --git a/README.md b/README.md index cc53a16..dc5f80d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Test Workflow](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/actions/workflows/run-tests.yml/badge.svg)](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/actions/workflows/run-tests.yml) +![CmdScale Project](https://github.com/cmdscale/.github/raw/main/profile/assets/CmdShield.svg) +[![Test Workflow](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/actions/workflows/run-tests.yml/badge.svg)](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/actions/workflows/run-tests.yml) [![GitHub release (latest by date)](https://img.shields.io/github/v/tag/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB)](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/tags) [![GitHub issues](https://img.shields.io/github/issues/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB)](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/issues) [![GitHub license](https://img.shields.io/github/license/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB)](https://github.com/cmdscale/CmdScale.EntityFrameworkCore.TimescaleDB/blob/main/LICENSE) @@ -25,17 +26,6 @@ Seamlessly define and manage **TimescaleDB hypertables** using standard EF Core - **Chunk Time Interval**: Configure chunk intervals to balance performance and storage efficiency. - **Compression & Chunk Skipping**: Enable TimescaleDB's native compression and configure chunk skipping to improve query performance. -### High-Performance Data Ingestion - -For time-series workloads where ingestion speed is critical, the package provides a highly optimized **bulk copy utility**. This method bypasses the standard `SaveChanges()` change tracker and leverages **PostgreSQL's native COPY command** for maximum throughput. - -- **Blazing Fast**: Ingest hundreds of thousands of records per second. -- **Parallelism**: Automatically distributes the workload across multiple concurrent workers. -- **Configurable**: Easily configure batch sizes, worker counts, and column mappings. -- **Generic**: Works with any POCO, with automatic mapping of properties to table columns. - - - --- ## 📦 NuGet Packages @@ -171,10 +161,10 @@ To build and publish the core libraries to a local NuGet feed for testing, use t ```powershell # Publish the design-time package -./Publish-Local.ps1 -ProjectName "CmdScale.EntityFrameworkCore.TimescaleDB.Design" +.\Scripts\Publish-Local.ps1 -ProjectName "CmdScale.EntityFrameworkCore.TimescaleDB.Design" # Publish the runtime package -./Publish-Local.ps1 -ProjectName "CmdScale.EntityFrameworkCore.TimescaleDB" +.\Scripts\Publish-Local.ps1 -ProjectName "CmdScale.EntityFrameworkCore.TimescaleDB" ``` By default, this script outputs the `.nupkg` files to: