From d54c5222353bacbb4be9400393522790531b7082 Mon Sep 17 00:00:00 2001 From: Spruce Weber-Milne Date: Wed, 8 Jul 2026 09:27:33 -0700 Subject: [PATCH 1/2] Add api request debug and improve table truncate --- server/Helpers/LoggingMiddlewareHelper.cs | 20 +++++- server/Import/FlatFileImportService.cs | 88 ++++++++++++++++++++++- server/Program.cs | 1 + 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/server/Helpers/LoggingMiddlewareHelper.cs b/server/Helpers/LoggingMiddlewareHelper.cs index c9e740b..d6b4a53 100644 --- a/server/Helpers/LoggingMiddlewareHelper.cs +++ b/server/Helpers/LoggingMiddlewareHelper.cs @@ -4,6 +4,24 @@ namespace Server.Helpers; public static class LoggingMiddlewareHelper { + public static void UseApiFailureLogging(this WebApplication app) + { + app.Use(async (ctx, next) => + { + await next(); + + if (ctx.Request.Path.StartsWithSegments("/api") && ctx.Response.StatusCode >= 400) + { + app.Logger.LogWarning( + "API request completed with status {StatusCode}: {Method} {Path}. ContentLength={ContentLength}.", + ctx.Response.StatusCode, + ctx.Request.Method, + ctx.Request.Path, + ctx.Request.ContentLength); + } + }); + } + /// /// Adds request context enrichment middleware that includes trace info, user info, and client details in log scope /// @@ -43,4 +61,4 @@ public static void UseRequestContextLogging(this WebApplication app) } }); } -} \ No newline at end of file +} diff --git a/server/Import/FlatFileImportService.cs b/server/Import/FlatFileImportService.cs index fc6539e..3c63a0a 100644 --- a/server/Import/FlatFileImportService.cs +++ b/server/Import/FlatFileImportService.cs @@ -1,4 +1,5 @@ using System.Data; +using System.Diagnostics; using System.Globalization; using System.Security.Claims; using System.Text.Json; @@ -69,7 +70,20 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'." return await ValidationAsync(definition.Id, file?.FileName, 0, fileErrors, [], user, startedAt, cancellationToken); } + var importStopwatch = Stopwatch.StartNew(); + logger.LogInformation( + "Import started for dataset {DatasetId} from file {Filename} with {FileSizeBytes} bytes.", + definition.Id, + file!.FileName, + file.Length); + var parseResult = ParseFile(definition, file!); + logger.LogInformation( + "Import parsed {RowCount} rows for dataset {DatasetId} in {ElapsedMilliseconds} ms.", + parseResult.Rows.Count, + definition.Id, + importStopwatch.ElapsedMilliseconds); + if (parseResult.FileErrors.Count > 0 || parseResult.Rows.Any(row => row.Errors.Count > 0 || row.CellErrors.Count > 0)) { return await ValidationAsync( @@ -85,7 +99,13 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'." try { + var persistenceStopwatch = Stopwatch.StartNew(); await ReplaceTargetTableAsync(definition, parseResult.ParsedRows, cancellationToken); + logger.LogInformation( + "Import persisted {RowCount} rows for dataset {DatasetId} in {ElapsedMilliseconds} ms.", + parseResult.ParsedRows.Count, + definition.Id, + persistenceStopwatch.ElapsedMilliseconds); } catch (StagingValidationException ex) { @@ -127,6 +147,12 @@ [new ImportFileError("database_validation_failed", "The database rejected the im startedAt, cancellationToken); + logger.LogInformation( + "Import completed for dataset {DatasetId} with {RowCount} rows in {ElapsedMilliseconds} ms.", + definition.Id, + parseResult.Rows.Count, + importStopwatch.ElapsedMilliseconds); + return new ImportSucceeded(new ImportSuccessResponse( definition.Id, file!.FileName, @@ -767,6 +793,12 @@ private async Task ReplaceTargetTableAsync( IReadOnlyList rows, CancellationToken cancellationToken) { + if (definition.UniqueKeys.Count == 0) + { + await ReplaceTargetTableDirectlyAsync(definition, rows, cancellationToken); + return; + } + var connectionString = DataDbConnection.Resolve( configuration, dataDbContext.Database.GetConnectionString()); @@ -810,6 +842,39 @@ await connection.ExecuteAsync(new CommandDefinition( await transaction.CommitAsync(cancellationToken); } + private async Task ReplaceTargetTableDirectlyAsync( + ImportDatasetDefinition definition, + IReadOnlyList rows, + CancellationToken cancellationToken) + { + var connectionString = DataDbConnection.Resolve( + configuration, + dataDbContext.Database.GetConnectionString()); + + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(cancellationToken); + + await TruncateTargetTableAsync(connection, transaction, definition, cancellationToken); + + var table = CreateDataTable(definition, rows); + using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.TableLock, transaction)) + { + bulkCopy.DestinationTableName = TargetTableSql(definition); + bulkCopy.BatchSize = Math.Max(rows.Count, 1); + bulkCopy.BulkCopyTimeout = DatabaseCommandTimeoutSeconds; + + foreach (var column in definition.Columns) + { + bulkCopy.ColumnMappings.Add(column.TargetColumn, column.TargetColumn); + } + + await bulkCopy.WriteToServerAsync(table, cancellationToken); + } + + await transaction.CommitAsync(cancellationToken); + } + private static Task DropTempTableIfExistsAsync( SqlConnection connection, SqlTransaction transaction, @@ -822,6 +887,19 @@ private static Task DropTempTableIfExistsAsync( cancellationToken: cancellationToken)); } + private static Task TruncateTargetTableAsync( + SqlConnection connection, + SqlTransaction transaction, + ImportDatasetDefinition definition, + CancellationToken cancellationToken) + { + return connection.ExecuteAsync(new CommandDefinition( + $"TRUNCATE TABLE {TargetTableSql(definition)};", + transaction: transaction, + commandTimeout: DatabaseCommandTimeoutSeconds, + cancellationToken: cancellationToken)); + } + private static async Task RunStagingValidationAsync( SqlConnection connection, SqlTransaction transaction, @@ -948,17 +1026,21 @@ CREATE TABLE {TempTableName} private static string ReplaceTableSql(ImportDatasetDefinition definition) { - var targetTable = $"{Quote(definition.SchemaName)}.{Quote(definition.TableName)}"; var columns = string.Join(", ", definition.Columns.Select(column => Quote(column.TargetColumn))); return $""" - DELETE FROM {targetTable}; + TRUNCATE TABLE {TargetTableSql(definition)}; - INSERT INTO {targetTable} ({columns}) + INSERT INTO {TargetTableSql(definition)} ({columns}) SELECT {columns} FROM {TempTableName}; """; } + private static string TargetTableSql(ImportDatasetDefinition definition) + { + return $"{Quote(definition.SchemaName)}.{Quote(definition.TableName)}"; + } + private static string SqlType(ImportColumn column) { return column.Type switch diff --git a/server/Program.cs b/server/Program.cs index 5b698f2..dda14fe 100644 --- a/server/Program.cs +++ b/server/Program.cs @@ -128,6 +128,7 @@ await StartupLogging.RunAsync(app, async cancellationToken => } + app.UseApiFailureLogging(); app.UseAuthentication(); app.UseAuthorization(); From b8ac8776c70fa583ff4f9a58b9183b4dceb8c00b Mon Sep 17 00:00:00 2001 From: Spruce Weber-Milne Date: Wed, 8 Jul 2026 09:46:41 -0700 Subject: [PATCH 2/2] Address coderabbit issues --- server/Helpers/LoggingMiddlewareHelper.cs | 15 ++++++++++++++- server/Import/FlatFileImportService.cs | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/server/Helpers/LoggingMiddlewareHelper.cs b/server/Helpers/LoggingMiddlewareHelper.cs index d6b4a53..2bda801 100644 --- a/server/Helpers/LoggingMiddlewareHelper.cs +++ b/server/Helpers/LoggingMiddlewareHelper.cs @@ -8,7 +8,20 @@ public static void UseApiFailureLogging(this WebApplication app) { app.Use(async (ctx, next) => { - await next(); + try + { + await next(); + } + catch (Exception ex) when (ctx.Request.Path.StartsWithSegments("/api")) + { + app.Logger.LogError( + ex, + "API request failed with an unhandled exception: {Method} {Path}. ContentLength={ContentLength}.", + ctx.Request.Method, + ctx.Request.Path, + ctx.Request.ContentLength); + throw; + } if (ctx.Request.Path.StartsWithSegments("/api") && ctx.Response.StatusCode >= 400) { diff --git a/server/Import/FlatFileImportService.cs b/server/Import/FlatFileImportService.cs index 3c63a0a..b74f75e 100644 --- a/server/Import/FlatFileImportService.cs +++ b/server/Import/FlatFileImportService.cs @@ -74,7 +74,7 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'." logger.LogInformation( "Import started for dataset {DatasetId} from file {Filename} with {FileSizeBytes} bytes.", definition.Id, - file!.FileName, + Path.GetFileName(file!.FileName), file.Length); var parseResult = ParseFile(definition, file!);