diff --git a/server/Helpers/LoggingMiddlewareHelper.cs b/server/Helpers/LoggingMiddlewareHelper.cs
index c9e740b..2bda801 100644
--- a/server/Helpers/LoggingMiddlewareHelper.cs
+++ b/server/Helpers/LoggingMiddlewareHelper.cs
@@ -4,6 +4,37 @@ namespace Server.Helpers;
public static class LoggingMiddlewareHelper
{
+ public static void UseApiFailureLogging(this WebApplication app)
+ {
+ app.Use(async (ctx, next) =>
+ {
+ try
+ {
+ await next();
+ }
+ catch (Exception ex) when (ctx.Request.Path.StartsWithSegments("/api"))
+ {
+ app.Logger.LogError(
+ ex,
+ "API request failed with an unhandled exception: {Method} {Path}. ContentLength={ContentLength}.",
+ ctx.Request.Method,
+ ctx.Request.Path,
+ ctx.Request.ContentLength);
+ throw;
+ }
+
+ if (ctx.Request.Path.StartsWithSegments("/api") && ctx.Response.StatusCode >= 400)
+ {
+ app.Logger.LogWarning(
+ "API request completed with status {StatusCode}: {Method} {Path}. ContentLength={ContentLength}.",
+ ctx.Response.StatusCode,
+ ctx.Request.Method,
+ ctx.Request.Path,
+ ctx.Request.ContentLength);
+ }
+ });
+ }
+
///
/// Adds request context enrichment middleware that includes trace info, user info, and client details in log scope
///
@@ -43,4 +74,4 @@ public static void UseRequestContextLogging(this WebApplication app)
}
});
}
-}
\ No newline at end of file
+}
diff --git a/server/Import/FlatFileImportService.cs b/server/Import/FlatFileImportService.cs
index fc6539e..b74f75e 100644
--- a/server/Import/FlatFileImportService.cs
+++ b/server/Import/FlatFileImportService.cs
@@ -1,4 +1,5 @@
using System.Data;
+using System.Diagnostics;
using System.Globalization;
using System.Security.Claims;
using System.Text.Json;
@@ -69,7 +70,20 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'."
return await ValidationAsync(definition.Id, file?.FileName, 0, fileErrors, [], user, startedAt, cancellationToken);
}
+ var importStopwatch = Stopwatch.StartNew();
+ logger.LogInformation(
+ "Import started for dataset {DatasetId} from file {Filename} with {FileSizeBytes} bytes.",
+ definition.Id,
+ Path.GetFileName(file!.FileName),
+ file.Length);
+
var parseResult = ParseFile(definition, file!);
+ logger.LogInformation(
+ "Import parsed {RowCount} rows for dataset {DatasetId} in {ElapsedMilliseconds} ms.",
+ parseResult.Rows.Count,
+ definition.Id,
+ importStopwatch.ElapsedMilliseconds);
+
if (parseResult.FileErrors.Count > 0 || parseResult.Rows.Any(row => row.Errors.Count > 0 || row.CellErrors.Count > 0))
{
return await ValidationAsync(
@@ -85,7 +99,13 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'."
try
{
+ var persistenceStopwatch = Stopwatch.StartNew();
await ReplaceTargetTableAsync(definition, parseResult.ParsedRows, cancellationToken);
+ logger.LogInformation(
+ "Import persisted {RowCount} rows for dataset {DatasetId} in {ElapsedMilliseconds} ms.",
+ parseResult.ParsedRows.Count,
+ definition.Id,
+ persistenceStopwatch.ElapsedMilliseconds);
}
catch (StagingValidationException ex)
{
@@ -127,6 +147,12 @@ [new ImportFileError("database_validation_failed", "The database rejected the im
startedAt,
cancellationToken);
+ logger.LogInformation(
+ "Import completed for dataset {DatasetId} with {RowCount} rows in {ElapsedMilliseconds} ms.",
+ definition.Id,
+ parseResult.Rows.Count,
+ importStopwatch.ElapsedMilliseconds);
+
return new ImportSucceeded(new ImportSuccessResponse(
definition.Id,
file!.FileName,
@@ -767,6 +793,12 @@ private async Task ReplaceTargetTableAsync(
IReadOnlyList rows,
CancellationToken cancellationToken)
{
+ if (definition.UniqueKeys.Count == 0)
+ {
+ await ReplaceTargetTableDirectlyAsync(definition, rows, cancellationToken);
+ return;
+ }
+
var connectionString = DataDbConnection.Resolve(
configuration,
dataDbContext.Database.GetConnectionString());
@@ -810,6 +842,39 @@ await connection.ExecuteAsync(new CommandDefinition(
await transaction.CommitAsync(cancellationToken);
}
+ private async Task ReplaceTargetTableDirectlyAsync(
+ ImportDatasetDefinition definition,
+ IReadOnlyList rows,
+ CancellationToken cancellationToken)
+ {
+ var connectionString = DataDbConnection.Resolve(
+ configuration,
+ dataDbContext.Database.GetConnectionString());
+
+ await using var connection = new SqlConnection(connectionString);
+ await connection.OpenAsync(cancellationToken);
+ await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(cancellationToken);
+
+ await TruncateTargetTableAsync(connection, transaction, definition, cancellationToken);
+
+ var table = CreateDataTable(definition, rows);
+ using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.TableLock, transaction))
+ {
+ bulkCopy.DestinationTableName = TargetTableSql(definition);
+ bulkCopy.BatchSize = Math.Max(rows.Count, 1);
+ bulkCopy.BulkCopyTimeout = DatabaseCommandTimeoutSeconds;
+
+ foreach (var column in definition.Columns)
+ {
+ bulkCopy.ColumnMappings.Add(column.TargetColumn, column.TargetColumn);
+ }
+
+ await bulkCopy.WriteToServerAsync(table, cancellationToken);
+ }
+
+ await transaction.CommitAsync(cancellationToken);
+ }
+
private static Task DropTempTableIfExistsAsync(
SqlConnection connection,
SqlTransaction transaction,
@@ -822,6 +887,19 @@ private static Task DropTempTableIfExistsAsync(
cancellationToken: cancellationToken));
}
+ private static Task TruncateTargetTableAsync(
+ SqlConnection connection,
+ SqlTransaction transaction,
+ ImportDatasetDefinition definition,
+ CancellationToken cancellationToken)
+ {
+ return connection.ExecuteAsync(new CommandDefinition(
+ $"TRUNCATE TABLE {TargetTableSql(definition)};",
+ transaction: transaction,
+ commandTimeout: DatabaseCommandTimeoutSeconds,
+ cancellationToken: cancellationToken));
+ }
+
private static async Task RunStagingValidationAsync(
SqlConnection connection,
SqlTransaction transaction,
@@ -948,17 +1026,21 @@ CREATE TABLE {TempTableName}
private static string ReplaceTableSql(ImportDatasetDefinition definition)
{
- var targetTable = $"{Quote(definition.SchemaName)}.{Quote(definition.TableName)}";
var columns = string.Join(", ", definition.Columns.Select(column => Quote(column.TargetColumn)));
return $"""
- DELETE FROM {targetTable};
+ TRUNCATE TABLE {TargetTableSql(definition)};
- INSERT INTO {targetTable} ({columns})
+ INSERT INTO {TargetTableSql(definition)} ({columns})
SELECT {columns}
FROM {TempTableName};
""";
}
+ private static string TargetTableSql(ImportDatasetDefinition definition)
+ {
+ return $"{Quote(definition.SchemaName)}.{Quote(definition.TableName)}";
+ }
+
private static string SqlType(ImportColumn column)
{
return column.Type switch
diff --git a/server/Program.cs b/server/Program.cs
index 5b698f2..dda14fe 100644
--- a/server/Program.cs
+++ b/server/Program.cs
@@ -128,6 +128,7 @@ await StartupLogging.RunAsync(app, async cancellationToken =>
}
+ app.UseApiFailureLogging();
app.UseAuthentication();
app.UseAuthorization();