Add api request debug and improve table truncation

server/Helpers/LoggingMiddlewareHelper.cs

@@ -4,6 +4,37 @@ namespace Server.Helpers;
 
 public static class LoggingMiddlewareHelper
 {
+    public static void UseApiFailureLogging(this WebApplication app)
+    {
+        app.Use(async (ctx, next) =>
+        {
+            try
+            {
+                await next();
+            }
+            catch (Exception ex) when (ctx.Request.Path.StartsWithSegments("/api"))
+            {
+                app.Logger.LogError(
+                    ex,
+                    "API request failed with an unhandled exception: {Method} {Path}. ContentLength={ContentLength}.",
+                    ctx.Request.Method,
+                    ctx.Request.Path,
+                    ctx.Request.ContentLength);
+                throw;
+            }
+
+            if (ctx.Request.Path.StartsWithSegments("/api") && ctx.Response.StatusCode >= 400)
+            {
+                app.Logger.LogWarning(
+                    "API request completed with status {StatusCode}: {Method} {Path}. ContentLength={ContentLength}.",
+                    ctx.Response.StatusCode,
+                    ctx.Request.Method,
+                    ctx.Request.Path,
+                    ctx.Request.ContentLength);
+            }
+        });
+    }
+
     /// <summary>
     /// Adds request context enrichment middleware that includes trace info, user info, and client details in log scope
     /// </summary>
@@ -43,4 +74,4 @@ public static void UseRequestContextLogging(this WebApplication app)
             }
         });
     }
-}
+}

server/Import/FlatFileImportService.cs

@@ -1,4 +1,5 @@
 using System.Data;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Claims;
 using System.Text.Json;
@@ -69,7 +70,20 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'."
             return await ValidationAsync(definition.Id, file?.FileName, 0, fileErrors, [], user, startedAt, cancellationToken);
         }
 
+        var importStopwatch = Stopwatch.StartNew();
+        logger.LogInformation(
+            "Import started for dataset {DatasetId} from file {Filename} with {FileSizeBytes} bytes.",
+            definition.Id,
+            Path.GetFileName(file!.FileName),
+            file.Length);
+
         var parseResult = ParseFile(definition, file!);
+        logger.LogInformation(
+            "Import parsed {RowCount} rows for dataset {DatasetId} in {ElapsedMilliseconds} ms.",
+            parseResult.Rows.Count,
+            definition.Id,
+            importStopwatch.ElapsedMilliseconds);
+
         if (parseResult.FileErrors.Count > 0 || parseResult.Rows.Any(row => row.Errors.Count > 0 || row.CellErrors.Count > 0))
         {
             return await ValidationAsync(
@@ -85,7 +99,13 @@ [new ImportFileError("unknown_dataset", $"Unknown import dataset '{datasetId}'."
 
         try
         {
+            var persistenceStopwatch = Stopwatch.StartNew();
             await ReplaceTargetTableAsync(definition, parseResult.ParsedRows, cancellationToken);
+            logger.LogInformation(
+                "Import persisted {RowCount} rows for dataset {DatasetId} in {ElapsedMilliseconds} ms.",
+                parseResult.ParsedRows.Count,
+                definition.Id,
+                persistenceStopwatch.ElapsedMilliseconds);
         }
         catch (StagingValidationException ex)
         {
@@ -127,6 +147,12 @@ [new ImportFileError("database_validation_failed", "The database rejected the im
             startedAt,
             cancellationToken);
 
+        logger.LogInformation(
+            "Import completed for dataset {DatasetId} with {RowCount} rows in {ElapsedMilliseconds} ms.",
+            definition.Id,
+            parseResult.Rows.Count,
+            importStopwatch.ElapsedMilliseconds);
+
         return new ImportSucceeded(new ImportSuccessResponse(
             definition.Id,
             file!.FileName,
@@ -767,6 +793,12 @@ private async Task ReplaceTargetTableAsync(
         IReadOnlyList<ParsedImportRow> rows,
         CancellationToken cancellationToken)
     {
+        if (definition.UniqueKeys.Count == 0)
+        {
+            await ReplaceTargetTableDirectlyAsync(definition, rows, cancellationToken);
+            return;
+        }
+
         var connectionString = DataDbConnection.Resolve(
             configuration,
             dataDbContext.Database.GetConnectionString());
@@ -810,6 +842,39 @@ await connection.ExecuteAsync(new CommandDefinition(
         await transaction.CommitAsync(cancellationToken);
     }
 
+    private async Task ReplaceTargetTableDirectlyAsync(
+        ImportDatasetDefinition definition,
+        IReadOnlyList<ParsedImportRow> rows,
+        CancellationToken cancellationToken)
+    {
+        var connectionString = DataDbConnection.Resolve(
+            configuration,
+            dataDbContext.Database.GetConnectionString());
+
+        await using var connection = new SqlConnection(connectionString);
+        await connection.OpenAsync(cancellationToken);
+        await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(cancellationToken);
+
+        await TruncateTargetTableAsync(connection, transaction, definition, cancellationToken);
+
+        var table = CreateDataTable(definition, rows);
+        using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints | SqlBulkCopyOptions.TableLock, transaction))
+        {
+            bulkCopy.DestinationTableName = TargetTableSql(definition);
+            bulkCopy.BatchSize = Math.Max(rows.Count, 1);
+            bulkCopy.BulkCopyTimeout = DatabaseCommandTimeoutSeconds;
+
+            foreach (var column in definition.Columns)
+            {
+                bulkCopy.ColumnMappings.Add(column.TargetColumn, column.TargetColumn);
+            }
+
+            await bulkCopy.WriteToServerAsync(table, cancellationToken);
+        }
+
+        await transaction.CommitAsync(cancellationToken);
+    }
+
     private static Task DropTempTableIfExistsAsync(
         SqlConnection connection,
         SqlTransaction transaction,
@@ -822,6 +887,19 @@ private static Task DropTempTableIfExistsAsync(
             cancellationToken: cancellationToken));
     }
 
+    private static Task TruncateTargetTableAsync(
+        SqlConnection connection,
+        SqlTransaction transaction,
+        ImportDatasetDefinition definition,
+        CancellationToken cancellationToken)
+    {
+        return connection.ExecuteAsync(new CommandDefinition(
+            $"TRUNCATE TABLE {TargetTableSql(definition)};",
+            transaction: transaction,
+            commandTimeout: DatabaseCommandTimeoutSeconds,
+            cancellationToken: cancellationToken));
+    }
+
     private static async Task RunStagingValidationAsync(
         SqlConnection connection,
         SqlTransaction transaction,
@@ -948,17 +1026,21 @@ CREATE TABLE {TempTableName}
 
     private static string ReplaceTableSql(ImportDatasetDefinition definition)
     {
-        var targetTable = $"{Quote(definition.SchemaName)}.{Quote(definition.TableName)}";
         var columns = string.Join(", ", definition.Columns.Select(column => Quote(column.TargetColumn)));
         return $"""
-            DELETE FROM {targetTable};
+            TRUNCATE TABLE {TargetTableSql(definition)};
 
-            INSERT INTO {targetTable} ({columns})
+            INSERT INTO {TargetTableSql(definition)} ({columns})
             SELECT {columns}
             FROM {TempTableName};
             """;
     }
 
+    private static string TargetTableSql(ImportDatasetDefinition definition)
+    {
+        return $"{Quote(definition.SchemaName)}.{Quote(definition.TableName)}";
+    }
+
     private static string SqlType(ImportColumn column)
     {
         return column.Type switch

server/Program.cs

@@ -128,6 +128,7 @@ await StartupLogging.RunAsync(app, async cancellationToken =>
     }
 
 
+    app.UseApiFailureLogging();
     app.UseAuthentication();
     app.UseAuthorization();