Fix data mapping regression and increase import timeout

server.core/Data/AppDbContext.cs

@@ -6,64 +6,17 @@ namespace Server.Core.Data;
 public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
 {
     public const string AppSchema = "app";
-    public const string DataSchema = "data";
     public const string MigrationsHistoryTable = "__EFMigrationsHistory";
 
     public DbSet<ImportLog> ImportLogs => Set<ImportLog>();
     public DbSet<User> Users => Set<User>();
 
-    public DbSet<ChartStringSegment> ChartStringSegments => Set<ChartStringSegment>();
-
-    public DbSet<DepartmentHierarchy> DepartmentHierarchies => Set<DepartmentHierarchy>();
-    public DbSet<AccountHierarchy> AccountHierarchies => Set<AccountHierarchy>();
-    public DbSet<FundHierarchy> FundHierarchies => Set<FundHierarchy>();
-    public DbSet<ActivityHierarchy> ActivityHierarchies => Set<ActivityHierarchy>();
-
     protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
         base.OnModelCreating(modelBuilder);
 
         modelBuilder.HasDefaultSchema(AppSchema);
         modelBuilder.Entity<ImportLog>().ToTable("ImportLog", AppSchema);
         modelBuilder.Entity<User>().ToTable("Users", AppSchema);
-
-        modelBuilder.Entity<ChartStringSegment>(entity =>
-        {
-            entity.ToTable("ChartStringSegments", DataSchema, table => table.ExcludeFromMigrations());
-            entity.HasKey(segment => new { segment.SegmentType, segment.Code });
-            entity.Property(segment => segment.SegmentType).HasConversion<string>().HasMaxLength(20);
-            entity.Property(segment => segment.Code).HasMaxLength(50);
-            entity.Property(segment => segment.Description).HasMaxLength(300);
-            entity.Property(segment => segment.Sfn).HasMaxLength(10);
-        });
-
-        ConfigureHierarchy<DepartmentHierarchy>(modelBuilder, "DepartmentHierarchy");
-        ConfigureHierarchy<AccountHierarchy>(modelBuilder, "AccountHierarchy");
-        ConfigureHierarchy<FundHierarchy>(modelBuilder, "FundHierarchy");
-        ConfigureHierarchy<ActivityHierarchy>(modelBuilder, "ActivityHierarchy");
-    }
-
-    private static void ConfigureHierarchy<T>(ModelBuilder modelBuilder, string table)
-        where T : class, ISegmentHierarchy
-    {
-        modelBuilder.Entity<T>(entity =>
-        {
-            entity.ToTable(table, DataSchema, t => t.ExcludeFromMigrations());
-            entity.HasKey(e => e.Code);
-
-            foreach (var property in entity.Metadata.GetProperties()
-                         .Where(p => p.ClrType == typeof(string)))
-            {
-                var maxLength = property.Name switch
-                {
-                    // Match ChartStringSegment.Code so joins/lookups on Code stay aligned.
-                    nameof(ISegmentHierarchy.Code) => 50,
-                    "Description" => 1000,
-                    _ when property.Name.EndsWith("Name") => 1000,
-                    _ => 20,
-                };
-                property.SetMaxLength(maxLength);
-            }
-        });
     }
 }

server.core/Data/ChartStringSegmentSeed.cs

@@ -21,7 +21,7 @@ public static class ChartStringSegmentSeed
     private static readonly string[] FundSfnPool =
         ["201", "202", "203", "205", "220", "221", "223", "Multiple"];
 
-    public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct = default)
+    public static async Task EnsureSeededAsync(DataDbContext db, CancellationToken ct = default)
     {
         if (await db.ChartStringSegments.AnyAsync(ct))
         {

server.core/Data/DataDbConnection.cs

@@ -6,6 +6,7 @@ public static class DataDbConnection
 {
     public const string EnvironmentVariableName = "DATA_DB_CONNECTION";
     public const string ConnectionStringName = "DataConnection";
+    public const int ImportCommandTimeoutSeconds = 600;
 
     public static string Resolve(IConfiguration configuration, string? fallbackConnectionString)
     {

server.core/Data/DataDbContext.cs

@@ -0,0 +1,62 @@
+using Microsoft.EntityFrameworkCore;
+using Server.Core.Domain;
+
+namespace Server.Core.Data;
+
+public class DataDbContext(DbContextOptions<DataDbContext> options) : DbContext(options)
+{
+    public const string DataSchema = "data";
+
+    public DbSet<ChartStringSegment> ChartStringSegments => Set<ChartStringSegment>();
+
+    public DbSet<DepartmentHierarchy> DepartmentHierarchies => Set<DepartmentHierarchy>();
+    public DbSet<AccountHierarchy> AccountHierarchies => Set<AccountHierarchy>();
+    public DbSet<FundHierarchy> FundHierarchies => Set<FundHierarchy>();
+    public DbSet<ActivityHierarchy> ActivityHierarchies => Set<ActivityHierarchy>();
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating(modelBuilder);
+
+        modelBuilder.HasDefaultSchema(DataSchema);
+
+        modelBuilder.Entity<ChartStringSegment>(entity =>
+        {
+            entity.ToTable("ChartStringSegments", DataSchema, table => table.ExcludeFromMigrations());
+            entity.HasKey(segment => new { segment.SegmentType, segment.Code });
+            entity.Property(segment => segment.SegmentType).HasConversion<string>().HasMaxLength(20);
+            entity.Property(segment => segment.Code).HasMaxLength(50);
+            entity.Property(segment => segment.Description).HasMaxLength(300);
+            entity.Property(segment => segment.Sfn).HasMaxLength(10);
+        });
+
+        ConfigureHierarchy<DepartmentHierarchy>(modelBuilder, "DepartmentHierarchy");
+        ConfigureHierarchy<AccountHierarchy>(modelBuilder, "AccountHierarchy");
+        ConfigureHierarchy<FundHierarchy>(modelBuilder, "FundHierarchy");
+        ConfigureHierarchy<ActivityHierarchy>(modelBuilder, "ActivityHierarchy");
+    }
+
+    private static void ConfigureHierarchy<T>(ModelBuilder modelBuilder, string table)
+        where T : class, ISegmentHierarchy
+    {
+        modelBuilder.Entity<T>(entity =>
+        {
+            entity.ToTable(table, DataSchema, mapping => mapping.ExcludeFromMigrations());
+            entity.HasKey(e => e.Code);
+
+            foreach (var property in entity.Metadata.GetProperties()
+                         .Where(p => p.ClrType == typeof(string)))
+            {
+                var maxLength = property.Name switch
+                {
+                    // Match ChartStringSegment.Code so joins/lookups on Code stay aligned.
+                    nameof(ISegmentHierarchy.Code) => 50,
+                    "Description" => 1000,
+                    _ when property.Name.EndsWith("Name") => 1000,
+                    _ => 20,
+                };
+                property.SetMaxLength(maxLength);
+            }
+        });
+    }
+}

server.core/Data/DbInitializer.cs

@@ -10,11 +10,13 @@ public interface IDbInitializer
 public class DbInitializer : IDbInitializer
 {
     private readonly AppDbContext _db;
+    private readonly DataDbContext _dataDb;
     private readonly ILogger<DbInitializer> _logger;
 
-    public DbInitializer(AppDbContext db, ILogger<DbInitializer> logger)
+    public DbInitializer(AppDbContext db, DataDbContext dataDb, ILogger<DbInitializer> logger)
     {
         _db = db;
+        _dataDb = dataDb;
         _logger = logger;
     }
 
@@ -37,8 +39,8 @@ public async Task InitializeAsync(bool includeDevSeed, CancellationToken cancell
     private async Task SeedDevelopmentAsync(CancellationToken ct)
     {
         // Hierarchy first: chart-string segments are derived from the hierarchy tables.
-        await HierarchySeed.EnsureSeededAsync(_db, ct);
-        await ChartStringSegmentSeed.EnsureSeededAsync(_db, ct);
+        await HierarchySeed.EnsureSeededAsync(_dataDb, ct);
+        await ChartStringSegmentSeed.EnsureSeededAsync(_dataDb, ct);
     }
 
     // just a placeholder for any production-safe seeding

server.core/Data/HierarchySeed.cs

@@ -12,7 +12,7 @@ namespace Server.Core.Data;
 /// </summary>
 public static class HierarchySeed
 {
-    public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct = default)
+    public static async Task EnsureSeededAsync(DataDbContext db, CancellationToken ct = default)
     {
         if (!await db.AccountHierarchies.AnyAsync(ct))
         {

server.core/Import/PgmProjectsImportService.cs

@@ -13,7 +13,7 @@ public sealed class PgmProjectsImportService : IPgmProjectsImportService
 
     public const string RemoteLinkedServer = "AE_Redshift_PROD";
 
-    private const int CommandTimeoutSeconds = 600;
+    private const int CommandTimeoutSeconds = DataDbConnection.ImportCommandTimeoutSeconds;
     private const string DestinationTable = "[data].[PGMProjects]";
 
     // source reader column -> destination table column.
@@ -58,16 +58,16 @@ private static readonly (string Source, string Destination)[] ColumnMappings =
         ("contract_admins", "ContractAdmins"),
     ];
 
-    private readonly AppDbContext _dbContext;
+    private readonly DataDbContext _dataDbContext;
     private readonly IConfiguration _configuration;
     private readonly ILogger<PgmProjectsImportService> _logger;
 
     public PgmProjectsImportService(
-        AppDbContext dbContext,
+        DataDbContext dataDbContext,
         IConfiguration configuration,
         ILogger<PgmProjectsImportService> logger)
     {
-        _dbContext = dbContext;
+        _dataDbContext = dataDbContext;
         _configuration = configuration;
         _logger = logger;
     }
@@ -86,7 +86,7 @@ public async Task<PgmProjectsImportResult> ImportAsync(DateOnly reportDate, Canc
 
         var destinationConnectionString = DataDbConnection.Resolve(
             _configuration,
-            _dbContext.Database.GetConnectionString());
+            _dataDbContext.Database.GetConnectionString());
 
         _logger.LogInformation("Importing PGM projects for report date {ReportDate}", reportDate);
 

server.core/createMigration.sh

@@ -12,6 +12,6 @@ MIGRATION_NAME=$1
 ROOT_DIR="$(dirname "$0")/.."
 
 echo "📦 Creating migration '$MIGRATION_NAME'..."
-dotnet ef migrations add "$MIGRATION_NAME" -p "$ROOT_DIR/server.core" -s "$ROOT_DIR/server"
+dotnet ef migrations add "$MIGRATION_NAME" --context AppDbContext -p "$ROOT_DIR/server.core" -s "$ROOT_DIR/server"
 
 echo "✅ Migration created successfully."

server/Controllers/ChartStringSegmentsController.cs

@@ -8,9 +8,9 @@ namespace Server.Controllers;
 
 public class ChartStringSegmentsController : ApiControllerBase
 {
-    private readonly AppDbContext _db;
+    private readonly DataDbContext _db;
 
-    public ChartStringSegmentsController(AppDbContext db)
+    public ChartStringSegmentsController(DataDbContext db)
     {
         _db = db;
     }

server/Import/FlatFileImportService.cs

@@ -34,10 +34,12 @@ public sealed record ImportValidationFailed(ImportValidationResponse Response) :
 
 public sealed class FlatFileImportService(
     AppDbContext dbContext,
+    DataDbContext dataDbContext,
     IFlatFileImportRegistry registry,
     IConfiguration configuration,
     ILogger<FlatFileImportService> logger) : IFlatFileImportService
 {
+    private const int DatabaseCommandTimeoutSeconds = DataDbConnection.ImportCommandTimeoutSeconds;
     private const string TempTableName = "#FlatFileImportRows";
     private const string StatusSucceeded = "Succeeded";
     private const string StatusValidationFailed = "ValidationFailed";
@@ -767,7 +769,7 @@ private async Task ReplaceTargetTableAsync(
     {
         var connectionString = DataDbConnection.Resolve(
             configuration,
-            dbContext.Database.GetConnectionString());
+            dataDbContext.Database.GetConnectionString());
 
         await using var connection = new SqlConnection(connectionString);
         await connection.OpenAsync(cancellationToken);
@@ -778,13 +780,15 @@ private async Task ReplaceTargetTableAsync(
         await connection.ExecuteAsync(new CommandDefinition(
             CreateTempTableSql(definition),
             transaction: transaction,
+            commandTimeout: DatabaseCommandTimeoutSeconds,
             cancellationToken: cancellationToken));
 
         var table = CreateDataTable(definition, rows);
         using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.CheckConstraints, transaction))
         {
             bulkCopy.DestinationTableName = TempTableName;
             bulkCopy.BatchSize = Math.Max(rows.Count, 1);
+            bulkCopy.BulkCopyTimeout = DatabaseCommandTimeoutSeconds;
 
             bulkCopy.ColumnMappings.Add("ImportRowNumber", "ImportRowNumber");
             foreach (var column in definition.Columns)
@@ -800,6 +804,7 @@ await connection.ExecuteAsync(new CommandDefinition(
         await connection.ExecuteAsync(new CommandDefinition(
             ReplaceTableSql(definition),
             transaction: transaction,
+            commandTimeout: DatabaseCommandTimeoutSeconds,
             cancellationToken: cancellationToken));
 
         await transaction.CommitAsync(cancellationToken);
@@ -813,6 +818,7 @@ private static Task DropTempTableIfExistsAsync(
         return connection.ExecuteAsync(new CommandDefinition(
             $"DROP TABLE IF EXISTS {TempTableName};",
             transaction: transaction,
+            commandTimeout: DatabaseCommandTimeoutSeconds,
             cancellationToken: cancellationToken));
     }
 
@@ -830,6 +836,7 @@ private static async Task RunStagingValidationAsync(
             var duplicateRowNumbers = await connection.QueryAsync<int>(new CommandDefinition(
                 DuplicateKeyValidationSql(uniqueKey, definition.Columns),
                 transaction: transaction,
+                commandTimeout: DatabaseCommandTimeoutSeconds,
                 cancellationToken: cancellationToken));
 
             foreach (var rowNumber in duplicateRowNumbers)

server/Program.cs

@@ -75,9 +75,13 @@
     .MigrationsAssembly("server.core")
     .MigrationsHistoryTable(AppDbContext.MigrationsHistoryTable, AppDbContext.AppSchema)));
 
+var dataConn = DataDbConnection.Resolve(builder.Configuration, conn);
+builder.Services.AddDbContextPool<DataDbContext>(o => o.UseSqlServer(dataConn));
+
 builder.Services
     .AddHealthChecks()
-    .AddDbContextCheck<AppDbContext>();
+    .AddDbContextCheck<AppDbContext>()
+    .AddDbContextCheck<DataDbContext>("data_db");
 
 // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
 builder.Services.AddEndpointsApiExplorer();

tests/server.tests/ChartStringSegments/ChartStringSegmentMappingTests.cs

@@ -10,7 +10,7 @@ public class ChartStringSegmentMappingTests
     [Fact]
     public void Maps_to_data_schema_with_composite_key()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
 
         var entityType = db.Model.FindEntityType(typeof(ChartStringSegment));
 
@@ -24,7 +24,7 @@ public void Maps_to_data_schema_with_composite_key()
     [Fact]
     public void Stores_segment_type_as_string()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
 
         var property = db.Model.FindEntityType(typeof(ChartStringSegment))!
             .FindProperty(nameof(ChartStringSegment.SegmentType));
@@ -35,7 +35,7 @@ public void Stores_segment_type_as_string()
     [Fact]
     public void Sfn_column_allows_ten_characters()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
 
         var property = db.Model.FindEntityType(typeof(ChartStringSegment))!
             .FindProperty(nameof(ChartStringSegment.Sfn));

tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs

@@ -10,7 +10,7 @@ public class ChartStringSegmentSeedTests
     [Fact]
     public async Task EnsureSeeded_derives_one_segment_per_hierarchy_row()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
         await HierarchySeed.EnsureSeededAsync(db);
 
         await ChartStringSegmentSeed.EnsureSeededAsync(db);
@@ -27,7 +27,7 @@ await db.DepartmentHierarchies.CountAsync() +
     [Fact]
     public async Task EnsureSeeded_loads_ern_codes_from_csv()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
         await HierarchySeed.EnsureSeededAsync(db);
 
         await ChartStringSegmentSeed.EnsureSeededAsync(db);
@@ -52,7 +52,7 @@ public async Task EnsureSeeded_loads_ern_codes_from_csv()
     [Fact]
     public async Task EnsureSeeded_maps_segment_types_to_their_hierarchy()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
         await HierarchySeed.EnsureSeededAsync(db);
 
         await ChartStringSegmentSeed.EnsureSeededAsync(db);
@@ -67,7 +67,7 @@ public async Task EnsureSeeded_maps_segment_types_to_their_hierarchy()
     [Fact]
     public async Task EnsureSeeded_is_idempotent()
     {
-        using var db = TestDbContextFactory.CreateInMemory();
+        using var db = TestDbContextFactory.CreateDataInMemory();
         await HierarchySeed.EnsureSeededAsync(db);
 
         await ChartStringSegmentSeed.EnsureSeededAsync(db);