perf: batch the raw data export instead of holding it in memory

eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn.Test/AnswerFilterHelperUTests.cs

@@ -24,6 +24,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 
 namespace InsightDashboard.Pn.Test;
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -32,6 +33,7 @@ namespace InsightDashboard.Pn.Test;
 using Infrastructure.Models.Dashboards;
 using Microsoft.EntityFrameworkCore;
 using Microting.eForm.Infrastructure.Constants;
+using Microting.eForm.Infrastructure.Data.Entities;
 using Microting.InsightDashboardBase.Infrastructure.Data.Entities;
 using NUnit.Framework;
 
@@ -270,4 +272,174 @@ public async Task LocationFilter_PrefersSiteOverTag()
                 "A location must win over a tag rather than intersecting with it.");
         }
     }
+
+    /// <summary>
+    /// The batching the export depends on: seeking through answers in small windows
+    /// must yield every answer exactly once, in the same order a single unbatched
+    /// read would. This is what Skip/Take could not guarantee cheaply, and it is the
+    /// whole point of the keyset rewrite.
+    /// </summary>
+    [Test]
+    public async Task KeysetPaging_StitchesBatchesIntoTheFullSetExactlyOnce()
+    {
+        var answers = DbContext.Answers
+            .AsNoTracking()
+            .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed);
+
+        var expected = await RawDataPaging.OrderNewestFirst(answers)
+            .Select(x => x.Id)
+            .ToListAsync();
+
+        Assert.That(expected, Is.Not.Empty, "Seed data has no answers to page through.");
+
+        const int batchSize = 3;
+        var stitched = new List<int>();
+        DateTime? cursorFinishedAt = null;
+        int? cursorId = null;
+        var guard = 0;
+
+        while (true)
+        {
+            var batch = await RawDataPaging
+                .AfterCursor(answers, cursorFinishedAt, cursorId)
+                .Take(batchSize)
+                .Select(x => new { x.Id, x.FinishedAt })
+                .ToListAsync();
+
+            if (batch.Count == 0)
+            {
+                break;
+            }
+
+            stitched.AddRange(batch.Select(x => x.Id));
+            cursorFinishedAt = batch[^1].FinishedAt;
+            cursorId = batch[^1].Id;
+
+            Assert.That(++guard, Is.LessThan(expected.Count + 10),
+                "The cursor stopped advancing - paging would loop forever.");
+        }
+
+        Assert.That(stitched, Is.EqualTo(expected),
+            "Stitched batches must equal a single ordered read, in order.");
+        Assert.That(stitched.Distinct().Count(), Is.EqualTo(stitched.Count),
+            "No answer may appear in two batches.");
+    }
+
+    /// <summary>
+    /// Answers sharing a FinishedAt are exactly where paging breaks without a total
+    /// order, so they must not straddle a batch boundary incorrectly.
+    /// </summary>
+    [Test]
+    public async Task KeysetPaging_HandlesAnswersSharingATimestamp()
+    {
+        var duplicated = await DbContext.Answers
+            .AsNoTracking()
+            .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed)
+            .GroupBy(x => x.FinishedAt)
+            .Where(g => g.Count() > 1)
+            .Select(g => g.Key)
+            .FirstOrDefaultAsync();
+
+        if (duplicated == default)
+        {
+            Assert.Ignore("Seed data has no answers sharing a FinishedAt.");
+        }
+
+        var answers = DbContext.Answers
+            .AsNoTracking()
+            .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed)
+            .Where(x => x.FinishedAt == duplicated);
+
+        var expected = await RawDataPaging.OrderNewestFirst(answers).Select(x => x.Id).ToListAsync();
+
+        var stitched = new List<int>();
+        DateTime? cursorFinishedAt = null;
+        int? cursorId = null;
+
+        while (true)
+        {
+            var batch = await RawDataPaging
+                .AfterCursor(answers, cursorFinishedAt, cursorId)
+                .Take(1)
+                .Select(x => new { x.Id, x.FinishedAt })
+                .ToListAsync();
+
+            if (batch.Count == 0)
+            {
+                break;
+            }
+
+            stitched.Add(batch[0].Id);
+            cursorFinishedAt = batch[0].FinishedAt;
+            cursorId = batch[0].Id;
+        }
+
+        Assert.That(stitched, Is.EqualTo(expected),
+            "One-row batches through a timestamp collision must still cover it exactly once.");
+    }
+
+    /// <summary>
+    /// The seeded database happens to contain no two answers sharing a FinishedAt,
+    /// so the test above skips and the timestamp-collision case - the exact reason
+    /// the Id tie-breaker exists - would go unverified. This pins the predicate
+    /// itself against an in-memory set, where the collision can be constructed.
+    ///
+    /// It deliberately complements rather than replaces the database test: that one
+    /// proves the expression translates to SQL, this one proves it is correct when
+    /// timestamps collide.
+    /// </summary>
+    [Test]
+    public void KeysetPaging_TieBreaksOnIdWhenTimestampsCollide()
+    {
+        var shared = new DateTime(2026, 3, 2, 8, 14, 22);
+
+        var answers = new List<Answer>
+        {
+            new() { Id = 1, FinishedAt = shared },
+            new() { Id = 2, FinishedAt = shared },
+            new() { Id = 3, FinishedAt = shared },
+            new() { Id = 4, FinishedAt = shared.AddMinutes(-1) },
+        }.AsQueryable();
+
+        var expected = new[] { 3, 2, 1, 4 };
+
+        var stitched = new List<int>();
+        DateTime? cursorFinishedAt = null;
+        int? cursorId = null;
+
+        // One row at a time, so every step crosses the collision.
+        for (var i = 0; i < expected.Length + 2; i++)
+        {
+            var batch = RawDataPaging
+                .AfterCursor(answers, cursorFinishedAt, cursorId)
+                .Take(1)
+                .ToList();
+
+            if (batch.Count == 0)
+            {
+                break;
+            }
+
+            stitched.Add(batch[0].Id);
+            cursorFinishedAt = batch[0].FinishedAt;
+            cursorId = batch[0].Id;
+        }
+
+        Assert.That(stitched, Is.EqualTo(expected),
+            "Answers sharing a timestamp must be returned newest-id-first, exactly once each.");
+    }
+
+    [Test]
+    public void KeysetPaging_WithoutACursorStartsAtTheNewest()
+    {
+        var answers = new List<Answer>
+        {
+            new() { Id = 1, FinishedAt = new DateTime(2026, 1, 1) },
+            new() { Id = 2, FinishedAt = new DateTime(2026, 3, 1) },
+        }.AsQueryable();
+
+        var first = RawDataPaging.AfterCursor(answers, null, null).First();
+
+        Assert.That(first.Id, Is.EqualTo(2));
+    }
 }

eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn.Test/RawDataExportUTests.cs

@@ -0,0 +1,206 @@
+/*
+The MIT License (MIT)
+
+Copyright (c) 2007 - 2021 Microting A/S
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+namespace InsightDashboard.Pn.Test;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using DocumentFormat.OpenXml.Packaging;
+using DocumentFormat.OpenXml.Spreadsheet;
+using Infrastructure.Models.RawData;
+using NUnit.Framework;
+using Services.RawDataExcelService;
+
+/// <summary>
+/// Covers the streaming xlsx writer that replaced the whole-sheet DOM. These need
+/// no database: each writes a workbook to a temp file and reads it back.
+/// </summary>
+[TestFixture]
+public class RawDataExportUTests
+{
+    private string _file;
+
+    private static List<RawDataColumnModel> Columns() =>
+    [
+        new() { Field = "id", Header = "Id", Kind = RawDataColumnKinds.Answer },
+        new() { Field = "siteName", Header = "Site", Kind = RawDataColumnKinds.Answer },
+        new() { Field = "q7", Header = "1 - Tilfredshed", Kind = RawDataColumnKinds.Smiley },
+    ];
+
+    [SetUp]
+    public void SetUp() =>
+        _file = Path.Combine(Path.GetTempPath(), $"rawdata-{Guid.NewGuid():N}.xlsx");
+
+    [TearDown]
+    public void TearDown()
+    {
+        if (File.Exists(_file))
+        {
+            File.Delete(_file);
+        }
+    }
+
+    private static List<List<string>> ReadSheet(string path)
+    {
+        using var document = SpreadsheetDocument.Open(path, false);
+        var worksheetPart = document.WorkbookPart!.WorksheetParts.First();
+
+        return worksheetPart.Worksheet
+            .Descendants<Row>()
+            .Select(row => row.Elements<Cell>()
+                .Select(cell => cell.CellValue?.Text ?? string.Empty)
+                .ToList())
+            .ToList();
+    }
+
+    [Test]
+    public void Writer_WritesHeaderAndEveryRowStreamedToIt()
+    {
+        var service = new RawDataExcelService(null);
+
+        using (var writer = service.CreateWriter(_file, Columns()))
+        {
+            // Comfortably more than one export batch, so this exercises the case
+            // the batching exists for.
+            for (var i = 1; i <= 5000; i++)
+            {
+                writer.WriteRow(new Dictionary<string, object>
+                {
+                    ["id"] = i,
+                    ["siteName"] = $"Site {i}",
+                    ["q7"] = "Glad (75)",
+                });
+            }
+
+            writer.Complete();
+        }
+
+        var sheet = ReadSheet(_file);
+
+        Assert.That(sheet.Count, Is.EqualTo(5001), "Header plus every streamed row.");
+        Assert.That(sheet[0], Is.EqualTo(new List<string> { "Id", "Site", "1 - Tilfredshed" }));
+        Assert.That(sheet[1], Is.EqualTo(new List<string> { "1", "Site 1", "Glad (75)" }));
+        Assert.That(sheet[5000], Is.EqualTo(new List<string> { "5000", "Site 5000", "Glad (75)" }));
+    }
+
+    [Test]
+    public void Writer_KeepsColumnsAlignedWhenValuesAreMissing()
+    {
+        var service = new RawDataExcelService(null);
+
+        using (var writer = service.CreateWriter(_file, Columns()))
+        {
+            // No siteName. The cell must be skipped at its own reference rather
+            // than shifted left, or every later column in the row would be wrong.
+            writer.WriteRow(new Dictionary<string, object> { ["id"] = 1, ["q7"] = "Sur (25)" });
+            writer.Complete();
+        }
+
+        using var document = SpreadsheetDocument.Open(_file, false);
+        var cells = document.WorkbookPart!.WorksheetParts.First().Worksheet
+            .Descendants<Row>().Last().Elements<Cell>().ToList();
+
+        Assert.That(cells.Count, Is.EqualTo(2));
+        Assert.That(cells[0].CellReference?.Value, Is.EqualTo("A2"));
+        Assert.That(cells[1].CellReference?.Value, Is.EqualTo("C2"),
+            "A missing value must leave a gap rather than shifting later columns.");
+    }
+
+    [Test]
+    public void Writer_StripsControlCharactersThatWouldCorruptTheWorkbook()
+    {
+        var service = new RawDataExcelService(null);
+
+        // A vertical tab is illegal in XML. Written raw, Excel refuses the file.
+        var hostile = "Bad\vvalue";
+
+        using (var writer = service.CreateWriter(_file, Columns()))
+        {
+            writer.WriteRow(new Dictionary<string, object>
+            {
+                ["id"] = 1,
+                ["siteName"] = "Site",
+                ["q7"] = hostile,
+            });
+            writer.Complete();
+        }
+
+        // Reopening is itself the assertion: an illegal character throws here.
+        var sheet = ReadSheet(_file);
+
+        Assert.That(sheet.Count, Is.EqualTo(2));
+        Assert.That(sheet[1][2], Is.EqualTo("Badvalue"));
+        Assert.That(sheet[1][2], Does.Not.Contain('\v'));
+    }
+
+    [Test]
+    public void Writer_KeepsTabsAndNewlinesWhichAreLegal()
+    {
+        var service = new RawDataExcelService(null);
+
+        using (var writer = service.CreateWriter(_file, Columns()))
+        {
+            writer.WriteRow(new Dictionary<string, object>
+            {
+                ["id"] = 1,
+                ["q7"] = "line one\nline two",
+            });
+            writer.Complete();
+        }
+
+        var sheet = ReadSheet(_file);
+        Assert.That(sheet[1][1], Does.Contain("line one"));
+        Assert.That(sheet[1][1], Does.Contain("line two"));
+    }
+
+    [Test]
+    public void Writer_ProducesAnEmptyButValidWorkbookWhenNothingMatches()
+    {
+        var service = new RawDataExcelService(null);
+
+        using (var writer = service.CreateWriter(_file, Columns()))
+        {
+            writer.Complete();
+        }
+
+        var sheet = ReadSheet(_file);
+        Assert.That(sheet.Count, Is.EqualTo(1), "Header only.");
+    }
+
+    [Test]
+    public void Writer_DisposingWithoutCompletingDoesNotThrow()
+    {
+        var service = new RawDataExcelService(null);
+
+        // The failure path: the service abandons the writer and deletes the
+        // partial file. Disposing must not mask the original exception.
+        Assert.DoesNotThrow(() =>
+        {
+            using var writer = service.CreateWriter(_file, Columns());
+            writer.WriteRow(new Dictionary<string, object> { ["id"] = 1 });
+        });
+    }
+}