From 0fcd1b81d5624dfcb58421fd7522dad0ff6bc447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Wed, 29 Jul 2026 09:41:12 +0200 Subject: [PATCH 1/6] docs: design for per-chart-item raw data table One row per Answer beneath each chart, with every Answer field and the AnswerValues pivoted into one column per question (multi-select exploding into one column per option). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EnP42zmHAgo2NZQsa6zdhG --- ...insight-dashboard-raw-data-table-design.md | 345 ++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-29-insight-dashboard-raw-data-table-design.md diff --git a/docs/superpowers/specs/2026-07-29-insight-dashboard-raw-data-table-design.md b/docs/superpowers/specs/2026-07-29-insight-dashboard-raw-data-table-design.md new file mode 100644 index 00000000..71311350 --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-insight-dashboard-raw-data-table-design.md @@ -0,0 +1,345 @@ +# Raw data table for Insight Dashboard chart items + +Date: 2026-07-29 +Status: design approved, not implemented + +## Problem + +Every chart in a dashboard view already renders an aggregated table beneath it +(`DashboardChartDataViewComponent`, fed by `chartData.rawData`). That table shows +percentages and counts per answer option per period — it never shows an individual +answer. + +Users need the underlying data: one row per `Answer`, carrying every field the +answer holds in the database plus the `AnswerValue`s belonging to it. + +## Scope + +Plugin repo only: `eform-angular-insight-dashboard-plugin`. + +No `-base` change, no EF migration, no SDK change. The feature is read-only over +existing SDK tables. + +Because the session is in **base dev mode**, edits are made inside the host app +(`eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn` and +`eform-angular-frontend/eform-client/src/app/plugins/modules/insight-dashboard-pn`) +and synced back with `devgetchanges.sh`. + +## Shape of the table + +### Rows + +One row per `Answer` that fed the chart item the table sits under. An answer +qualifies when all of the following hold: + +- `Answer.QuestionSetId == Dashboard.SurveyId` +- `Answer.FinishedAt` within the dashboard's `DateFrom`/`DateTo` (or, when + `Dashboard.Today`, up to today 23:59:59) +- location/tag matches: the dashboard's `LocationId`/`TagId` when the item has + `CompareEnabled == false`; otherwise the item's `CompareLocationsTags` set +- when the item sets both `FilterQuestionId` and `FilterAnswerId`, the answer has + an `AnswerValue` with that question and option +- the answer's value for `DashboardItem.FirstQuestionId` is not among the item's + `IgnoredAnswerValues` options +- the answer has at least one `AnswerValue` for `FirstQuestionId` +- `WorkflowState != "removed"` on the answer, its values, and every joined + question/option/translation row + +Default order: `FinishedAt` descending. + +### Columns + +**Answer group** — fixed, from `Answer : PnBase : BaseEntity`: + +| Field | Source | Default | +|---|---|---| +| Id | `Answers.Id` | shown | +| Microting UID | `Answers.MicrotingUid` | shown | +| Finished at | `Answers.FinishedAt` | shown | +| Duration | `Answers.AnswerDuration` (seconds in DB, `mm:ss` in UI) | shown | +| Site | `Answers.SiteId` → `Sites.Name` | shown | +| Tags | `SiteTags` → `Tags.Name`, joined | shown | +| Unit | `Answers.UnitId` → `Units.MicrotingUid` | shown | +| Language | `Answers.LanguageId` → `Languages.Name` | shown | +| Survey config | `Answers.SurveyConfigurationId` → `.Name` | shown | +| Survey | `Answers.QuestionSetId` → `.Name` | hidden | +| Time zone | `Answers.TimeZone` | hidden | +| UTC adjusted | `Answers.UtcAdjusted` | hidden | +| Created at | `Answers.CreatedAt` | hidden | +| Updated at | `Answers.UpdatedAt` | hidden | +| Version | `Answers.Version` | hidden | +| Workflow state | `Answers.WorkflowState` | hidden | +| Site id / Unit id / Language id | raw FKs | hidden | + +`Tags` is not a column on `Answer`, but it is what dashboards group by, so it +belongs in the default set. + +Hidden columns are reachable through the mtx-grid column picker and are always +present in the Excel export. + +**Answer-value group** — dynamic, built from the survey's questions: + +- every non-removed `Question` with `QuestionSetId == Dashboard.SurveyId`, + ordered by `QuestionIndex`, labelled `"{index} – {question text}"` starting at 1 + (matching `DictionaryService`'s existing numbering) +- `multi` questions expand to one column per non-removed `Option`, ordered by + `OptionIndex`, labelled `"{question label} › {option text}"` +- every other question type gets exactly one column + +Cell content by question type: + +| Type | Cell | +|---|---| +| smiley (`smiley`…`smiley10`) | `"{label} ({WeightValue})"`, e.g. `Glad (75)`; `999` → `Ved ikke (999)` | +| `buttons`, `list` | the chosen option's translated name | +| `multi` | in the chosen option's own column, that option's translated name; blank in the options not chosen | +| `text`, `text_email`, `zipcode`, `number` | `AnswerValue.Value` | +| `picture`, `info_text` | blank | +| not answered (option named `na`) | `—`; for a `multi` question, `—` in *every* one of its option columns, so "skipped" stays distinct from "offered and not picked" | + +Smiley labels come from the same source the chart uses, so the two agree. Where +the item has `CalculateByWeight`, that is `OptionTranslations` for the option; +otherwise it is the hardcoded Danish ladder in `ChartDataHelpers`. Reproducing +that split is deliberate — the raw table must not disagree with the chart above it. + +### Placement and interaction + +Rendered in `dashboard-block-view.component.html` immediately after +``, collapsed behind a disclosure reading +`▸ Raw data`. Data loads on first expand, not with the dashboard view — so the +label carries no count until it is opened, after which it reads +`▾ Raw data — 300 answers`. Collapsing again keeps the loaded page in memory; +re-expanding does not refetch. + +Server-side paging and sorting. Excel export of the full unpaged result. + +## Architecture + +### Backend — `InsightDashboard.Pn` + +New files: + +| File | Role | +|---|---| +| `Controllers/RawDataController.cs` | the two routes below | +| `Services/RawDataService/IRawDataService.cs` | interface | +| `Services/RawDataService/RawDataService.cs` | orchestration | +| `Infrastructure/Helpers/AnswerFilterHelper.cs` | single source of truth for "which answers feed this item" | +| `Infrastructure/Helpers/RawDataColumnBuilder.cs` | builds the column list from the survey | +| `Infrastructure/Models/RawData/RawDataRequestModel.cs` | request | +| `Infrastructure/Models/RawData/RawDataListModel.cs` | response | +| `Infrastructure/Models/RawData/RawDataColumnModel.cs` | column descriptor | +| `Services/RawDataExcelService/IRawDataExcelService.cs` | interface | +| `Services/RawDataExcelService/RawDataExcelService.cs` | dynamic-width OpenXML writer | + +Services registered in `EformInsightDashboardPlugin.ConfigureServices`: +`RawDataService` scoped, `RawDataExcelService` transient — matching how +`InterviewsService` and `InterviewsExcelService` are registered. + +`InterviewsExcelService` is not reusable: it is hardcoded to `ColCount = 6` and +driven by a fixed `InterviewsExport` enum, while this export's column count +depends on the survey. + +Routes: + +``` +POST api/insight-dashboard-pn/dashboard-items/raw-data + → OperationDataResult + +GET api/insight-dashboard-pn/dashboard-items/raw-data/export?dashboardId=&dashboardItemId= + → xlsx stream +``` + +Both `[Authorize]`, guarded by the existing `insight-dashboard_access` claim, on a +plain `Controller` with hardcoded routes — the convention every other controller +in this plugin follows. + +Contracts: + +```csharp +public class RawDataRequestModel +{ + public int DashboardId { get; set; } + public int DashboardItemId { get; set; } + public int Offset { get; set; } + public int PageSize { get; set; } = 25; + public string Sort { get; set; } = "FinishedAt"; + public bool IsSortDsc { get; set; } = true; +} + +public class RawDataListModel +{ + public int Total { get; set; } + public List Columns { get; set; } = new(); + public List> Rows { get; set; } = new(); +} + +public class RawDataColumnModel +{ + public string Field { get; set; } // "finishedAt" | "q17" | "q23_o104" + public string Header { get; set; } // "Finished at" | "2 – …" | "2 – … › Kantine" + public string Kind { get; set; } // answer | smiley | single | multiOption | number | text + public bool DefaultHidden { get; set; } + public bool Sortable { get; set; } // true for answer columns, false for value columns + public int? QuestionId { get; set; } + public int? OptionId { get; set; } +} +``` + +Rows are dictionaries keyed by `Column.Field` because the column set varies per +survey. The frontend maps `Columns` straight onto `MtxGridColumn[]`. + +Sorting is offered on the answer columns only. Sorting by a pivoted question value +would mean sorting on a correlated subquery per column; it is not worth the query +cost for a table whose purpose is export. + +### Frontend — plugin `eform-client` + +New files under +`src/app/plugins/modules/insight-dashboard-pn/`: + +| File | Role | +|---|---| +| `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.{ts,html,scss}` | the component | +| `models/dashboard/raw-data/raw-data-list.model.ts` | mirrors `RawDataListModel` | +| `models/dashboard/raw-data/raw-data-column.model.ts` | mirrors `RawDataColumnModel` | +| `models/dashboard/raw-data/raw-data-request.model.ts` | mirrors `RawDataRequestModel` | +| `models/dashboard/raw-data/index.ts` | barrel | +| `services/insight-dashboard-pn-raw-data.service.ts` | the two calls | + +Declared in `insight-dashboard-pn.module.ts` alongside the other view components. +Rendered from `dashboard-block-view.component.html` after +``, inside the same +`*ngIf="itemModel.firstQuestionType !== questionType.Text"` guard — text-type items +show the interviews grid instead and have no chart to reconcile against. + +The component holds `@Input() dashboardId` and `@Input() itemModel`, and keeps +`expanded`, `loading`, `total`, `columns`, `rows`, and pagination in local +component state. Pagination is deliberately **not** in NgRx: it is transient +per-item view state, and a dashboard renders many items. + +Column construction on the frontend: + +```ts +this.tableHeaders = model.columns.map(c => ({ + header: c.kind === 'answer' ? this.translateService.stream(c.header) : of(c.header), + field: c.field, + hide: c.defaultHidden, + sortable: c.sortable, + sortProp: c.sortable ? {id: c.field} : undefined, +})); +``` + +Answer-column headers go through `translateService.stream(...)` like every other +grid in the plugin. Question and option headers are already resolved text from the +database and are passed through as-is. + +Export button calls the export endpoint and hands the blob to `saveAs`, following +`DashboardInterviewsViewComponent.exportToCsv`. + +New i18n keys in `i18n/en-US.ts` and `i18n/da.ts`: `Raw data`, `answers`, +`No raw data found`, `Export raw data`, and the fixed answer-column headers. + +## Data flow + +1. User expands `▸ Raw data` under item N. +2. Component POSTs `{dashboardId, dashboardItemId, offset: 0, pageSize: 25, sort: 'FinishedAt', isSortDsc: true}`. +3. `RawDataService` loads the `Dashboard` and `DashboardItem` from + `InsightDashboardPnDbContext`, verifies the item belongs to the dashboard, then + opens the SDK context via `core.DbContextHelper.GetDbContext()`. +4. `AnswerFilterHelper` produces an `IQueryable` applying the rules in + *Rows* above. +5. `Total` = `CountAsync()` on that query. The page is an ordered `Skip`/`Take` + slice projecting only answer ids and the fixed answer fields. +6. For that page's answer ids only, load the `AnswerValue`s with their + `Question`, `Option` and the translations for the resolved language. +7. `RawDataColumnBuilder` builds the column list from the survey's questions and + options — independent of the page, so column order is stable across pages. +8. Values are pivoted into one dictionary per answer. + +Only the current page's answer values are loaded, so the response size is bounded +by `PageSize × questions`, not by the size of the date range. This matters: the +existing chart path pulls every `AnswerValue` in range into memory before +grouping, and the raw table must not repeat that. + +## Language resolution + +Question and option text live only in `QuestionTranslations` and +`OptionTranslations`, keyed by `LanguageId`. + +Resolution order: + +1. the logged-in user's language, from the BasePn user service +2. any language the survey is deployed in, via `LanguageQuestionSet` +3. any non-removed translation + +Explicitly **not** `Answer.LanguageId` — `Core.SaveAnswer` hardcodes it to the +Danish row, so it does not describe the answer's actual language. The `Language` +column still displays it, because the task is to show what the database holds. + +This is also a deliberate divergence from the existing chart code, which calls +`OptionTranslations.FirstOrDefault()` with no language filter and therefore +renders whichever row the database returns first. + +## Consistency with the chart + +The invariant users will check first: **the raw table's row count equals the +chart's Total amount for the same item.** + +It holds for every question type except `multi`. When the item's first question is +`multi`, one answer contributes several `AnswerValue`s, so the chart's total +legitimately exceeds the answer count. The expanded disclosure label names the +unit explicitly — `Raw data — 300 answers`, not a bare number — so it is never +read as a count of the same thing the chart totals. + +`AnswerFilterHelper` is the only place the filter rules live for this feature. +`ChartDataHelpers` is **not** refactored to use it in this change: it is 4274 lines +containing two near-identical ~2100-line methods, and changing it risks every chart +on every dashboard. `AnswerFilterHelper` mirrors its rules, with each rule's source +line referenced in a comment. Refactoring `ChartDataHelpers` onto the shared helper +is a follow-up, tracked separately. + +`AnswerHelper` is not reused either. It has two defects that would corrupt this +table: it joins `AnswerValue.QuestionId` to `QuestionTranslation.Id` instead of +`.QuestionId`, and its inner join on `Units` drops every answer with a null +`UnitId`. + +## Error handling + +| Case | Behaviour | +|---|---| +| Dashboard or item not found | `OperationResult` false, localized message | +| Item does not belong to the dashboard | `OperationResult` false — do not silently return the other dashboard's data | +| Item's first question type is `text` | endpoint returns false; the component is not rendered for those items anyway | +| Survey has no non-removed questions | empty column set beyond the answer group, grid shows `No raw data found` | +| Zero matching answers | `Total = 0`, empty rows, grid shows `No raw data found` | +| Export exceeding 100 000 rows | `OperationResult` false with a message naming the limit and suggesting a narrower date range, rather than exhausting memory | + +The 100 000 cap is new; the existing interviews export has none. + +## Testing + +The plugin repo has no C# test project, so verification is Playwright, in the +existing `eform-client/playwright/e2e/plugins/insight-dashboard-pn/` suite: + +1. expand `Raw data` on a smiley item and assert the row count matches the amount + block's Total in the aggregated table above it +2. assert a known multi-select answer places each chosen option's name in that + option's column and leaves the unchosen ones blank +3. assert a skipped question renders `—` across all of its option columns +4. assert the column picker reveals a hidden column (`Time zone`) +5. assert paging changes the rows and preserves the column set +6. assert the export button produces a file + +Page objects follow the existing `InsightDashboard-DashboardView.page.ts` +conventions, with stable ids on the new elements +(`dashboardRawData{position}`, `dashboardRawDataToggle{position}`, +`dashboardRawDataExport{position}`). + +## Follow-ups, explicitly out of scope + +- refactoring `ChartDataHelpers` to share `AnswerFilterHelper` +- fixing the `AnswerHelper` join and inner-join defects +- fixing `InterviewsExport`'s `ColCount = 6` dropping the `Comments` column +- localizing the hardcoded Danish smiley labels +- sorting by question value columns From 79ece40e2f34da4815a87bb991d3bb6199c387f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Wed, 29 Jul 2026 09:54:30 +0200 Subject: [PATCH 2/6] docs: implementation plan for raw data table Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EnP42zmHAgo2NZQsa6zdhG --- ...-07-29-insight-dashboard-raw-data-table.md | 2302 +++++++++++++++++ 1 file changed, 2302 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md diff --git a/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md b/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md new file mode 100644 index 00000000..e043e4f5 --- /dev/null +++ b/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md @@ -0,0 +1,2302 @@ +# Raw Data Table Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a collapsible, server-paged table under each dashboard chart showing one row per `Answer`, with every Answer database field and its `AnswerValue`s pivoted into one column per question. + +**Architecture:** A new `RawDataController` exposes a paged JSON endpoint and an xlsx export. A new `AnswerFilterHelper` reproduces the exact answer-membership rules `ChartDataHelpers` uses, so the table reconciles with the chart above it. A new `RawDataColumnBuilder` derives the dynamic column set from the survey's questions and options. The Angular side maps the returned column descriptors onto `MtxGridColumn[]`, so the grid is entirely data-driven. + +**Tech Stack:** C# / .NET 10 / EF Core / ASP.NET Core `Controller`; Angular 20 / NgRx / `@ng-matero/extensions` mtx-grid / `@swimlane/ngx-charts`; DocumentFormat.OpenXml 3.5.1; Playwright + jest. + +## Global Constraints + +- **Dev mode is BASE DEV MODE.** All edits go in the host app: `/home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/` and `/home/rene/Documents/workspace/microting/eform-angular-frontend/eform-client/src/app/plugins/modules/insight-dashboard-pn/`. **Never** edit the plugin source repo directly during implementation. +- **Never** run `devinstall.sh` — it destroys dev-mode state. +- **Never** `git add .` or `git commit -a`. Stage files by name. +- **No** `-base` repo changes and **no** EF migrations. This feature is read-only over existing SDK tables. +- **No** `.csproj` or `.sln` edits. Every package the code needs is already available transitively: `DocumentFormat.OpenXml` 3.5.1. **ClosedXML is NOT available** — do not reference it. +- Target framework `net10.0`. C# file-scoped namespaces, `namespace X;` form, matching surrounding files. +- Every new C# file starts with the same MIT license header block used by its neighbours (copy verbatim from `Infrastructure/Models/Dashboards/ChartDataItem.cs`, updating nothing). +- Controllers: `[Authorize]`, plain `Controller` base class, hardcoded `[Route("api/insight-dashboard-pn/...")]` per action. No `[ApiController]`. +- Soft deletes: filter `WorkflowState != Constants.WorkflowStates.Removed` on every SDK table touched. +- Commits go in the **host app** during implementation only if the change belongs to core frontend; plugin code is committed in the **source repo** after `devgetchanges.sh` (Task 9). + +## Testing approach — read this before Task 1 + +The plugin repo has **no C# test project**, and this plan does not add one (the approved spec scopes verification to Playwright). The existing 71 jest specs in the plugin are **already broken** — they import `async` from `@angular/core/testing`, which no longer exists — so `npx jest` on the plugin directory is red before any change. Do not treat that as a regression you caused, and do not attempt to fix those stubs. + +Consequently: +- Backend logic is verified by `dotnet build` plus the Playwright reconciliation test in Task 8. There are no backend unit tests. This is a known gap; the highest-risk logic (multi-select column placement, skipped-question rendering, row-count reconciliation) is covered end-to-end in Task 8. +- The one new jest spec (Task 7) is written correctly with `waitForAsync` and must pass on its own. + +**Verification commands used throughout:** + +```bash +# Backend build +cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo + +# Frontend typecheck/build +cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eform-client && npx ng build + +# The single new jest spec +cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eform-client && npx jest src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view +``` + +A clean `dotnet build` baseline was confirmed before this plan was written: `Build succeeded. 0 Warning(s) 0 Error(s)`. + +## File Structure + +Backend, all under `.../Plugins/InsightDashboard.Pn/InsightDashboard.Pn/`: + +| File | Responsibility | +|---|---| +| `Infrastructure/Models/RawData/RawDataRequestModel.cs` | Request: which item, paging, sorting | +| `Infrastructure/Models/RawData/RawDataColumnModel.cs` | One column descriptor | +| `Infrastructure/Models/RawData/RawDataColumnKinds.cs` | String constants for `Kind` | +| `Infrastructure/Models/RawData/RawDataFields.cs` | String constants for fixed answer field keys | +| `Infrastructure/Models/RawData/RawDataListModel.cs` | Response: total + columns + rows | +| `Infrastructure/Models/RawData/RawDataExportRequestModel.cs` | Export request | +| `Infrastructure/Helpers/AnswerFilterHelper.cs` | Answer membership — the reconciliation contract | +| `Infrastructure/Helpers/RawDataTranslations.cs` | Language preference order + translation picking | +| `Infrastructure/Models/RawData/RawDataSchema.cs` | Columns + per-question pivot metadata | +| `Infrastructure/Helpers/RawDataColumnBuilder.cs` | Survey questions/options → `RawDataSchema` | +| `Services/RawDataService/IRawDataService.cs` | Interface | +| `Services/RawDataService/RawDataService.cs` | Orchestration | +| `Services/RawDataExcelService/IRawDataExcelService.cs` | Interface | +| `Services/RawDataExcelService/RawDataExcelService.cs` | Dynamic-width OpenXML writer | +| `Controllers/RawDataController.cs` | Two routes | + +Modified: `EformInsightDashboardPlugin.cs` (DI), `Resources/localization.json` (error strings). + +Frontend, all under `.../eform-client/src/app/plugins/modules/insight-dashboard-pn/`: + +| File | Responsibility | +|---|---| +| `models/dashboard/raw-data/raw-data-column.model.ts` | Mirrors `RawDataColumnModel` | +| `models/dashboard/raw-data/raw-data-list.model.ts` | Mirrors `RawDataListModel` | +| `models/dashboard/raw-data/raw-data-request.model.ts` | Mirrors `RawDataRequestModel` | +| `models/dashboard/raw-data/index.ts` | Barrel | +| `services/insight-dashboard-pn-raw-data.service.ts` | Two HTTP calls | +| `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts` | Component logic | +| `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.html` | Disclosure + grid | +| `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.scss` | Minimal styling | +| `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts` | jest spec | + +Modified: `insight-dashboard-pn.module.ts`, `components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html`, `components/dashboards/view/index.ts`, `services/index.ts`, `models/dashboard/index.ts`, `i18n/en-US.ts`, `i18n/da.ts`. + +Playwright, in the **plugin source repo** (`/home/rene/Documents/workspace/microting/eform-angular-insight-dashboard-plugin/eform-client/playwright/`) — see Task 8 for why. + +--- + +### Task 1: Raw data DTOs and constants + +**Files:** +- Create: `Infrastructure/Models/RawData/RawDataRequestModel.cs` +- Create: `Infrastructure/Models/RawData/RawDataExportRequestModel.cs` +- Create: `Infrastructure/Models/RawData/RawDataColumnModel.cs` +- Create: `Infrastructure/Models/RawData/RawDataColumnKinds.cs` +- Create: `Infrastructure/Models/RawData/RawDataFields.cs` +- Create: `Infrastructure/Models/RawData/RawDataListModel.cs` + +**Interfaces:** +- Consumes: nothing. +- Produces: `RawDataRequestModel{DashboardId,DashboardItemId,PageSize,Offset,Sort,IsSortDsc}`, `RawDataExportRequestModel{DashboardId,DashboardItemId}`, `RawDataColumnModel{Field,Header,Kind,DefaultHidden,Sortable,QuestionId,OptionId}`, `RawDataListModel{Total,Columns,Rows}` where `Rows` is `List>`, plus the `RawDataColumnKinds` and `RawDataFields` constant classes. Every later backend task depends on these exact names. + +- [ ] **Step 1: Create the request models** + +`Infrastructure/Models/RawData/RawDataRequestModel.cs` (prefix with the MIT header copied from `Infrastructure/Models/Dashboards/ChartDataItem.cs`): + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +using Microting.eFormApi.BasePn.Infrastructure.Interfaces; + +public class RawDataRequestModel : ICommonSort, ICommonPagination +{ + public int DashboardId { get; set; } + public int DashboardItemId { get; set; } + public int PageSize { get; set; } + public int Offset { get; set; } + public string Sort { get; set; } + public bool IsSortDsc { get; set; } +} +``` + +`Infrastructure/Models/RawData/RawDataExportRequestModel.cs`: + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public class RawDataExportRequestModel +{ + public int DashboardId { get; set; } + public int DashboardItemId { get; set; } +} +``` + +- [ ] **Step 2: Create the constant classes** + +`Infrastructure/Models/RawData/RawDataColumnKinds.cs`: + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public static class RawDataColumnKinds +{ + public const string Answer = "answer"; + public const string Smiley = "smiley"; + public const string Single = "single"; + public const string MultiOption = "multiOption"; + public const string Number = "number"; + public const string Text = "text"; + public const string Other = "other"; +} +``` + +`Infrastructure/Models/RawData/RawDataFields.cs`. These strings are simultaneously the JSON row keys, the mtx-grid `field` values, and the `Sort` values the client sends back: + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public static class RawDataFields +{ + public const string Id = "id"; + public const string MicrotingUid = "microtingUid"; + public const string FinishedAt = "finishedAt"; + public const string AnswerDuration = "answerDuration"; + public const string SiteName = "siteName"; + public const string TagNames = "tagNames"; + public const string UnitMicrotingUid = "unitMicrotingUid"; + public const string LanguageName = "languageName"; + public const string SurveyConfigurationName = "surveyConfigurationName"; + public const string QuestionSetName = "questionSetName"; + public const string TimeZone = "timeZone"; + public const string UtcAdjusted = "utcAdjusted"; + public const string CreatedAt = "createdAt"; + public const string UpdatedAt = "updatedAt"; + public const string Version = "version"; + public const string WorkflowState = "workflowState"; + public const string SiteId = "siteId"; + public const string UnitId = "unitId"; + public const string LanguageId = "languageId"; +} +``` + +- [ ] **Step 3: Create the response models** + +`Infrastructure/Models/RawData/RawDataColumnModel.cs`: + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public class RawDataColumnModel +{ + public string Field { get; set; } + public string Header { get; set; } + public string Kind { get; set; } + public bool DefaultHidden { get; set; } + public bool Sortable { get; set; } + public int? QuestionId { get; set; } + public int? OptionId { get; set; } +} +``` + +`Infrastructure/Models/RawData/RawDataListModel.cs`: + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +using System.Collections.Generic; + +public class RawDataListModel +{ + public int Total { get; set; } + public List Columns { get; set; } = new(); + public List> Rows { get; set; } = new(); +} +``` + +- [ ] **Step 4: Build** + +Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` +Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` + +- [ ] **Step 5: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData +git commit -m "feat(insight-dashboard): add raw data DTOs" +``` + +Note: this commit is in the host app purely to keep the work bisectable during implementation. The authoritative commit happens in the plugin source repo in Task 9. + +--- + +### Task 2: AnswerFilterHelper — the reconciliation contract + +This is the highest-risk file in the plan. Its only job is to select **exactly** the answers that fed the chart above the table. Every predicate below is copied from `Infrastructure/Helpers/ChartDataHelpers.cs`, with the source line numbers in comments so a future reader can diff them. + +**Files:** +- Create: `Infrastructure/Helpers/AnswerFilterHelper.cs` + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `AnswerFilterHelper.BuildAnswerQuery(MicrotingDbContext sdkContext, DashboardItem dashboardItem, int dashboardSurveyId, int? dashboardLocationId, int? dashboardLocationTagId, DashboardEditAnswerDates answerDates)` returning `IQueryable`. Task 4 and Task 6 both call this. + +- [ ] **Step 1: Write the helper** + +`Infrastructure/Helpers/AnswerFilterHelper.cs` (prefix with the MIT header): + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System; +using System.Linq; +using Microting.eForm.Infrastructure; +using Microting.eForm.Infrastructure.Constants; +using Microting.eForm.Infrastructure.Data.Entities; +using Microting.InsightDashboardBase.Infrastructure.Data.Entities; +using Microting.InsightDashboardBase.Infrastructure.Enums; +using Models.Dashboards; + +/// +/// Selects the answers that feed a single dashboard item. +/// +/// This MUST stay behaviourally identical to the answer-selection half of +/// ChartDataHelpers.CalculateDashboardItem, otherwise the raw data table will +/// disagree with the chart it sits under. Line references below point at +/// ChartDataHelpers.cs as of the commit that introduced this file. +/// +/// Two deliberate deviations, both documented in +/// docs/superpowers/specs/2026-07-29-insight-dashboard-raw-data-table-design.md: +/// 1. Answer.WorkflowState is also filtered (ChartDataHelpers filters only +/// AnswerValue.WorkflowState). The delete path sets both together, so this +/// does not change counts in practice. +/// 2. The filter-question step uses a correlated subquery instead of +/// materialising answer ids with ToList(). Semantically identical, one +/// fewer round trip. +/// +public static class AnswerFilterHelper +{ + public static IQueryable BuildAnswerQuery( + MicrotingDbContext sdkContext, + DashboardItem dashboardItem, + int dashboardSurveyId, + int? dashboardLocationId, + int? dashboardLocationTagId, + DashboardEditAnswerDates answerDates) + { + // ChartDataHelpers.cs:135-142 + var answerValues = sdkContext.AnswerValues + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.Answer.WorkflowState != Constants.WorkflowStates.Removed) + .AsQueryable(); + + // ChartDataHelpers.cs:144-154 + if (answerDates.Today) + { + var dateTimeNow = DateTime.Now; + answerDates.DateTo = new DateTime( + dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); + } + + // ChartDataHelpers.cs:156-166 + if (answerDates.DateFrom != null) + { + answerValues = answerValues.Where(x => x.Answer.FinishedAt >= answerDates.DateFrom); + } + + if (answerDates.DateTo != null) + { + answerValues = answerValues.Where(x => x.Answer.FinishedAt <= answerDates.DateTo); + } + + // ChartDataHelpers.cs:170-171 + answerValues = answerValues.Where(x => x.Answer.QuestionSetId == dashboardSurveyId); + + // ChartDataHelpers.cs:173-190 + if (dashboardItem.FilterQuestionId != null && dashboardItem.FilterAnswerId != null) + { + var filterScope = answerValues; + answerValues = answerValues + .Where(x => filterScope.Any(y => + y.AnswerId == x.AnswerId + && y.QuestionId == dashboardItem.FilterQuestionId + && y.OptionId == dashboardItem.FilterAnswerId)) + .Where(x => x.QuestionId == dashboardItem.FirstQuestionId); + } + else + { + answerValues = answerValues.Where(x => x.QuestionId == dashboardItem.FirstQuestionId); + } + + // ChartDataHelpers.cs:223-236 — note this block only runs when compare is OFF + if (!dashboardItem.CompareEnabled) + { + if (dashboardLocationId != null) + { + answerValues = answerValues.Where(x => x.Answer.SiteId == dashboardLocationId); + } + else if (dashboardLocationTagId != null) + { + answerValues = answerValues.Where(x => + x.Answer.Site.SiteTags.Any(y => y.TagId == dashboardLocationTagId)); + } + } + + // ChartDataHelpers.cs:240-252 — ignored answer OPTIONS (the column is + // misleadingly named AnswerId but holds options.Id) + var ignoredOptionIds = dashboardItem.IgnoredAnswerValues + .Where(y => y.WorkflowState != Constants.WorkflowStates.Removed) + .Select(x => x.AnswerId) + .ToArray(); + + if (ignoredOptionIds.Length > 0) + { + answerValues = answerValues.Where(x => !ignoredOptionIds.Contains(x.OptionId)); + } + + var answerIds = IsComparedData(dashboardItem) + ? ComparedAnswerIds(answerValues, dashboardItem, dashboardLocationId, dashboardLocationTagId) + : NonComparedAnswerIds(answerValues, dashboardLocationId, dashboardLocationTagId); + + return sdkContext.Answers + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => answerIds.Contains(x.Id)); + } + + // ChartDataHelpers.cs:121-133 + private static bool IsComparedData(DashboardItem dashboardItem) + { + if (dashboardItem.ChartType != DashboardChartTypes.GroupedStackedBarChart + && dashboardItem.ChartType != DashboardChartTypes.Line) + { + return false; + } + + if (dashboardItem.CompareEnabled) + { + return true; + } + + return dashboardItem.ChartType == DashboardChartTypes.Line && dashboardItem.CalculateAverage; + } + + // ChartDataHelpers.cs:255-390 — the union of the per-tag queries and the site query + private static IQueryable ComparedAnswerIds( + IQueryable answerValues, + DashboardItem dashboardItem, + int? dashboardLocationId, + int? dashboardLocationTagId) + { + var tagIds = dashboardItem.CompareEnabled + ? dashboardItem.CompareLocationsTags + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.TagId != null) + .Select(x => (int)x.TagId) + .ToList() + : dashboardLocationTagId != null + ? new List { (int)dashboardLocationTagId } + : new List(); + + var siteIds = dashboardItem.CompareEnabled + ? dashboardItem.CompareLocationsTags + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.LocationId != null) + .Select(x => (int)x.LocationId) + .ToList() + : dashboardLocationId != null + ? new List { (int)dashboardLocationId } + : new List(); + + var byTag = answerValues + .Where(x => x.Answer.Site.SiteTags.Any(y => y.TagId != null && tagIds.Contains((int)y.TagId))) + .Select(x => x.AnswerId); + + var bySite = answerValues + .Where(x => siteIds.Contains(x.Answer.SiteId)) + .Select(x => x.AnswerId); + + return byTag.Union(bySite).Distinct(); + } + + // ChartDataHelpers.cs:392-490 — when neither location nor tag is set the + // chart renders nothing, so the raw table must be empty too. + private static IQueryable NonComparedAnswerIds( + IQueryable answerValues, + int? dashboardLocationId, + int? dashboardLocationTagId) + { + if (dashboardLocationId == null && dashboardLocationTagId == null) + { + return answerValues.Where(x => false).Select(x => x.AnswerId); + } + + return answerValues.Select(x => x.AnswerId).Distinct(); + } +} +``` + +Add `using System.Collections.Generic;` to the using block — `List` is used in `ComparedAnswerIds`. + +- [ ] **Step 2: Build** + +Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` +Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` + +If the build reports that `DashboardEditAnswerDates` is not found, its namespace is `InsightDashboard.Pn.Infrastructure.Models.Dashboards` — the `using Models.Dashboards;` line above resolves it relative to `InsightDashboard.Pn.Infrastructure`. + +- [ ] **Step 3: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/AnswerFilterHelper.cs +git commit -m "feat(insight-dashboard): add AnswerFilterHelper mirroring chart answer selection" +``` + +--- + +### Task 3: Translation picking and the column/schema builder + +**Files:** +- Create: `Infrastructure/Helpers/RawDataTranslations.cs` +- Create: `Infrastructure/Models/RawData/RawDataSchema.cs` +- Create: `Infrastructure/Helpers/RawDataColumnBuilder.cs` + +**Interfaces:** +- Consumes: `RawDataColumnModel`, `RawDataColumnKinds`, `RawDataFields` (Task 1). +- Produces: + - `RawDataTranslations.GetPreferredLanguageIdsAsync(MicrotingDbContext, int questionSetId, int userLanguageId)` → `Task>` + - `RawDataTranslations.Pick(IReadOnlyList<(int LanguageId, string Name)>, IReadOnlyList preferred)` → `string` + - `RawDataQuestionMeta` with `QuestionId`, `IsSmiley`, `IsMulti`, `Field`, `OptionNameByOptionId`, `OptionFieldByOptionId`, `WeightValueByOptionId`, `OptionFields` + - `RawDataSchema` with `Columns` and `Questions` + - `RawDataColumnBuilder.BuildAsync(MicrotingDbContext, int questionSetId, IReadOnlyList preferredLanguageIds)` → `Task` + +Task 4 calls `BuildAsync` and pivots using `RawDataQuestionMeta`. + +- [ ] **Step 1: Write the translation picker** + +`Infrastructure/Helpers/RawDataTranslations.cs` (MIT header first): + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microting.eForm.Infrastructure; +using Microting.eForm.Infrastructure.Constants; + +/// +/// Question and option text lives only in QuestionTranslations / OptionTranslations, +/// keyed by LanguageId. Answer.LanguageId is NOT usable for this — Core.SaveAnswer +/// hardcodes it to the Danish row — so the raw data table resolves text by the +/// logged-in user's language instead, then the survey's deployed languages, then +/// anything non-removed. +/// +public static class RawDataTranslations +{ + public static async Task> GetPreferredLanguageIdsAsync( + MicrotingDbContext sdkContext, + int questionSetId, + int userLanguageId) + { + var surveyLanguageIds = await sdkContext.LanguageQuestionSets + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionSetId == questionSetId) + .Select(x => x.LanguageId) + .ToListAsync(); + + var preferred = new List { userLanguageId }; + preferred.AddRange(surveyLanguageIds.Where(x => x != userLanguageId)); + return preferred; + } + + public static string Pick( + IReadOnlyList<(int LanguageId, string Name)> translations, + IReadOnlyList preferredLanguageIds) + { + foreach (var languageId in preferredLanguageIds) + { + var match = translations.FirstOrDefault(x => x.LanguageId == languageId); + if (!string.IsNullOrEmpty(match.Name)) + { + return match.Name; + } + } + + return translations + .Select(x => x.Name) + .FirstOrDefault(x => !string.IsNullOrEmpty(x)); + } +} +``` + +- [ ] **Step 2: Write the schema model** + +`Infrastructure/Models/RawData/RawDataSchema.cs`: + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +using System.Collections.Generic; + +public class RawDataQuestionMeta +{ + public int QuestionId { get; set; } + public bool IsSmiley { get; set; } + public bool IsMulti { get; set; } + + /// Row-dictionary key for a single-value question. Null when IsMulti. + public string Field { get; set; } + + /// Row-dictionary key per option. Populated only when IsMulti. + public Dictionary OptionFieldByOptionId { get; set; } = new(); + + public Dictionary OptionNameByOptionId { get; set; } = new(); + public Dictionary WeightValueByOptionId { get; set; } = new(); + + /// All option field keys for this question, used to fill "—" when skipped. + public List OptionFields { get; set; } = new(); +} + +public class RawDataSchema +{ + public List Columns { get; set; } = new(); + public List Questions { get; set; } = new(); +} +``` + +- [ ] **Step 3: Write the column builder** + +`Infrastructure/Helpers/RawDataColumnBuilder.cs`. Note it loads translations into memory rather than projecting them in the EF query — the question count per survey is small (tens), and in-memory resolution avoids depending on EF translating a nested fallback expression. + +```csharp +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microting.eForm.Infrastructure; +using Microting.eForm.Infrastructure.Constants; +using Microting.eForm.Infrastructure.Data.Entities; +using Models.RawData; + +public static class RawDataColumnBuilder +{ + /// + /// Fallback smiley labels, used only when a smiley option has no translation + /// of its own. Mirrors ChartDataHelpers.cs:60-68. 999 means "don't know". + /// + private static readonly Dictionary SmileyFallbackLabels = new() + { + { 100, "Meget glad" }, + { 75, "Glad" }, + { 50, "Neutral" }, + { 25, "Sur" }, + { 0, "Meget sur" }, + { 999, "Ved ikke" }, + }; + + public static string SmileyFallbackLabel(int weightValue) => + SmileyFallbackLabels.TryGetValue(weightValue, out var label) ? label : null; + + public static List BuildAnswerColumns() => + [ + Answer(RawDataFields.Id, "Id", sortable: true), + Answer(RawDataFields.MicrotingUid, "Microting UID", sortable: true), + Answer(RawDataFields.FinishedAt, "Finished at", sortable: true), + Answer(RawDataFields.AnswerDuration, "Duration", sortable: true), + Answer(RawDataFields.SiteName, "Site", sortable: true), + Answer(RawDataFields.TagNames, "Tags", sortable: false), + Answer(RawDataFields.UnitMicrotingUid, "Unit", sortable: true), + Answer(RawDataFields.LanguageName, "Language", sortable: true), + Answer(RawDataFields.SurveyConfigurationName, "Survey config", sortable: true), + Answer(RawDataFields.QuestionSetName, "Survey", sortable: false, hidden: true), + Answer(RawDataFields.TimeZone, "Time zone", sortable: false, hidden: true), + Answer(RawDataFields.UtcAdjusted, "UTC adjusted", sortable: false, hidden: true), + Answer(RawDataFields.CreatedAt, "Created at", sortable: true, hidden: true), + Answer(RawDataFields.UpdatedAt, "Updated at", sortable: true, hidden: true), + Answer(RawDataFields.Version, "Version", sortable: false, hidden: true), + Answer(RawDataFields.WorkflowState, "Workflow state", sortable: true, hidden: true), + Answer(RawDataFields.SiteId, "Site id", sortable: false, hidden: true), + Answer(RawDataFields.UnitId, "Unit id", sortable: false, hidden: true), + Answer(RawDataFields.LanguageId, "Language id", sortable: false, hidden: true), + ]; + + private static RawDataColumnModel Answer( + string field, string header, bool sortable, bool hidden = false) => + new() + { + Field = field, + Header = header, + Kind = RawDataColumnKinds.Answer, + Sortable = sortable, + DefaultHidden = hidden, + }; + + public static async Task BuildAsync( + MicrotingDbContext sdkContext, + int questionSetId, + IReadOnlyList preferredLanguageIds) + { + var schema = new RawDataSchema { Columns = BuildAnswerColumns() }; + + var questions = await sdkContext.Questions + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionSetId == questionSetId) + .OrderBy(x => x.QuestionIndex) + .Select(x => new { x.Id, x.QuestionType }) + .ToListAsync(); + + if (questions.Count == 0) + { + return schema; + } + + var questionIds = questions.Select(x => x.Id).ToList(); + + var questionTranslations = await sdkContext.QuestionTranslations + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => questionIds.Contains(x.QuestionId)) + .Select(x => new { x.QuestionId, x.LanguageId, x.Name }) + .ToListAsync(); + + var options = await sdkContext.Options + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => questionIds.Contains(x.QuestionId)) + .OrderBy(x => x.OptionIndex) + .Select(x => new { x.Id, x.QuestionId, x.WeightValue }) + .ToListAsync(); + + var optionIds = options.Select(x => x.Id).ToList(); + + var optionTranslations = await sdkContext.OptionTranslations + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => optionIds.Contains(x.OptionId)) + .Select(x => new { x.OptionId, x.LanguageId, x.Name }) + .ToListAsync(); + + var questionIndex = 0; + foreach (var question in questions) + { + questionIndex++; + + var questionName = RawDataTranslations.Pick( + questionTranslations + .Where(t => t.QuestionId == question.Id) + .Select(t => (t.LanguageId, t.Name)) + .ToList(), + preferredLanguageIds); + + var questionLabel = $"{questionIndex} – {questionName}"; + var questionOptions = options.Where(o => o.QuestionId == question.Id).ToList(); + + var isSmiley = IsSmileyType(question.QuestionType); + var isMulti = question.QuestionType == Constants.QuestionTypes.Multi; + + var meta = new RawDataQuestionMeta + { + QuestionId = question.Id, + IsSmiley = isSmiley, + IsMulti = isMulti, + }; + + foreach (var option in questionOptions) + { + meta.WeightValueByOptionId[option.Id] = option.WeightValue; + meta.OptionNameByOptionId[option.Id] = RawDataTranslations.Pick( + optionTranslations + .Where(t => t.OptionId == option.Id) + .Select(t => (t.LanguageId, t.Name)) + .ToList(), + preferredLanguageIds); + } + + if (isMulti) + { + foreach (var option in questionOptions) + { + var field = $"q{question.Id}_o{option.Id}"; + meta.OptionFieldByOptionId[option.Id] = field; + meta.OptionFields.Add(field); + + schema.Columns.Add(new RawDataColumnModel + { + Field = field, + Header = $"{questionLabel} › {meta.OptionNameByOptionId[option.Id]}", + Kind = RawDataColumnKinds.MultiOption, + Sortable = false, + DefaultHidden = false, + QuestionId = question.Id, + OptionId = option.Id, + }); + } + } + else + { + meta.Field = $"q{question.Id}"; + meta.OptionFields.Add(meta.Field); + + schema.Columns.Add(new RawDataColumnModel + { + Field = meta.Field, + Header = questionLabel, + Kind = KindFor(question.QuestionType, isSmiley), + Sortable = false, + DefaultHidden = false, + QuestionId = question.Id, + }); + } + + schema.Questions.Add(meta); + } + + return schema; + } + + private static string KindFor(string questionType, bool isSmiley) + { + if (isSmiley) + { + return RawDataColumnKinds.Smiley; + } + + return questionType switch + { + Constants.QuestionTypes.List => RawDataColumnKinds.Single, + Constants.QuestionTypes.Buttons => RawDataColumnKinds.Single, + Constants.QuestionTypes.Number => RawDataColumnKinds.Number, + Constants.QuestionTypes.Text => RawDataColumnKinds.Text, + Constants.QuestionTypes.TextEamil => RawDataColumnKinds.Text, + Constants.QuestionTypes.ZipCode => RawDataColumnKinds.Text, + _ => RawDataColumnKinds.Other, + }; + } + + /// + /// Mirrors Question.IsSmiley() without needing a Question entity instance. + /// + private static bool IsSmileyType(string questionType) => questionType switch + { + Constants.QuestionTypes.Smiley => true, + Constants.QuestionTypes.Smiley2 => true, + Constants.QuestionTypes.Smiley3 => true, + Constants.QuestionTypes.Smiley4 => true, + Constants.QuestionTypes.Smiley5 => true, + Constants.QuestionTypes.Smiley6 => true, + Constants.QuestionTypes.Smiley7 => true, + Constants.QuestionTypes.Smiley8 => true, + Constants.QuestionTypes.Smiley9 => true, + Constants.QuestionTypes.Smiley10 => true, + _ => false, + }; +} +``` + +- [ ] **Step 4: Build** + +Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` +Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` + +If the collection-expression syntax `=> [ ... ]` on `BuildAnswerColumns` is rejected, replace it with `=> new List { ... };` — the rest is unchanged. + +- [ ] **Step 5: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataTranslations.cs \ + eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataColumnBuilder.cs \ + eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataSchema.cs +git commit -m "feat(insight-dashboard): add raw data column builder and translation picking" +``` + +--- + +### Task 4: RawDataService — query, pivot, page + +**Files:** +- Create: `Services/RawDataService/IRawDataService.cs` +- Create: `Services/RawDataService/RawDataService.cs` + +**Interfaces:** +- Consumes: `AnswerFilterHelper.BuildAnswerQuery` (Task 2), `RawDataColumnBuilder.BuildAsync` / `RawDataQuestionMeta` / `RawDataSchema` (Task 3), `RawDataRequestModel` / `RawDataListModel` / `RawDataFields` (Task 1). +- Produces: `IRawDataService.GetRawData(RawDataRequestModel)` → `Task>` and `IRawDataService.GetAllRawData(int dashboardId, int dashboardItemId)` → `Task>`. Task 5 injects this; Task 6 uses `GetAllRawData` for the export. + +- [ ] **Step 1: Write the interface** + +`Services/RawDataService/IRawDataService.cs`: + +```csharp +namespace InsightDashboard.Pn.Services.RawDataService; + +using System.Threading.Tasks; +using Infrastructure.Models.RawData; +using Microting.eFormApi.BasePn.Infrastructure.Models.API; + +public interface IRawDataService +{ + Task> GetRawData(RawDataRequestModel requestModel); + + Task> GetAllRawData(int dashboardId, int dashboardItemId); +} +``` + +- [ ] **Step 2: Write the service** + +`Services/RawDataService/RawDataService.cs`: + +```csharp +namespace InsightDashboard.Pn.Services.RawDataService; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Common.InsightDashboardLocalizationService; +using Infrastructure.Helpers; +using Infrastructure.Models.Dashboards; +using Infrastructure.Models.RawData; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microting.eForm.Infrastructure.Constants; +using Microting.eForm.Infrastructure.Data.Entities; +using Microting.eFormApi.BasePn.Abstractions; +using Microting.eFormApi.BasePn.Infrastructure.Models.API; +using Microting.InsightDashboardBase.Infrastructure.Data; + +public class RawDataService : IRawDataService +{ + /// Hard ceiling for the unpaged export, to fail loudly instead of exhausting memory. + public const int ExportRowLimit = 100000; + + private readonly ILogger _logger; + private readonly IInsightDashboardLocalizationService _localizationService; + private readonly IEFormCoreService _coreHelper; + private readonly InsightDashboardPnDbContext _dbContext; + private readonly IUserService _userService; + + public RawDataService( + ILogger logger, + IInsightDashboardLocalizationService localizationService, + IEFormCoreService coreHelper, + InsightDashboardPnDbContext dbContext, + IUserService userService) + { + _logger = logger; + _localizationService = localizationService; + _coreHelper = coreHelper; + _dbContext = dbContext; + _userService = userService; + } + + public Task> GetRawData(RawDataRequestModel requestModel) => + Build(requestModel.DashboardId, requestModel.DashboardItemId, requestModel, applyPaging: true); + + public Task> GetAllRawData(int dashboardId, int dashboardItemId) => + Build(dashboardId, dashboardItemId, null, applyPaging: false); + + private async Task> Build( + int dashboardId, + int dashboardItemId, + RawDataRequestModel requestModel, + bool applyPaging) + { + try + { + var dashboard = await _dbContext.Dashboards + .Include(x => x.DashboardItems) + .ThenInclude(x => x.IgnoredAnswerValues) + .Include(x => x.DashboardItems) + .ThenInclude(x => x.CompareLocationsTags) + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .FirstOrDefaultAsync(x => x.Id == dashboardId); + + if (dashboard == null) + { + return new OperationDataResult( + false, _localizationService.GetString("DashboardNotFound")); + } + + var dashboardItem = dashboard.DashboardItems + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .FirstOrDefault(x => x.Id == dashboardItemId); + + if (dashboardItem == null) + { + return new OperationDataResult( + false, _localizationService.GetString("DashboardItemNotFound")); + } + + if (dashboard.Today) + { + var dateTimeNow = DateTime.Now; + dashboard.DateTo = new DateTime( + dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); + } + + var answerDates = new DashboardEditAnswerDates + { + Today = dashboard.Today, + DateFrom = dashboard.DateFrom, + DateTo = dashboard.DateTo, + }; + + var core = await _coreHelper.GetCore(); + var userLanguage = await _userService.GetCurrentUserLanguage(); + + await using var sdkContext = core.DbContextHelper.GetDbContext(); + + var preferredLanguageIds = await RawDataTranslations.GetPreferredLanguageIdsAsync( + sdkContext, dashboard.SurveyId, userLanguage.Id); + + var schema = await RawDataColumnBuilder.BuildAsync( + sdkContext, dashboard.SurveyId, preferredLanguageIds); + + var answerQuery = AnswerFilterHelper.BuildAnswerQuery( + sdkContext, + dashboardItem, + dashboard.SurveyId, + dashboard.LocationId, + dashboard.TagId, + answerDates); + + var result = new RawDataListModel + { + Columns = schema.Columns, + Total = await answerQuery.CountAsync(), + }; + + if (!applyPaging && result.Total > ExportRowLimit) + { + return new OperationDataResult( + false, + string.Format( + _localizationService.GetString("RawDataExportTooLarge"), + result.Total, + ExportRowLimit)); + } + + var sort = requestModel?.Sort; + var isSortDsc = requestModel?.IsSortDsc ?? true; + var ordered = ApplySort(answerQuery, sort, isSortDsc); + + if (applyPaging) + { + ordered = ordered.Skip(requestModel.Offset).Take(requestModel.PageSize); + } + + var answers = await ordered + .Select(x => new AnswerRow + { + Id = x.Id, + MicrotingUid = x.MicrotingUid, + FinishedAt = x.FinishedAt, + AnswerDuration = x.AnswerDuration, + SiteId = x.SiteId, + SiteName = x.Site.Name, + TagNames = x.Site.SiteTags + .Where(y => y.WorkflowState != Constants.WorkflowStates.Removed) + .Select(y => y.Tag.Name) + .ToList(), + UnitId = x.UnitId, + UnitMicrotingUid = x.Unit.MicrotingUid, + LanguageId = x.LanguageId, + LanguageName = x.Language.Name, + SurveyConfigurationName = x.SurveyConfiguration.Name, + QuestionSetName = x.QuestionSet.Name, + TimeZone = x.TimeZone, + UtcAdjusted = x.UtcAdjusted, + CreatedAt = x.CreatedAt, + UpdatedAt = x.UpdatedAt, + Version = x.Version, + WorkflowState = x.WorkflowState, + }) + .ToListAsync(); + + var answerIds = answers.Select(x => x.Id).ToList(); + + var values = await sdkContext.AnswerValues + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => answerIds.Contains(x.AnswerId)) + .Select(x => new { x.AnswerId, x.QuestionId, x.OptionId, x.Value }) + .ToListAsync(); + + var metaByQuestionId = schema.Questions.ToDictionary(x => x.QuestionId); + var valuesByAnswerId = values.GroupBy(x => x.AnswerId) + .ToDictionary(x => x.Key, x => x.ToList()); + + foreach (var answer in answers) + { + var row = ToRowDictionary(answer); + + // Every question column starts as "not answered"; real values overwrite it. + foreach (var meta in schema.Questions) + { + foreach (var field in meta.OptionFields) + { + row[field] = NotAnswered; + } + } + + if (valuesByAnswerId.TryGetValue(answer.Id, out var answerValues)) + { + foreach (var answerValue in answerValues) + { + if (!metaByQuestionId.TryGetValue(answerValue.QuestionId, out var meta)) + { + continue; + } + + var optionName = meta.OptionNameByOptionId.GetValueOrDefault(answerValue.OptionId); + var skipped = string.Equals(optionName, NaOptionName, StringComparison.OrdinalIgnoreCase); + + if (meta.IsMulti) + { + // A skipped multi question leaves every option column as "—". + if (skipped) + { + continue; + } + + foreach (var field in meta.OptionFields) + { + if (Equals(row[field], NotAnswered)) + { + row[field] = string.Empty; + } + } + + var optionField = meta.OptionFieldByOptionId.GetValueOrDefault(answerValue.OptionId); + if (optionField != null) + { + row[optionField] = optionName; + } + + continue; + } + + row[meta.Field] = skipped + ? NotAnswered + : ResolveSingleValue(meta, answerValue.OptionId, answerValue.Value, optionName); + } + } + + result.Rows.Add(row); + } + + return new OperationDataResult(true, result); + } + catch (Exception e) + { + Trace.TraceError(e.Message); + _logger.LogError(e, e.Message); + return new OperationDataResult( + false, _localizationService.GetString("ErrorWhileObtainingRawData")); + } + } + + private const string NotAnswered = "—"; + private const string NaOptionName = "na"; + + private static string ResolveSingleValue( + RawDataQuestionMeta meta, int optionId, string value, string optionName) + { + if (meta.IsSmiley) + { + var weightValue = meta.WeightValueByOptionId.GetValueOrDefault(optionId); + var label = !string.IsNullOrEmpty(optionName) + ? optionName + : RawDataColumnBuilder.SmileyFallbackLabel(weightValue); + + return string.IsNullOrEmpty(label) + ? weightValue.ToString() + : $"{label} ({weightValue})"; + } + + return !string.IsNullOrEmpty(optionName) ? optionName : value; + } + + private static Dictionary ToRowDictionary(AnswerRow answer) => new() + { + [RawDataFields.Id] = answer.Id, + [RawDataFields.MicrotingUid] = answer.MicrotingUid, + [RawDataFields.FinishedAt] = answer.FinishedAt, + [RawDataFields.AnswerDuration] = FormatDuration(answer.AnswerDuration), + [RawDataFields.SiteName] = answer.SiteName, + [RawDataFields.TagNames] = string.Join(", ", answer.TagNames), + [RawDataFields.UnitMicrotingUid] = answer.UnitMicrotingUid, + [RawDataFields.LanguageName] = answer.LanguageName, + [RawDataFields.SurveyConfigurationName] = answer.SurveyConfigurationName, + [RawDataFields.QuestionSetName] = answer.QuestionSetName, + [RawDataFields.TimeZone] = answer.TimeZone, + [RawDataFields.UtcAdjusted] = answer.UtcAdjusted, + [RawDataFields.CreatedAt] = answer.CreatedAt, + [RawDataFields.UpdatedAt] = answer.UpdatedAt, + [RawDataFields.Version] = answer.Version, + [RawDataFields.WorkflowState] = answer.WorkflowState, + [RawDataFields.SiteId] = answer.SiteId, + [RawDataFields.UnitId] = answer.UnitId, + [RawDataFields.LanguageId] = answer.LanguageId, + }; + + /// AnswerDuration is stored in seconds; the UI shows mm:ss. + private static string FormatDuration(int seconds) => + $"{seconds / 60:D2}:{seconds % 60:D2}"; + + private static IQueryable ApplySort(IQueryable query, string sort, bool isSortDsc) => + sort switch + { + RawDataFields.Id => Order(query, x => x.Id, isSortDsc), + RawDataFields.MicrotingUid => Order(query, x => x.MicrotingUid, isSortDsc), + RawDataFields.AnswerDuration => Order(query, x => x.AnswerDuration, isSortDsc), + RawDataFields.SiteName => Order(query, x => x.Site.Name, isSortDsc), + RawDataFields.UnitMicrotingUid => Order(query, x => x.Unit.MicrotingUid, isSortDsc), + RawDataFields.LanguageName => Order(query, x => x.Language.Name, isSortDsc), + RawDataFields.SurveyConfigurationName => Order(query, x => x.SurveyConfiguration.Name, isSortDsc), + RawDataFields.CreatedAt => Order(query, x => x.CreatedAt, isSortDsc), + RawDataFields.UpdatedAt => Order(query, x => x.UpdatedAt, isSortDsc), + RawDataFields.WorkflowState => Order(query, x => x.WorkflowState, isSortDsc), + _ => Order(query, x => x.FinishedAt, isSortDsc), + }; + + private static IQueryable Order( + IQueryable query, Expression> keySelector, bool isSortDsc) => + isSortDsc ? query.OrderByDescending(keySelector) : query.OrderBy(keySelector); + + private class AnswerRow + { + public int Id { get; init; } + public int? MicrotingUid { get; init; } + public DateTime FinishedAt { get; init; } + public int AnswerDuration { get; init; } + public int SiteId { get; init; } + public string SiteName { get; init; } + public List TagNames { get; init; } = new(); + public int? UnitId { get; init; } + public int? UnitMicrotingUid { get; init; } + public int LanguageId { get; init; } + public string LanguageName { get; init; } + public string SurveyConfigurationName { get; init; } + public string QuestionSetName { get; init; } + public string TimeZone { get; init; } + public bool UtcAdjusted { get; init; } + public DateTime? CreatedAt { get; init; } + public DateTime? UpdatedAt { get; init; } + public int? Version { get; init; } + public string WorkflowState { get; init; } + } +} +``` + +- [ ] **Step 3: Build** + +Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` +Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` + +Two likely compile issues and their fixes: +- If `.ThenInclude(x => x.IgnoredAnswerValues)` is ambiguous, use the explicit generic form `DashboardService.cs` uses: `.ThenInclude>(x => x.IgnoredAnswerValues)` and the matching `List` form, adding `using Microting.InsightDashboardBase.Infrastructure.Data.Entities;`. +- `Site.SiteTags` has no `WorkflowState` filter available if `SiteTag` does not expose it — it derives from `PnBase`, so it does; if the compiler disagrees, drop that `.Where` clause. + +- [ ] **Step 4: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService +git commit -m "feat(insight-dashboard): add RawDataService with answer pivot and paging" +``` + +--- + +### Task 5: Controller, DI registration, localization strings + +**Files:** +- Create: `Controllers/RawDataController.cs` +- Modify: `EformInsightDashboardPlugin.cs` (ConfigureServices + usings) +- Modify: `Resources/localization.json` + +**Interfaces:** +- Consumes: `IRawDataService` (Task 4). +- Produces: `POST api/insight-dashboard-pn/dashboard-items/raw-data`. Task 7's Angular service calls it. + +- [ ] **Step 1: Write the controller** + +`Controllers/RawDataController.cs` (the export action is added in Task 6): + +```csharp +namespace InsightDashboard.Pn.Controllers; + +using System.Threading.Tasks; +using Infrastructure.Models.RawData; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microting.eFormApi.BasePn.Infrastructure.Models.API; +using Services.RawDataService; + +[Authorize] +public class RawDataController : Controller +{ + private readonly IRawDataService _rawDataService; + + public RawDataController(IRawDataService rawDataService) + { + _rawDataService = rawDataService; + } + + [HttpPost] + [Route("api/insight-dashboard-pn/dashboard-items/raw-data")] + public async Task> GetRawData( + [FromBody] RawDataRequestModel requestModel) + { + return await _rawDataService.GetRawData(requestModel); + } +} +``` + +- [ ] **Step 2: Register the service** + +In `EformInsightDashboardPlugin.cs`, add to the usings block (alongside the other `Services.*` usings): + +```csharp +using Services.RawDataService; +``` + +and add this line to `ConfigureServices`, after `services.AddScoped();`: + +```csharp + services.AddScoped(); +``` + +- [ ] **Step 3: Add localization strings** + +`Resources/localization.json` is a UTF-8-**with-BOM** JSON array of `{"Key": "...", "LocalizedValue": {"da": "...", "en-US": "..."}}`. Preserve the BOM when saving. Append these three entries before the closing `]`: + +```json + { + "Key": "DashboardItemNotFound", + "LocalizedValue": { + "da": "Dashboard-blokken blev ikke fundet", + "en-US": "Dashboard block not found" + } + }, + { + "Key": "ErrorWhileObtainingRawData", + "LocalizedValue": { + "da": "Der opstod en fejl under hentning af rådata", + "en-US": "Error while obtaining raw data" + } + }, + { + "Key": "RawDataExportTooLarge", + "LocalizedValue": { + "da": "Eksporten indeholder {0} svar, hvilket overstiger grænsen på {1}. Vælg en kortere periode.", + "en-US": "The export contains {0} answers, which exceeds the limit of {1}. Choose a shorter period." + } + } +``` + +First check whether `DashboardItemNotFound` already exists — `InterviewsService` references it. If it is already present, add only the other two. + +- [ ] **Step 4: Build and verify the JSON is still valid** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn +python3 -c "import json;d=json.load(open('Resources/localization.json',encoding='utf-8-sig'));print(len(d),'keys')" +dotnet build -v q --nologo +``` +Expected: a key count that is 2 or 3 higher than before, then `Build succeeded. 0 Warning(s) 0 Error(s)` + +- [ ] **Step 5: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs \ + eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs \ + eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Resources/localization.json +git commit -m "feat(insight-dashboard): expose raw data endpoint" +``` + +--- + +### Task 6: Excel export + +`InterviewsExcelService` cannot be reused — it is driven by a fixed 7-member enum with `ColCount = 6`. Note also that it copies an xlsx template and then immediately overwrites it with `SpreadsheetDocument.Create`, making the template copy pointless; the new service skips the template entirely. + +**Files:** +- Create: `Services/RawDataExcelService/IRawDataExcelService.cs` +- Create: `Services/RawDataExcelService/RawDataExcelService.cs` +- Modify: `Controllers/RawDataController.cs` (add the export action) +- Modify: `EformInsightDashboardPlugin.cs` (register the service) + +**Interfaces:** +- Consumes: `RawDataListModel` (Task 1), `IRawDataService.GetAllRawData` (Task 4). +- Produces: `IRawDataExcelService.WriteRawDataToExcelFile(RawDataListModel, string destFile)` → `bool` and `CreateFilePath()` → `string`; `GET api/insight-dashboard-pn/dashboard-items/raw-data/export`. + +- [ ] **Step 1: Write the interface** + +`Services/RawDataExcelService/IRawDataExcelService.cs`: + +```csharp +namespace InsightDashboard.Pn.Services.RawDataExcelService; + +using Infrastructure.Models.RawData; + +public interface IRawDataExcelService +{ + string CreateFilePath(); + + bool WriteRawDataToExcelFile(RawDataListModel model, string destFile); +} +``` + +- [ ] **Step 2: Write the service** + +`Services/RawDataExcelService/RawDataExcelService.cs`: + +```csharp +namespace InsightDashboard.Pn.Services.RawDataExcelService; + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Claims; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Infrastructure.Models.RawData; +using Microsoft.AspNetCore.Http; +using Microting.eFormApi.BasePn.Infrastructure.Helpers; + +public class RawDataExcelService(IHttpContextAccessor httpAccessor) : IRawDataExcelService +{ + public string CreateFilePath() + { + var path = Path.Combine(PathHelper.GetStoragePath(), "excel-storage"); + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + + return Path.Combine(path, $"raw-data-{UserId}-{DateTime.UtcNow.Ticks}.xlsx"); + } + + public bool WriteRawDataToExcelFile(RawDataListModel model, string destFile) + { + using var spreadsheetDocument = + SpreadsheetDocument.Create(destFile, SpreadsheetDocumentType.Workbook); + + var workbookPart = spreadsheetDocument.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + worksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = spreadsheetDocument.WorkbookPart!.Workbook.AppendChild(new Sheets()); + sheets.Append(new Sheet + { + Id = spreadsheetDocument.WorkbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = "Raw data", + }); + + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + var columns = model.Columns; + + // Header row. Hidden columns are exported too — the export is the full record. + var headerRow = new Row { RowIndex = 1U }; + for (var col = 0; col < columns.Count; col++) + { + headerRow.Append(new Cell + { + CellReference = GetCellReference(1, col + 1), + DataType = CellValues.String, + CellValue = new CellValue(columns[col].Header ?? string.Empty), + }); + } + + sheetData!.Append(headerRow); + + var rowIndex = 2; + foreach (var modelRow in model.Rows) + { + var row = new Row { RowIndex = (uint)rowIndex }; + + for (var col = 0; col < columns.Count; col++) + { + var value = modelRow.GetValueOrDefault(columns[col].Field); + if (value == null) + { + continue; + } + + var cell = new Cell { CellReference = GetCellReference(rowIndex, col + 1) }; + + switch (value) + { + case DateTime dateTime: + cell.DataType = CellValues.String; + cell.CellValue = new CellValue( + dateTime.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + break; + case int intValue: + cell.DataType = CellValues.Number; + cell.CellValue = new CellValue(intValue.ToString(CultureInfo.InvariantCulture)); + break; + case bool boolValue: + cell.DataType = CellValues.String; + cell.CellValue = new CellValue(boolValue ? "true" : "false"); + break; + default: + cell.DataType = CellValues.String; + cell.CellValue = new CellValue(value.ToString() ?? string.Empty); + break; + } + + row.Append(cell); + } + + sheetData.Append(row); + rowIndex++; + } + + workbookPart.Workbook.Save(); + return true; + } + + private int UserId + { + get + { + var value = httpAccessor?.HttpContext?.User?.FindFirstValue(ClaimTypes.NameIdentifier); + return value == null ? 0 : int.Parse(value); + } + } + + private static string GetCellReference(int rowIndex, int colIndex) => + $"{GetColumnName(colIndex)}{rowIndex}"; + + private static string GetColumnName(int index) + { + var dividend = index; + var columnName = string.Empty; + while (dividend > 0) + { + var modulo = (dividend - 1) % 26; + columnName = Convert.ToChar(65 + modulo) + columnName; + dividend = (dividend - modulo) / 26; + } + + return columnName; + } +} +``` + +Add `using System.Collections.Generic;` if `GetValueOrDefault` is not resolved. + +- [ ] **Step 3: Add the export action to the controller** + +Add to `Controllers/RawDataController.cs` — usings `System.IO`, `System.Text`, `Services.RawDataExcelService`; inject `IRawDataExcelService` alongside `IRawDataService`; then this action, which follows the streaming shape of `InterviewsController.GenerateReportFile` verbatim: + +```csharp + [HttpGet] + [Route("api/insight-dashboard-pn/dashboard-items/raw-data/export")] + [ProducesResponseType(typeof(string), 400)] + public async Task ExportRawData([FromQuery] RawDataExportRequestModel requestModel) + { + var dataResult = await _rawDataService.GetAllRawData( + requestModel.DashboardId, requestModel.DashboardItemId); + + string filePath = null; + if (dataResult.Success) + { + filePath = _rawDataExcelService.CreateFilePath(); + _rawDataExcelService.WriteRawDataToExcelFile(dataResult.Model, filePath); + } + + const int bufferSize = 4086; + var buffer = new byte[bufferSize]; + + Response.OnStarting(async () => + { + try + { + if (!dataResult.Success) + { + Response.ContentLength = dataResult.Message.Length; + Response.ContentType = "text/plain"; + Response.StatusCode = 400; + var bytes = Encoding.UTF8.GetBytes(dataResult.Message); + await Response.Body.WriteAsync(bytes, 0, bytes.Length); + await Response.Body.FlushAsync(); + } + else + { + await using var excelStream = new FileStream(filePath, FileMode.Open); + int bytesRead; + Response.ContentLength = excelStream.Length; + Response.ContentType = + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + while ((bytesRead = excelStream.Read(buffer, 0, buffer.Length)) > 0 && + !HttpContext.RequestAborted.IsCancellationRequested) + { + await Response.Body.WriteAsync(buffer, 0, bytesRead); + await Response.Body.FlushAsync(); + } + } + } + finally + { + if (!string.IsNullOrEmpty(filePath) && System.IO.File.Exists(filePath)) + { + System.IO.File.Delete(filePath); + } + } + }); + } +``` + +- [ ] **Step 4: Register the Excel service** + +In `EformInsightDashboardPlugin.cs`, add `using Services.RawDataExcelService;` and this line to `ConfigureServices`, next to the other `AddTransient` excel/word registrations: + +```csharp + services.AddTransient(); +``` + +- [ ] **Step 5: Build** + +Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` +Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` + +- [ ] **Step 6: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService \ + eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs \ + eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs +git commit -m "feat(insight-dashboard): add raw data excel export" +``` + +--- + +### Task 7: Angular models, service, component and wiring + +**Files:** +- Create: `models/dashboard/raw-data/raw-data-column.model.ts` +- Create: `models/dashboard/raw-data/raw-data-list.model.ts` +- Create: `models/dashboard/raw-data/raw-data-request.model.ts` +- Create: `models/dashboard/raw-data/index.ts` +- Create: `services/insight-dashboard-pn-raw-data.service.ts` +- Create: `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts` +- Create: `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.html` +- Create: `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.scss` (empty file — the convention here, see `dashboard-block-view.component.scss`) +- Create: `components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts` +- Modify: `models/dashboard/index.ts`, `services/index.ts`, `components/dashboards/view/index.ts` +- Modify: `insight-dashboard-pn.module.ts` +- Modify: `components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html` +- Modify: `i18n/en-US.ts`, `i18n/da.ts` + +All paths relative to `/home/rene/Documents/workspace/microting/eform-angular-frontend/eform-client/src/app/plugins/modules/insight-dashboard-pn/`. + +**Interfaces:** +- Consumes: `POST api/insight-dashboard-pn/dashboard-items/raw-data` (Task 5), `GET .../raw-data/export` (Task 6). +- Produces: ``, plus DOM ids `dashboardRawDataToggle{position}`, `dashboardRawData{position}`, `dashboardRawDataExport{position}` that Task 8 targets. + +- [ ] **Step 1: Create the models** + +`models/dashboard/raw-data/raw-data-column.model.ts`: + +```typescript +export class RawDataColumnModel { + field: string; + header: string; + kind: string; + defaultHidden: boolean; + sortable: boolean; + questionId: number | null; + optionId: number | null; +} +``` + +`models/dashboard/raw-data/raw-data-list.model.ts`: + +```typescript +import {RawDataColumnModel} from './raw-data-column.model'; + +export class RawDataListModel { + total = 0; + columns: RawDataColumnModel[] = []; + rows: any[] = []; +} +``` + +Rows are `any[]` deliberately: the key set is decided by the survey at runtime, so there is no meaningful static type. + +`models/dashboard/raw-data/raw-data-request.model.ts`: + +```typescript +export class RawDataRequestModel { + dashboardId: number; + dashboardItemId: number; + pageSize = 25; + offset = 0; + sort = 'finishedAt'; + isSortDsc = true; +} + +export class RawDataExportRequestModel { + dashboardId: number; + dashboardItemId: number; +} +``` + +`models/dashboard/raw-data/index.ts`: + +```typescript +export * from './raw-data-column.model'; +export * from './raw-data-list.model'; +export * from './raw-data-request.model'; +``` + +Add `export * from './raw-data';` to `models/dashboard/index.ts`. + +- [ ] **Step 2: Create the service** + +`services/insight-dashboard-pn-raw-data.service.ts`: + +```typescript +import {inject, Injectable} from '@angular/core'; +import {Observable} from 'rxjs'; +import {RawDataExportRequestModel, RawDataListModel, RawDataRequestModel} from '../models'; +import {OperationDataResult} from 'src/app/common/models'; +import {ApiBaseService} from 'src/app/common/services'; + +const RawDataMethods = { + RawData: 'api/insight-dashboard-pn/dashboard-items/raw-data', + Export: 'api/insight-dashboard-pn/dashboard-items/raw-data/export', +}; + +@Injectable() +export class InsightDashboardPnRawDataService { + private apiBaseService = inject(ApiBaseService); + + getRawData(model: RawDataRequestModel): Observable> { + return this.apiBaseService.post(RawDataMethods.RawData, model); + } + + exportToExcel(model: RawDataExportRequestModel): Observable { + return this.apiBaseService.getBlobData(RawDataMethods.Export, model); + } +} +``` + +Add `export * from './insight-dashboard-pn-raw-data.service';` to `services/index.ts`. + +- [ ] **Step 3: Create the component TypeScript** + +`components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts`: + +```typescript +import {Component, inject, Input, OnDestroy} from '@angular/core'; +import {AutoUnsubscribe} from 'ngx-auto-unsubscribe'; +import {Observable, of, Subscription} from 'rxjs'; +import {Sort} from '@angular/material/sort'; +import {MtxGridColumn} from '@ng-matero/extensions/grid'; +import {TranslateService} from '@ngx-translate/core'; +import {saveAs} from 'file-saver'; +import {PaginationModel} from 'src/app/common/models'; +import {updateTableSort} from 'src/app/common/helpers'; +import {InsightDashboardPnRawDataService} from '../../../../services'; +import {DashboardViewItemModel, DashboardViewModel, RawDataColumnModel} from '../../../../models'; + +@AutoUnsubscribe() +@Component({ + selector: 'app-dashboard-raw-data-view', + templateUrl: './dashboard-raw-data-view.component.html', + styleUrls: ['./dashboard-raw-data-view.component.scss'], + standalone: false, +}) +export class DashboardRawDataViewComponent implements OnDestroy { + private translateService = inject(TranslateService); + private rawDataService = inject(InsightDashboardPnRawDataService); + + @Input() dashboardViewModel: DashboardViewModel = new DashboardViewModel(); + @Input() itemModel: DashboardViewItemModel = new DashboardViewItemModel(); + + expanded = false; + loading = false; + loaded = false; + total = 0; + rows: any[] = []; + tableHeaders: MtxGridColumn[] = []; + pagination: PaginationModel = new PaginationModel(0, 25, 0); + sort = 'finishedAt'; + isSortDsc = true; + + getRawDataSub$: Subscription; + exportSub$: Subscription; + + get sortDirection(): 'asc' | 'desc' { + return this.isSortDsc ? 'desc' : 'asc'; + } + + toggle() { + this.expanded = !this.expanded; + if (this.expanded && !this.loaded) { + this.getRawData(); + } + } + + getRawData() { + this.loading = true; + this.getRawDataSub$ = this.rawDataService + .getRawData({ + dashboardId: this.dashboardViewModel.id, + dashboardItemId: this.itemModel.id, + offset: this.pagination.offset, + pageSize: this.pagination.pageSize, + sort: this.sort, + isSortDsc: this.isSortDsc, + }) + .subscribe((data) => { + this.loading = false; + if (data && data.success && data.model) { + this.loaded = true; + this.total = data.model.total; + this.pagination = {...this.pagination, total: data.model.total}; + this.rows = data.model.rows; + this.tableHeaders = data.model.columns.map((column) => this.toGridColumn(column)); + } + }); + } + + // Answer column headers are translation keys; question and option headers are + // already-resolved text from the database and must not be run through translate. + private toGridColumn(column: RawDataColumnModel): MtxGridColumn { + const header: Observable = column.kind === 'answer' + ? this.translateService.stream(column.header) + : of(column.header); + + const gridColumn: MtxGridColumn = { + header: header, + field: column.field, + hide: column.defaultHidden, + sortable: column.sortable, + }; + + if (column.sortable) { + gridColumn.sortProp = {id: column.field}; + } + + return gridColumn; + } + + sortTable(sort: Sort) { + const updated = updateTableSort(sort.active, this.sort, this.isSortDsc); + this.sort = updated.sort; + this.isSortDsc = updated.isSortDsc; + this.getRawData(); + } + + onPaginationChanged(pagination: PaginationModel) { + this.pagination = { + ...this.pagination, + pageSize: pagination.pageSize, + offset: pagination.offset, + }; + this.getRawData(); + } + + exportToExcel() { + this.exportSub$ = this.rawDataService + .exportToExcel({ + dashboardId: this.dashboardViewModel.id, + dashboardItemId: this.itemModel.id, + }) + .subscribe((data) => { + saveAs(new Blob([data]), `${this.dashboardViewModel.dashboardName}_raw_data.xlsx`); + }); + } + + ngOnDestroy(): void {} +} +``` + +- [ ] **Step 4: Create the component template** + +`components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.html`: + +```html +
+ +
+ + + + + + + + + + + + + +``` + +Create `dashboard-raw-data-view.component.scss` as an empty file. + +- [ ] **Step 5: Export the component and register it** + +Add to `components/dashboards/view/index.ts`: + +```typescript +export * from './dashboard-raw-data-view/dashboard-raw-data-view.component'; +``` + +In `insight-dashboard-pn.module.ts`: +- add `DashboardRawDataViewComponent` to the import list from `'./components'` +- add `DashboardRawDataViewComponent,` to `declarations` +- add `InsightDashboardPnRawDataService` to the import list from `'./services'` and to `providers` + +- [ ] **Step 6: Render it under each chart** + +In `components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html`, inside the existing ``, add the new component immediately after ``: + +```html + + + + + + + +``` + +Text-type items keep showing only the interviews grid — they have no chart to reconcile against, and the backend rejects raw-data requests for them. + +- [ ] **Step 7: Add i18n keys** + +Append to the object in `i18n/en-US.ts` (before the closing `};`): + +```typescript + 'Raw data': 'Raw data', + 'answers': 'answers', + 'No raw data found': 'No raw data found', + 'Export raw data': 'Export raw data', + 'Microting UID': 'Microting UID', + 'Finished at': 'Finished at', + 'Duration': 'Duration', + 'Site': 'Site', + 'Tags': 'Tags', + 'Unit': 'Unit', + 'Survey config': 'Survey config', + 'Time zone': 'Time zone', + 'UTC adjusted': 'UTC adjusted', + 'Created at': 'Created at', + 'Updated at': 'Updated at', + 'Version': 'Version', + 'Workflow state': 'Workflow state', + 'Site id': 'Site id', + 'Unit id': 'Unit id', + 'Language id': 'Language id', +``` + +Append the same keys to `i18n/da.ts`: + +```typescript + 'Raw data': 'Rådata', + 'answers': 'svar', + 'No raw data found': 'Ingen rådata fundet', + 'Export raw data': 'Eksportér rådata', + 'Microting UID': 'Microting UID', + 'Finished at': 'Afsluttet', + 'Duration': 'Varighed', + 'Site': 'Lokation', + 'Tags': 'Tags', + 'Unit': 'Enhed', + 'Survey config': 'Undersøgelseskonfiguration', + 'Time zone': 'Tidszone', + 'UTC adjusted': 'UTC-justeret', + 'Created at': 'Oprettet', + 'Updated at': 'Opdateret', + 'Version': 'Version', + 'Workflow state': 'Workflow-status', + 'Site id': 'Lokations-id', + 'Unit id': 'Enheds-id', + 'Language id': 'Sprog-id', +``` + +`Id`, `Language`, `Survey` and `Question` already exist in both files — do not duplicate them. Check before appending; a duplicate key is a silent overwrite. + +- [ ] **Step 8: Write the failing jest spec** + +`components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts`. Written with `waitForAsync`, unlike the 71 pre-existing broken stubs that use the removed `async`: + +```typescript +import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; +import {NO_ERRORS_SCHEMA, Pipe, PipeTransform} from '@angular/core'; +import {of} from 'rxjs'; +import {TranslateService} from '@ngx-translate/core'; +import {DashboardRawDataViewComponent} from './dashboard-raw-data-view.component'; +import {InsightDashboardPnRawDataService} from '../../../../services'; + +@Pipe({name: 'translate', standalone: false}) +class MockTranslatePipe implements PipeTransform { + transform(value: string): string { + return value; + } +} + +describe('DashboardRawDataViewComponent', () => { + let component: DashboardRawDataViewComponent; + let fixture: ComponentFixture; + + const response = { + success: true, + model: { + total: 2, + columns: [ + {field: 'finishedAt', header: 'Finished at', kind: 'answer', defaultHidden: false, sortable: true, questionId: null, optionId: null}, + {field: 'timeZone', header: 'Time zone', kind: 'answer', defaultHidden: true, sortable: false, questionId: null, optionId: null}, + {field: 'q7_o21', header: '2 – Områder › Kantine', kind: 'multiOption', defaultHidden: false, sortable: false, questionId: 7, optionId: 21}, + ], + rows: [ + {finishedAt: '2026-03-02T08:14:22', timeZone: 'Europe/Copenhagen', q7_o21: 'Kantine'}, + {finishedAt: '2026-03-03T07:22:11', timeZone: 'Europe/Copenhagen', q7_o21: ''}, + ], + }, + }; + + const rawDataServiceMock = { + getRawData: jest.fn(() => of(response)), + exportToExcel: jest.fn(() => of(new Blob())), + }; + + beforeEach(waitForAsync(() => { + TestBed.configureTestingModule({ + declarations: [DashboardRawDataViewComponent, MockTranslatePipe], + providers: [ + {provide: InsightDashboardPnRawDataService, useValue: rawDataServiceMock}, + { + provide: TranslateService, + useValue: { + stream: (key: string) => of(key), + get: (key: string) => of(key), + instant: (key: string) => key, + }, + }, + ], + schemas: [NO_ERRORS_SCHEMA], + }).compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(DashboardRawDataViewComponent); + component = fixture.componentInstance; + component.dashboardViewModel = {id: 3, dashboardName: 'Test'} as any; + component.itemModel = {id: 9, position: 1} as any; + fixture.detectChanges(); + }); + + it('should create without loading data', () => { + expect(component).toBeTruthy(); + expect(component.expanded).toBe(false); + expect(rawDataServiceMock.getRawData).not.toHaveBeenCalled(); + }); + + it('loads data on first expand and does not refetch on re-expand', () => { + component.toggle(); + + expect(component.expanded).toBe(true); + expect(component.total).toBe(2); + expect(component.rows.length).toBe(2); + expect(rawDataServiceMock.getRawData).toHaveBeenCalledTimes(1); + + component.toggle(); + component.toggle(); + + expect(rawDataServiceMock.getRawData).toHaveBeenCalledTimes(1); + }); + + it('maps columns, hiding defaults and marking only answer columns sortable', () => { + component.toggle(); + + const [finishedAt, timeZone, optionColumn] = component.tableHeaders; + + expect(finishedAt.field).toBe('finishedAt'); + expect(finishedAt.sortable).toBe(true); + expect(finishedAt.sortProp).toEqual({id: 'finishedAt'}); + + expect(timeZone.hide).toBe(true); + + expect(optionColumn.sortable).toBe(false); + expect(optionColumn.sortProp).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 9: Run the spec — expect failure first** + +Run: +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eform-client +npx jest src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view +``` +Expected before Steps 3-4 are complete: FAIL, "Cannot find module './dashboard-raw-data-view.component'". If Steps 3-4 are already done, expect PASS — in that case, deliberately break one assertion, re-run to confirm the spec actually exercises the component, then restore it. + +- [ ] **Step 10: Run the spec and the build to verify green** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eform-client +npx jest src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view +npx ng build +``` +Expected: `Tests: 3 passed`, then a successful Angular build. + +- [ ] **Step 11: Commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-frontend +git add eform-client/src/app/plugins/modules/insight-dashboard-pn +git commit -m "feat(insight-dashboard): add raw data table under each chart" +``` + +--- + +### Task 8: Playwright end-to-end coverage + +**Important:** `devgetchanges.sh` does **not** sync `eform-client/playwright/`, and the host app has no insight-dashboard Playwright specs. This suite lives only in the plugin source repo, so — unlike every other task — these files are authored directly at +`/home/rene/Documents/workspace/microting/eform-angular-insight-dashboard-plugin/eform-client/playwright/e2e/plugins/insight-dashboard-pn/`. + +**Files:** +- Modify: `InsightDashboard-DashboardView.page.ts` (add locators) +- Create: `c/insight-dashboard-raw-data.spec.ts` + +**Interfaces:** +- Consumes: DOM ids from Task 7 (`dashboardRawDataToggle{position}`, `dashboardRawData{position}`, `dashboardRawDataExport{position}`) and the existing `dataViewAmount{n}` ids on the aggregated table. +- Produces: nothing consumed downstream. + +- [ ] **Step 1: Add locators to the page object** + +Append these methods to the `InsightDashboardDashboardViewPage` class in `InsightDashboard-DashboardView.page.ts`, matching the existing `page.locator('#id')` style: + +```typescript + public rawDataToggle(rowNum: number) { + return this.page.locator(`#dashboardRawDataToggle${rowNum + 1}`); + } + + public rawDataGrid(rowNum: number) { + return this.page.locator(`#dashboardRawData${rowNum + 1}`); + } + + public rawDataExportButton(rowNum: number) { + return this.page.locator(`#dashboardRawDataExport${rowNum + 1}`); + } + + public rawDataRows(rowNum: number) { + return this.rawDataGrid(rowNum).locator('tbody tr.mat-row, tbody tr.mdc-data-table__row'); + } + + public rawDataHeaders(rowNum: number) { + return this.rawDataGrid(rowNum).locator('thead th'); + } +``` + +`position` is 1-based on the item model while the existing page-object helpers take a 0-based index, hence the `+ 1`. + +- [ ] **Step 2: Write the spec** + +Create `c/insight-dashboard-raw-data.spec.ts`. Model its dashboard setup on the existing `c/insight-dashboard-multi.multi.spec.ts` — open that file first and reuse its `test.beforeEach` login/navigation and its dashboard-creation helper calls verbatim, changing only the assertions below. The reconciliation assertion is the point of this test: + +```typescript +import {expect, test} from '@playwright/test'; +import {InsightDashboardDashboardViewPage} from '../InsightDashboard-DashboardView.page'; + +test.describe('Insight dashboard raw data table', () => { + test('is collapsed until opened, then reconciles with the chart total', async ({page}) => { + const viewPage = new InsightDashboardDashboardViewPage(page); + + // The grid must not exist before the toggle is clicked — data loads lazily. + await expect(viewPage.rawDataGrid(0)).toHaveCount(0); + + await viewPage.rawDataToggle(0).click(); + await expect(viewPage.rawDataGrid(0)).toBeVisible(); + + // The aggregated table's bolded Total row, last cell, is the chart's total amount. + const amountTotalRow = viewPage.rawChartDataAmountValueRow(0, 0, 0).last(); + const totalCells = amountTotalRow.locator('td'); + const chartTotalText = await totalCells.last().textContent(); + const chartTotal = Number(chartTotalText?.trim()); + + const toggleText = await viewPage.rawDataToggle(0).textContent(); + const reportedAnswers = Number(toggleText?.match(/(\d+)/)?.[1]); + + // Equal for every question type except multi, where one answer contributes + // several answer values and the chart total legitimately exceeds the row count. + expect(reportedAnswers).toBe(chartTotal); + }); + + test('places each chosen multi-select option in its own column', async ({page}) => { + const viewPage = new InsightDashboardDashboardViewPage(page); + + await viewPage.rawDataToggle(0).click(); + await expect(viewPage.rawDataGrid(0)).toBeVisible(); + + const headers = await viewPage.rawDataHeaders(0).allTextContents(); + const optionHeaders = headers.filter((h) => h.includes('›')); + expect(optionHeaders.length).toBeGreaterThan(0); + + // Every option header carries its question label as a prefix. + for (const header of optionHeaders) { + expect(header.split('›')[0].trim().length).toBeGreaterThan(0); + } + + const rowCount = await viewPage.rawDataRows(0).count(); + expect(rowCount).toBeGreaterThan(0); + }); +}); +``` + +- [ ] **Step 3: Run the spec** + +Run the insight-dashboard Playwright suite the same way the existing `c/` specs are run in this repo (check `package.json` / CI config in the plugin repo for the exact playwright project name before inventing a command). +Expected: both tests pass against a running dev stack. + +If the stack is not running, this step is blocked — record that in the task and hand back to the user rather than marking the task done. + +- [ ] **Step 4: Commit in the plugin source repo** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-insight-dashboard-plugin +git add "eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts" \ + "eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts" +git commit -m "test(insight-dashboard): cover raw data table" +``` + +--- + +### Task 9: Review, sync back, and commit the plugin + +**Files:** no new files; this task moves the host-app work into the plugin source repo. + +- [ ] **Step 1: Request code review** + +Use the `superpowers:requesting-code-review` skill against the full diff of the host app's plugin directories. Address any findings before continuing. + +- [ ] **Step 2: Confirm the browser behaviour with the user** + +Ask the user to open a dashboard, expand `Raw data` under a chart, and confirm: the row count matches the chart, the column picker reveals `Time zone`, paging works, and the Excel export downloads. Do not proceed until they confirm — the backend has no unit tests, so this is the real verification gate. + +- [ ] **Step 3: Sync back to the plugin repo** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-insight-dashboard-plugin +./devgetchanges.sh +``` + +This wholesale-replaces `eform-client/src/app/plugins/modules/insight-dashboard-pn` and `eFormAPI/Plugins/InsightDashboard.Pn` in the plugin repo with the host-app copies. + +- [ ] **Step 4: Discard build and config artifacts** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-insight-dashboard-plugin +git checkout -- $(git diff --name-only | grep -E '\.(csproj|conf\.ts|xlsx|docx)$') 2>/dev/null || true +git status --short +``` + +- [ ] **Step 5: Compare against intent** + +Review `git status` line by line. The expected additions are exactly: +- `eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/` (6 files) +- `eFormAPI/.../Infrastructure/Helpers/AnswerFilterHelper.cs`, `RawDataTranslations.cs`, `RawDataColumnBuilder.cs` +- `eFormAPI/.../Services/RawDataService/` (2 files), `Services/RawDataExcelService/` (2 files) +- `eFormAPI/.../Controllers/RawDataController.cs` +- `eform-client/src/.../models/dashboard/raw-data/` (4 files) +- `eform-client/src/.../services/insight-dashboard-pn-raw-data.service.ts` +- `eform-client/src/.../components/dashboards/view/dashboard-raw-data-view/` (4 files) + +The expected modifications are exactly: `EformInsightDashboardPlugin.cs`, `Resources/localization.json`, `insight-dashboard-pn.module.ts`, `dashboard-block-view.component.html`, `i18n/en-US.ts`, `i18n/da.ts`, and the three `index.ts` barrels. + +Run `git checkout --` on anything else that appears. + +- [ ] **Step 6: Show the user the final status and commit** + +```bash +cd /home/rene/Documents/workspace/microting/eform-angular-insight-dashboard-plugin +git status --short +``` + +Present that output to the user for confirmation. Only after they approve, stage the files **by name** (never `git add .`) and commit: + +```bash +git commit -m "feat: add raw data table under each dashboard chart" +``` + +--- + +## Self-review + +**Spec coverage.** Rows definition → Task 2. Answer column inventory → Task 3 `BuildAnswerColumns`. Dynamic question columns, `QuestionIndex` ordering, `"N – text"` labels, multi expansion by `OptionIndex` → Task 3. Cell content per question type, including smiley `label (weight)` and the `—` skipped convention → Task 4 `ResolveSingleValue` and the pivot loop. Placement and lazy collapse → Task 7 Steps 4 and 6. Server paging and sorting → Task 4 `ApplySort` plus Task 7 Steps 3-4. Excel export with hidden columns included and the 100 000 cap → Task 6 and `RawDataService.ExportRowLimit`. Language resolution → Task 3 `RawDataTranslations`. Error handling table → Task 4's guards plus Task 5's localization entries. Reconciliation invariant → Task 2's comments and Task 8's first test. + +Two spec items are deliberately narrowed, and both are called out where they occur: the spec's error table lists a distinct behaviour for text-type items, which Task 7 Step 6 handles by not rendering the component at all rather than by returning a specific message; and the spec's testing section listed six Playwright assertions, of which Task 8 implements the two that carry real risk (reconciliation, multi-select placement) — the column-picker, paging and export assertions are covered by the jest spec and the manual gate in Task 9 Step 2 instead. Anyone wanting the full six should add them to Task 8. + +**Placeholder scan.** No TBDs, no "add error handling", no "similar to Task N". Every code step carries the actual code. Task 8 Step 3 deliberately does not invent a test command and instead directs the implementer to read the repo's config — inventing one would be worse than saying so. + +**Type consistency.** `RawDataListModel.Rows` is `List>` in Tasks 1, 4 and 6 and `any[]` in Task 7. `RawDataColumnModel` field names match between C# (PascalCase) and TypeScript (camelCase) under default ASP.NET JSON casing. `RawDataQuestionMeta.OptionFields` is populated for both the multi branch and the single branch in Task 3 and consumed by the "not answered" pre-fill in Task 4. `AnswerFilterHelper.BuildAnswerQuery` is non-async and returns `IQueryable` in both Task 2 and its Task 4 call site. `IRawDataService` exposes both `GetRawData` and `GetAllRawData`, used by Task 5 and Task 6 respectively. From 2daca3f93d2a1965b3b6c98a8d73557fe7aa9ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Wed, 29 Jul 2026 10:00:12 +0200 Subject: [PATCH 3/6] docs: correct plan to forbid commits inside the host app CLAUDE.md forbids committing plugin code under eform-angular-frontend; the plan wrongly instructed a commit per backend task there. Plugin commits happen only in this repo after devgetchanges.sh. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EnP42zmHAgo2NZQsa6zdhG --- ...-07-29-insight-dashboard-raw-data-table.md | 76 +++++-------------- 1 file changed, 20 insertions(+), 56 deletions(-) diff --git a/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md b/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md index e043e4f5..89d31757 100644 --- a/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md +++ b/docs/superpowers/plans/2026-07-29-insight-dashboard-raw-data-table.md @@ -19,7 +19,7 @@ - Every new C# file starts with the same MIT license header block used by its neighbours (copy verbatim from `Infrastructure/Models/Dashboards/ChartDataItem.cs`, updating nothing). - Controllers: `[Authorize]`, plain `Controller` base class, hardcoded `[Route("api/insight-dashboard-pn/...")]` per action. No `[ApiController]`. - Soft deletes: filter `WorkflowState != Constants.WorkflowStates.Removed` on every SDK table touched. -- Commits go in the **host app** during implementation only if the change belongs to core frontend; plugin code is committed in the **source repo** after `devgetchanges.sh` (Task 9). +- **Never commit inside `eform-angular-frontend`.** Its `eFormAPI/Plugins/` and `eform-client/src/app/plugins/modules/` trees are working copies; CLAUDE.md forbids committing them there. All plugin commits happen in the plugin source repo after `devgetchanges.sh` (Task 9). ## Testing approach — read this before Task 1 @@ -219,17 +219,11 @@ public class RawDataListModel - [ ] **Step 4: Build** Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` -Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` +Expected: `Build succeeded.` with `0 Error(s)`. One pre-existing CS0618 warning in ChartDataHelpers.cs:4271 is expected on a clean rebuild and is not a regression. -- [ ] **Step 5: Commit** +- [ ] **Step 5: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData -git commit -m "feat(insight-dashboard): add raw data DTOs" -``` - -Note: this commit is in the host app purely to keep the work bisectable during implementation. The authoritative commit happens in the plugin source repo in Task 9. +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- @@ -445,17 +439,13 @@ Add `using System.Collections.Generic;` to the using block — `List` is us - [ ] **Step 2: Build** Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` -Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` +Expected: `Build succeeded.` with `0 Error(s)`. One pre-existing CS0618 warning in ChartDataHelpers.cs:4271 is expected on a clean rebuild and is not a regression. If the build reports that `DashboardEditAnswerDates` is not found, its namespace is `InsightDashboard.Pn.Infrastructure.Models.Dashboards` — the `using Models.Dashboards;` line above resolves it relative to `InsightDashboard.Pn.Infrastructure`. -- [ ] **Step 3: Commit** +- [ ] **Step 3: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/AnswerFilterHelper.cs -git commit -m "feat(insight-dashboard): add AnswerFilterHelper mirroring chart answer selection" -``` +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- @@ -807,19 +797,13 @@ public static class RawDataColumnBuilder - [ ] **Step 4: Build** Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` -Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` +Expected: `Build succeeded.` with `0 Error(s)`. One pre-existing CS0618 warning in ChartDataHelpers.cs:4271 is expected on a clean rebuild and is not a regression. If the collection-expression syntax `=> [ ... ]` on `BuildAnswerColumns` is rejected, replace it with `=> new List { ... };` — the rest is unchanged. -- [ ] **Step 5: Commit** +- [ ] **Step 5: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataTranslations.cs \ - eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataColumnBuilder.cs \ - eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataSchema.cs -git commit -m "feat(insight-dashboard): add raw data column builder and translation picking" -``` +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- @@ -1205,19 +1189,15 @@ public class RawDataService : IRawDataService - [ ] **Step 3: Build** Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` -Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` +Expected: `Build succeeded.` with `0 Error(s)`. One pre-existing CS0618 warning in ChartDataHelpers.cs:4271 is expected on a clean rebuild and is not a regression. Two likely compile issues and their fixes: - If `.ThenInclude(x => x.IgnoredAnswerValues)` is ambiguous, use the explicit generic form `DashboardService.cs` uses: `.ThenInclude>(x => x.IgnoredAnswerValues)` and the matching `List` form, adding `using Microting.InsightDashboardBase.Infrastructure.Data.Entities;`. - `Site.SiteTags` has no `WorkflowState` filter available if `SiteTag` does not expose it — it derives from `PnBase`, so it does; if the compiler disagrees, drop that `.Where` clause. -- [ ] **Step 4: Commit** +- [ ] **Step 4: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService -git commit -m "feat(insight-dashboard): add RawDataService with answer pivot and paging" -``` +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- @@ -1319,15 +1299,9 @@ dotnet build -v q --nologo ``` Expected: a key count that is 2 or 3 higher than before, then `Build succeeded. 0 Warning(s) 0 Error(s)` -- [ ] **Step 5: Commit** +- [ ] **Step 5: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs \ - eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs \ - eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Resources/localization.json -git commit -m "feat(insight-dashboard): expose raw data endpoint" -``` +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- @@ -1580,17 +1554,11 @@ In `EformInsightDashboardPlugin.cs`, add `using Services.RawDataExcelService;` a - [ ] **Step 5: Build** Run: `cd /home/rene/Documents/workspace/microting/eform-angular-frontend/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn && dotnet build -v q --nologo` -Expected: `Build succeeded. 0 Warning(s) 0 Error(s)` +Expected: `Build succeeded.` with `0 Error(s)`. One pre-existing CS0618 warning in ChartDataHelpers.cs:4271 is expected on a clean rebuild and is not a regression. -- [ ] **Step 6: Commit** +- [ ] **Step 6: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService \ - eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs \ - eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs -git commit -m "feat(insight-dashboard): add raw data excel export" -``` +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- @@ -2110,13 +2078,9 @@ npx ng build ``` Expected: `Tests: 3 passed`, then a successful Angular build. -- [ ] **Step 11: Commit** +- [ ] **Step 11: Leave uncommitted** -```bash -cd /home/rene/Documents/workspace/microting/eform-angular-frontend -git add eform-client/src/app/plugins/modules/insight-dashboard-pn -git commit -m "feat(insight-dashboard): add raw data table under each chart" -``` +**Do not commit here.** Plugin code inside `eform-angular-frontend` is a working copy, and committing it there is forbidden by CLAUDE.md. Leave the change uncommitted; it reaches git in Task 9 via `devgetchanges.sh` and a commit in the plugin source repo. --- From 847b5ea73373e9d5e9a0d5537aaba6a2af604e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Wed, 29 Jul 2026 11:13:03 +0200 Subject: [PATCH 4/6] feat: add raw data table under each dashboard chart Each chart already renders an aggregated table of percentages and counts. This adds a second, collapsible table showing one row per Answer, with every field the Answer holds in the database plus its AnswerValues pivoted into one column per question. Multi-select questions expand to one column per option, each cell carrying the chosen option's own name. Backend: - RawDataController exposes a paged endpoint and an xlsx export. - AnswerFilterHelper reproduces the answer-selection rules of ChartDataHelpers.CalculateDashboardItem predicate for predicate, with each one annotated with its source line, so the table's row count reconciles with the chart above it. Two documented deviations: it also filters Answer.WorkflowState, and it replaces the materialised filter-question id list with an equivalent correlated subquery. - RawDataColumnBuilder derives the column set from the survey's questions and options; RawDataTranslations resolves question and option text by the logged-in user's language, since Answer.LanguageId is hardcoded to Danish on import and is not trustworthy. - Cell values are resolved by question kind rather than by option name: the SDK auto-generates options whose translation is the type keyword ("smiley1", "number", "text", "next"), so smileys read from the weight ladder and free-text/number read from AnswerValue.Value. - Paging always breaks ties on Id; without it, answers sharing a FinishedAt could be duplicated and skipped across pages. - Text-type items are rejected: ChartDataHelpers filters them down a different branch that this helper deliberately does not mirror. Frontend: - dashboard-raw-data-view renders an mtx-grid whose columns are built from the server's column descriptors, with server-side paging and sorting, a column picker for the audit fields, and an Excel export. Data loads on first expand so the dashboard response does not grow. Excel export is capped at 25000 rows and fails with a localized message rather than exhausting memory; batching the id lookup would lift it. Verified: dotnet build and ng build clean, 6 jest tests passing (checked non-vacuous by mutation). The Playwright specs added here have not yet been executed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EnP42zmHAgo2NZQsa6zdhG --- .../Controllers/RawDataController.cs | 150 ++++++ .../EformInsightDashboardPlugin.cs | 4 + .../Helpers/AnswerFilterHelper.cs | 214 +++++++++ .../Helpers/RawDataColumnBuilder.cs | 250 ++++++++++ .../Helpers/RawDataTranslations.cs | 77 ++++ .../Models/RawData/RawDataColumnKinds.cs | 36 ++ .../Models/RawData/RawDataColumnModel.cs | 36 ++ .../RawData/RawDataExportRequestModel.cs | 31 ++ .../Models/RawData/RawDataFields.cs | 48 ++ .../Models/RawData/RawDataListModel.cs | 34 ++ .../Models/RawData/RawDataRequestModel.cs | 37 ++ .../Models/RawData/RawDataSchema.cs | 59 +++ .../Resources/localization.json | 30 +- .../IRawDataExcelService.cs | 34 ++ .../RawDataExcelService.cs | 167 +++++++ .../RawDataService/IRawDataService.cs | 36 ++ .../Services/RawDataService/RawDataService.cs | 426 ++++++++++++++++++ .../InsightDashboard-DashboardView.page.ts | 40 ++ .../c/insight-dashboard-raw-data.spec.ts | 114 +++++ .../dashboard-block-view.component.html | 4 + .../dashboard-raw-data-view.component.html | 52 +++ .../dashboard-raw-data-view.component.scss | 0 .../dashboard-raw-data-view.component.spec.ts | 147 ++++++ .../dashboard-raw-data-view.component.ts | 151 +++++++ .../components/dashboards/view/index.ts | 1 + .../modules/insight-dashboard-pn/i18n/da.ts | 22 +- .../insight-dashboard-pn/i18n/en-US.ts | 22 +- .../insight-dashboard-pn.module.ts | 4 + .../models/dashboard/index.ts | 1 + .../models/dashboard/raw-data/index.ts | 3 + .../raw-data/raw-data-column.model.ts | 9 + .../dashboard/raw-data/raw-data-list.model.ts | 8 + .../raw-data/raw-data-request.model.ts | 13 + .../insight-dashboard-pn/services/index.ts | 1 + .../insight-dashboard-pn-raw-data.service.ts | 29 ++ 35 files changed, 2287 insertions(+), 3 deletions(-) create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/AnswerFilterHelper.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataColumnBuilder.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataTranslations.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnKinds.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnModel.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataExportRequestModel.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataFields.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataListModel.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataRequestModel.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataSchema.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/IRawDataExcelService.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/RawDataExcelService.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/IRawDataService.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs create mode 100644 eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.html create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.scss create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/index.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-column.model.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-list.model.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-request.model.ts create mode 100644 eform-client/src/app/plugins/modules/insight-dashboard-pn/services/insight-dashboard-pn-raw-data.service.ts diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs new file mode 100644 index 00000000..e8a0442e --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Controllers/RawDataController.cs @@ -0,0 +1,150 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Controllers; + +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Infrastructure.Models.RawData; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Microting.eFormApi.BasePn.Infrastructure.Models.API; +using Services.Common.InsightDashboardLocalizationService; +using Services.RawDataExcelService; +using Services.RawDataService; + +[Authorize] +public class RawDataController : Controller +{ + private readonly IRawDataService _rawDataService; + private readonly IRawDataExcelService _rawDataExcelService; + private readonly IInsightDashboardLocalizationService _localizationService; + private readonly ILogger _logger; + + public RawDataController( + IRawDataService rawDataService, + IRawDataExcelService rawDataExcelService, + IInsightDashboardLocalizationService localizationService, + ILogger logger) + { + _rawDataService = rawDataService; + _rawDataExcelService = rawDataExcelService; + _localizationService = localizationService; + _logger = logger; + } + + [HttpPost] + [Route("api/insight-dashboard-pn/dashboard-items/raw-data")] + public async Task> GetRawData( + [FromBody] RawDataRequestModel requestModel) + { + return await _rawDataService.GetRawData(requestModel); + } + + /// + /// Download the full, unpaged raw data set as xlsx. + /// + /// Return excel blob + /// Error message + [HttpGet] + [Route("api/insight-dashboard-pn/dashboard-items/raw-data/export")] + [ProducesResponseType(typeof(string), 400)] + public async Task ExportRawData([FromQuery] RawDataExportRequestModel requestModel) + { + var dataResult = await _rawDataService.GetAllRawData( + requestModel.DashboardId, requestModel.DashboardItemId); + + string filePath = null; + if (dataResult.Success) + { + // Write before OnStarting registers, so a failure here can still be + // reported as a localized 400 rather than surfacing as a bare 500 with + // a half-written file orphaned in excel-storage. + try + { + filePath = _rawDataExcelService.CreateFilePath(); + if (!_rawDataExcelService.WriteRawDataToExcelFile(dataResult.Model, filePath)) + { + throw new Exception($"Error while writing excel file {filePath}"); + } + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + + if (!string.IsNullOrEmpty(filePath) && System.IO.File.Exists(filePath)) + { + System.IO.File.Delete(filePath); + } + + filePath = null; + dataResult = new OperationDataResult( + false, _localizationService.GetString("ErrorWhileGeneratingRawDataExport")); + } + } + + const int bufferSize = 4086; + var buffer = new byte[bufferSize]; + + Response.OnStarting(async () => + { + try + { + if (!dataResult.Success) + { + var bytes = Encoding.UTF8.GetBytes(dataResult.Message); + Response.ContentLength = bytes.Length; + Response.ContentType = "text/plain"; + Response.StatusCode = 400; + await Response.Body.WriteAsync(bytes, 0, bytes.Length); + await Response.Body.FlushAsync(); + } + else + { + await using var excelStream = new FileStream(filePath, FileMode.Open); + int bytesRead; + Response.ContentLength = excelStream.Length; + Response.ContentType = + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + while ((bytesRead = excelStream.Read(buffer, 0, buffer.Length)) > 0 && + !HttpContext.RequestAborted.IsCancellationRequested) + { + await Response.Body.WriteAsync(buffer, 0, bytesRead); + await Response.Body.FlushAsync(); + } + } + } + finally + { + if (!string.IsNullOrEmpty(filePath) && System.IO.File.Exists(filePath)) + { + System.IO.File.Delete(filePath); + } + } + }); + } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs index 92081893..eac7f9dc 100644 --- a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/EformInsightDashboardPlugin.cs @@ -55,6 +55,8 @@ namespace InsightDashboard.Pn; using Services.DictionaryService; using Services.InterviewsExcelService; using Services.InterviewsService; +using Services.RawDataExcelService; +using Services.RawDataService; using Services.SurveysService; using Services.WordService; @@ -84,7 +86,9 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); } diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/AnswerFilterHelper.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/AnswerFilterHelper.cs new file mode 100644 index 00000000..795f401c --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/AnswerFilterHelper.cs @@ -0,0 +1,214 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.EntityFrameworkCore; +using Microting.eForm.Infrastructure; +using Microting.eForm.Infrastructure.Constants; +using Microting.eForm.Infrastructure.Data.Entities; +using Microting.InsightDashboardBase.Infrastructure.Data.Entities; +using Microting.InsightDashboardBase.Infrastructure.Enums; +using Models.Dashboards; + +/// +/// Selects the answers that feed a single dashboard item. +/// +/// This MUST stay behaviourally identical to the answer-selection half of +/// ChartDataHelpers.CalculateDashboardItem, otherwise the raw data table will +/// disagree with the chart it sits under. Line references below point at +/// ChartDataHelpers.cs as of the commit that introduced this file. +/// +/// Two deliberate deviations: +/// 1. Answer.WorkflowState is also filtered (ChartDataHelpers filters only +/// AnswerValue.WorkflowState). The delete path sets both together, so this +/// does not change counts in practice. +/// 2. The filter-question step uses a correlated subquery instead of +/// materialising answer ids with ToList(). Semantically identical, one +/// fewer round trip. +/// +public static class AnswerFilterHelper +{ + public static IQueryable BuildAnswerQuery( + MicrotingDbContext sdkContext, + DashboardItem dashboardItem, + int dashboardSurveyId, + int? dashboardLocationId, + int? dashboardLocationTagId, + DashboardEditAnswerDates answerDates) + { + // ChartDataHelpers.cs:135-142 + var answerValues = sdkContext.AnswerValues + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.Answer.WorkflowState != Constants.WorkflowStates.Removed) + .AsQueryable(); + + // ChartDataHelpers.cs:144-154 + if (answerDates.Today) + { + var dateTimeNow = DateTime.Now; + answerDates.DateTo = new DateTime( + dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); + } + + // ChartDataHelpers.cs:156-166 + if (answerDates.DateFrom != null) + { + answerValues = answerValues.Where(x => x.Answer.FinishedAt >= answerDates.DateFrom); + } + + if (answerDates.DateTo != null) + { + answerValues = answerValues.Where(x => x.Answer.FinishedAt <= answerDates.DateTo); + } + + // ChartDataHelpers.cs:170-171 + answerValues = answerValues.Where(x => x.Answer.QuestionSetId == dashboardSurveyId); + + // ChartDataHelpers.cs:173-190 + if (dashboardItem.FilterQuestionId != null && dashboardItem.FilterAnswerId != null) + { + var filterScope = answerValues; + answerValues = answerValues + .Where(x => filterScope.Any(y => + y.AnswerId == x.AnswerId + && y.QuestionId == dashboardItem.FilterQuestionId + && y.OptionId == dashboardItem.FilterAnswerId)) + .Where(x => x.QuestionId == dashboardItem.FirstQuestionId); + } + else + { + answerValues = answerValues.Where(x => x.QuestionId == dashboardItem.FirstQuestionId); + } + + // ChartDataHelpers.cs:223-236 - this block only runs when compare is OFF + if (!dashboardItem.CompareEnabled) + { + if (dashboardLocationId != null) + { + answerValues = answerValues.Where(x => x.Answer.SiteId == dashboardLocationId); + } + else if (dashboardLocationTagId != null) + { + answerValues = answerValues.Where(x => + x.Answer.Site.SiteTags.Any(y => y.TagId == dashboardLocationTagId)); + } + } + + // ChartDataHelpers.cs:240-252 - ignored answer OPTIONS. The column is + // misleadingly named AnswerId but holds options.Id. + var ignoredOptionIds = dashboardItem.IgnoredAnswerValues + .Where(y => y.WorkflowState != Constants.WorkflowStates.Removed) + .Select(x => x.AnswerId) + .ToArray(); + + if (ignoredOptionIds.Length > 0) + { + answerValues = answerValues.Where(x => !ignoredOptionIds.Contains(x.OptionId)); + } + + var answerIds = IsComparedData(dashboardItem) + ? ComparedAnswerIds(answerValues, dashboardItem, dashboardLocationId, dashboardLocationTagId) + : NonComparedAnswerIds(answerValues, dashboardLocationId, dashboardLocationTagId); + + return sdkContext.Answers + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => answerIds.Contains(x.Id)); + } + + // ChartDataHelpers.cs:121-133 + private static bool IsComparedData(DashboardItem dashboardItem) + { + if (dashboardItem.ChartType != DashboardChartTypes.GroupedStackedBarChart + && dashboardItem.ChartType != DashboardChartTypes.Line) + { + return false; + } + + if (dashboardItem.CompareEnabled) + { + return true; + } + + return dashboardItem.ChartType == DashboardChartTypes.Line && dashboardItem.CalculateAverage; + } + + // ChartDataHelpers.cs:255-390 - union of the per-tag queries and the site query + private static IQueryable ComparedAnswerIds( + IQueryable answerValues, + DashboardItem dashboardItem, + int? dashboardLocationId, + int? dashboardLocationTagId) + { + var tagIds = dashboardItem.CompareEnabled + ? dashboardItem.CompareLocationsTags + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.TagId != null) + .Select(x => (int)x.TagId) + .ToList() + : dashboardLocationTagId != null + ? new List { (int)dashboardLocationTagId } + : new List(); + + var siteIds = dashboardItem.CompareEnabled + ? dashboardItem.CompareLocationsTags + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.LocationId != null) + .Select(x => (int)x.LocationId) + .ToList() + : dashboardLocationId != null + ? new List { (int)dashboardLocationId } + : new List(); + + var byTag = answerValues + .Where(x => x.Answer.Site.SiteTags.Any(y => y.TagId != null && tagIds.Contains((int)y.TagId))) + .Select(x => x.AnswerId); + + var bySite = answerValues + .Where(x => siteIds.Contains(x.Answer.SiteId)) + .Select(x => x.AnswerId); + + return byTag.Union(bySite).Distinct(); + } + + // ChartDataHelpers.cs:392-490 - when neither location nor tag is set the + // chart renders nothing, so the raw table must be empty too. + private static IQueryable NonComparedAnswerIds( + IQueryable answerValues, + int? dashboardLocationId, + int? dashboardLocationTagId) + { + if (dashboardLocationId == null && dashboardLocationTagId == null) + { + return answerValues.Where(x => false).Select(x => x.AnswerId); + } + + return answerValues.Select(x => x.AnswerId).Distinct(); + } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataColumnBuilder.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataColumnBuilder.cs new file mode 100644 index 00000000..af325cbb --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataColumnBuilder.cs @@ -0,0 +1,250 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microting.eForm.Infrastructure; +using Microting.eForm.Infrastructure.Constants; +using Models.RawData; + +public static class RawDataColumnBuilder +{ + /// + /// Fallback smiley labels, used only when a smiley option has no translation + /// of its own. Mirrors ChartDataHelpers.cs:60-68. 999 means "don't know". + /// + private static readonly Dictionary SmileyFallbackLabels = new() + { + { 100, "Meget glad" }, + { 75, "Glad" }, + { 50, "Neutral" }, + { 25, "Sur" }, + { 0, "Meget sur" }, + { 999, "Ved ikke" }, + }; + + public static string SmileyFallbackLabel(int weightValue) => + SmileyFallbackLabels.TryGetValue(weightValue, out var label) ? label : null; + + public static List BuildAnswerColumns() => new() + { + AnswerColumn(RawDataFields.Id, "Id", sortable: true), + AnswerColumn(RawDataFields.MicrotingUid, "Microting UID", sortable: true), + AnswerColumn(RawDataFields.FinishedAt, "Finished at", sortable: true), + AnswerColumn(RawDataFields.AnswerDuration, "Duration", sortable: true), + AnswerColumn(RawDataFields.SiteName, "Site", sortable: true), + AnswerColumn(RawDataFields.TagNames, "Tags", sortable: false), + AnswerColumn(RawDataFields.UnitMicrotingUid, "Unit", sortable: true), + AnswerColumn(RawDataFields.LanguageName, "Language", sortable: true), + AnswerColumn(RawDataFields.SurveyConfigurationName, "Survey config", sortable: true), + AnswerColumn(RawDataFields.QuestionSetName, "Survey", sortable: false, hidden: true), + AnswerColumn(RawDataFields.TimeZone, "Time zone", sortable: false, hidden: true), + AnswerColumn(RawDataFields.UtcAdjusted, "UTC adjusted", sortable: false, hidden: true), + AnswerColumn(RawDataFields.CreatedAt, "Created at", sortable: true, hidden: true), + AnswerColumn(RawDataFields.UpdatedAt, "Updated at", sortable: true, hidden: true), + AnswerColumn(RawDataFields.Version, "Version", sortable: false, hidden: true), + AnswerColumn(RawDataFields.WorkflowState, "Workflow state", sortable: true, hidden: true), + AnswerColumn(RawDataFields.SiteId, "Site id", sortable: false, hidden: true), + AnswerColumn(RawDataFields.UnitId, "Unit id", sortable: false, hidden: true), + AnswerColumn(RawDataFields.LanguageId, "Language id", sortable: false, hidden: true), + }; + + private static RawDataColumnModel AnswerColumn( + string field, string header, bool sortable, bool hidden = false) => + new() + { + Field = field, + Header = header, + Kind = RawDataColumnKinds.Answer, + Sortable = sortable, + DefaultHidden = hidden, + }; + + public static async Task BuildAsync( + MicrotingDbContext sdkContext, + int questionSetId, + IReadOnlyList preferredLanguageIds) + { + var schema = new RawDataSchema { Columns = BuildAnswerColumns() }; + + var questions = await sdkContext.Questions + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionSetId == questionSetId) + .OrderBy(x => x.QuestionIndex) + .Select(x => new { x.Id, x.QuestionType }) + .ToListAsync(); + + if (questions.Count == 0) + { + return schema; + } + + var questionIds = questions.Select(x => x.Id).ToList(); + + var questionTranslations = await sdkContext.QuestionTranslations + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => questionIds.Contains(x.QuestionId)) + .Select(x => new { x.QuestionId, x.LanguageId, x.Name }) + .ToListAsync(); + + var options = await sdkContext.Options + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => questionIds.Contains(x.QuestionId)) + .OrderBy(x => x.OptionIndex) + .Select(x => new { x.Id, x.QuestionId, x.WeightValue }) + .ToListAsync(); + + var optionIds = options.Select(x => x.Id).ToList(); + + var optionTranslations = await sdkContext.OptionTranslations + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => optionIds.Contains(x.OptionId)) + .Select(x => new { x.OptionId, x.LanguageId, x.Name }) + .ToListAsync(); + + var questionNumber = 0; + foreach (var question in questions) + { + questionNumber++; + + var questionName = RawDataTranslations.Pick( + questionTranslations + .Where(t => t.QuestionId == question.Id) + .Select(t => (t.LanguageId, t.Name)) + .ToList(), + preferredLanguageIds); + + var questionLabel = $"{questionNumber} – {questionName}"; + var questionOptions = options.Where(o => o.QuestionId == question.Id).ToList(); + + var isSmiley = IsSmileyType(question.QuestionType); + var isMulti = question.QuestionType == Constants.QuestionTypes.Multi; + + var meta = new RawDataQuestionMeta + { + QuestionId = question.Id, + IsSmiley = isSmiley, + IsMulti = isMulti, + QuestionType = question.QuestionType, + Kind = isMulti ? RawDataColumnKinds.MultiOption : KindFor(question.QuestionType, isSmiley), + }; + + foreach (var option in questionOptions) + { + meta.WeightValueByOptionId[option.Id] = option.WeightValue; + meta.OptionNameByOptionId[option.Id] = RawDataTranslations.Pick( + optionTranslations + .Where(t => t.OptionId == option.Id) + .Select(t => (t.LanguageId, t.Name)) + .ToList(), + preferredLanguageIds); + } + + if (isMulti) + { + foreach (var option in questionOptions) + { + var field = $"q{question.Id}_o{option.Id}"; + meta.OptionFieldByOptionId[option.Id] = field; + meta.OptionFields.Add(field); + + schema.Columns.Add(new RawDataColumnModel + { + Field = field, + Header = $"{questionLabel} › {meta.OptionNameByOptionId[option.Id]}", + Kind = RawDataColumnKinds.MultiOption, + Sortable = false, + DefaultHidden = false, + QuestionId = question.Id, + OptionId = option.Id, + }); + } + } + else + { + meta.Field = $"q{question.Id}"; + meta.OptionFields.Add(meta.Field); + + schema.Columns.Add(new RawDataColumnModel + { + Field = meta.Field, + Header = questionLabel, + Kind = KindFor(question.QuestionType, isSmiley), + Sortable = false, + DefaultHidden = false, + QuestionId = question.Id, + }); + } + + schema.Questions.Add(meta); + } + + return schema; + } + + private static string KindFor(string questionType, bool isSmiley) + { + if (isSmiley) + { + return RawDataColumnKinds.Smiley; + } + + return questionType switch + { + Constants.QuestionTypes.List => RawDataColumnKinds.Single, + Constants.QuestionTypes.Buttons => RawDataColumnKinds.Single, + Constants.QuestionTypes.Number => RawDataColumnKinds.Number, + Constants.QuestionTypes.Text => RawDataColumnKinds.Text, + Constants.QuestionTypes.TextEamil => RawDataColumnKinds.Text, + Constants.QuestionTypes.ZipCode => RawDataColumnKinds.Text, + _ => RawDataColumnKinds.Other, + }; + } + + /// + /// Mirrors Question.IsSmiley() without needing a Question entity instance. + /// + private static bool IsSmileyType(string questionType) => questionType switch + { + Constants.QuestionTypes.Smiley => true, + Constants.QuestionTypes.Smiley2 => true, + Constants.QuestionTypes.Smiley3 => true, + Constants.QuestionTypes.Smiley4 => true, + Constants.QuestionTypes.Smiley5 => true, + Constants.QuestionTypes.Smiley6 => true, + Constants.QuestionTypes.Smiley7 => true, + Constants.QuestionTypes.Smiley8 => true, + Constants.QuestionTypes.Smiley9 => true, + Constants.QuestionTypes.Smiley10 => true, + _ => false, + }; +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataTranslations.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataTranslations.cs new file mode 100644 index 00000000..883a6cf1 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataTranslations.cs @@ -0,0 +1,77 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microting.eForm.Infrastructure; +using Microting.eForm.Infrastructure.Constants; + +/// +/// Question and option text lives only in QuestionTranslations / OptionTranslations, +/// keyed by LanguageId. Answer.LanguageId is NOT usable for this - Core.SaveAnswer +/// hardcodes it to the Danish row - so the raw data table resolves text by the +/// logged-in user's language, then the survey's deployed languages, then anything +/// non-removed. +/// +public static class RawDataTranslations +{ + public static async Task> GetPreferredLanguageIdsAsync( + MicrotingDbContext sdkContext, + int questionSetId, + int userLanguageId) + { + var surveyLanguageIds = await sdkContext.LanguageQuestionSets + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionSetId == questionSetId) + .Select(x => x.LanguageId) + .ToListAsync(); + + var preferred = new List { userLanguageId }; + preferred.AddRange(surveyLanguageIds.Where(x => x != userLanguageId)); + return preferred; + } + + public static string Pick( + IReadOnlyList<(int LanguageId, string Name)> translations, + IReadOnlyList preferredLanguageIds) + { + foreach (var languageId in preferredLanguageIds) + { + var match = translations.FirstOrDefault(x => x.LanguageId == languageId); + if (!string.IsNullOrEmpty(match.Name)) + { + return match.Name; + } + } + + return translations + .Select(x => x.Name) + .FirstOrDefault(x => !string.IsNullOrEmpty(x)); + } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnKinds.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnKinds.cs new file mode 100644 index 00000000..0a1cba51 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnKinds.cs @@ -0,0 +1,36 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public static class RawDataColumnKinds +{ + public const string Answer = "answer"; + public const string Smiley = "smiley"; + public const string Single = "single"; + public const string MultiOption = "multiOption"; + public const string Number = "number"; + public const string Text = "text"; + public const string Other = "other"; +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnModel.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnModel.cs new file mode 100644 index 00000000..e4a08b07 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataColumnModel.cs @@ -0,0 +1,36 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public class RawDataColumnModel +{ + public string Field { get; set; } + public string Header { get; set; } + public string Kind { get; set; } + public bool DefaultHidden { get; set; } + public bool Sortable { get; set; } + public int? QuestionId { get; set; } + public int? OptionId { get; set; } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataExportRequestModel.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataExportRequestModel.cs new file mode 100644 index 00000000..8cccab33 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataExportRequestModel.cs @@ -0,0 +1,31 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public class RawDataExportRequestModel +{ + public int DashboardId { get; set; } + public int DashboardItemId { get; set; } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataFields.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataFields.cs new file mode 100644 index 00000000..e02455f9 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataFields.cs @@ -0,0 +1,48 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +public static class RawDataFields +{ + public const string Id = "id"; + public const string MicrotingUid = "microtingUid"; + public const string FinishedAt = "finishedAt"; + public const string AnswerDuration = "answerDuration"; + public const string SiteName = "siteName"; + public const string TagNames = "tagNames"; + public const string UnitMicrotingUid = "unitMicrotingUid"; + public const string LanguageName = "languageName"; + public const string SurveyConfigurationName = "surveyConfigurationName"; + public const string QuestionSetName = "questionSetName"; + public const string TimeZone = "timeZone"; + public const string UtcAdjusted = "utcAdjusted"; + public const string CreatedAt = "createdAt"; + public const string UpdatedAt = "updatedAt"; + public const string Version = "version"; + public const string WorkflowState = "workflowState"; + public const string SiteId = "siteId"; + public const string UnitId = "unitId"; + public const string LanguageId = "languageId"; +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataListModel.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataListModel.cs new file mode 100644 index 00000000..c7e22668 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataListModel.cs @@ -0,0 +1,34 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +using System.Collections.Generic; + +public class RawDataListModel +{ + public int Total { get; set; } + public List Columns { get; set; } = new(); + public List> Rows { get; set; } = new(); +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataRequestModel.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataRequestModel.cs new file mode 100644 index 00000000..05141c94 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataRequestModel.cs @@ -0,0 +1,37 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +using Microting.eFormApi.BasePn.Infrastructure.Interfaces; + +public class RawDataRequestModel : ICommonSort, ICommonPagination +{ + public int DashboardId { get; set; } + public int DashboardItemId { get; set; } + public int PageSize { get; set; } + public int Offset { get; set; } + public string Sort { get; set; } + public bool IsSortDsc { get; set; } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataSchema.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataSchema.cs new file mode 100644 index 00000000..dd0ac4ab --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Models/RawData/RawDataSchema.cs @@ -0,0 +1,59 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Models.RawData; + +using System.Collections.Generic; + +public class RawDataQuestionMeta +{ + public int QuestionId { get; set; } + public bool IsSmiley { get; set; } + public bool IsMulti { get; set; } + + /// Raw SDK question type, e.g. "text", "number", "list". + public string QuestionType { get; set; } + + /// One of RawDataColumnKinds; decides how a cell value is resolved. + public string Kind { get; set; } + + /// Row-dictionary key for a single-value question. Null when IsMulti. + public string Field { get; set; } + + /// Row-dictionary key per option. Populated only when IsMulti. + public Dictionary OptionFieldByOptionId { get; set; } = new(); + + public Dictionary OptionNameByOptionId { get; set; } = new(); + + public Dictionary WeightValueByOptionId { get; set; } = new(); + + /// All field keys for this question, used to pre-fill "not answered". + public List OptionFields { get; set; } = new(); +} + +public class RawDataSchema +{ + public List Columns { get; set; } = new(); + public List Questions { get; set; } = new(); +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Resources/localization.json b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Resources/localization.json index a97ffe5b..2afa071e 100644 --- a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Resources/localization.json +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Resources/localization.json @@ -439,5 +439,33 @@ "da": "Svar værdier ikke fundet", "en-US": "Answer values not found" } + }, + { + "Key": "ErrorWhileObtainingRawData", + "LocalizedValue": { + "da": "Der opstod en fejl under hentning af rådata", + "en-US": "Error while obtaining raw data" + } + }, + { + "Key": "RawDataExportTooLarge", + "LocalizedValue": { + "da": "Eksporten indeholder {0} svar, hvilket overstiger grænsen på {1}. Vælg en kortere periode.", + "en-US": "The export contains {0} answers, which exceeds the limit of {1}. Choose a shorter period." + } + }, + { + "Key": "RawDataNotAvailableForTextQuestions", + "LocalizedValue": { + "da": "Rådata er ikke tilgængelige for fritekstspørgsmål", + "en-US": "Raw data is not available for free-text questions" + } + }, + { + "Key": "ErrorWhileGeneratingRawDataExport", + "LocalizedValue": { + "da": "Der opstod en fejl under generering af rådata-eksporten", + "en-US": "Error while generating the raw data export" + } } -] \ No newline at end of file +] diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/IRawDataExcelService.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/IRawDataExcelService.cs new file mode 100644 index 00000000..52294435 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/IRawDataExcelService.cs @@ -0,0 +1,34 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Services.RawDataExcelService; + +using Infrastructure.Models.RawData; + +public interface IRawDataExcelService +{ + string CreateFilePath(); + + bool WriteRawDataToExcelFile(RawDataListModel model, string destFile); +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/RawDataExcelService.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/RawDataExcelService.cs new file mode 100644 index 00000000..bab8274c --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataExcelService/RawDataExcelService.cs @@ -0,0 +1,167 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Services.RawDataExcelService; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Claims; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Infrastructure.Models.RawData; +using Microsoft.AspNetCore.Http; +using Microting.eFormApi.BasePn.Infrastructure.Helpers; + +/// +/// Writes the raw data table to xlsx. Unlike InterviewsExcelService this does not +/// copy a template first - that service creates a fresh SpreadsheetDocument over +/// the copied template anyway, so the copy is dead weight - and the column count +/// here is decided by the survey rather than a fixed enum. +/// +public class RawDataExcelService(IHttpContextAccessor httpAccessor) : IRawDataExcelService +{ + public string CreateFilePath() + { + var path = Path.Combine(PathHelper.GetStoragePath(), "excel-storage"); + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + + return Path.Combine(path, $"raw-data-{UserId}-{DateTime.UtcNow.Ticks}.xlsx"); + } + + public bool WriteRawDataToExcelFile(RawDataListModel model, string destFile) + { + using var spreadsheetDocument = + SpreadsheetDocument.Create(destFile, SpreadsheetDocumentType.Workbook); + + var workbookPart = spreadsheetDocument.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + worksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = spreadsheetDocument.WorkbookPart!.Workbook.AppendChild(new Sheets()); + sheets.Append(new Sheet + { + Id = spreadsheetDocument.WorkbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = "Raw data", + }); + + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + var columns = model.Columns; + + // Header row. Columns hidden in the UI are exported too - the export is + // the complete record. + var headerRow = new Row { RowIndex = 1U }; + for (var col = 0; col < columns.Count; col++) + { + headerRow.Append(new Cell + { + CellReference = GetCellReference(1, col + 1), + DataType = CellValues.String, + CellValue = new CellValue(columns[col].Header ?? string.Empty), + }); + } + + sheetData!.Append(headerRow); + + var rowIndex = 2; + foreach (var modelRow in model.Rows) + { + var row = new Row { RowIndex = (uint)rowIndex }; + + for (var col = 0; col < columns.Count; col++) + { + var value = modelRow.GetValueOrDefault(columns[col].Field); + if (value == null) + { + continue; + } + + var cell = new Cell { CellReference = GetCellReference(rowIndex, col + 1) }; + + switch (value) + { + case DateTime dateTime: + cell.DataType = CellValues.String; + cell.CellValue = new CellValue( + dateTime.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + break; + case int intValue: + cell.DataType = CellValues.Number; + cell.CellValue = new CellValue(intValue.ToString(CultureInfo.InvariantCulture)); + break; + case bool boolValue: + cell.DataType = CellValues.String; + cell.CellValue = new CellValue(boolValue ? "true" : "false"); + break; + default: + cell.DataType = CellValues.String; + cell.CellValue = new CellValue(value.ToString() ?? string.Empty); + break; + } + + row.Append(cell); + } + + sheetData.Append(row); + rowIndex++; + } + + workbookPart.Workbook.Save(); + return true; + } + + private int UserId + { + get + { + var value = httpAccessor?.HttpContext?.User?.FindFirstValue(ClaimTypes.NameIdentifier); + return value == null ? 0 : int.Parse(value); + } + } + + private static string GetCellReference(int rowIndex, int colIndex) => + $"{GetColumnName(colIndex)}{rowIndex}"; + + private static string GetColumnName(int index) + { + var dividend = index; + var columnName = string.Empty; + while (dividend > 0) + { + var modulo = (dividend - 1) % 26; + columnName = Convert.ToChar(65 + modulo) + columnName; + dividend = (dividend - modulo) / 26; + } + + return columnName; + } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/IRawDataService.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/IRawDataService.cs new file mode 100644 index 00000000..33344f33 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/IRawDataService.cs @@ -0,0 +1,36 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Services.RawDataService; + +using System.Threading.Tasks; +using Infrastructure.Models.RawData; +using Microting.eFormApi.BasePn.Infrastructure.Models.API; + +public interface IRawDataService +{ + Task> GetRawData(RawDataRequestModel requestModel); + + Task> GetAllRawData(int dashboardId, int dashboardItemId); +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs new file mode 100644 index 00000000..f428dbb0 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs @@ -0,0 +1,426 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Services.RawDataService; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Common.InsightDashboardLocalizationService; +using Infrastructure.Helpers; +using Infrastructure.Models.Dashboards; +using Infrastructure.Models.RawData; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microting.eForm.Infrastructure.Constants; +using Microting.eForm.Infrastructure.Data.Entities; +using Microting.eFormApi.BasePn.Abstractions; +using Microting.eFormApi.BasePn.Infrastructure.Models.API; +using Microting.InsightDashboardBase.Infrastructure.Data; +using Microting.InsightDashboardBase.Infrastructure.Data.Entities; + +public class RawDataService : IRawDataService +{ + /// + /// Hard ceiling for the unpaged export. Deliberately well below the point where + /// the process would struggle: the export materialises every answer id into an + /// IN(...) set, every AnswerValue for those answers, and one dictionary per row. + /// Batching the id lookup would let this rise; until then the cap must stay where + /// the whole set comfortably fits in memory. + /// + public const int ExportRowLimit = 25000; + + private const string NotAnswered = "—"; + private const string NaOptionName = "na"; + + private readonly ILogger _logger; + private readonly IInsightDashboardLocalizationService _localizationService; + private readonly IEFormCoreService _coreHelper; + private readonly InsightDashboardPnDbContext _dbContext; + private readonly IUserService _userService; + + public RawDataService( + ILogger logger, + IInsightDashboardLocalizationService localizationService, + IEFormCoreService coreHelper, + InsightDashboardPnDbContext dbContext, + IUserService userService) + { + _logger = logger; + _localizationService = localizationService; + _coreHelper = coreHelper; + _dbContext = dbContext; + _userService = userService; + } + + public Task> GetRawData(RawDataRequestModel requestModel) => + Build(requestModel.DashboardId, requestModel.DashboardItemId, requestModel, applyPaging: true); + + public Task> GetAllRawData(int dashboardId, int dashboardItemId) => + Build(dashboardId, dashboardItemId, null, applyPaging: false); + + private async Task> Build( + int dashboardId, + int dashboardItemId, + RawDataRequestModel requestModel, + bool applyPaging) + { + try + { + var dashboard = await _dbContext.Dashboards + .Include(x => x.DashboardItems) + .ThenInclude>(x => x.IgnoredAnswerValues) + .Include(x => x.DashboardItems) + .ThenInclude>(x => x.CompareLocationsTags) + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .FirstOrDefaultAsync(x => x.Id == dashboardId); + + if (dashboard == null) + { + return new OperationDataResult( + false, _localizationService.GetString("DashboardNotFound")); + } + + var dashboardItem = dashboard.DashboardItems + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .FirstOrDefault(x => x.Id == dashboardItemId); + + if (dashboardItem == null) + { + return new OperationDataResult( + false, _localizationService.GetString("DashboardItemNotFound")); + } + + if (dashboard.Today) + { + var dateTimeNow = DateTime.Now; + dashboard.DateTo = new DateTime( + dateTimeNow.Year, dateTimeNow.Month, dateTimeNow.Day, 23, 59, 59); + } + + var answerDates = new DashboardEditAnswerDates + { + Today = dashboard.Today, + DateFrom = dashboard.DateFrom, + DateTo = dashboard.DateTo, + }; + + var core = await _coreHelper.GetCore(); + var userLanguage = await _userService.GetCurrentUserLanguage(); + + await using var sdkContext = core.DbContextHelper.GetDbContext(); + + // Text items render the interviews grid, not a chart, and ChartDataHelpers + // filters them down a different branch (it applies the location filter + // regardless of CompareEnabled). AnswerFilterHelper does not mirror that + // branch, so refuse rather than return a set that matches nothing on screen. + var firstQuestionType = await sdkContext.Questions + .AsNoTracking() + .Where(x => x.Id == dashboardItem.FirstQuestionId) + .Select(x => x.QuestionType) + .FirstOrDefaultAsync(); + + if (firstQuestionType == Constants.QuestionTypes.Text) + { + return new OperationDataResult( + false, _localizationService.GetString("RawDataNotAvailableForTextQuestions")); + } + + var preferredLanguageIds = await RawDataTranslations.GetPreferredLanguageIdsAsync( + sdkContext, dashboard.SurveyId, userLanguage.Id); + + var schema = await RawDataColumnBuilder.BuildAsync( + sdkContext, dashboard.SurveyId, preferredLanguageIds); + + var answerQuery = AnswerFilterHelper.BuildAnswerQuery( + sdkContext, + dashboardItem, + dashboard.SurveyId, + dashboard.LocationId, + dashboard.TagId, + answerDates); + + var result = new RawDataListModel + { + Columns = schema.Columns, + Total = await answerQuery.CountAsync(), + }; + + if (!applyPaging && result.Total > ExportRowLimit) + { + return new OperationDataResult( + false, + string.Format( + _localizationService.GetString("RawDataExportTooLarge"), + result.Total, + ExportRowLimit)); + } + + var ordered = ApplySort(answerQuery, requestModel?.Sort, requestModel?.IsSortDsc ?? true); + + if (applyPaging) + { + ordered = ordered.Skip(requestModel.Offset).Take(requestModel.PageSize); + } + + var answers = await ordered + .Select(x => new AnswerRow + { + Id = x.Id, + MicrotingUid = x.MicrotingUid, + FinishedAt = x.FinishedAt, + AnswerDuration = x.AnswerDuration, + SiteId = x.SiteId, + SiteName = x.Site.Name, + TagNames = x.Site.SiteTags + .Where(y => y.WorkflowState != Constants.WorkflowStates.Removed) + .Select(y => y.Tag.Name) + .ToList(), + UnitId = x.UnitId, + UnitMicrotingUid = x.Unit.MicrotingUid, + LanguageId = x.LanguageId, + LanguageName = x.Language.Name, + SurveyConfigurationName = x.SurveyConfiguration.Name, + QuestionSetName = x.QuestionSet.Name, + TimeZone = x.TimeZone, + UtcAdjusted = x.UtcAdjusted, + CreatedAt = x.CreatedAt, + UpdatedAt = x.UpdatedAt, + Version = x.Version, + WorkflowState = x.WorkflowState, + }) + .ToListAsync(); + + var answerIds = answers.Select(x => x.Id).ToList(); + + var values = await sdkContext.AnswerValues + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => answerIds.Contains(x.AnswerId)) + .Select(x => new { x.AnswerId, x.QuestionId, x.OptionId, x.Value }) + .ToListAsync(); + + var metaByQuestionId = schema.Questions.ToDictionary(x => x.QuestionId); + var valuesByAnswerId = values + .GroupBy(x => x.AnswerId) + .ToDictionary(x => x.Key, x => x.ToList()); + + foreach (var answer in answers) + { + var row = ToRowDictionary(answer); + + // Every question column starts as "not answered"; real values overwrite it. + foreach (var meta in schema.Questions) + { + foreach (var field in meta.OptionFields) + { + row[field] = NotAnswered; + } + } + + if (valuesByAnswerId.TryGetValue(answer.Id, out var answerValues)) + { + foreach (var answerValue in answerValues) + { + if (!metaByQuestionId.TryGetValue(answerValue.QuestionId, out var meta)) + { + continue; + } + + var optionName = meta.OptionNameByOptionId.GetValueOrDefault(answerValue.OptionId); + var skipped = string.Equals(optionName, NaOptionName, StringComparison.OrdinalIgnoreCase); + + if (meta.IsMulti) + { + // A skipped multi question leaves every option column as "not answered". + if (skipped) + { + continue; + } + + // The option was removed from the survey after this answer was + // given, so it has no column. Leave the question untouched rather + // than blanking its columns, which would assert the respondent + // was offered these options and picked none. + var optionField = meta.OptionFieldByOptionId.GetValueOrDefault(answerValue.OptionId); + if (optionField == null) + { + continue; + } + + foreach (var field in meta.OptionFields) + { + if (Equals(row[field], NotAnswered)) + { + row[field] = string.Empty; + } + } + + row[optionField] = optionName; + continue; + } + + row[meta.Field] = skipped + ? NotAnswered + : ResolveSingleValue(meta, answerValue.OptionId, answerValue.Value, optionName); + } + } + + result.Rows.Add(row); + } + + return new OperationDataResult(true, result); + } + catch (Exception e) + { + Trace.TraceError(e.Message); + _logger.LogError(e, e.Message); + return new OperationDataResult( + false, _localizationService.GetString("ErrorWhileObtainingRawData")); + } + } + + /// + /// Resolves one cell. The option's own translated name is only meaningful for + /// choice questions: the SDK auto-generates options whose translation IS the + /// type keyword - "smiley1".."smiley6", "number", "text", "next", "na" + /// (Question.GenerateSmileyOptions / GenerateSpecialQuestionTypes). Preferring + /// the option name for every type would therefore render "text" instead of the + /// answer, and "smiley1" instead of "Meget glad". + /// + /// Core.SaveAnswer only overwrites AnswerValue.Value with the option name for + /// buttons/list/multi; for every other type Value holds the real answer. + /// + private static string ResolveSingleValue( + RawDataQuestionMeta meta, int optionId, string value, string optionName) + { + switch (meta.Kind) + { + case RawDataColumnKinds.Smiley: + { + // Label from the weight ladder, matching what the chart plots + // (ChartDataHelpers.cs:60-68), not from the "smileyN" translation. + var weightValue = meta.WeightValueByOptionId.GetValueOrDefault(optionId); + var label = RawDataColumnBuilder.SmileyFallbackLabel(weightValue); + + return string.IsNullOrEmpty(label) + ? weightValue.ToString() + : $"{label} ({weightValue})"; + } + + case RawDataColumnKinds.Single: + // buttons / list - the option translation is the real answer text. + return !string.IsNullOrEmpty(optionName) ? optionName : value; + + case RawDataColumnKinds.Number: + case RawDataColumnKinds.Text: + // Free input lives in Value; an empty Value means the question was skipped. + return string.IsNullOrEmpty(value) ? NotAnswered : value; + + default: + // picture / info_text carry no answer, only a synthetic "next" option. + return string.Empty; + } + } + + private static Dictionary ToRowDictionary(AnswerRow answer) => new() + { + [RawDataFields.Id] = answer.Id, + [RawDataFields.MicrotingUid] = answer.MicrotingUid, + [RawDataFields.FinishedAt] = answer.FinishedAt, + [RawDataFields.AnswerDuration] = FormatDuration(answer.AnswerDuration), + [RawDataFields.SiteName] = answer.SiteName, + [RawDataFields.TagNames] = string.Join(", ", answer.TagNames), + [RawDataFields.UnitMicrotingUid] = answer.UnitMicrotingUid, + [RawDataFields.LanguageName] = answer.LanguageName, + [RawDataFields.SurveyConfigurationName] = answer.SurveyConfigurationName, + [RawDataFields.QuestionSetName] = answer.QuestionSetName, + [RawDataFields.TimeZone] = answer.TimeZone, + [RawDataFields.UtcAdjusted] = answer.UtcAdjusted, + [RawDataFields.CreatedAt] = answer.CreatedAt, + [RawDataFields.UpdatedAt] = answer.UpdatedAt, + [RawDataFields.Version] = answer.Version, + [RawDataFields.WorkflowState] = answer.WorkflowState, + [RawDataFields.SiteId] = answer.SiteId, + [RawDataFields.UnitId] = answer.UnitId, + [RawDataFields.LanguageId] = answer.LanguageId, + }; + + /// AnswerDuration is stored in seconds; the UI shows mm:ss. + private static string FormatDuration(int seconds) => + $"{seconds / 60:D2}:{seconds % 60:D2}"; + + private static IQueryable ApplySort(IQueryable query, string sort, bool isSortDsc) => + sort switch + { + RawDataFields.Id => Order(query, x => x.Id, isSortDsc), + RawDataFields.MicrotingUid => Order(query, x => x.MicrotingUid, isSortDsc), + RawDataFields.AnswerDuration => Order(query, x => x.AnswerDuration, isSortDsc), + RawDataFields.SiteName => Order(query, x => x.Site.Name, isSortDsc), + RawDataFields.UnitMicrotingUid => Order(query, x => x.Unit.MicrotingUid, isSortDsc), + RawDataFields.LanguageName => Order(query, x => x.Language.Name, isSortDsc), + RawDataFields.SurveyConfigurationName => Order(query, x => x.SurveyConfiguration.Name, isSortDsc), + RawDataFields.CreatedAt => Order(query, x => x.CreatedAt, isSortDsc), + RawDataFields.UpdatedAt => Order(query, x => x.UpdatedAt, isSortDsc), + RawDataFields.WorkflowState => Order(query, x => x.WorkflowState, isSortDsc), + _ => Order(query, x => x.FinishedAt, isSortDsc), + }; + + /// + /// Always breaks ties on Id. Without it, answers sharing a FinishedAt (common - + /// they arrive in batches) can be ordered differently per page, so paging would + /// silently duplicate and skip rows. + /// + private static IQueryable Order( + IQueryable query, Expression> keySelector, bool isSortDsc) => + isSortDsc + ? query.OrderByDescending(keySelector).ThenByDescending(x => x.Id) + : query.OrderBy(keySelector).ThenBy(x => x.Id); + + private class AnswerRow + { + public int Id { get; init; } + public int? MicrotingUid { get; init; } + public DateTime FinishedAt { get; init; } + public int AnswerDuration { get; init; } + public int SiteId { get; init; } + public string SiteName { get; init; } + public List TagNames { get; init; } = new(); + public int? UnitId { get; init; } + public int? UnitMicrotingUid { get; init; } + public int LanguageId { get; init; } + public string LanguageName { get; init; } + public string SurveyConfigurationName { get; init; } + public string QuestionSetName { get; init; } + public string TimeZone { get; init; } + public bool UtcAdjusted { get; init; } + public DateTime? CreatedAt { get; init; } + public DateTime? UpdatedAt { get; init; } + public int? Version { get; init; } + public string WorkflowState { get; init; } + } +} diff --git a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts index 7860309e..ecc4171c 100644 --- a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts +++ b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts @@ -170,4 +170,44 @@ export class InsightDashboardDashboardViewPage { expect(format(dateFromInTable, 'dd.MM.yyyy')).toBe(format(dateFrom, 'dd.MM.yyyy')); expect(format(dateToInTable, 'dd.MM.yyyy')).toBe(format(dateTo, 'dd.MM.yyyy')); } + + // ---- Raw data table (appended to InsightDashboardDashboardViewPage) ---- + // `position` on the item model is 1-based while the existing helpers here take + // a 0-based index, hence the +1. + + public rawDataToggle(rowNum: number) { + return this.page.locator(`#dashboardRawDataToggle${rowNum + 1}`); + } + + public rawDataGrid(rowNum: number) { + return this.page.locator(`#dashboardRawData${rowNum + 1}`); + } + + public rawDataExportButton(rowNum: number) { + return this.page.locator(`#dashboardRawDataExport${rowNum + 1}`); + } + + public rawDataRows(rowNum: number) { + return this.rawDataGrid(rowNum).locator('tbody tr'); + } + + public rawDataHeaders(rowNum: number) { + return this.rawDataGrid(rowNum).locator('thead th'); + } + + /** Reads the "— N svar" suffix the toggle shows once data has loaded. */ + async rawDataAnswerCount(rowNum: number): Promise { + const text = await this.rawDataToggle(rowNum).textContent(); + const match = text?.match(/(\d+)/); + return match ? Number(match[1]) : null; + } + + /** Last cell of the aggregated table's final amount row, i.e. the chart total. */ + async chartAmountTotal(rowNum: number, rawDataNum = 0): Promise { + const totalRow = this.rawChartDataAmountValueRow(rowNum, rawDataNum, 0).last(); + const cells = totalRow.locator('td'); + const text = await cells.last().textContent(); + const parsed = Number(text?.trim()); + return Number.isNaN(parsed) ? null : parsed; + } } diff --git a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts new file mode 100644 index 00000000..e38b9ecb --- /dev/null +++ b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts @@ -0,0 +1,114 @@ +import { test, expect } from '@playwright/test'; +import { LoginPage } from '../../../Page objects/Login.page'; +import { InsightDashboardPage } from '../InsightDashboard.page'; +import { InsightDashboardDashboardsPage } from '../InsightDashboard-Dashboards.page'; +import { InsightDashboardDashboardViewPage } from '../InsightDashboard-DashboardView.page'; +import { + InsightDashboardDashboardEditPage, + DashboardTestConfigEditModel, +} from '../InsightDashboard-DashboardEdit.page'; +import { dashboardMultiChartItems } from '../ChartData/DashboardMultiChart.data'; + +const dashboardConfig: DashboardTestConfigEditModel = { + locationTagName: 'Location 1', + dateRange: { + yearFrom: 2016, + monthFrom: 1, + dayFrom: 1, + yearTo: 2020, + monthTo: 6, + dayTo: 14, + }, + today: true, +}; + +test.describe('InSight Dashboard - Raw data table', () => { + let page: any; + let insightDashboardPage: InsightDashboardPage; + let dashboardsPage: InsightDashboardDashboardsPage; + let dashboardEditPage: InsightDashboardDashboardEditPage; + let dashboardsViewPage: InsightDashboardDashboardViewPage; + + test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); + const loginPage = new LoginPage(page); + insightDashboardPage = new InsightDashboardPage(page); + dashboardsPage = new InsightDashboardDashboardsPage(page); + dashboardEditPage = new InsightDashboardDashboardEditPage(page); + dashboardsViewPage = new InsightDashboardDashboardViewPage(page); + await loginPage.open('/auth'); + await loginPage.login(); + await insightDashboardPage.goToDashboards(); + await dashboardsPage.createDashboard('Raw data'); + await dashboardEditPage.setDashboardSettings(dashboardConfig); + await dashboardEditPage.generateItems(dashboardMultiChartItems); + await dashboardEditPage.dashboardUpdateSaveBtn.click(); + await page.locator('#spinner-animation').waitFor({ state: 'hidden', timeout: 40000 }); + await page.waitForTimeout(1000); + }); + + test.afterAll(async () => { + await insightDashboardPage.goToDashboards(); + await dashboardsPage.clearTable(); + await page.close(); + }); + + test('does not load the grid until the toggle is clicked', async () => { + // Lazy loading is the whole point of the disclosure: the dashboard view + // response must not carry raw data for every chart on the page. + await expect(dashboardsViewPage.rawDataGrid(0)).toHaveCount(0); + + await dashboardsViewPage.rawDataToggle(0).click(); + + await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + expect(await dashboardsViewPage.rawDataRows(0).count()).toBeGreaterThan(0); + }); + + test('reports an answer count that reconciles with the chart', async () => { + await dashboardsViewPage.rawDataToggle(0).click(); + await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + + const answerCount = await dashboardsViewPage.rawDataAnswerCount(0); + expect(answerCount).not.toBeNull(); + + // The aggregated table's amount block ends in a bolded Total row whose last + // cell is the chart's total. For a `multi` first question one answer + // contributes several answer values, so the chart total is >= the number of + // answers; for every other question type the two are equal. This fixture + // uses a multi question, hence the inequality rather than strict equality. + const chartTotal = await dashboardsViewPage.chartAmountTotal(0); + expect(chartTotal).not.toBeNull(); + expect(answerCount).toBeGreaterThan(0); + expect(answerCount).toBeLessThanOrEqual(chartTotal as number); + }); + + test('gives every multi-select option its own question-prefixed column', async () => { + await dashboardsViewPage.rawDataToggle(0).click(); + await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + + const headers = await dashboardsViewPage.rawDataHeaders(0).allTextContents(); + const optionHeaders = headers.filter((header) => header.includes('›')); + + expect(optionHeaders.length).toBeGreaterThan(0); + + for (const header of optionHeaders) { + const [questionPart, optionPart] = header.split('›'); + expect(questionPart.trim().length).toBeGreaterThan(0); + expect(optionPart.trim().length).toBeGreaterThan(0); + // Question columns are labelled "N - question text". + expect(questionPart.trim()).toMatch(/^\d+\s/); + } + }); + + test('exposes hidden answer columns through the column menu', async () => { + await dashboardsViewPage.rawDataToggle(0).click(); + await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + + const visibleBefore = await dashboardsViewPage.rawDataHeaders(0).count(); + expect(visibleBefore).toBeGreaterThan(0); + + // Time zone ships hidden by default, so it must not be in the header row. + const headers = await dashboardsViewPage.rawDataHeaders(0).allTextContents(); + expect(headers.some((header) => header.trim() === 'Tidszone')).toBe(false); + }); +}); diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html index f729a1cf..5a433794 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-block-view/dashboard-block-view.component.html @@ -51,6 +51,10 @@ [itemModel]="itemModel"> + + + + + + + + + + + + + + + + + diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.scss b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.scss new file mode 100644 index 00000000..e69de29b diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts new file mode 100644 index 00000000..92d8490c --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.spec.ts @@ -0,0 +1,147 @@ +import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; +import {NO_ERRORS_SCHEMA, Pipe, PipeTransform} from '@angular/core'; +import {of} from 'rxjs'; +import {TranslateService} from '@ngx-translate/core'; +import {DashboardRawDataViewComponent} from './dashboard-raw-data-view.component'; +import {InsightDashboardPnRawDataService} from '../../../../services'; + +@Pipe({name: 'translate', standalone: false}) +class MockTranslatePipe implements PipeTransform { + transform(value: string): string { + return value; + } +} + +describe('DashboardRawDataViewComponent', () => { + let component: DashboardRawDataViewComponent; + let fixture: ComponentFixture; + + const response = { + success: true, + model: { + total: 2, + columns: [ + { + field: 'finishedAt', header: 'Finished at', kind: 'answer', + defaultHidden: false, sortable: true, questionId: null, optionId: null, + }, + { + field: 'timeZone', header: 'Time zone', kind: 'answer', + defaultHidden: true, sortable: false, questionId: null, optionId: null, + }, + { + field: 'q7_o21', header: '2 – Områder › Kantine', kind: 'multiOption', + defaultHidden: false, sortable: false, questionId: 7, optionId: 21, + }, + ], + rows: [ + {finishedAt: '2026-03-02T08:14:22', timeZone: 'Europe/Copenhagen', q7_o21: 'Kantine'}, + {finishedAt: '2026-03-03T07:22:11', timeZone: 'Europe/Copenhagen', q7_o21: ''}, + ], + }, + }; + + const rawDataServiceMock = { + getRawData: jest.fn(() => of(response)), + exportToExcel: jest.fn(() => of(new Blob())), + }; + + beforeEach(waitForAsync(() => { + rawDataServiceMock.getRawData.mockClear(); + TestBed.configureTestingModule({ + declarations: [DashboardRawDataViewComponent, MockTranslatePipe], + providers: [ + {provide: InsightDashboardPnRawDataService, useValue: rawDataServiceMock}, + { + provide: TranslateService, + useValue: { + stream: (key: string) => of(key), + get: (key: string) => of(key), + instant: (key: string) => key, + }, + }, + ], + schemas: [NO_ERRORS_SCHEMA], + }).compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(DashboardRawDataViewComponent); + component = fixture.componentInstance; + component.dashboardViewModel = {id: 3, dashboardName: 'Test'} as any; + component.itemModel = {id: 9, position: 1} as any; + fixture.detectChanges(); + }); + + it('should create without loading data', () => { + expect(component).toBeTruthy(); + expect(component.expanded).toBe(false); + expect(rawDataServiceMock.getRawData).not.toHaveBeenCalled(); + }); + + it('loads data on first expand and does not refetch on re-expand', () => { + component.toggle(); + + expect(component.expanded).toBe(true); + expect(component.total).toBe(2); + expect(component.rows.length).toBe(2); + expect(rawDataServiceMock.getRawData).toHaveBeenCalledTimes(1); + + component.toggle(); + component.toggle(); + + expect(rawDataServiceMock.getRawData).toHaveBeenCalledTimes(1); + }); + + it('maps columns, hiding defaults and marking only answer columns sortable', () => { + component.toggle(); + + const [finishedAt, timeZone, optionColumn] = component.tableHeaders; + + expect(finishedAt.field).toBe('finishedAt'); + expect(finishedAt.sortable).toBe(true); + expect(finishedAt.sortProp).toEqual({id: 'finishedAt'}); + + expect(timeZone.hide).toBe(true); + + expect(optionColumn.sortable).toBe(false); + expect(optionColumn.sortProp).toBeUndefined(); + }); + + it('returns to the first page when the sort changes', () => { + component.toggle(); + component.onPaginationChanged({total: 2, pageSize: 25, offset: 50} as any); + expect(component.pagination.offset).toBe(50); + + component.sortTable({active: 'siteName', direction: 'asc'} as any); + + expect(component.pagination.offset).toBe(0); + expect(component.sort).toBe('siteName'); + }); + + it('refetches from page one when the item model changes while expanded', () => { + component.toggle(); + component.onPaginationChanged({total: 2, pageSize: 25, offset: 50} as any); + expect(rawDataServiceMock.getRawData).toHaveBeenCalledTimes(2); + + component.ngOnChanges({itemModel: {} as any}); + + // Still expanded, so the stale rows are replaced rather than just dropped. + expect(component.pagination.offset).toBe(0); + expect(rawDataServiceMock.getRawData).toHaveBeenCalledTimes(3); + expect(component.loaded).toBe(true); + }); + + it('drops loaded rows when the item model changes while collapsed', () => { + component.toggle(); + expect(component.rows.length).toBe(2); + component.toggle(); + + component.ngOnChanges({itemModel: {} as any}); + + expect(component.loaded).toBe(false); + expect(component.rows.length).toBe(0); + expect(component.tableHeaders.length).toBe(0); + expect(component.total).toBe(0); + }); +}); diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts new file mode 100644 index 00000000..8aad1a23 --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/dashboard-raw-data-view/dashboard-raw-data-view.component.ts @@ -0,0 +1,151 @@ +import {Component, inject, Input, OnChanges, OnDestroy, SimpleChanges} from '@angular/core'; +import {AutoUnsubscribe} from 'ngx-auto-unsubscribe'; +import {Observable, of, Subscription} from 'rxjs'; +import {Sort} from '@angular/material/sort'; +import {MtxGridColumn} from '@ng-matero/extensions/grid'; +import {TranslateService} from '@ngx-translate/core'; +import {saveAs} from 'file-saver'; +import {PaginationModel} from 'src/app/common/models'; +import {updateTableSort} from 'src/app/common/helpers'; +import {InsightDashboardPnRawDataService} from '../../../../services'; +import { + DashboardViewItemModel, + DashboardViewModel, + RawDataColumnModel, +} from '../../../../models'; + +@AutoUnsubscribe() +@Component({ + selector: 'app-dashboard-raw-data-view', + templateUrl: './dashboard-raw-data-view.component.html', + styleUrls: ['./dashboard-raw-data-view.component.scss'], + standalone: false, +}) +export class DashboardRawDataViewComponent implements OnChanges, OnDestroy { + private translateService = inject(TranslateService); + private rawDataService = inject(InsightDashboardPnRawDataService); + + @Input() dashboardViewModel: DashboardViewModel = new DashboardViewModel(); + @Input() itemModel: DashboardViewItemModel = new DashboardViewItemModel(); + + expanded = false; + loading = false; + loaded = false; + total = 0; + rows: any[] = []; + tableHeaders: MtxGridColumn[] = []; + pagination: PaginationModel = new PaginationModel(0, 25, 0); + sort = 'finishedAt'; + isSortDsc = true; + + getRawDataSub$: Subscription; + exportSub$: Subscription; + + get sortDirection(): 'asc' | 'desc' { + return this.isSortDsc ? 'desc' : 'asc'; + } + + ngOnChanges(changes: SimpleChanges) { + // The dashboard view re-fetches when the period or date range changes, handing + // us a new itemModel. Anything already loaded is now stale. + if (changes.itemModel || changes.dashboardViewModel) { + this.loaded = false; + this.rows = []; + this.tableHeaders = []; + this.total = 0; + this.pagination = new PaginationModel(0, this.pagination.pageSize, 0); + + if (this.expanded) { + this.getRawData(); + } + } + } + + toggle() { + this.expanded = !this.expanded; + if (this.expanded && !this.loaded) { + this.getRawData(); + } + } + + getRawData() { + this.loading = true; + this.getRawDataSub$ = this.rawDataService + .getRawData({ + dashboardId: this.dashboardViewModel.id, + dashboardItemId: this.itemModel.id, + offset: this.pagination.offset, + pageSize: this.pagination.pageSize, + sort: this.sort, + isSortDsc: this.isSortDsc, + }) + .subscribe((data) => { + this.loading = false; + if (data && data.success && data.model) { + this.loaded = true; + this.total = data.model.total; + this.pagination = {...this.pagination, total: data.model.total}; + this.rows = data.model.rows; + this.tableHeaders = data.model.columns.map((column) => + this.toGridColumn(column) + ); + } + }); + } + + // Answer column headers are translation keys. Question and option headers are + // already-resolved text from the database and must not go through translate. + private toGridColumn(column: RawDataColumnModel): MtxGridColumn { + const header: Observable = + column.kind === 'answer' + ? this.translateService.stream(column.header) + : of(column.header); + + const gridColumn: MtxGridColumn = { + header: header, + field: column.field, + hide: column.defaultHidden, + sortable: column.sortable, + }; + + if (column.sortable) { + gridColumn.sortProp = {id: column.field}; + } + + return gridColumn; + } + + sortTable(sort: Sort) { + const updated = updateTableSort(sort.active, this.sort, this.isSortDsc); + this.sort = updated.sort; + this.isSortDsc = updated.isSortDsc; + // A new ordering makes the current offset meaningless - go back to page one. + this.pagination = {...this.pagination, offset: 0}; + this.getRawData(); + } + + onPaginationChanged(pagination: PaginationModel) { + this.pagination = { + ...this.pagination, + pageSize: pagination.pageSize, + offset: pagination.offset, + }; + this.getRawData(); + } + + exportToExcel() { + this.exportSub$ = this.rawDataService + .exportToExcel({ + dashboardId: this.dashboardViewModel.id, + dashboardItemId: this.itemModel.id, + }) + .subscribe((data) => { + saveAs( + new Blob([data]), + `${this.dashboardViewModel.dashboardName}_raw_data.xlsx` + ); + }); + } + + ngOnDestroy(): void {} +} diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/index.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/index.ts index 3469a78a..4aa6a5b6 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/index.ts +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/components/dashboards/view/index.ts @@ -3,3 +3,4 @@ export * from './dashboard-block-view/dashboard-block-view.component'; export * from './dashboard-chart-view/dashboard-chart-view.component'; export * from './dashboard-interviews-view/dashboard-interviews-view.component'; export * from './dashboard-chart-data-view/dashboard-chart-data-view.component'; +export * from './dashboard-raw-data-view/dashboard-raw-data-view.component'; diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/da.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/da.ts index 2bfe0e4a..5d758e1c 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/da.ts +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/da.ts @@ -94,5 +94,25 @@ export const da = { 'Select answer values to ignore in calculation': 'Vælg svarmuligheder som ikke ønskes vist i diagram', 'Compare Location/Report tag': 'Sammenlign lokationer', 'When you want to show the average score for the options': 'Når du ønsker at vise den gennemsnitlige score på dine svarmuligheder.', - 'When you want to compare locations (eg. center, department etc.': 'Når du ønsker at sammenligne lokationer (fx centre, afsnit etc.) i samme diagram.' + 'When you want to compare locations (eg. center, department etc.': 'Når du ønsker at sammenligne lokationer (fx centre, afsnit etc.) i samme diagram.', + 'Raw data': 'Rådata', + 'answers': 'svar', + 'No raw data found': 'Ingen rådata fundet', + 'Export raw data': 'Eksportér rådata', + 'Id': 'Id', + 'Microting UID': 'Microting UID', + 'Duration': 'Varighed', + 'Site': 'Lokation', + 'Tags': 'Tags', + 'Unit': 'Enhed', + 'Language': 'Sprog', + 'Survey config': 'Undersøgelseskonfiguration', + 'Survey': 'Undersøgelse', + 'UTC adjusted': 'UTC-justeret', + 'Created at': 'Oprettet', + 'Version': 'Version', + 'Workflow state': 'Workflow-status', + 'Site id': 'Lokations-id', + 'Unit id': 'Enheds-id', + 'Language id': 'Sprog-id', }; diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/en-US.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/en-US.ts index 9f645c3e..1fd6194f 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/en-US.ts +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/i18n/en-US.ts @@ -83,5 +83,25 @@ export const enUS = { 'Export to Word': 'Export to Word', 'AdvancedPie': 'Advanced Pie', 'PieGrid': 'Pie Grid', - 'HorizontalBarStackedGrouped': 'Horizontal Bar Stacked Grouped' + 'HorizontalBarStackedGrouped': 'Horizontal Bar Stacked Grouped', + 'Raw data': 'Raw data', + 'answers': 'answers', + 'No raw data found': 'No raw data found', + 'Export raw data': 'Export raw data', + 'Id': 'Id', + 'Microting UID': 'Microting UID', + 'Duration': 'Duration', + 'Site': 'Site', + 'Tags': 'Tags', + 'Unit': 'Unit', + 'Language': 'Language', + 'Survey config': 'Survey config', + 'Survey': 'Survey', + 'UTC adjusted': 'UTC adjusted', + 'Created at': 'Created at', + 'Version': 'Version', + 'Workflow state': 'Workflow state', + 'Site id': 'Site id', + 'Unit id': 'Unit id', + 'Language id': 'Language id', }; diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/insight-dashboard-pn.module.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/insight-dashboard-pn.module.ts index f7ff66d1..01166312 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/insight-dashboard-pn.module.ts +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/insight-dashboard-pn.module.ts @@ -9,6 +9,7 @@ import { InsightDashboardPnDashboardDictionariesService, InsightDashboardPnDashboardItemsService, InsightDashboardPnDashboardsService, + InsightDashboardPnRawDataService, InsightDashboardPnSettingsService, InsightDashboardPnSurveyConfigsService, } from './services'; @@ -29,6 +30,7 @@ import { DashboardInterviewsViewComponent, DashboardItemEditComponent, DashboardNewComponent, + DashboardRawDataViewComponent, DashboardsPageComponent, DashboardViewComponent, InsightDashboardSettingsComponent, @@ -111,6 +113,7 @@ import {MatMenu, MatMenuItem, MatMenuTrigger} from "@angular/material/menu"; AnswerValuesTableComponent, AnswerDeleteModalComponent, DashboardInterviewsViewComponent, + DashboardRawDataViewComponent, ], providers: [ InsightDashboardPnSettingsService, @@ -119,6 +122,7 @@ import {MatMenu, MatMenuItem, MatMenuTrigger} from "@angular/material/menu"; InsightDashboardPnDashboardDictionariesService, InsightDashboardPnDashboardItemsService, InsightDashboardPnAnswersService, + InsightDashboardPnRawDataService, ], }) export class InsightDashboardPnModule { diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/index.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/index.ts index f6b9f722..1f397cda 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/index.ts +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/index.ts @@ -7,3 +7,4 @@ export * from './dashboard-chart-data.model'; export * from './dashboard-answer-dates.model'; export * from './dashboard-item'; export * from './dashboard-view'; +export * from './raw-data'; diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/index.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/index.ts new file mode 100644 index 00000000..1a81c61b --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/index.ts @@ -0,0 +1,3 @@ +export * from './raw-data-column.model'; +export * from './raw-data-list.model'; +export * from './raw-data-request.model'; diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-column.model.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-column.model.ts new file mode 100644 index 00000000..b9caf9cd --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-column.model.ts @@ -0,0 +1,9 @@ +export class RawDataColumnModel { + field: string; + header: string; + kind: string; + defaultHidden: boolean; + sortable: boolean; + questionId: number | null; + optionId: number | null; +} diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-list.model.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-list.model.ts new file mode 100644 index 00000000..c26a228b --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-list.model.ts @@ -0,0 +1,8 @@ +import {RawDataColumnModel} from './raw-data-column.model'; + +export class RawDataListModel { + total = 0; + columns: RawDataColumnModel[] = []; + // The key set is decided by the survey at runtime, so there is no useful static type. + rows: any[] = []; +} diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-request.model.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-request.model.ts new file mode 100644 index 00000000..c817b90c --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/models/dashboard/raw-data/raw-data-request.model.ts @@ -0,0 +1,13 @@ +export class RawDataRequestModel { + dashboardId: number; + dashboardItemId: number; + pageSize = 25; + offset = 0; + sort = 'finishedAt'; + isSortDsc = true; +} + +export class RawDataExportRequestModel { + dashboardId: number; + dashboardItemId: number; +} diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/index.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/index.ts index f0bbb328..6039b8f6 100644 --- a/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/index.ts +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/index.ts @@ -4,3 +4,4 @@ export * from './insight-dashboard-pn-survey-configs.service'; export * from './insight-dashboard-pn-dashboard-dictionaries.service'; export * from './insight-dashboard-pn-dashboard-items.service'; export * from './insight-dashboard-pn-answers.service'; +export * from './insight-dashboard-pn-raw-data.service'; diff --git a/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/insight-dashboard-pn-raw-data.service.ts b/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/insight-dashboard-pn-raw-data.service.ts new file mode 100644 index 00000000..2d13f478 --- /dev/null +++ b/eform-client/src/app/plugins/modules/insight-dashboard-pn/services/insight-dashboard-pn-raw-data.service.ts @@ -0,0 +1,29 @@ +import {inject, Injectable} from '@angular/core'; +import {Observable} from 'rxjs'; +import { + RawDataExportRequestModel, + RawDataListModel, + RawDataRequestModel, +} from '../models'; +import {OperationDataResult} from 'src/app/common/models'; +import {ApiBaseService} from 'src/app/common/services'; + +const RawDataMethods = { + RawData: 'api/insight-dashboard-pn/dashboard-items/raw-data', + Export: 'api/insight-dashboard-pn/dashboard-items/raw-data/export', +}; + +@Injectable() +export class InsightDashboardPnRawDataService { + private apiBaseService = inject(ApiBaseService); + + getRawData( + model: RawDataRequestModel + ): Observable> { + return this.apiBaseService.post(RawDataMethods.RawData, model); + } + + exportToExcel(model: RawDataExportRequestModel): Observable { + return this.apiBaseService.getBlobData(RawDataMethods.Export, model); + } +} From ec2b7efa1960d530354aa56b80801d98547c652b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Wed, 29 Jul 2026 11:26:36 +0200 Subject: [PATCH 5/6] test: verify the raw data table's rows and cells in CI The PR previously relied on Playwright specs that had never been run, and on nothing at all for the backend. This adds NUnit coverage to InsightDashboard.Pn.Test, which the insight-dashboard-test-dotnet check already runs against a 420_SDK database seeded from 420_SDK.sql. RawData_RowSet_MatchesChartAnswers asserts the table's row set equals the answers the chart plotted, for every dashboard fixture already used by ChartDataUTests. The expected set is restated independently from the spec rather than copied from AnswerFilterHelper, so the two cannot drift together, and every returned answer is checked to actually carry a value for the measured question. RawData_Columns_CoverEveryQuestion asserts one column per question in QuestionIndex order, multi questions expanded to one column per option with the question label as prefix, and the audit columns present but hidden by default. RawData_Cells_ShowAnswersNotOptionKeywords guards the defect found in review: the SDK names auto-generated options after the question type ("smiley1", "number", "text", "next"), so a naive implementation renders the keyword instead of the answer. To make cell resolution testable, the private ResolveSingleValue moves out of RawDataService into a public RawDataValueResolver, which also owns the not-answered sentinel and the skipped-question check. Behaviour is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EnP42zmHAgo2NZQsa6zdhG --- .../InsightDashboard.Pn.Test/RawDataUTests.cs | 391 ++++++++++++++++++ .../Helpers/RawDataValueResolver.cs | 86 ++++ .../Services/RawDataService/RawDataService.cs | 51 +-- 3 files changed, 481 insertions(+), 47 deletions(-) create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn.Test/RawDataUTests.cs create mode 100644 eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataValueResolver.cs diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn.Test/RawDataUTests.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn.Test/RawDataUTests.cs new file mode 100644 index 00000000..011642e6 --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn.Test/RawDataUTests.cs @@ -0,0 +1,391 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Test; + +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Base; +using Helpers; +using Infrastructure.Helpers; +using Infrastructure.Models.RawData; +using Microsoft.EntityFrameworkCore; +using Microting.eForm.Infrastructure.Constants; +using NUnit.Framework; + +/// +/// Verifies the raw data table shows what it is supposed to show. +/// +/// Two things are checked here that nothing else covers: +/// 1. The table's row set reconciles with the chart above it - that is the +/// entire contract of AnswerFilterHelper. +/// 2. Cells render the answer, not the SDK's synthetic option keyword. +/// +[TestFixture] +public class RawDataUTests : DbTestFixture +{ + private const int DanishLanguageId = 1; + + private static IReadOnlyList PreferredLanguages => new[] { DanishLanguageId }; + + /// + /// The row count must equal the number of distinct answers the chart plotted. + /// Compared against an independently written query rather than against + /// AnswerFilterHelper itself, so the two cannot drift together. + /// + [Test] + public async Task RawData_RowSet_MatchesChartAnswers() + { + CultureInfo.CurrentCulture = new CultureInfo("da"); + await DatabaseHelper.AddTotalTag(DbContext); + + var dashboardViews = DashboardHelpers.GetChartDataDashBoards(); + var comparisons = 0; + + foreach (var (dashboardView, templateName) in dashboardViews) + { + foreach (var itemViewModel in dashboardView.Items) + { + var dashboardItem = DashboardHelpers.GetDashboardItemFromModel(itemViewModel); + + // Text items are deliberately not supported by the raw data table. + var questionType = await DbContext.Questions + .AsNoTracking() + .Where(x => x.Id == dashboardItem.FirstQuestionId) + .Select(x => x.QuestionType) + .FirstOrDefaultAsync(); + + if (questionType == Constants.QuestionTypes.Text) + { + continue; + } + + var answerDates = new Infrastructure.Models.Dashboards.DashboardEditAnswerDates + { + Today = dashboardView.AnswerDates.Today, + DateFrom = dashboardView.AnswerDates.DateFrom, + DateTo = dashboardView.AnswerDates.DateTo, + }; + + var actual = await AnswerFilterHelper + .BuildAnswerQuery( + DbContext, + dashboardItem, + dashboardView.SurveyId, + dashboardView.LocationId, + dashboardView.TagId, + answerDates) + .Select(x => x.Id) + .ToListAsync(); + + var expected = await ExpectedAnswerIds( + dashboardItem, + dashboardView.SurveyId, + dashboardView.LocationId, + dashboardView.TagId, + answerDates); + + Assert.That( + actual.OrderBy(x => x), + Is.EqualTo(expected.OrderBy(x => x)), + $"Row set diverged from the chart's answers for template {templateName}, " + + $"item {itemViewModel.Position} (question {dashboardItem.FirstQuestionId})."); + + // Every returned answer must genuinely carry a value for the + // measured question, otherwise it could not have fed the chart. + if (actual.Count > 0) + { + var withoutFirstQuestion = await DbContext.Answers + .AsNoTracking() + .Where(x => actual.Contains(x.Id)) + .Where(x => !DbContext.AnswerValues.Any(v => + v.AnswerId == x.Id + && v.QuestionId == dashboardItem.FirstQuestionId + && v.WorkflowState != Constants.WorkflowStates.Removed)) + .CountAsync(); + + Assert.That(withoutFirstQuestion, Is.Zero, + $"{withoutFirstQuestion} answers have no value for the measured question " + + $"in template {templateName}."); + } + + comparisons++; + } + } + + Assert.That(comparisons, Is.GreaterThan(0), + "No dashboard items were compared - the fixtures did not load."); + } + + /// + /// Independent restatement of "which answers feed this item", written from the + /// spec rather than copied from AnswerFilterHelper. + /// + private async Task> ExpectedAnswerIds( + Microting.InsightDashboardBase.Infrastructure.Data.Entities.DashboardItem dashboardItem, + int surveyId, + int? locationId, + int? tagId, + Infrastructure.Models.Dashboards.DashboardEditAnswerDates answerDates) + { + var values = DbContext.AnswerValues + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.Answer.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.Answer.QuestionSetId == surveyId) + .Where(x => x.QuestionId == dashboardItem.FirstQuestionId); + + if (answerDates.DateFrom != null) + { + values = values.Where(x => x.Answer.FinishedAt >= answerDates.DateFrom); + } + + if (answerDates.DateTo != null) + { + values = values.Where(x => x.Answer.FinishedAt <= answerDates.DateTo); + } + + if (dashboardItem.FilterQuestionId != null && dashboardItem.FilterAnswerId != null) + { + var filtered = await DbContext.AnswerValues + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionId == dashboardItem.FilterQuestionId) + .Where(x => x.OptionId == dashboardItem.FilterAnswerId) + .Select(x => x.AnswerId) + .ToListAsync(); + + values = values.Where(x => filtered.Contains(x.AnswerId)); + } + + var ignored = dashboardItem.IgnoredAnswerValues + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Select(x => x.AnswerId) + .ToList(); + + if (ignored.Count > 0) + { + values = values.Where(x => !ignored.Contains(x.OptionId)); + } + + var isCompared = + (dashboardItem.ChartType == Microting.InsightDashboardBase.Infrastructure.Enums.DashboardChartTypes.GroupedStackedBarChart + || dashboardItem.ChartType == Microting.InsightDashboardBase.Infrastructure.Enums.DashboardChartTypes.Line) + && (dashboardItem.CompareEnabled + || (dashboardItem.ChartType == Microting.InsightDashboardBase.Infrastructure.Enums.DashboardChartTypes.Line + && dashboardItem.CalculateAverage)); + + if (isCompared) + { + var tagIds = dashboardItem.CompareEnabled + ? dashboardItem.CompareLocationsTags + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.TagId != null) + .Select(x => (int)x.TagId).ToList() + : tagId != null ? new List { (int)tagId } : new List(); + + var siteIds = dashboardItem.CompareEnabled + ? dashboardItem.CompareLocationsTags + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.LocationId != null) + .Select(x => (int)x.LocationId).ToList() + : locationId != null ? new List { (int)locationId } : new List(); + + var byTag = await values + .Where(x => x.Answer.Site.SiteTags.Any(y => y.TagId != null && tagIds.Contains((int)y.TagId))) + .Select(x => x.AnswerId).ToListAsync(); + + var bySite = await values + .Where(x => siteIds.Contains(x.Answer.SiteId)) + .Select(x => x.AnswerId).ToListAsync(); + + return byTag.Concat(bySite).Distinct().ToList(); + } + + if (!dashboardItem.CompareEnabled) + { + if (locationId != null) + { + values = values.Where(x => x.Answer.SiteId == locationId); + } + else if (tagId != null) + { + values = values.Where(x => x.Answer.Site.SiteTags.Any(y => y.TagId == tagId)); + } + } + + if (locationId == null && tagId == null) + { + return new List(); + } + + return await values.Select(x => x.AnswerId).Distinct().ToListAsync(); + } + + /// + /// Columns must cover every question, numbered and ordered by QuestionIndex, + /// with multi questions expanded to one column per option. + /// + [Test] + public async Task RawData_Columns_CoverEveryQuestion() + { + var surveyId = await DbContext.Questions + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Select(x => x.QuestionSetId) + .FirstOrDefaultAsync(); + + Assert.That(surveyId, Is.GreaterThan(0), "No question set found in the seeded database."); + + var schema = await RawDataColumnBuilder.BuildAsync(DbContext, surveyId, PreferredLanguages); + + var answerColumns = schema.Columns + .Where(x => x.Kind == RawDataColumnKinds.Answer).ToList(); + + // The fixed half of the table. + Assert.That(answerColumns.Select(x => x.Field), Does.Contain(RawDataFields.Id)); + Assert.That(answerColumns.Select(x => x.Field), Does.Contain(RawDataFields.FinishedAt)); + Assert.That(answerColumns.Select(x => x.Field), Does.Contain(RawDataFields.SiteName)); + + // Audit fields exist but stay out of the way. + var timeZone = answerColumns.Single(x => x.Field == RawDataFields.TimeZone); + Assert.That(timeZone.DefaultHidden, Is.True, "Time zone should be hidden by default."); + + var finishedAt = answerColumns.Single(x => x.Field == RawDataFields.FinishedAt); + Assert.That(finishedAt.DefaultHidden, Is.False); + Assert.That(finishedAt.Sortable, Is.True); + + var questions = await DbContext.Questions + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionSetId == surveyId) + .OrderBy(x => x.QuestionIndex) + .Select(x => new { x.Id, x.QuestionType }) + .ToListAsync(); + + Assert.That(schema.Questions.Count, Is.EqualTo(questions.Count), + "Every non-removed question must produce metadata."); + + Assert.That( + schema.Questions.Select(x => x.QuestionId), + Is.EqualTo(questions.Select(x => x.Id)), + "Questions must keep QuestionIndex order."); + + // Question columns are numbered from 1 in QuestionIndex order. + var questionColumns = schema.Columns + .Where(x => x.Kind != RawDataColumnKinds.Answer).ToList(); + + Assert.That(questionColumns, Is.Not.Empty); + Assert.That(questionColumns.All(x => x.Sortable), Is.False, + "Pivoted question columns are not sortable."); + + foreach (var meta in schema.Questions) + { + var columns = questionColumns.Where(x => x.QuestionId == meta.QuestionId).ToList(); + Assert.That(columns, Is.Not.Empty, $"Question {meta.QuestionId} produced no column."); + + if (meta.IsMulti) + { + var optionCount = await DbContext.Options + .AsNoTracking() + .Where(x => x.WorkflowState != Constants.WorkflowStates.Removed) + .Where(x => x.QuestionId == meta.QuestionId) + .CountAsync(); + + Assert.That(columns.Count, Is.EqualTo(optionCount), + $"Multi question {meta.QuestionId} must expand to one column per option."); + Assert.That(columns.All(x => x.Header.Contains('›')), Is.True, + "Option columns carry their question label as a prefix."); + Assert.That(columns.All(x => x.OptionId != null), Is.True); + } + else + { + Assert.That(columns.Count, Is.EqualTo(1), + $"Non-multi question {meta.QuestionId} must produce exactly one column."); + } + } + } + + /// + /// The defect this guards against: the SDK names auto-generated options after + /// the question type, so a naive implementation renders "text" and "smiley1" + /// instead of the answer. + /// + [Test] + public void RawData_Cells_ShowAnswersNotOptionKeywords() + { + var smiley = new RawDataQuestionMeta + { + QuestionId = 1, + IsSmiley = true, + Kind = RawDataColumnKinds.Smiley, + WeightValueByOptionId = new Dictionary { { 10, 75 }, { 11, 999 } }, + }; + + // "smiley2" is the option's actual translation in the database. + Assert.That( + RawDataValueResolver.ResolveSingleValue(smiley, 10, null, "smiley2"), + Is.EqualTo("Glad (75)")); + + Assert.That( + RawDataValueResolver.ResolveSingleValue(smiley, 11, null, "smiley6"), + Is.EqualTo("Ved ikke (999)")); + + var text = new RawDataQuestionMeta { QuestionId = 2, Kind = RawDataColumnKinds.Text }; + + // "text" is the synthetic option name; the answer lives in Value. + Assert.That( + RawDataValueResolver.ResolveSingleValue(text, 20, "Toiletterne trænger", "text"), + Is.EqualTo("Toiletterne trænger")); + + Assert.That( + RawDataValueResolver.ResolveSingleValue(text, 20, "", "text"), + Is.EqualTo(RawDataValueResolver.NotAnswered)); + + var number = new RawDataQuestionMeta { QuestionId = 3, Kind = RawDataColumnKinds.Number }; + + Assert.That( + RawDataValueResolver.ResolveSingleValue(number, 30, "12", "number"), + Is.EqualTo("12")); + + // Single-select is the one type where the option name IS the answer. + var single = new RawDataQuestionMeta { QuestionId = 4, Kind = RawDataColumnKinds.Single }; + + Assert.That( + RawDataValueResolver.ResolveSingleValue(single, 40, "Produktion", "Produktion"), + Is.EqualTo("Produktion")); + + // picture / info_text carry no answer at all. + var other = new RawDataQuestionMeta { QuestionId = 5, Kind = RawDataColumnKinds.Other }; + + Assert.That( + RawDataValueResolver.ResolveSingleValue(other, 50, null, "next"), + Is.EqualTo(string.Empty)); + + Assert.That(RawDataValueResolver.IsSkipped("na"), Is.True); + Assert.That(RawDataValueResolver.IsSkipped("Kantine"), Is.False); + } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataValueResolver.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataValueResolver.cs new file mode 100644 index 00000000..46f3826f --- /dev/null +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Infrastructure/Helpers/RawDataValueResolver.cs @@ -0,0 +1,86 @@ +/* +The MIT License (MIT) + +Copyright (c) 2007 - 2021 Microting A/S + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +namespace InsightDashboard.Pn.Infrastructure.Helpers; + +using System; +using System.Collections.Generic; +using Models.RawData; + +/// +/// Turns one AnswerValue into the text a raw-data cell shows. +/// +/// The option's own translated name is only meaningful for choice questions. +/// The SDK auto-generates options whose translation IS the type keyword - +/// "smiley1".."smiley6", "number", "text", "next", "na" (see +/// Question.GenerateSmileyOptions / GenerateSpecialQuestionTypes). Preferring the +/// option name for every type would therefore render "text" instead of the typed +/// answer, and "smiley1" instead of "Meget glad". +/// +/// Core.SaveAnswer only overwrites AnswerValue.Value with the option name for +/// buttons/list/multi; for every other type Value holds the real answer. +/// +public static class RawDataValueResolver +{ + /// Shown when a question was not answered. + public const string NotAnswered = "—"; + + /// Name of the synthetic option the SDK creates for a skipped question. + public const string NaOptionName = "na"; + + public static bool IsSkipped(string optionName) => + string.Equals(optionName, NaOptionName, StringComparison.OrdinalIgnoreCase); + + public static string ResolveSingleValue( + RawDataQuestionMeta meta, int optionId, string value, string optionName) + { + switch (meta.Kind) + { + case RawDataColumnKinds.Smiley: + { + // Label from the weight ladder, matching what the chart plots + // (ChartDataHelpers.cs:60-68), not from the "smileyN" translation. + var weightValue = meta.WeightValueByOptionId.GetValueOrDefault(optionId); + var label = RawDataColumnBuilder.SmileyFallbackLabel(weightValue); + + return string.IsNullOrEmpty(label) + ? weightValue.ToString() + : $"{label} ({weightValue})"; + } + + case RawDataColumnKinds.Single: + // buttons / list - the option translation is the real answer text. + return !string.IsNullOrEmpty(optionName) ? optionName : value; + + case RawDataColumnKinds.Number: + case RawDataColumnKinds.Text: + // Free input lives in Value; an empty Value means the question was skipped. + return string.IsNullOrEmpty(value) ? NotAnswered : value; + + default: + // picture / info_text carry no answer, only a synthetic "next" option. + return string.Empty; + } + } +} diff --git a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs index f428dbb0..8555a97e 100644 --- a/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs +++ b/eFormAPI/Plugins/InsightDashboard.Pn/InsightDashboard.Pn/Services/RawDataService/RawDataService.cs @@ -54,8 +54,7 @@ public class RawDataService : IRawDataService /// public const int ExportRowLimit = 25000; - private const string NotAnswered = "—"; - private const string NaOptionName = "na"; + private const string NotAnswered = RawDataValueResolver.NotAnswered; private readonly ILogger _logger; private readonly IInsightDashboardLocalizationService _localizationService; @@ -252,7 +251,7 @@ private async Task> Build( } var optionName = meta.OptionNameByOptionId.GetValueOrDefault(answerValue.OptionId); - var skipped = string.Equals(optionName, NaOptionName, StringComparison.OrdinalIgnoreCase); + var skipped = RawDataValueResolver.IsSkipped(optionName); if (meta.IsMulti) { @@ -286,7 +285,8 @@ private async Task> Build( row[meta.Field] = skipped ? NotAnswered - : ResolveSingleValue(meta, answerValue.OptionId, answerValue.Value, optionName); + : RawDataValueResolver.ResolveSingleValue( + meta, answerValue.OptionId, answerValue.Value, optionName); } } @@ -304,49 +304,6 @@ private async Task> Build( } } - /// - /// Resolves one cell. The option's own translated name is only meaningful for - /// choice questions: the SDK auto-generates options whose translation IS the - /// type keyword - "smiley1".."smiley6", "number", "text", "next", "na" - /// (Question.GenerateSmileyOptions / GenerateSpecialQuestionTypes). Preferring - /// the option name for every type would therefore render "text" instead of the - /// answer, and "smiley1" instead of "Meget glad". - /// - /// Core.SaveAnswer only overwrites AnswerValue.Value with the option name for - /// buttons/list/multi; for every other type Value holds the real answer. - /// - private static string ResolveSingleValue( - RawDataQuestionMeta meta, int optionId, string value, string optionName) - { - switch (meta.Kind) - { - case RawDataColumnKinds.Smiley: - { - // Label from the weight ladder, matching what the chart plots - // (ChartDataHelpers.cs:60-68), not from the "smileyN" translation. - var weightValue = meta.WeightValueByOptionId.GetValueOrDefault(optionId); - var label = RawDataColumnBuilder.SmileyFallbackLabel(weightValue); - - return string.IsNullOrEmpty(label) - ? weightValue.ToString() - : $"{label} ({weightValue})"; - } - - case RawDataColumnKinds.Single: - // buttons / list - the option translation is the real answer text. - return !string.IsNullOrEmpty(optionName) ? optionName : value; - - case RawDataColumnKinds.Number: - case RawDataColumnKinds.Text: - // Free input lives in Value; an empty Value means the question was skipped. - return string.IsNullOrEmpty(value) ? NotAnswered : value; - - default: - // picture / info_text carry no answer, only a synthetic "next" option. - return string.Empty; - } - } - private static Dictionary ToRowDictionary(AnswerRow answer) => new() { [RawDataFields.Id] = answer.Id, From 44f1b912fc19656ebd8c1f827a18b04791e0bdf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Schultz=20Madsen?= Date: Wed, 29 Jul 2026 11:45:56 +0200 Subject: [PATCH 6/6] test: fix the raw data Playwright spec CI ran these for the first time and three of four failed. All three were defects in the test code, not the feature - the run showed the table rendering 137 answers correctly. - Every test shares one page from beforeAll, so each unconditional toggle click alternated the disclosure open and shut. The third test therefore collapsed what the second had opened and could not find the grid. Expanding now goes through ensureExpanded(), which only clicks when the grid is absent. - Row count was asserted the instant the grid became visible, before the response had rendered any rows. Now waits on the first row with an auto-retrying assertion. - chartAmountTotal() read the last cell of the wrong row, yielding 5 against 137 actual answers. Rather than repair a fragile scrape, the comparison is dropped: RawDataUTests.RawData_RowSet_MatchesChartAnswers already reconciles answer sets against the database, which is both stricter and stable. The Playwright spec now checks only that the UI is self-consistent, and the unused helper is removed. The multi-select column check is skipped when the seed survey has no multi question, since that is a property of the fixture rather than of the feature, and the expansion itself is asserted in RawDataUTests. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EnP42zmHAgo2NZQsa6zdhG --- .../InsightDashboard-DashboardView.page.ts | 9 --- .../c/insight-dashboard-raw-data.spec.ts | 72 +++++++++++-------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts index ecc4171c..22ed7531 100644 --- a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts +++ b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/InsightDashboard-DashboardView.page.ts @@ -201,13 +201,4 @@ export class InsightDashboardDashboardViewPage { const match = text?.match(/(\d+)/); return match ? Number(match[1]) : null; } - - /** Last cell of the aggregated table's final amount row, i.e. the chart total. */ - async chartAmountTotal(rowNum: number, rawDataNum = 0): Promise { - const totalRow = this.rawChartDataAmountValueRow(rowNum, rawDataNum, 0).last(); - const cells = totalRow.locator('td'); - const text = await cells.last().textContent(); - const parsed = Number(text?.trim()); - return Number.isNaN(parsed) ? null : parsed; - } } diff --git a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts index e38b9ecb..8b847bb4 100644 --- a/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts +++ b/eform-client/playwright/e2e/plugins/insight-dashboard-pn/c/insight-dashboard-raw-data.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { expect, test } from '@playwright/test'; import { LoginPage } from '../../../Page objects/Login.page'; import { InsightDashboardPage } from '../InsightDashboard.page'; import { InsightDashboardDashboardsPage } from '../InsightDashboard-Dashboards.page'; @@ -22,6 +22,9 @@ const dashboardConfig: DashboardTestConfigEditModel = { today: true, }; +// All tests here share one page created in beforeAll, so the disclosure keeps +// whatever state the previous test left it in. Every test therefore goes through +// ensureExpanded() instead of clicking blindly, which would toggle it shut again. test.describe('InSight Dashboard - Raw data table', () => { let page: any; let insightDashboardPage: InsightDashboardPage; @@ -29,6 +32,15 @@ test.describe('InSight Dashboard - Raw data table', () => { let dashboardEditPage: InsightDashboardDashboardEditPage; let dashboardsViewPage: InsightDashboardDashboardViewPage; + const ensureExpanded = async (item: number) => { + if ((await dashboardsViewPage.rawDataGrid(item).count()) === 0) { + await dashboardsViewPage.rawDataToggle(item).click(); + } + await expect(dashboardsViewPage.rawDataGrid(item)).toBeVisible(); + // Rows arrive with the response, not with the grid, so wait for them. + await expect(dashboardsViewPage.rawDataRows(item).first()).toBeVisible({ timeout: 30000 }); + }; + test.beforeAll(async ({ browser }) => { page = await browser.newPage(); const loginPage = new LoginPage(page); @@ -53,43 +65,46 @@ test.describe('InSight Dashboard - Raw data table', () => { await page.close(); }); - test('does not load the grid until the toggle is clicked', async () => { - // Lazy loading is the whole point of the disclosure: the dashboard view - // response must not carry raw data for every chart on the page. + test('does not render the grid until the toggle is clicked', async () => { + // Lazy loading is the point of the disclosure: the dashboard view response + // must not carry raw data for every chart on the page. await expect(dashboardsViewPage.rawDataGrid(0)).toHaveCount(0); await dashboardsViewPage.rawDataToggle(0).click(); await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); - expect(await dashboardsViewPage.rawDataRows(0).count()).toBeGreaterThan(0); + await expect(dashboardsViewPage.rawDataRows(0).first()).toBeVisible({ timeout: 30000 }); }); - test('reports an answer count that reconciles with the chart', async () => { - await dashboardsViewPage.rawDataToggle(0).click(); - await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + test('reports an answer count matching the rows it renders', async () => { + await ensureExpanded(0); const answerCount = await dashboardsViewPage.rawDataAnswerCount(0); expect(answerCount).not.toBeNull(); - - // The aggregated table's amount block ends in a bolded Total row whose last - // cell is the chart's total. For a `multi` first question one answer - // contributes several answer values, so the chart total is >= the number of - // answers; for every other question type the two are equal. This fixture - // uses a multi question, hence the inequality rather than strict equality. - const chartTotal = await dashboardsViewPage.chartAmountTotal(0); - expect(chartTotal).not.toBeNull(); - expect(answerCount).toBeGreaterThan(0); - expect(answerCount).toBeLessThanOrEqual(chartTotal as number); + expect(answerCount as number).toBeGreaterThan(0); + + // Exact reconciliation against the chart is asserted in + // InsightDashboard.Pn.Test/RawDataUTests.cs, which compares answer sets + // against the database rather than scraping the rendered table. Here we only + // check the UI is self-consistent: one page of rows, never more than the + // page size, and never more than the reported total. + const rowCount = await dashboardsViewPage.rawDataRows(0).count(); + expect(rowCount).toBeGreaterThan(0); + expect(rowCount).toBeLessThanOrEqual(25); + expect(rowCount).toBeLessThanOrEqual(answerCount as number); }); test('gives every multi-select option its own question-prefixed column', async () => { - await dashboardsViewPage.rawDataToggle(0).click(); - await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + await ensureExpanded(0); const headers = await dashboardsViewPage.rawDataHeaders(0).allTextContents(); const optionHeaders = headers.filter((header) => header.includes('›')); - expect(optionHeaders.length).toBeGreaterThan(0); + // Whether this survey contains a multi question is a property of the seed + // data, not of the feature. That multi questions expand to one column per + // option is asserted against the database in RawDataUTests.cs; here we only + // validate the rendered label format when such columns are present. + test.skip(optionHeaders.length === 0, 'Seed survey has no multi-select question.'); for (const header of optionHeaders) { const [questionPart, optionPart] = header.split('›'); @@ -100,15 +115,14 @@ test.describe('InSight Dashboard - Raw data table', () => { } }); - test('exposes hidden answer columns through the column menu', async () => { - await dashboardsViewPage.rawDataToggle(0).click(); - await expect(dashboardsViewPage.rawDataGrid(0)).toBeVisible(); + test('keeps audit columns out of the default view', async () => { + await ensureExpanded(0); - const visibleBefore = await dashboardsViewPage.rawDataHeaders(0).count(); - expect(visibleBefore).toBeGreaterThan(0); - - // Time zone ships hidden by default, so it must not be in the header row. const headers = await dashboardsViewPage.rawDataHeaders(0).allTextContents(); - expect(headers.some((header) => header.trim() === 'Tidszone')).toBe(false); + const trimmed = headers.map((header) => header.trim()); + + // Time zone ships hidden by default; the visible header row must not carry it. + expect(trimmed).not.toContain('Tidszone'); + expect(trimmed.some((header) => header.length > 0)).toBe(true); }); });