From 802118fdcfb87253690b84f65615b26974b39f22 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Tue, 7 Jul 2026 12:42:49 -0400 Subject: [PATCH 01/15] Add expense import tables, sprocs and project list (#31) Database layer for the expense data import pipeline: - ChartSegments: consolidated reference table for the eight ae_dwh erp_* segment sources - AETransactions and UcPathTransactions: raw transaction detail with persisted exclusion columns for rules step 2 does not own (ExcludedByDate, ExcludedByPurpose, AccountNotInAE) - Projects: materialized project list (ActiveProjects x AllProjects x PGMProjects) built by BuildProjects, snapshotting the step 1 outcome - v_PgmProjectSfnBuckets: CFDA-to-SFN-bucket logic shared with GetProjectList's PgmClassified CTE - SeedChartStringSegments: insert-only seeding of unclassified segment codes (including the new Ern type) from imported transactions - ClassifyTransactions: stamps date, purpose and account-not-in-AE exclusions; fails closed on missing preconditions and lingering 204 SFN disagreements Co-Authored-By: Claude Fable 5 --- .../Indexes/IX_AETransactions_Project.sql | 2 + .../Indexes/IX_UcPathTransactions_Account.sql | 2 + .../Indexes/IX_UcPathTransactions_Project.sql | 2 + .../data/StoredProcedures/BuildProjects.sql | 68 ++++++++++ .../StoredProcedures/ClassifyTransactions.sql | 125 ++++++++++++++++++ .../SeedChartStringSegments.sql | 63 +++++++++ database/data/Tables/AETransactions.sql | 50 +++++++ database/data/Tables/ChartSegments.sql | 21 +++ database/data/Tables/ChartStringSegments.sql | 2 +- database/data/Tables/Projects.sql | 25 ++++ database/data/Tables/UcPathTransactions.sql | 47 +++++++ .../data/Views/v_PgmProjectSfnBuckets.sql | 32 +++++ 12 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 database/data/Indexes/IX_AETransactions_Project.sql create mode 100644 database/data/Indexes/IX_UcPathTransactions_Account.sql create mode 100644 database/data/Indexes/IX_UcPathTransactions_Project.sql create mode 100644 database/data/StoredProcedures/BuildProjects.sql create mode 100644 database/data/StoredProcedures/ClassifyTransactions.sql create mode 100644 database/data/StoredProcedures/SeedChartStringSegments.sql create mode 100644 database/data/Tables/AETransactions.sql create mode 100644 database/data/Tables/ChartSegments.sql create mode 100644 database/data/Tables/Projects.sql create mode 100644 database/data/Tables/UcPathTransactions.sql create mode 100644 database/data/Views/v_PgmProjectSfnBuckets.sql diff --git a/database/data/Indexes/IX_AETransactions_Project.sql b/database/data/Indexes/IX_AETransactions_Project.sql new file mode 100644 index 0000000..fc75ccf --- /dev/null +++ b/database/data/Indexes/IX_AETransactions_Project.sql @@ -0,0 +1,2 @@ +CREATE NONCLUSTERED INDEX [IX_AETransactions_Project] + ON [data].[AETransactions] ([Project]); diff --git a/database/data/Indexes/IX_UcPathTransactions_Account.sql b/database/data/Indexes/IX_UcPathTransactions_Account.sql new file mode 100644 index 0000000..3754ef9 --- /dev/null +++ b/database/data/Indexes/IX_UcPathTransactions_Account.sql @@ -0,0 +1,2 @@ +CREATE NONCLUSTERED INDEX [IX_UcPathTransactions_Account] + ON [data].[UcPathTransactions] ([Account]); diff --git a/database/data/Indexes/IX_UcPathTransactions_Project.sql b/database/data/Indexes/IX_UcPathTransactions_Project.sql new file mode 100644 index 0000000..5442a69 --- /dev/null +++ b/database/data/Indexes/IX_UcPathTransactions_Project.sql @@ -0,0 +1,2 @@ +CREATE NONCLUSTERED INDEX [IX_UcPathTransactions_Project] + ON [data].[UcPathTransactions] ([Project]); diff --git a/database/data/StoredProcedures/BuildProjects.sql b/database/data/StoredProcedures/BuildProjects.sql new file mode 100644 index 0000000..865378d --- /dev/null +++ b/database/data/StoredProcedures/BuildProjects.sql @@ -0,0 +1,68 @@ +CREATE PROCEDURE [data].[BuildProjects] +AS +BEGIN + SET NOCOUNT ON; + + -- Materializes the cycle's consolidated project list from ActiveProjects, + -- AllProjects and PGMProjects: one row per NIFA project x AE project pair + -- (a NIFA project with no PGM match is one row with NULL AE fields). + -- Runs as an import stage after step 1 settles the project list; downstream + -- consumers (classify sproc 204 carve-out, expense views, associations) + -- read this table instead of re-deriving the joins. + + IF NOT EXISTS (SELECT 1 FROM [data].[ActiveProjects]) + THROW 50000, 'ActiveProjects is empty; complete Project Identification before building the project list.', 1; + + DELETE FROM [data].[Projects]; + + INSERT INTO [data].[Projects] + ( + [AccessionNumber], + [NifaProjectNumber], + [NifaAwardNumber], + [Title], + [ProjectStartDate], + [ProjectEndDate], + [ProjectDirector], + [UcpEmployeeId], + [Is204], + [NifaSfn], + [AEProjectNumber], + [SponsorAwardNumber], + [PgmSfnBucket], + [PrincipalInvestigatorNames] + ) + SELECT + a.[AccessionNumber], + a.[ProjectNumber], + ap.[AwardNumber], + ap.[Title], + ap.[ProjectStartDate], + ap.[ProjectEndDate], + a.[ProjectDirector], + a.[UcpEmployeeId], + a.[Is204], + CASE + WHEN a.[ProjectNumber] LIKE '%-H' THEN '201' + WHEN a.[ProjectNumber] LIKE '%-RR' THEN '202' + WHEN a.[ProjectNumber] LIKE '%-CG' THEN '204' + WHEN a.[ProjectNumber] LIKE '%-AH' THEN '205' + ELSE 'UNKNOWN' -- unrecognized suffix: fail closed, never silently classified + END, + pc.[ProjectNumber], + pc.[SponsorAwardNumber], + pc.[PgmSfnBucket], + pgm.[PrincipalInvestigatorNames] + FROM [data].[ActiveProjects] a + LEFT JOIN [data].[AllProjects] ap + ON ap.[ProjectNumber] = a.[ProjectNumber] + LEFT JOIN [data].[v_PgmProjectSfnBuckets] pc + ON ap.[AwardNumber] IS NOT NULL + AND pc.[AwardKey] = REPLACE(ap.[AwardNumber], '-', '') + LEFT JOIN [data].[PGMProjects] pgm + ON pgm.[ProjectId] = pc.[ProjectId] + WHERE ISNULL(a.[ExcludeFromUi], 0) = 0; + + -- Row count for the import run stage. + SELECT COUNT(*) AS ProjectRowsBuilt FROM [data].[Projects]; +END diff --git a/database/data/StoredProcedures/ClassifyTransactions.sql b/database/data/StoredProcedures/ClassifyTransactions.sql new file mode 100644 index 0000000..6cc1b60 --- /dev/null +++ b/database/data/StoredProcedures/ClassifyTransactions.sql @@ -0,0 +1,125 @@ +CREATE PROCEDURE [data].[ClassifyTransactions] + @cycleStart DATE, + @cycleEnd DATE +AS +BEGIN + SET NOCOUNT ON; + + -- Stamps the persisted exclusion columns on the imported transactions: the + -- rules step 2 classification does not own. Step 2 based exclusions (fund, + -- account, financial dept, activity, ern) are derived at read time from + -- ChartStringSegments and are never stamped here. + + IF @cycleStart IS NULL OR @cycleEnd IS NULL + THROW 50000, '@cycleStart and @cycleEnd are required.', 1; + + IF @cycleStart > @cycleEnd + THROW 50000, '@cycleStart must not be after @cycleEnd.', 1; + + -- AccountNotInAE compares against the AE chart of accounts; an empty + -- reference table would flag every UCPath row, so fail loudly instead. + IF NOT EXISTS (SELECT 1 FROM [data].[ChartSegments] WHERE [SegmentName] = 'Account') + THROW 50000, 'ChartSegments has no Account rows; run the segment reference import first.', 1; + + IF NOT EXISTS (SELECT 1 FROM [data].[Projects]) + THROW 50000, 'Projects is empty; run BuildProjects first.', 1; + + -- SFN mismatches on 204 projects are resolved during Project Identification + -- (GetProjectList reports them); none should remain by import time. + IF EXISTS + ( + SELECT 1 FROM [data].[Projects] + WHERE [PgmSfnBucket] IS NOT NULL + AND ( + ([NifaSfn] = '204' AND [PgmSfnBucket] <> '204') + OR ([NifaSfn] <> '204' AND [PgmSfnBucket] = '204') + ) + ) + THROW 50000, 'Projects has 204 SFN disagreements between NIFA and PGM; resolve them in Project Identification first.', 1; + + -- AE projects mapped to a 204 NIFA project. Rows on these projects are + -- reportable regardless of purpose (the 204 carve-out). Any reportable 204 + -- project is mapped through ActiveProjects/AllProjects by the time the + -- import runs, so the materialized project list is the complete source. + SELECT DISTINCT [AEProjectNumber] + INTO #Projects204 + FROM [data].[Projects] + WHERE [NifaSfn] = '204' + AND [AEProjectNumber] IS NOT NULL; + + CREATE UNIQUE CLUSTERED INDEX [IX_Projects204] ON #Projects204 ([AEProjectNumber]); + + -- ExcludedByDate, AE: by accounting period membership, consistent with how + -- the rows are pulled (period_name list). The cycle months are generated as + -- period names ('Oct-24' style, culture pinned) the same way the import + -- builds its pull list; a row is in the cycle iff its period is in this + -- set, so buffer periods and NULLs are excluded by date. + DECLARE @cyclePeriods TABLE ([PeriodName] NVARCHAR(30) PRIMARY KEY); + + DECLARE @month DATE = DATEFROMPARTS(YEAR(@cycleStart), MONTH(@cycleStart), 1); + WHILE @month <= @cycleEnd + BEGIN + INSERT INTO @cyclePeriods VALUES (FORMAT(@month, 'MMM-yy', 'en-US')); + SET @month = DATEADD(MONTH, 1, @month); + END; + + UPDATE t + SET [ExcludedByDate] = CASE WHEN cp.[PeriodName] IS NULL THEN 1 ELSE 0 END + FROM [data].[AETransactions] t + LEFT JOIN @cyclePeriods cp ON cp.[PeriodName] = t.[PeriodName]; + + -- ExcludedByDate, UCPath: by pay period end date (authoritative; fiscal + -- year/period are unreliable on payroll corrections). Missing dates fail + -- closed as excluded. + UPDATE [data].[UcPathTransactions] + SET [ExcludedByDate] = + CASE + WHEN [PayPeriodEndDate] IS NULL THEN 1 + WHEN CAST([PayPeriodEndDate] AS DATE) BETWEEN @cycleStart AND @cycleEnd THEN 0 + ELSE 1 + END; + + -- ExcludedByPurpose: hardcoded 2025 list, with carve-outs for fund 13U02 + -- (state AES funds, reported regardless of purpose) and 204 projects. + UPDATE t + SET [ExcludedByPurpose] = + CASE + WHEN t.[Purpose] IN ('00', '40', '43', '60', '61', '62', '72', '76', '78', '80') + AND ISNULL(t.[Fund], '') <> '13U02' + AND p204.[AEProjectNumber] IS NULL + THEN 1 + ELSE 0 + END + FROM [data].[AETransactions] t + LEFT JOIN #Projects204 p204 ON p204.[AEProjectNumber] = t.[Project]; + + UPDATE t + SET [ExcludedByPurpose] = + CASE + WHEN t.[Purpose] IN ('00', '40', '43', '60', '61', '62', '72', '76', '78', '80') + AND ISNULL(t.[Fund], '') <> '13U02' + AND p204.[AEProjectNumber] IS NULL + THEN 1 + ELSE 0 + END + FROM [data].[UcPathTransactions] t + LEFT JOIN #Projects204 p204 ON p204.[AEProjectNumber] = t.[Project]; + + -- AccountNotInAE: UCPath accounts that do not exist in the AE chart of + -- accounts (this should not happen but does). Missing accounts fail closed. + UPDATE t + SET [AccountNotInAE] = + CASE + WHEN t.[Account] IS NULL THEN 1 + WHEN cs.[Code] IS NULL THEN 1 + ELSE 0 + END + FROM [data].[UcPathTransactions] t + LEFT JOIN [data].[ChartSegments] cs + ON cs.[SegmentName] = 'Account' AND cs.[Code] = t.[Account]; + + -- Row counts for the import run stage. + SELECT + (SELECT COUNT(*) FROM [data].[AETransactions]) AS AeRowsClassified, + (SELECT COUNT(*) FROM [data].[UcPathTransactions]) AS UcPathRowsClassified; +END diff --git a/database/data/StoredProcedures/SeedChartStringSegments.sql b/database/data/StoredProcedures/SeedChartStringSegments.sql new file mode 100644 index 0000000..8b282b9 --- /dev/null +++ b/database/data/StoredProcedures/SeedChartStringSegments.sql @@ -0,0 +1,63 @@ +CREATE PROCEDURE [data].[SeedChartStringSegments] +AS +BEGIN + SET NOCOUNT ON; + + -- Insert any segment value present in the imported transactions but missing + -- from ChartStringSegments, as an unclassified row (IncludeInReport NULL). + -- Existing rows and their classifications are never modified: unclassified + -- fails closed downstream until someone classifies the code in step 2. + -- + -- Descriptions come from the ChartSegments reference data where available. + -- Ern codes (UCPath DOS/earnings codes) have no local description source; + -- the UCPath import fills those in from the PeopleSoft earnings table. + -- 'XXX' is the placeholder DOS code on fringe rows, which carry no FTE, so + -- classifying it is meaningless and it is not seeded. + + DECLARE @inserted TABLE ([SegmentType] NVARCHAR(20) NOT NULL); + + WITH TransactionValues AS + ( + SELECT 'Fund' AS SegmentType, [Fund] AS Code FROM [data].[AETransactions] WHERE [Fund] IS NOT NULL + UNION + SELECT 'Fund', [Fund] FROM [data].[UcPathTransactions] WHERE [Fund] IS NOT NULL + UNION + SELECT 'Account', [Account] FROM [data].[AETransactions] WHERE [Account] IS NOT NULL + UNION + SELECT 'Account', [Account] FROM [data].[UcPathTransactions] WHERE [Account] IS NOT NULL + UNION + SELECT 'FinancialDepartment', [FinancialDepartment] FROM [data].[AETransactions] WHERE [FinancialDepartment] IS NOT NULL + UNION + SELECT 'FinancialDepartment', [FinancialDepartment] FROM [data].[UcPathTransactions] WHERE [FinancialDepartment] IS NOT NULL + UNION + SELECT 'Activity', [Activity] FROM [data].[AETransactions] WHERE [Activity] IS NOT NULL + UNION + SELECT 'Activity', [Activity] FROM [data].[UcPathTransactions] WHERE [Activity] IS NOT NULL + UNION + SELECT 'Ern', [DosCode] FROM [data].[UcPathTransactions] WHERE [DosCode] IS NOT NULL AND [DosCode] <> 'XXX' + ) + INSERT INTO [data].[ChartStringSegments] ([SegmentType], [Code], [Description]) + OUTPUT inserted.[SegmentType] INTO @inserted ([SegmentType]) + SELECT + tv.SegmentType, + tv.Code, + LEFT(cs.[Description], 300) + FROM TransactionValues tv + LEFT JOIN [data].[ChartSegments] cs + ON cs.[SegmentName] = tv.SegmentType AND cs.[Code] = tv.Code + WHERE NOT EXISTS + ( + SELECT 1 + FROM [data].[ChartStringSegments] existing + WHERE existing.[SegmentType] = tv.SegmentType + AND existing.[Code] = tv.Code + ); + + -- One row per segment type with the number of newly seeded codes + -- (types with nothing new are omitted). + SELECT + CAST([SegmentType] AS NVARCHAR(20)) AS SegmentType, + COUNT(*) AS InsertedCount + FROM @inserted + GROUP BY [SegmentType]; +END diff --git a/database/data/Tables/AETransactions.sql b/database/data/Tables/AETransactions.sql new file mode 100644 index 0000000..0e77e36 --- /dev/null +++ b/database/data/Tables/AETransactions.sql @@ -0,0 +1,50 @@ +CREATE TABLE [data].[AETransactions] +( + [Id] BIGINT NOT NULL IDENTITY(1, 1), + + -- Chart string segments (column names aligned with UcPathTransactions for a future union) + [Entity] NVARCHAR(50) NULL, + [Fund] NVARCHAR(50) NULL, + [FinancialDepartment] NVARCHAR(50) NULL, + [Account] NVARCHAR(50) NULL, + [Purpose] NVARCHAR(50) NULL, + [Program] NVARCHAR(50) NULL, + [Project] NVARCHAR(50) NULL, + [Activity] NVARCHAR(50) NULL, + + [EntityDescription] NVARCHAR(400) NULL, + [FundDescription] NVARCHAR(400) NULL, + [FinancialDepartmentDescription] NVARCHAR(400) NULL, + [AccountDescription] NVARCHAR(400) NULL, + [PurposeDescription] NVARCHAR(400) NULL, + [ProgramDescription] NVARCHAR(400) NULL, + [ProjectDescription] NVARCHAR(400) NULL, + [ActivityDescription] NVARCHAR(400) NULL, + + [DocumentType] NVARCHAR(10) NULL, + [AccountingSequenceNumber] BIGINT NULL, + [TrackingNo] NVARCHAR(300) NULL, + [Reference] NVARCHAR(300) NULL, + [JournalLineDescription] NVARCHAR(400) NULL, + [JournalAcctDate] DATE NULL, + [JournalName] NVARCHAR(200) NULL, + [JournalReference] NVARCHAR(160) NULL, + [PeriodName] NVARCHAR(30) NULL, + [JournalBatchName] NVARCHAR(200) NULL, + [JournalSource] NVARCHAR(50) NULL, + [JournalCategory] NVARCHAR(50) NULL, + [BatchStatus] NVARCHAR(2) NULL, + + [Amount] DECIMAL(19, 4) NULL, -- actual_amount; import filters to actual_flag = 'A' + [CommitmentAmount] DECIMAL(19, 4) NULL, + [ObligationAmount] DECIMAL(19, 4) NULL, + [EtlLoadDt] DATETIME2(7) NULL, + + -- Persisted exclusions for rules not owned by step 2 classification. + -- NULL = not yet classified (fails closed downstream). + [ExcludedByDate] BIT NULL, + [ExcludedByPurpose] BIT NULL, + + [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_AETransactions_LoadedAt] DEFAULT (SYSUTCDATETIME()), + CONSTRAINT [PK_AETransactions] PRIMARY KEY CLUSTERED ([Id]) +); diff --git a/database/data/Tables/ChartSegments.sql b/database/data/Tables/ChartSegments.sql new file mode 100644 index 0000000..1ebecfe --- /dev/null +++ b/database/data/Tables/ChartSegments.sql @@ -0,0 +1,21 @@ +CREATE TABLE [data].[ChartSegments] +( + [SegmentName] NVARCHAR(30) NOT NULL, -- Entity | Fund | FinancialDepartment | Account | Purpose | Program | Project | Activity + [Code] NVARCHAR(300) NOT NULL, + [ValueId] BIGINT NULL, + [Description] NVARCHAR(500) NULL, + [ValueDesc] NVARCHAR(800) NULL, + [HierarchyDepth] INT NULL, + [SummaryFlag] NVARCHAR(60) NULL, + [EnabledFlag] NVARCHAR(4) NULL, + [StartDateActive] DATE NULL, + [EndDateActive] DATE NULL, + [ParentLevel0Code] NVARCHAR(200) NULL, + [ParentLevel1Code] NVARCHAR(200) NULL, + [ParentLevel2Code] NVARCHAR(200) NULL, + [ParentLevel3Code] NVARCHAR(200) NULL, + [ParentLevel4Code] NVARCHAR(200) NULL, + [ParentLevel5Code] NVARCHAR(200) NULL, + [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_ChartSegments_LoadedAt] DEFAULT (SYSUTCDATETIME()), + CONSTRAINT [PK_ChartSegments] PRIMARY KEY CLUSTERED ([SegmentName], [Code]) +); diff --git a/database/data/Tables/ChartStringSegments.sql b/database/data/Tables/ChartStringSegments.sql index 66a07d0..3a3d7d3 100644 --- a/database/data/Tables/ChartStringSegments.sql +++ b/database/data/Tables/ChartStringSegments.sql @@ -1,6 +1,6 @@ CREATE TABLE [data].[ChartStringSegments] ( - [SegmentType] NVARCHAR(20) NOT NULL, -- FinancialDepartment | Account | Fund | Activity + [SegmentType] NVARCHAR(20) NOT NULL, -- FinancialDepartment | Account | Fund | Activity | Ern [Code] NVARCHAR(50) NOT NULL, [Description] NVARCHAR(300) NULL, [IncludeInReport] BIT NULL, -- NULL = unclassified, needs review diff --git a/database/data/Tables/Projects.sql b/database/data/Tables/Projects.sql new file mode 100644 index 0000000..8bcfcd6 --- /dev/null +++ b/database/data/Tables/Projects.sql @@ -0,0 +1,25 @@ +CREATE TABLE [data].[Projects] +( + [Id] BIGINT NOT NULL IDENTITY(1, 1), + + -- NIFA side (from ActiveProjects joined to AllProjects) + [AccessionNumber] NVARCHAR(7) NOT NULL, + [NifaProjectNumber] NVARCHAR(20) NOT NULL, + [NifaAwardNumber] NVARCHAR(16) NULL, + [Title] NVARCHAR(MAX) NULL, + [ProjectStartDate] DATE NULL, + [ProjectEndDate] DATE NULL, + [ProjectDirector] NVARCHAR(200) NULL, + [UcpEmployeeId] NVARCHAR(8) NULL, + [Is204] BIT NOT NULL, + [NifaSfn] NVARCHAR(7) NULL, -- from project number suffix: 201 | 202 | 204 | 205 | UNKNOWN + + -- AE side (from PGMProjects via award number match; NULL when no PGM match) + [AEProjectNumber] NVARCHAR(50) NULL, + [SponsorAwardNumber] NVARCHAR(100) NULL, + [PgmSfnBucket] NVARCHAR(10) NULL, -- CFDA-derived: HATCH | 203 | 204 | 205 | NON-NIFA | NULL + [PrincipalInvestigatorNames] NVARCHAR(MAX) NULL, + + [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_Projects_LoadedAt] DEFAULT (SYSUTCDATETIME()), + CONSTRAINT [PK_Projects] PRIMARY KEY CLUSTERED ([Id]) +); diff --git a/database/data/Tables/UcPathTransactions.sql b/database/data/Tables/UcPathTransactions.sql new file mode 100644 index 0000000..ef819fc --- /dev/null +++ b/database/data/Tables/UcPathTransactions.sql @@ -0,0 +1,47 @@ +CREATE TABLE [data].[UcPathTransactions] +( + -- Composite natural key from the source: journal id, journal line, addl seq, + -- emplid, empl_rcd, erncd ('XXX' for fringe rows), run id. + [LaborTransactionId] NVARCHAR(125) NOT NULL, + + -- Chart string segments (column names aligned with AETransactions for a future union) + [Entity] NVARCHAR(50) NULL, + [Fund] NVARCHAR(50) NULL, + [FinancialDepartment] NVARCHAR(50) NULL, + [ParentDepartment] NVARCHAR(50) NULL, + [Account] NVARCHAR(50) NULL, + [Purpose] NVARCHAR(50) NULL, + [Program] NVARCHAR(50) NULL, + [Project] NVARCHAR(50) NULL, + [Activity] NVARCHAR(50) NULL, + + [FinanceDocTypeCd] NVARCHAR(4) NULL, + [DosCode] NVARCHAR(3) NULL, -- ERNCD for salary rows, 'XXX' for fringe rows + [EmployeeId] NVARCHAR(10) NULL, + [EmployeeName] NVARCHAR(100) NULL, + [PositionNumber] NVARCHAR(8) NULL, + [EffDt] DATETIME2(7) NULL, + [JobCode] NVARCHAR(4) NULL, + [RateTypeCd] NVARCHAR(1) NULL, + [Hours] DECIMAL(18, 6) NULL, + [Amount] DECIMAL(19, 4) NULL, + [PayRate] DECIMAL(17, 4) NULL, + [CalculatedFte] DECIMAL(9, 6) NULL, -- hours / hours in federal fiscal year (2088 or 2096) + [PayPeriodEndDate] DATETIME2(7) NULL, -- authoritative date for cycle-window logic (fiscal year/period unreliable on corrections) + [FringeBenefitSalaryCd] NVARCHAR(1) NULL, -- 'S' salary, 'F' fringe + [PaidPercent] DECIMAL(7, 4) NULL, + [ErnDerivedPercent] DECIMAL(7, 4) NULL, + [FiscalYear] INT NULL, -- PeopleSoft fiscal bookkeeping, kept for QA totals by period + [Period] NVARCHAR(2) NULL, + [EmpRcd] SMALLINT NULL, + [EffSeq] SMALLINT NULL, + + -- Persisted exclusions for rules not owned by step 2 classification. + -- NULL = not yet classified (fails closed downstream). + [ExcludedByDate] BIT NULL, + [ExcludedByPurpose] BIT NULL, + [AccountNotInAE] BIT NULL, + + [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_UcPathTransactions_LoadedAt] DEFAULT (SYSUTCDATETIME()), + CONSTRAINT [PK_UcPathTransactions] PRIMARY KEY CLUSTERED ([LaborTransactionId]) +); diff --git a/database/data/Views/v_PgmProjectSfnBuckets.sql b/database/data/Views/v_PgmProjectSfnBuckets.sql new file mode 100644 index 0000000..0ef8d0f --- /dev/null +++ b/database/data/Views/v_PgmProjectSfnBuckets.sql @@ -0,0 +1,32 @@ +CREATE VIEW [data].[v_PgmProjectSfnBuckets] +AS +-- Each PGM award classified to an SFN bucket from its CFDA (same logic as the +-- PgmClassified CTE in [data].[GetProjectList]). The CFDA is zero-padded to +-- NN.NNN first because the warehouse stores 10.310 as the number 10.31, then +-- joined to the ALN catalog. When the catalog has no match (or is not yet +-- loaded) the bucket is NULL. +SELECT + pgm.ProjectId, + pgm.ProjectNumber, + pgm.SponsorAwardNumber, + REPLACE(pgm.SponsorAwardNumber, '-', '') AS AwardKey, + CASE + WHEN aln.ProgramNumber IS NULL THEN NULL + WHEN aln.ProgramNumber = '10.203' THEN 'HATCH' -- Hatch, matches NIFA 201/202 + WHEN aln.ProgramNumber = '10.202' THEN '203' -- McIntire-Stennis + WHEN aln.ProgramNumber IN ('10.205', '10.207') THEN '205' -- Evans-Allen / Animal Health + WHEN aln.FederalAgency030 LIKE 'NATIONAL INSTITUTE OF FOOD AND AGRICULTURE%' THEN '204' -- NIFA competitive + ELSE 'NON-NIFA' + END AS PgmSfnBucket +FROM [data].[PGMProjects] pgm +OUTER APPLY +( + SELECT CASE + WHEN CHARINDEX('.', pgm.Cfda) > 0 + THEN LEFT(pgm.Cfda, CHARINDEX('.', pgm.Cfda)) + + LEFT(SUBSTRING(pgm.Cfda, CHARINDEX('.', pgm.Cfda) + 1, 10) + '000', 3) + ELSE pgm.Cfda + END AS PaddedCfda +) p +LEFT JOIN [data].[AssistanceListingNumbers] aln + ON aln.ProgramNumber = p.PaddedCfda; From 1e96c5791ba1ef11de355d7ffc237df1a2d87567 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:18:50 -0400 Subject: [PATCH 02/15] Emit letter hierarchy level keys for all segment types Co-Authored-By: Claude Fable 5 --- server.core/Domain/AccountHierarchy.cs | 12 +++++----- server.core/Domain/ActivityHierarchy.cs | 12 +++++----- server.core/Domain/FundHierarchy.cs | 12 +++++----- .../ChartStringSegmentsControllerTests.cs | 10 ++++---- .../HierarchyMappingTests.cs | 23 +++++++++++++++++++ 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/server.core/Domain/AccountHierarchy.cs b/server.core/Domain/AccountHierarchy.cs index a54784d..608ced2 100644 --- a/server.core/Domain/AccountHierarchy.cs +++ b/server.core/Domain/AccountHierarchy.cs @@ -21,12 +21,12 @@ public IReadOnlyList Levels() => [ .. new (string Level, string? Code, string? Name)[] { - ("0", ParentLevel0Code, ParentLevel0Name), - ("1", ParentLevel1Code, ParentLevel1Name), - ("2", ParentLevel2Code, ParentLevel2Name), - ("3", ParentLevel3Code, ParentLevel3Name), - ("4", ParentLevel4Code, ParentLevel4Name), - ("5", ParentLevel5Code, ParentLevel5Name), + ("A", ParentLevel0Code, ParentLevel0Name), + ("B", ParentLevel1Code, ParentLevel1Name), + ("C", ParentLevel2Code, ParentLevel2Name), + ("D", ParentLevel3Code, ParentLevel3Name), + ("E", ParentLevel4Code, ParentLevel4Name), + ("F", ParentLevel5Code, ParentLevel5Name), } .Where(l => l.Code is not null) .Select(l => new HierarchyLevel(l.Level, l.Code!, l.Name)), diff --git a/server.core/Domain/ActivityHierarchy.cs b/server.core/Domain/ActivityHierarchy.cs index 7988a95..6716b68 100644 --- a/server.core/Domain/ActivityHierarchy.cs +++ b/server.core/Domain/ActivityHierarchy.cs @@ -21,12 +21,12 @@ public IReadOnlyList Levels() => [ .. new (string Level, string? Code, string? Name)[] { - ("0", ParentLevel0Code, ParentLevel0Name), - ("1", ParentLevel1Code, ParentLevel1Name), - ("2", ParentLevel2Code, ParentLevel2Name), - ("3", ParentLevel3Code, ParentLevel3Name), - ("4", ParentLevel4Code, ParentLevel4Name), - ("5", ParentLevel5Code, ParentLevel5Name), + ("A", ParentLevel0Code, ParentLevel0Name), + ("B", ParentLevel1Code, ParentLevel1Name), + ("C", ParentLevel2Code, ParentLevel2Name), + ("D", ParentLevel3Code, ParentLevel3Name), + ("E", ParentLevel4Code, ParentLevel4Name), + ("F", ParentLevel5Code, ParentLevel5Name), } .Where(l => l.Code is not null) .Select(l => new HierarchyLevel(l.Level, l.Code!, l.Name)), diff --git a/server.core/Domain/FundHierarchy.cs b/server.core/Domain/FundHierarchy.cs index cde659f..f6faabf 100644 --- a/server.core/Domain/FundHierarchy.cs +++ b/server.core/Domain/FundHierarchy.cs @@ -21,12 +21,12 @@ public IReadOnlyList Levels() => [ .. new (string Level, string? Code, string? Name)[] { - ("0", ParentLevel0Code, ParentLevel0Name), - ("1", ParentLevel1Code, ParentLevel1Name), - ("2", ParentLevel2Code, ParentLevel2Name), - ("3", ParentLevel3Code, ParentLevel3Name), - ("4", ParentLevel4Code, ParentLevel4Name), - ("5", ParentLevel5Code, ParentLevel5Name), + ("A", ParentLevel0Code, ParentLevel0Name), + ("B", ParentLevel1Code, ParentLevel1Name), + ("C", ParentLevel2Code, ParentLevel2Name), + ("D", ParentLevel3Code, ParentLevel3Name), + ("E", ParentLevel4Code, ParentLevel4Name), + ("F", ParentLevel5Code, ParentLevel5Name), } .Where(l => l.Code is not null) .Select(l => new HierarchyLevel(l.Level, l.Code!, l.Name)), diff --git a/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs b/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs index 5046a72..ea133d3 100644 --- a/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs +++ b/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs @@ -44,15 +44,15 @@ public async Task Get_includes_hierarchy_for_segment_with_matching_code() var ok = result.Result.Should().BeOfType().Subject; var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); dto.Hierarchy.Should().Equal( - new HierarchyLevelDto("0", "STATE", "State Funds"), - new HierarchyLevelDto("1", "APPROP", "Appropriations")); + new HierarchyLevelDto("A", "STATE", "State Funds"), + new HierarchyLevelDto("B", "APPROP", "Appropriations")); } [Theory] [InlineData(SegmentType.FinancialDepartment, "A")] - [InlineData(SegmentType.Account, "0")] - [InlineData(SegmentType.Fund, "0")] - [InlineData(SegmentType.Activity, "0")] + [InlineData(SegmentType.Account, "A")] + [InlineData(SegmentType.Fund, "A")] + [InlineData(SegmentType.Activity, "A")] public async Task Get_reads_hierarchy_from_the_matching_segment_types_own_table( SegmentType segmentType, string expectedLevel) { diff --git a/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs b/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs index 9e52b9d..c68330c 100644 --- a/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs +++ b/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs @@ -39,6 +39,29 @@ public void Department_levels_return_ordered_non_null_pairs() new HierarchyLevel("B", "DIV1", "Division One")); } + [Fact] + public void Fund_levels_use_letter_keys() + { + var fund = new FundHierarchy + { + Code = "45530", + ParentLevel0Code = "TOP", ParentLevel0Name = "Top", + ParentLevel1Code = "MID", ParentLevel1Name = "Mid", + }; + + fund.Levels().Select(l => l.Level).Should().Equal("A", "B"); + } + + [Fact] + public void Account_and_activity_levels_use_letter_keys() + { + var account = new AccountHierarchy { Code = "500000", ParentLevel0Code = "4X", ParentLevel5Code = "DEEP" }; + var activity = new ActivityHierarchy { Code = "000000", ParentLevel0Code = "ROOT" }; + + account.Levels().Select(l => l.Level).Should().Equal("A", "F"); + activity.Levels().Select(l => l.Level).Should().Equal("A"); + } + [Fact] public void Fund_level_properties_have_max_lengths() { From 4528a100b00b0095bbbd670d2c8d0f518aa6b959 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:20:28 -0400 Subject: [PATCH 03/15] Null self-referencing hierarchy levels at seed load Co-Authored-By: Claude Fable 5 --- server.core/Data/HierarchySeed.cs | 24 +++++++++++++--- .../ChartStringSegments/HierarchySeedTests.cs | 28 +++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/server.core/Data/HierarchySeed.cs b/server.core/Data/HierarchySeed.cs index 841bf8b..36f162b 100644 --- a/server.core/Data/HierarchySeed.cs +++ b/server.core/Data/HierarchySeed.cs @@ -16,17 +16,17 @@ public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct { if (!await db.AccountHierarchies.AnyAsync(ct)) { - db.AccountHierarchies.AddRange(SeedCsv.ReadRows("AccountHierarchy.csv", AccountFrom)); + db.AccountHierarchies.AddRange(SeedCsv.ReadRows("AccountHierarchy.csv", f => AccountFrom(WithoutSelfLevels(f)))); } if (!await db.FundHierarchies.AnyAsync(ct)) { - db.FundHierarchies.AddRange(SeedCsv.ReadRows("FundHierarchy.csv", FundFrom)); + db.FundHierarchies.AddRange(SeedCsv.ReadRows("FundHierarchy.csv", f => FundFrom(WithoutSelfLevels(f)))); } if (!await db.ActivityHierarchies.AnyAsync(ct)) { - db.ActivityHierarchies.AddRange(SeedCsv.ReadRows("ActivityHierarchy.csv", ActivityFrom)); + db.ActivityHierarchies.AddRange(SeedCsv.ReadRows("ActivityHierarchy.csv", f => ActivityFrom(WithoutSelfLevels(f)))); } if (!await db.DepartmentHierarchies.AnyAsync(ct)) @@ -37,6 +37,23 @@ public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct await db.SaveChangesAsync(ct); } + // A level cell that repeats the row's own code is a storage artifact of the + // warehouse's fixed-width hierarchy format and carries no information. + // Blank it (code and name) at load so stored rows are clean for every consumer. + private static string[] WithoutSelfLevels(string[] f) + { + for (var i = 2; i <= 12; i += 2) + { + if (i < f.Length && f[i].Trim() == f[0].Trim()) + { + f[i] = string.Empty; + if (i + 1 < f.Length) { f[i + 1] = string.Empty; } + } + } + + return f; + } + private static AccountHierarchy AccountFrom(string[] f) => new() { Code = f[0], @@ -89,7 +106,6 @@ private static List DepartmentRows() => ParentLevelDCode = SeedCsv.Nullable(f[0]), ParentLevelDName = SeedCsv.Nullable(f[1]), ParentLevelECode = SeedCsv.Nullable(f[2]), ParentLevelEName = SeedCsv.Nullable(f[3]), ParentLevelFCode = SeedCsv.Nullable(f[4]), ParentLevelFName = SeedCsv.Nullable(f[5]), - ParentLevelGCode = SeedCsv.Nullable(f[6]), ParentLevelGName = SeedCsv.Nullable(f[7]), }) .ToList(); } diff --git a/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs b/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs index 86d549f..a106fc2 100644 --- a/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs +++ b/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs @@ -23,7 +23,7 @@ public async Task EnsureSeeded_loads_rows_from_the_embedded_csvs() } [Fact] - public async Task EnsureSeeded_synthesizes_department_levels_a_through_g() + public async Task EnsureSeeded_synthesizes_department_levels_a_through_f() { using var db = TestDbContextFactory.CreateInMemory(); @@ -31,13 +31,35 @@ public async Task EnsureSeeded_synthesizes_department_levels_a_through_g() var department = await db.DepartmentHierarchies.FindAsync("ANUT006"); department.Should().NotBeNull(); - // Fabricated top levels plus the real D-G chain from the source file. + // Fabricated top levels plus the real D-F chain from the source file. + // Level G is omitted: it always repeats the row's own code. department!.Levels().Select(level => level.Level) - .Should().Equal("A", "B", "C", "D", "E", "F", "G"); + .Should().Equal("A", "B", "C", "D", "E", "F"); department.ParentLevelACode.Should().Be("UCD"); department.ParentLevelDCode.Should().Be("ACL100D"); } + [Fact] + public async Task Seeding_nulls_levels_that_repeat_the_rows_own_code() + { + using var db = TestDbContextFactory.CreateInMemory(); + + await HierarchySeed.EnsureSeededAsync(db); + + // The account CSV repeats the leaf code at the deepest level (e.g. 400900). + var account = db.AccountHierarchies.Single(a => a.Code == "400900"); + account.ParentLevel5Code.Should().BeNull(); + account.ParentLevel5Name.Should().BeNull(); + + // No account row keeps any level equal to its own code. + db.AccountHierarchies.AsEnumerable() + .Should().OnlyContain(a => a.Levels().All(l => l.Code != a.Code)); + + // Department rows no longer carry the self-referencing G level. + db.DepartmentHierarchies.AsEnumerable() + .Should().OnlyContain(d => d.Levels().All(l => l.Code != d.Code)); + } + [Fact] public async Task EnsureSeeded_is_idempotent() { From cc4696a9d6a873687baa2ca65195fcdad56aea1c Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:21:43 -0400 Subject: [PATCH 04/15] Add SFN catalog with descriptions, validate funds against it Co-Authored-By: Claude Fable 5 --- server.core/Data/ChartStringSegmentSeed.cs | 2 +- server/Models/ChartStringSegments/FundSfns.cs | 2 +- .../Models/ChartStringSegments/SfnCatalog.cs | 26 ++++++++++++++++ .../ChartStringSegments/SfnCatalogTests.cs | 31 +++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 server/Models/ChartStringSegments/SfnCatalog.cs create mode 100644 tests/server.tests/ChartStringSegments/SfnCatalogTests.cs diff --git a/server.core/Data/ChartStringSegmentSeed.cs b/server.core/Data/ChartStringSegmentSeed.cs index 381e09b..f698a92 100644 --- a/server.core/Data/ChartStringSegmentSeed.cs +++ b/server.core/Data/ChartStringSegmentSeed.cs @@ -19,7 +19,7 @@ public static class ChartStringSegmentSeed // "Multiple" marker). Used to give seeded, included funds a real SFN so the Fund // dropdown reflects a classified state rather than reading "Unset". private static readonly string[] FundSfnPool = - ["201", "202", "203", "205", "220", "221", "223", "Multiple"]; + ["201", "202", "203", "204", "205", "209", "219", "220", "221", "222", "223", "Multiple"]; public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct = default) { diff --git a/server/Models/ChartStringSegments/FundSfns.cs b/server/Models/ChartStringSegments/FundSfns.cs index 7bdb94a..7dd831d 100644 --- a/server/Models/ChartStringSegments/FundSfns.cs +++ b/server/Models/ChartStringSegments/FundSfns.cs @@ -5,7 +5,7 @@ public static class FundSfns public const string MultipleMarker = "Multiple"; public static readonly IReadOnlySet Codes = - new HashSet { "201", "202", "203", "205", "220", "221", "223" }; + SfnCatalog.Entries.Select(e => e.Code).ToHashSet(); public static bool IsValidForInclusion(string? sfn) => sfn is not null && (Codes.Contains(sfn) || sfn == MultipleMarker); diff --git a/server/Models/ChartStringSegments/SfnCatalog.cs b/server/Models/ChartStringSegments/SfnCatalog.cs new file mode 100644 index 0000000..564d150 --- /dev/null +++ b/server/Models/ChartStringSegments/SfnCatalog.cs @@ -0,0 +1,26 @@ +namespace Server.Models.ChartStringSegments; + +public sealed record SfnEntry(string Code, string Description); + +/// +/// The AD419 fund SFN catalog (State Funding Number codes and their line display +/// descriptors). Code and description are always separate fields; display composes +/// them. Source: AD419 report line descriptors. +/// +public static class SfnCatalog +{ + public static readonly IReadOnlyList Entries = + [ + new("201", "Hatch Funds"), + new("202", "Multi-State Research Funds"), + new("203", "McIntire-Stennis Funds"), + new("204", "Contracts, Grants, Research Coop Agreements"), + new("205", "OtherFunds(AnimalHealthSec1433,Evans-Allen)"), + new("209", "National Science Foundation"), + new("219", "USDA Contracts, Grants, Coop Agreements"), + new("220", "State Appropriations"), + new("221", "Self-Generated Funds"), + new("222", "Industry Grants and Agreements"), + new("223", "Other Non-Federal Funds"), + ]; +} diff --git a/tests/server.tests/ChartStringSegments/SfnCatalogTests.cs b/tests/server.tests/ChartStringSegments/SfnCatalogTests.cs new file mode 100644 index 0000000..90a7bea --- /dev/null +++ b/tests/server.tests/ChartStringSegments/SfnCatalogTests.cs @@ -0,0 +1,31 @@ +using FluentAssertions; +using Server.Models.ChartStringSegments; + +namespace Server.Tests.ChartStringSegments; + +public class SfnCatalogTests +{ + [Fact] + public void Catalog_contains_the_eleven_sfn_codes_in_order() + { + SfnCatalog.Entries.Select(e => e.Code).Should().Equal( + "201", "202", "203", "204", "205", "209", "219", "220", "221", "222", "223"); + } + + [Fact] + public void Descriptions_are_trimmed_and_nonempty() + { + SfnCatalog.Entries.Should().OnlyContain(e => + e.Description == e.Description.Trim() && e.Description.Length > 0); + } + + [Fact] + public void FundSfns_validation_is_catalog_backed() + { + FundSfns.IsValidForInclusion("204").Should().BeTrue(); + FundSfns.IsValidForInclusion("222").Should().BeTrue(); + FundSfns.IsValidForInclusion("Multiple").Should().BeTrue(); + FundSfns.IsValidForInclusion("999").Should().BeFalse(); + FundSfns.IsValidForInclusion(null).Should().BeFalse(); + } +} From 0a125453d014e898672019e707aad9b4852aac84 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:24:09 -0400 Subject: [PATCH 05/15] Add Purpose segment type with hierarchy table and wiring Co-Authored-By: Claude Fable 5 --- database/data/Tables/PurposeHierarchy.sql | 19 +++++++++++ server.core/Data/AppDbContext.cs | 2 ++ server.core/Domain/ChartStringSegment.cs | 1 + server.core/Domain/PurposeHierarchy.cs | 34 +++++++++++++++++++ .../ChartStringSegmentsController.cs | 2 ++ .../ChartStringSegmentsControllerTests.cs | 7 ++++ .../HierarchyMappingTests.cs | 13 +++++++ 7 files changed, 78 insertions(+) create mode 100644 database/data/Tables/PurposeHierarchy.sql create mode 100644 server.core/Domain/PurposeHierarchy.cs diff --git a/database/data/Tables/PurposeHierarchy.sql b/database/data/Tables/PurposeHierarchy.sql new file mode 100644 index 0000000..78532c2 --- /dev/null +++ b/database/data/Tables/PurposeHierarchy.sql @@ -0,0 +1,19 @@ +CREATE TABLE [data].[PurposeHierarchy] +( + [Code] NVARCHAR(50) NOT NULL, + [Description] NVARCHAR(1000) NULL, + [ParentLevel0Code] VARCHAR(20) NULL, + [ParentLevel0Name] NVARCHAR(1000) NULL, + [ParentLevel1Code] VARCHAR(20) NULL, + [ParentLevel1Name] NVARCHAR(1000) NULL, + [ParentLevel2Code] VARCHAR(20) NULL, + [ParentLevel2Name] NVARCHAR(1000) NULL, + [ParentLevel3Code] VARCHAR(20) NULL, + [ParentLevel3Name] NVARCHAR(1000) NULL, + [ParentLevel4Code] VARCHAR(20) NULL, + [ParentLevel4Name] NVARCHAR(1000) NULL, + [ParentLevel5Code] VARCHAR(20) NULL, + [ParentLevel5Name] NVARCHAR(1000) NULL, + [LoadedAt] DATETIME2(3) NOT NULL CONSTRAINT [DF_PurposeHierarchy_LoadedAt] DEFAULT (SYSUTCDATETIME()), + CONSTRAINT [PK_PurposeHierarchy] PRIMARY KEY CLUSTERED ([Code]) +); diff --git a/server.core/Data/AppDbContext.cs b/server.core/Data/AppDbContext.cs index 078bab5..0dc1589 100644 --- a/server.core/Data/AppDbContext.cs +++ b/server.core/Data/AppDbContext.cs @@ -18,6 +18,7 @@ public class AppDbContext(DbContextOptions options) : DbContext(op public DbSet AccountHierarchies => Set(); public DbSet FundHierarchies => Set(); public DbSet ActivityHierarchies => Set(); + public DbSet PurposeHierarchies => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -41,6 +42,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ConfigureHierarchy(modelBuilder, "AccountHierarchy"); ConfigureHierarchy(modelBuilder, "FundHierarchy"); ConfigureHierarchy(modelBuilder, "ActivityHierarchy"); + ConfigureHierarchy(modelBuilder, "PurposeHierarchy"); } private static void ConfigureHierarchy(ModelBuilder modelBuilder, string table) diff --git a/server.core/Domain/ChartStringSegment.cs b/server.core/Domain/ChartStringSegment.cs index b57e0f1..6d964af 100644 --- a/server.core/Domain/ChartStringSegment.cs +++ b/server.core/Domain/ChartStringSegment.cs @@ -7,6 +7,7 @@ public enum SegmentType Fund, Activity, Ern, + Purpose, } public class ChartStringSegment diff --git a/server.core/Domain/PurposeHierarchy.cs b/server.core/Domain/PurposeHierarchy.cs new file mode 100644 index 0000000..11a4510 --- /dev/null +++ b/server.core/Domain/PurposeHierarchy.cs @@ -0,0 +1,34 @@ +namespace Server.Core.Domain; + +public class PurposeHierarchy : ISegmentHierarchy +{ + public string Code { get; set; } = string.Empty; + public string? Description { get; set; } + public string? ParentLevel0Code { get; set; } + public string? ParentLevel0Name { get; set; } + public string? ParentLevel1Code { get; set; } + public string? ParentLevel1Name { get; set; } + public string? ParentLevel2Code { get; set; } + public string? ParentLevel2Name { get; set; } + public string? ParentLevel3Code { get; set; } + public string? ParentLevel3Name { get; set; } + public string? ParentLevel4Code { get; set; } + public string? ParentLevel4Name { get; set; } + public string? ParentLevel5Code { get; set; } + public string? ParentLevel5Name { get; set; } + + public IReadOnlyList Levels() => + [ + .. new (string Level, string? Code, string? Name)[] + { + ("A", ParentLevel0Code, ParentLevel0Name), + ("B", ParentLevel1Code, ParentLevel1Name), + ("C", ParentLevel2Code, ParentLevel2Name), + ("D", ParentLevel3Code, ParentLevel3Name), + ("E", ParentLevel4Code, ParentLevel4Name), + ("F", ParentLevel5Code, ParentLevel5Name), + } + .Where(l => l.Code is not null) + .Select(l => new HierarchyLevel(l.Level, l.Code!, l.Name)), + ]; +} diff --git a/server/Controllers/ChartStringSegmentsController.cs b/server/Controllers/ChartStringSegmentsController.cs index 03eb6dd..75406e0 100644 --- a/server/Controllers/ChartStringSegmentsController.cs +++ b/server/Controllers/ChartStringSegmentsController.cs @@ -25,6 +25,7 @@ public async Task>> Get(Cancel var accounts = await _db.AccountHierarchies.ToDictionaryAsync(h => h.Code, cancellationToken); var funds = await _db.FundHierarchies.ToDictionaryAsync(h => h.Code, cancellationToken); var activities = await _db.ActivityHierarchies.ToDictionaryAsync(h => h.Code, cancellationToken); + var purposes = await _db.PurposeHierarchies.ToDictionaryAsync(h => h.Code, cancellationToken); IReadOnlyList HierarchyFor(SegmentType type, string code) { @@ -34,6 +35,7 @@ IReadOnlyList HierarchyFor(SegmentType type, string code) SegmentType.Account => accounts.GetValueOrDefault(code), SegmentType.Fund => funds.GetValueOrDefault(code), SegmentType.Activity => activities.GetValueOrDefault(code), + SegmentType.Purpose => purposes.GetValueOrDefault(code), _ => null, }; diff --git a/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs b/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs index ea133d3..ce34789 100644 --- a/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs +++ b/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs @@ -53,6 +53,7 @@ public async Task Get_includes_hierarchy_for_segment_with_matching_code() [InlineData(SegmentType.Account, "A")] [InlineData(SegmentType.Fund, "A")] [InlineData(SegmentType.Activity, "A")] + [InlineData(SegmentType.Purpose, "A")] public async Task Get_reads_hierarchy_from_the_matching_segment_types_own_table( SegmentType segmentType, string expectedLevel) { @@ -86,6 +87,12 @@ public async Task Get_reads_hierarchy_from_the_matching_segment_types_own_table( Code = code, ParentLevel0Code = "X", ParentLevel0Name = "XName", }); break; + case SegmentType.Purpose: + db.PurposeHierarchies.Add(new PurposeHierarchy + { + Code = code, ParentLevel0Code = "X", ParentLevel0Name = "XName", + }); + break; } await db.SaveChangesAsync(); diff --git a/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs b/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs index c68330c..47ad71b 100644 --- a/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs +++ b/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs @@ -62,6 +62,19 @@ public void Account_and_activity_levels_use_letter_keys() activity.Levels().Select(l => l.Level).Should().Equal("A"); } + [Fact] + public void Purpose_levels_use_letter_keys() + { + var purpose = new PurposeHierarchy + { + Code = "44", + ParentLevel0Code = "1A", ParentLevel0Name = "Purpose Categories", + ParentLevel1Code = "1D", ParentLevel1Name = "Organized Research D", + }; + + purpose.Levels().Select(l => l.Level).Should().Equal("A", "B"); + } + [Fact] public void Fund_level_properties_have_max_lengths() { From 1d98bb3b4799fa397d78cb61e4f08bc19ec90edc Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:26:38 -0400 Subject: [PATCH 06/15] Seed purpose hierarchy and 2025 default classifications Co-Authored-By: Claude Fable 5 --- server.core/Data/ChartStringSegmentSeed.cs | 27 +++++++++++++++++++ server.core/Data/HierarchySeed.cs | 17 ++++++++++++ server.core/Data/Seed/PurposeHierarchy.csv | 19 +++++++++++++ .../ChartStringSegmentSeedTests.cs | 22 +++++++++++++++ .../ChartStringSegments/HierarchySeedTests.cs | 14 ++++++++++ 5 files changed, 99 insertions(+) create mode 100644 server.core/Data/Seed/PurposeHierarchy.csv diff --git a/server.core/Data/ChartStringSegmentSeed.cs b/server.core/Data/ChartStringSegmentSeed.cs index f698a92..51e9b90 100644 --- a/server.core/Data/ChartStringSegmentSeed.cs +++ b/server.core/Data/ChartStringSegmentSeed.cs @@ -34,6 +34,7 @@ public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct segments.AddRange(await BuildAsync(db.ActivityHierarchies, SegmentType.Activity, ct)); segments.AddRange(await BuildAsync(db.DepartmentHierarchies, SegmentType.FinancialDepartment, ct)); segments.AddRange(BuildErn()); + segments.AddRange(await BuildPurposeAsync(db, ct)); db.ChartStringSegments.AddRange(segments); await db.SaveChangesAsync(ct); @@ -61,6 +62,32 @@ private static List BuildErn() .ToList(); } + // Purpose classification defaults follow the 2025 processing rules: the known + // exclusion list is excluded, research purposes are included, and codes the 2025 + // process never ruled on stay unset so they surface for review (fail closed). + private static readonly IReadOnlySet ExcludedPurposes = + new HashSet { "00", "40", "43", "60", "61", "62", "72", "76", "78", "80" }; + + private static readonly IReadOnlySet IncludedPurposes = + new HashSet { "44", "45" }; + + private static async Task> BuildPurposeAsync(AppDbContext db, CancellationToken ct) + { + var rows = await db.PurposeHierarchies.ToListAsync(ct); + return rows + .OrderBy(row => row.Code, StringComparer.Ordinal) + .Select(row => new ChartStringSegment + { + SegmentType = SegmentType.Purpose, + Code = row.Code, + Description = row.Description, + IncludeInReport = ExcludedPurposes.Contains(row.Code) ? false + : IncludedPurposes.Contains(row.Code) ? true + : null, + }) + .ToList(); + } + private static async Task> BuildAsync( DbSet hierarchy, SegmentType type, diff --git a/server.core/Data/HierarchySeed.cs b/server.core/Data/HierarchySeed.cs index 36f162b..19a4fae 100644 --- a/server.core/Data/HierarchySeed.cs +++ b/server.core/Data/HierarchySeed.cs @@ -29,6 +29,11 @@ public static async Task EnsureSeededAsync(AppDbContext db, CancellationToken ct db.ActivityHierarchies.AddRange(SeedCsv.ReadRows("ActivityHierarchy.csv", f => ActivityFrom(WithoutSelfLevels(f)))); } + if (!await db.PurposeHierarchies.AnyAsync(ct)) + { + db.PurposeHierarchies.AddRange(SeedCsv.ReadRows("PurposeHierarchy.csv", f => PurposeFrom(WithoutSelfLevels(f)))); + } + if (!await db.DepartmentHierarchies.AnyAsync(ct)) { db.DepartmentHierarchies.AddRange(DepartmentRows()); @@ -90,6 +95,18 @@ private static string[] WithoutSelfLevels(string[] f) ParentLevel5Code = SeedCsv.Nullable(f[12]), ParentLevel5Name = SeedCsv.Nullable(f[13]), }; + private static PurposeHierarchy PurposeFrom(string[] f) => new() + { + Code = f[0], + Description = SeedCsv.Nullable(f[1]), + ParentLevel0Code = SeedCsv.Nullable(f[2]), ParentLevel0Name = SeedCsv.Nullable(f[3]), + ParentLevel1Code = SeedCsv.Nullable(f[4]), ParentLevel1Name = SeedCsv.Nullable(f[5]), + ParentLevel2Code = SeedCsv.Nullable(f[6]), ParentLevel2Name = SeedCsv.Nullable(f[7]), + ParentLevel3Code = SeedCsv.Nullable(f[8]), ParentLevel3Name = SeedCsv.Nullable(f[9]), + ParentLevel4Code = SeedCsv.Nullable(f[10]), ParentLevel4Name = SeedCsv.Nullable(f[11]), + ParentLevel5Code = SeedCsv.Nullable(f[12]), ParentLevel5Name = SeedCsv.Nullable(f[13]), + }; + // DepartmentSource.csv columns: // 0 D code, 1 D name, 2 E code, 3 E name, 4 F code, 5 F name, 6 G code, 7 G name, ... (fact columns ignored) private static List DepartmentRows() => diff --git a/server.core/Data/Seed/PurposeHierarchy.csv b/server.core/Data/Seed/PurposeHierarchy.csv new file mode 100644 index 0000000..938d9ff --- /dev/null +++ b/server.core/Data/Seed/PurposeHierarchy.csv @@ -0,0 +1,19 @@ +Code,Description,ParentLevel0Code,ParentLevel0Name,ParentLevel1Code,ParentLevel1Name,ParentLevel2Code,ParentLevel2Name,ParentLevel3Code,ParentLevel3Name,ParentLevel4Code,ParentLevel4Name,ParentLevel5Code,ParentLevel5Name +00,Purpose Default,1A,Purpose Categories,PD,Purpose Value Default,,,,,,,, +40,Instruction Dept Research,1A,Purpose Categories,1C,Instruction C,,,,,,,, +41,Summer Session,1A,Purpose Categories,1C,Instruction C,,,,,,,, +42,Teaching Hospitals,1A,Purpose Categories,1E,Teaching Hospitals B,,,,,,,, +43,Academic Support,1A,Purpose Categories,1B,Academic Support B,,,,,,,, +44,Research,1A,Purpose Categories,1D,Organized Research D,,,,,,,, +45,AES Research,1A,Purpose Categories,1D,Organized Research D,,,,,,,, +60,Libraries,1A,Purpose Categories,1B,Academic Support B,,,,,,,, +61,University Extension,1A,Purpose Categories,1C,Instruction C,,,,,,,, +62,Public Service,1A,Purpose Categories,1F,Public Service B,,,,,,,, +64,Operations Maintenance of Plant,1A,Purpose Categories,1G,Operations Maintenance of Plant B,,,,,,,, +65,Depreciation,1A,Purpose Categories,1H,Depreciation B,,,,,,,, +66,Impairment,1A,Purpose Categories,1I,Impairment B,,,,,,,, +68,Student Services,1A,Purpose Categories,1J,Student Services B,,,,,,,, +72,Institutional Support General Administration,1A,Purpose Categories,1L,Institutional Support General Administration B,,,,,,,, +76,Auxiliary Enterprises,1A,Purpose Categories,1M,Auxiliary Enterprises B,,,,,,,, +78,Student Financial Aid,1A,Purpose Categories,1K,Student Financial Aid B,,,,,,,, +80,Provisions for Allocations,1A,Purpose Categories,1P,Provisions for Allocations B,,,,,,,, diff --git a/tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs b/tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs index 4db68c0..cd7c88c 100644 --- a/tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs +++ b/tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs @@ -20,6 +20,7 @@ await db.AccountHierarchies.CountAsync() + await db.FundHierarchies.CountAsync() + await db.ActivityHierarchies.CountAsync() + await db.DepartmentHierarchies.CountAsync() + + await db.PurposeHierarchies.CountAsync() + db.ChartStringSegments.Count(segment => segment.SegmentType == SegmentType.Ern); (await db.ChartStringSegments.CountAsync()).Should().Be(expected); } @@ -64,6 +65,27 @@ public async Task EnsureSeeded_maps_segment_types_to_their_hierarchy() (await db.ChartStringSegments.FindAsync(SegmentType.Fund, "71549")).Should().NotBeNull(); } + [Fact] + public async Task Seeds_purpose_classifications_with_2025_defaults() + { + using var db = TestDbContextFactory.CreateInMemory(); + await HierarchySeed.EnsureSeededAsync(db); + + await ChartStringSegmentSeed.EnsureSeededAsync(db); + + var purposes = db.ChartStringSegments + .Where(s => s.SegmentType == SegmentType.Purpose) + .ToDictionary(s => s.Code, s => s.IncludeInReport); + + purposes.Should().HaveCount(18); + string[] excluded = ["00", "40", "43", "60", "61", "62", "72", "76", "78", "80"]; + string[] included = ["44", "45"]; + string[] unset = ["41", "42", "64", "65", "66", "68"]; + excluded.Should().OnlyContain(code => purposes[code] == false); + included.Should().OnlyContain(code => purposes[code] == true); + unset.Should().OnlyContain(code => purposes[code] == null); + } + [Fact] public async Task EnsureSeeded_is_idempotent() { diff --git a/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs b/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs index a106fc2..ec20ae6 100644 --- a/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs +++ b/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs @@ -39,6 +39,20 @@ public async Task EnsureSeeded_synthesizes_department_levels_a_through_f() department.ParentLevelDCode.Should().Be("ACL100D"); } + [Fact] + public async Task Seeds_purpose_hierarchy_leaves() + { + using var db = TestDbContextFactory.CreateInMemory(); + + await HierarchySeed.EnsureSeededAsync(db); + + db.PurposeHierarchies.Should().HaveCount(18); + var research = db.PurposeHierarchies.Single(p => p.Code == "44"); + research.Levels().Should().Equal( + new HierarchyLevel("A", "1A", "Purpose Categories"), + new HierarchyLevel("B", "1D", "Organized Research D")); + } + [Fact] public async Task Seeding_nulls_levels_that_repeat_the_rows_own_code() { From c20e2ff9f2f40aaaeb4afda0858f4397aa62e7c1 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:32:35 -0400 Subject: [PATCH 07/15] Suppress sort-objects lint for flat-file-ordered test fixture The test asserts the panel preserves flat-file column order, so the fixture's key order is meaningful and must stay unsorted. Co-Authored-By: Claude Fable 5 --- client/src/test/components/FlatFileImportPanel.test.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/test/components/FlatFileImportPanel.test.tsx b/client/src/test/components/FlatFileImportPanel.test.tsx index e6b9af8..3de16a0 100644 --- a/client/src/test/components/FlatFileImportPanel.test.tsx +++ b/client/src/test/components/FlatFileImportPanel.test.tsx @@ -601,11 +601,15 @@ describe('FlatFileImportPanel', () => { ], errors: [], rowNum: 2, + // Key order mirrors the flat file's column order, which the panel + // must preserve; keep it unsorted. + /* eslint-disable perfectionist/sort-objects */ values: { ProjectNumber: 'PRJ-1', AccessionNumber: 'A-2', OrganizationName: '', }, + /* eslint-enable perfectionist/sort-objects */ }, ], succeeded: false, From e1ab3bcc70a95a3434a33e77876c95d80988ca32 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:32:35 -0400 Subject: [PATCH 08/15] Add Purpose tab, per-tab headers and notes, SFN descriptions Co-Authored-By: Claude Fable 5 --- .../DataClassificationStage.tsx | 10 ++-- .../SegmentClassificationControl.tsx | 8 +-- .../dataClassification/SegmentGrid.tsx | 4 +- .../components/dataClassification/segments.ts | 57 +++++++++++++++++-- client/src/queries/chartStringSegments.ts | 23 +++++++- .../SegmentClassificationControl.test.tsx | 23 +++++++- .../dataClassification/SegmentGrid.test.tsx | 22 +++++-- .../dataClassification/segments.test.ts | 18 +++++- 8 files changed, 138 insertions(+), 27 deletions(-) diff --git a/client/src/components/dataClassification/DataClassificationStage.tsx b/client/src/components/dataClassification/DataClassificationStage.tsx index 36bc428..d9a8a1e 100644 --- a/client/src/components/dataClassification/DataClassificationStage.tsx +++ b/client/src/components/dataClassification/DataClassificationStage.tsx @@ -39,6 +39,8 @@ export function DataClassificationStage() { }; const gateOpen = allClassified(segments); + const activeTab = + SEGMENT_TABS.find((tab) => tab.type === activeType) ?? SEGMENT_TABS[0]; return (
@@ -71,16 +73,14 @@ export function DataClassificationStage() { })}
- {activeType === 'Ern' && ( + {activeTab.note && (
- - Note: ERN code classification affects FTE calculations only. It does not - affect dollar-amount calculations. - + {activeTab.note}
)} Unset - {FUND_SFNS.map((sfn) => ( - ))} diff --git a/client/src/components/dataClassification/SegmentGrid.tsx b/client/src/components/dataClassification/SegmentGrid.tsx index c3f7735..18e9344 100644 --- a/client/src/components/dataClassification/SegmentGrid.tsx +++ b/client/src/components/dataClassification/SegmentGrid.tsx @@ -26,10 +26,12 @@ function unsetFirstCodes(segments: ChartStringSegment[]): string[] { } export function SegmentGrid({ + classificationHeader, onClassify, segments, segmentType, }: { + classificationHeader: string; onClassify: ( segment: ChartStringSegment, includeInReport: boolean, @@ -85,7 +87,7 @@ export function SegmentGrid({ segment={row.original} /> ), - header: 'Classification', + header: classificationHeader, id: 'classification', }, ]; diff --git a/client/src/components/dataClassification/segments.ts b/client/src/components/dataClassification/segments.ts index d9b092c..900ed0c 100644 --- a/client/src/components/dataClassification/segments.ts +++ b/client/src/components/dataClassification/segments.ts @@ -3,12 +3,57 @@ import type { SegmentType, } from '@/queries/chartStringSegments.ts'; -export const SEGMENT_TABS: { label: string; type: SegmentType }[] = [ - { label: 'Financial Dept', type: 'FinancialDepartment' }, - { label: 'Natural Account', type: 'Account' }, - { label: 'Fund', type: 'Fund' }, - { label: 'Activity', type: 'Activity' }, - { label: 'ERN', type: 'Ern' }, +export interface SegmentTab { + classificationHeader: string; + label: string; + note: string; + slug: string; + type: SegmentType; +} + +export const SEGMENT_TABS: SegmentTab[] = [ + { + classificationHeader: 'Is AES?', + label: 'Financial Dept', + note: 'Departments marked AES have their expenses considered for the AD419 report.', + slug: 'financial-dept', + type: 'FinancialDepartment', + }, + { + classificationHeader: 'Include in AD419?', + label: 'Natural Account', + note: 'Whether expenses on this natural account are considered for the AD419 report.', + slug: 'natural-account', + type: 'Account', + }, + { + classificationHeader: 'SFN', + label: 'Fund', + note: 'Included funds must be assigned an SFN. If a fund does not map to a single SFN, select Multiple.', + slug: 'fund', + type: 'Fund', + }, + { + classificationHeader: 'Include in AD419?', + label: 'Activity', + note: 'Whether expenses on this activity are considered for the AD419 report.', + slug: 'activity', + type: 'Activity', + }, + { + classificationHeader: 'Include in AD419?', + label: 'Purpose', + note: 'All purposes are included for transactions with fund 13U02, regardless of classification.', + slug: 'purpose', + type: 'Purpose', + }, + { + classificationHeader: 'Include in FTE?', + label: 'ERN', + note: 'ERN code classification affects FTE calculations only. It does not affect dollar-amount calculations.', + slug: 'ern', + type: 'Ern', + }, ]; export function segmentsForType( diff --git a/client/src/queries/chartStringSegments.ts b/client/src/queries/chartStringSegments.ts index 8674bca..5ed4a44 100644 --- a/client/src/queries/chartStringSegments.ts +++ b/client/src/queries/chartStringSegments.ts @@ -10,6 +10,7 @@ export type SegmentType = | 'Account' | 'Fund' | 'Activity' + | 'Purpose' | 'Ern'; export interface HierarchyLevel { @@ -27,7 +28,27 @@ export interface ChartStringSegment { sfn: string | null; } -export const FUND_SFNS = ['201', '202', '203', '205', '220', '221', '223'] as const; +export interface SfnEntry { + code: string; + description: string; +} + +// Mirrors the server's SfnCatalog (code and description stored separately). +export const SFN_CATALOG: SfnEntry[] = [ + { code: '201', description: 'Hatch Funds' }, + { code: '202', description: 'Multi-State Research Funds' }, + { code: '203', description: 'McIntire-Stennis Funds' }, + { code: '204', description: 'Contracts, Grants, Research Coop Agreements' }, + { code: '205', description: 'OtherFunds(AnimalHealthSec1433,Evans-Allen)' }, + { code: '209', description: 'National Science Foundation' }, + { code: '219', description: 'USDA Contracts, Grants, Coop Agreements' }, + { code: '220', description: 'State Appropriations' }, + { code: '221', description: 'Self-Generated Funds' }, + { code: '222', description: 'Industry Grants and Agreements' }, + { code: '223', description: 'Other Non-Federal Funds' }, +]; + +export const FUND_SFNS = SFN_CATALOG.map((entry) => entry.code); export const SFN_MULTIPLE = 'Multiple'; const SEGMENTS_KEY = ['chartStringSegments'] as const; diff --git a/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx b/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx index 8068fd4..80a4b3a 100644 --- a/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx +++ b/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx @@ -34,12 +34,33 @@ describe('SegmentClassificationControl', () => { it('renders a dropdown with SFN options for a fund', () => { render(); - expect(screen.getByRole('option', { name: '201' })).toBeInTheDocument(); + expect(screen.getByRole('option', { name: '201 - Hatch Funds' })).toBeInTheDocument(); expect(screen.getByRole('option', { name: 'Multiple' })).toBeInTheDocument(); expect(screen.getByRole('option', { name: 'Excluded' })).toBeInTheDocument(); expect(screen.queryByRole('button', { name: 'Include' })).not.toBeInTheDocument(); }); + it('shows SFN descriptions in the fund dropdown', () => { + render( + + ); + expect( + screen.getByRole('option', { + name: '204 - Contracts, Grants, Research Coop Agreements', + }) + ).toBeInTheDocument(); + }); + it('calls onClassify with (true, sfn) when an SFN is selected', () => { const onClassify = vi.fn(); render(); diff --git a/client/src/test/components/dataClassification/SegmentGrid.test.tsx b/client/src/test/components/dataClassification/SegmentGrid.test.tsx index adfd935..62d9142 100644 --- a/client/src/test/components/dataClassification/SegmentGrid.test.tsx +++ b/client/src/test/components/dataClassification/SegmentGrid.test.tsx @@ -22,8 +22,8 @@ const fundSegments: ChartStringSegment[] = [ code: '45530', description: 'AES State Appropriations', hierarchy: [ - { code: 'STATE', level: '0', name: 'State Funds' }, - { code: 'APPROP', level: '1', name: 'Appropriations' }, + { code: 'STATE', level: 'A', name: 'State Funds' }, + { code: 'APPROP', level: 'B', name: 'Appropriations' }, ], includeInReport: true, segmentType: 'Fund', @@ -44,10 +44,10 @@ const accountSegments: ChartStringSegment[] = [ describe('SegmentGrid', () => { it('renders a column per hierarchy level with the code and hover title', () => { - render(); + render(); - expect(screen.getByRole('columnheader', { name: 'Level 0' })).toBeInTheDocument(); - expect(screen.getByRole('columnheader', { name: 'Level 1' })).toBeInTheDocument(); + expect(screen.getByRole('columnheader', { name: 'Level A' })).toBeInTheDocument(); + expect(screen.getByRole('columnheader', { name: 'Level B' })).toBeInTheDocument(); const state = screen.getByText('STATE'); expect(state).toHaveAttribute('data-tip', 'State Funds'); @@ -55,8 +55,15 @@ describe('SegmentGrid', () => { expect(screen.getByText('APPROP')).toHaveAttribute('data-tip', 'Appropriations'); }); + it('renders the passed classification header', () => { + render(); + + expect(screen.getByRole('columnheader', { name: 'Is AES?' })).toBeInTheDocument(); + expect(screen.queryByRole('columnheader', { name: 'Classification' })).not.toBeInTheDocument(); + }); + it('renders no level columns when no segment has a hierarchy', () => { - render(); + render(); expect(screen.queryByRole('columnheader', { name: /^Level / })).not.toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Include' })).toBeInTheDocument(); @@ -65,6 +72,7 @@ describe('SegmentGrid', () => { it('sorts unclassified rows to the top by default', () => { render( { it('keeps a row in place when it is classified in the same tab', () => { const { rerender } = render( { // BBB gets classified: order is frozen for the tab, so it stays above AAA. rerender( { - it('exposes the tabs in display order, including ERN', () => { - expect(SEGMENT_TABS.map((tab) => tab.type)).toEqual([ + it('defines six tabs with headers, notes, and slugs', () => { + expect(SEGMENT_TABS.map((t) => t.type)).toEqual([ 'FinancialDepartment', 'Account', 'Fund', 'Activity', + 'Purpose', 'Ern', ]); - expect(SEGMENT_TABS.at(-1)).toEqual({ label: 'ERN', type: 'Ern' }); + expect(SEGMENT_TABS.map((t) => t.classificationHeader)).toEqual([ + 'Is AES?', + 'Include in AD419?', + 'SFN', + 'Include in AD419?', + 'Include in AD419?', + 'Include in FTE?', + ]); + for (const tab of SEGMENT_TABS) { + expect(tab.note.length).toBeGreaterThan(0); + expect(tab.slug).toMatch(/^[a-z-]+$/); + } }); it('filters segments by type', () => { From e5dfcdd05f22afdda48e50da00634e9db5257ada Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:34:05 -0400 Subject: [PATCH 09/15] Add per-tab Export All for classification segments Co-Authored-By: Claude Fable 5 --- .../DataClassificationStage.tsx | 15 +++- .../dataClassification/exportSegments.ts | 79 +++++++++++++++++++ .../dataClassification/exportSegments.test.ts | 63 +++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 client/src/components/dataClassification/exportSegments.ts create mode 100644 client/src/test/components/dataClassification/exportSegments.test.ts diff --git a/client/src/components/dataClassification/DataClassificationStage.tsx b/client/src/components/dataClassification/DataClassificationStage.tsx index d9a8a1e..67ae278 100644 --- a/client/src/components/dataClassification/DataClassificationStage.tsx +++ b/client/src/components/dataClassification/DataClassificationStage.tsx @@ -5,7 +5,9 @@ import { segmentsForType, unclassifiedCount, } from './segments.ts'; +import { buildSegmentExport } from './exportSegments.ts'; import { SegmentGrid } from './SegmentGrid.tsx'; +import { ExportDataButton } from '@/shared/exportDataButton.tsx'; import { chartStringSegmentsQueryOptions, type ChartStringSegment, @@ -41,6 +43,8 @@ export function DataClassificationStage() { const gateOpen = allClassified(segments); const activeTab = SEGMENT_TABS.find((tab) => tab.type === activeType) ?? SEGMENT_TABS[0]; + const tabSegments = segmentsForType(segments, activeType); + const exportData = buildSegmentExport(tabSegments, activeTab); return (
@@ -79,10 +83,19 @@ export function DataClassificationStage() {
)} +
+ +
+ diff --git a/client/src/components/dataClassification/exportSegments.ts b/client/src/components/dataClassification/exportSegments.ts new file mode 100644 index 0000000..c1b100b --- /dev/null +++ b/client/src/components/dataClassification/exportSegments.ts @@ -0,0 +1,79 @@ +import type { SegmentTab } from './segments.ts'; +import type { CsvColumn } from '@/lib/csv.ts'; +import { + SFN_CATALOG, + type ChartStringSegment, +} from '@/queries/chartStringSegments.ts'; + +type ExportRow = Record; + +function classificationLabel(segment: ChartStringSegment): string { + if (segment.includeInReport === null) { + return 'Unset'; + } + return segment.includeInReport ? 'Included' : 'Excluded'; +} + +// Exports every row of the tab's segment type (intentionally ignores the grid +// search filter). SFN code and description stay separate columns; description +// resolves through the catalog. +export function buildSegmentExport( + segments: ChartStringSegment[], + tab: SegmentTab +): { columns: CsvColumn[]; filename: string; rows: ExportRow[] } { + const levelKeys = [ + ...new Set( + segments.flatMap((segment) => + segment.hierarchy.map((level) => level.level) + ) + ), + ].sort(); + + const columns: CsvColumn[] = [ + { header: 'Code', key: 'code' }, + { header: 'Name', key: 'name' }, + ...levelKeys.flatMap((levelKey): CsvColumn[] => [ + { header: `Level ${levelKey} Code`, key: `level${levelKey}Code` }, + { header: `Level ${levelKey} Name`, key: `level${levelKey}Name` }, + ]), + { header: 'Classification', key: 'classification' }, + ]; + + if (tab.type === 'Fund') { + columns.push( + { header: 'SFN', key: 'sfn' }, + { header: 'SFN Description', key: 'sfnDescription' } + ); + } + + const rows = segments.map((segment) => { + const row: ExportRow = { + classification: classificationLabel(segment), + code: segment.code, + name: segment.description ?? '', + }; + + for (const levelKey of levelKeys) { + const level = segment.hierarchy.find( + (candidate) => candidate.level === levelKey + ); + row[`level${levelKey}Code`] = level?.code ?? ''; + row[`level${levelKey}Name`] = level?.name ?? ''; + } + + if (tab.type === 'Fund') { + row.sfn = segment.sfn ?? ''; + row.sfnDescription = + SFN_CATALOG.find((entry) => entry.code === segment.sfn)?.description ?? + ''; + } + + return row; + }); + + return { + columns, + filename: `ad419-${tab.slug}-classification.csv`, + rows, + }; +} diff --git a/client/src/test/components/dataClassification/exportSegments.test.ts b/client/src/test/components/dataClassification/exportSegments.test.ts new file mode 100644 index 0000000..bed5d8e --- /dev/null +++ b/client/src/test/components/dataClassification/exportSegments.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { buildSegmentExport } from '@/components/dataClassification/exportSegments.ts'; +import { SEGMENT_TABS } from '@/components/dataClassification/segments.ts'; +import type { ChartStringSegment } from '@/queries/chartStringSegments.ts'; + +const fundTab = SEGMENT_TABS.find((tab) => tab.type === 'Fund')!; +const deptTab = SEGMENT_TABS.find((tab) => tab.type === 'FinancialDepartment')!; + +const fund: ChartStringSegment = { + code: '45530', + description: 'AES State Funds', + hierarchy: [ + { code: 'STATE', level: 'A', name: 'State Funds' }, + { code: 'APPROP', level: 'B', name: 'Appropriations' }, + ], + includeInReport: true, + segmentType: 'Fund', + sfn: '220', +}; + +describe('buildSegmentExport', () => { + it('builds level columns from the letters present and maps rows', () => { + const { columns, filename, rows } = buildSegmentExport([fund], fundTab); + + expect(filename).toBe('ad419-fund-classification.csv'); + expect(columns.map((c) => c.header)).toEqual([ + 'Code', + 'Name', + 'Level A Code', + 'Level A Name', + 'Level B Code', + 'Level B Name', + 'Classification', + 'SFN', + 'SFN Description', + ]); + expect(rows[0]).toMatchObject({ + classification: 'Included', + code: '45530', + levelACode: 'STATE', + levelAName: 'State Funds', + name: 'AES State Funds', + sfn: '220', + sfnDescription: 'State Appropriations', + }); + }); + + it('omits SFN columns for non-fund types and labels unset rows', () => { + const dept: ChartStringSegment = { + ...fund, + code: 'APLS001', + hierarchy: [], + includeInReport: null, + segmentType: 'FinancialDepartment', + sfn: null, + }; + + const { columns, rows } = buildSegmentExport([dept], deptTab); + + expect(columns.map((c) => c.header)).toEqual(['Code', 'Name', 'Classification']); + expect(rows[0].classification).toBe('Unset'); + }); +}); From 7b5be25b69949990ed2868a44924c0450b28667f Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 11:58:47 -0400 Subject: [PATCH 10/15] Move export button into the grid toolbar next to search Co-Authored-By: Claude Fable 5 --- .../DataClassificationStage.tsx | 17 ++++++++--------- .../dataClassification/SegmentGrid.tsx | 5 ++++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/client/src/components/dataClassification/DataClassificationStage.tsx b/client/src/components/dataClassification/DataClassificationStage.tsx index 67ae278..7db1ad4 100644 --- a/client/src/components/dataClassification/DataClassificationStage.tsx +++ b/client/src/components/dataClassification/DataClassificationStage.tsx @@ -83,20 +83,19 @@ export function DataClassificationStage() { )} -
- -
- + } />
diff --git a/client/src/components/dataClassification/SegmentGrid.tsx b/client/src/components/dataClassification/SegmentGrid.tsx index 18e9344..5b15858 100644 --- a/client/src/components/dataClassification/SegmentGrid.tsx +++ b/client/src/components/dataClassification/SegmentGrid.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useState, type ReactNode } from 'react'; import { SegmentClassificationControl } from './SegmentClassificationControl.tsx'; import type { ChartStringSegment, @@ -30,6 +30,7 @@ export function SegmentGrid({ onClassify, segments, segmentType, + tableActions, }: { classificationHeader: string; onClassify: ( @@ -39,6 +40,7 @@ export function SegmentGrid({ ) => void; segments: ChartStringSegment[]; segmentType: SegmentType; + tableActions?: ReactNode; }) { const levelKeys = [ ...new Set( @@ -115,6 +117,7 @@ export function SegmentGrid({ globalFilter="right" initialState={{ pagination: { pageSize: 25 } }} key={segmentType} + tableActions={tableActions} /> ); } From 099f4ec1711973b0d7aff1729481eef2bcccd040 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 15:52:36 -0400 Subject: [PATCH 11/15] Rename ChartStringSegments to SegmentClassifications throughout Disambiguates the classification table from the ChartSegments warehouse reference table: one holds what segments exist, the other how step 2 classified them. Renames the table, sproc, EF entity, controller, API route, and client module. Co-Authored-By: Claude Fable 5 --- .../DataClassificationStage.tsx | 10 +-- .../SegmentClassificationControl.tsx | 6 +- .../dataClassification/SegmentGrid.tsx | 16 ++--- .../dataClassification/exportSegments.ts | 8 +-- .../components/dataClassification/segments.ts | 12 ++-- ...gSegments.ts => segmentClassifications.ts} | 14 ++-- .../SegmentClassificationControl.test.tsx | 6 +- .../dataClassification/SegmentGrid.test.tsx | 8 +-- .../dataClassification/exportSegments.test.ts | 6 +- .../dataClassification/segments.test.ts | 4 +- .../dataClassification.test.tsx | 4 +- .../StoredProcedures/ClassifyTransactions.sql | 2 +- ...nts.sql => SeedSegmentClassifications.sql} | 8 +-- ...egments.sql => SegmentClassifications.sql} | 6 +- server.core/Data/DataDbContext.cs | 8 +-- server.core/Data/DbInitializer.cs | 2 +- ...ntSeed.cs => SegmentClassificationSeed.cs} | 20 +++--- ...ingSegment.cs => SegmentClassification.cs} | 2 +- ...cs => SegmentClassificationsController.cs} | 18 ++--- .../FundSfns.cs | 2 +- .../SegmentClassificationDtos.cs} | 4 +- .../SfnCatalog.cs | 2 +- .../HierarchyMappingTests.cs | 4 +- .../HierarchySeedTests.cs | 2 +- .../SegmentClassificationMappingTests.cs} | 18 ++--- .../SegmentClassificationSeedTests.cs} | 32 ++++----- .../SegmentClassificationsControllerTests.cs} | 70 +++++++++---------- .../SfnCatalogTests.cs | 4 +- 28 files changed, 149 insertions(+), 149 deletions(-) rename client/src/queries/{chartStringSegments.ts => segmentClassifications.ts} (86%) rename database/data/StoredProcedures/{SeedChartStringSegments.sql => SeedSegmentClassifications.sql} (90%) rename database/data/Tables/{ChartStringSegments.sql => SegmentClassifications.sql} (62%) rename server.core/Data/{ChartStringSegmentSeed.cs => SegmentClassificationSeed.cs} (88%) rename server.core/Domain/{ChartStringSegment.cs => SegmentClassification.cs} (91%) rename server/Controllers/{ChartStringSegmentsController.cs => SegmentClassificationsController.cs} (84%) rename server/Models/{ChartStringSegments => SegmentClassifications}/FundSfns.cs (87%) rename server/Models/{ChartStringSegments/ChartStringSegmentDtos.cs => SegmentClassifications/SegmentClassificationDtos.cs} (79%) rename server/Models/{ChartStringSegments => SegmentClassifications}/SfnCatalog.cs (95%) rename tests/server.tests/{ChartStringSegments => SegmentClassifications}/HierarchyMappingTests.cs (95%) rename tests/server.tests/{ChartStringSegments => SegmentClassifications}/HierarchySeedTests.cs (98%) rename tests/server.tests/{ChartStringSegments/ChartStringSegmentMappingTests.cs => SegmentClassifications/SegmentClassificationMappingTests.cs} (56%) rename tests/server.tests/{ChartStringSegments/ChartStringSegmentSeedTests.cs => SegmentClassifications/SegmentClassificationSeedTests.cs} (72%) rename tests/server.tests/{ChartStringSegments/ChartStringSegmentsControllerTests.cs => SegmentClassifications/SegmentClassificationsControllerTests.cs} (68%) rename tests/server.tests/{ChartStringSegments => SegmentClassifications}/SfnCatalogTests.cs (90%) diff --git a/client/src/components/dataClassification/DataClassificationStage.tsx b/client/src/components/dataClassification/DataClassificationStage.tsx index 7db1ad4..67ee75f 100644 --- a/client/src/components/dataClassification/DataClassificationStage.tsx +++ b/client/src/components/dataClassification/DataClassificationStage.tsx @@ -9,16 +9,16 @@ import { buildSegmentExport } from './exportSegments.ts'; import { SegmentGrid } from './SegmentGrid.tsx'; import { ExportDataButton } from '@/shared/exportDataButton.tsx'; import { - chartStringSegmentsQueryOptions, - type ChartStringSegment, + segmentClassificationsQueryOptions, + type SegmentClassification, useUpdateSegmentClassification, -} from '@/queries/chartStringSegments.ts'; +} from '@/queries/segmentClassifications.ts'; import { Link } from '@tanstack/react-router'; import { useQuery } from '@tanstack/react-query'; export function DataClassificationStage() { const { data: segments = [], isLoading } = useQuery( - chartStringSegmentsQueryOptions() + segmentClassificationsQueryOptions() ); const updateClassification = useUpdateSegmentClassification(); const [activeType, setActiveType] = useState(SEGMENT_TABS[0].type); @@ -28,7 +28,7 @@ export function DataClassificationStage() { } const handleClassify = ( - segment: ChartStringSegment, + segment: SegmentClassification, includeInReport: boolean, sfn: string | null ) => { diff --git a/client/src/components/dataClassification/SegmentClassificationControl.tsx b/client/src/components/dataClassification/SegmentClassificationControl.tsx index ba83799..9bca835 100644 --- a/client/src/components/dataClassification/SegmentClassificationControl.tsx +++ b/client/src/components/dataClassification/SegmentClassificationControl.tsx @@ -1,8 +1,8 @@ import { SFN_CATALOG, SFN_MULTIPLE, - type ChartStringSegment, -} from '@/queries/chartStringSegments.ts'; + type SegmentClassification, +} from '@/queries/segmentClassifications.ts'; const EXCLUDED = 'Excluded'; @@ -11,7 +11,7 @@ export function SegmentClassificationControl({ segment, }: { onClassify: (includeInReport: boolean, sfn: string | null) => void; - segment: ChartStringSegment; + segment: SegmentClassification; }) { if (segment.segmentType === 'Fund') { const value = diff --git a/client/src/components/dataClassification/SegmentGrid.tsx b/client/src/components/dataClassification/SegmentGrid.tsx index 5b15858..45096c7 100644 --- a/client/src/components/dataClassification/SegmentGrid.tsx +++ b/client/src/components/dataClassification/SegmentGrid.tsx @@ -1,13 +1,13 @@ import { useState, type ReactNode } from 'react'; import { SegmentClassificationControl } from './SegmentClassificationControl.tsx'; import type { - ChartStringSegment, + SegmentClassification, SegmentType, -} from '@/queries/chartStringSegments.ts'; +} from '@/queries/segmentClassifications.ts'; import { DataTable } from '@/shared/dataTable.tsx'; import type { ColumnDef } from '@tanstack/react-table'; -function classificationSortValue(segment: ChartStringSegment): string { +function classificationSortValue(segment: SegmentClassification): string { if (segment.includeInReport === null) { return ''; } @@ -15,7 +15,7 @@ function classificationSortValue(segment: ChartStringSegment): string { } // Codes ordered with unclassified (unset) rows first, otherwise stable. -function unsetFirstCodes(segments: ChartStringSegment[]): string[] { +function unsetFirstCodes(segments: SegmentClassification[]): string[] { return [...segments] .sort( (a, b) => @@ -34,11 +34,11 @@ export function SegmentGrid({ }: { classificationHeader: string; onClassify: ( - segment: ChartStringSegment, + segment: SegmentClassification, includeInReport: boolean, sfn: string | null ) => void; - segments: ChartStringSegment[]; + segments: SegmentClassification[]; segmentType: SegmentType; tableActions?: ReactNode; }) { @@ -50,7 +50,7 @@ export function SegmentGrid({ ), ].sort(); - const levelColumns: ColumnDef[] = levelKeys.map( + const levelColumns: ColumnDef[] = levelKeys.map( (levelKey) => ({ accessorFn: (segment) => segment.hierarchy.find((level) => level.level === levelKey)?.code ?? '', @@ -75,7 +75,7 @@ export function SegmentGrid({ }) ); - const columns: ColumnDef[] = [ + const columns: ColumnDef[] = [ { accessorKey: 'code', header: 'Code' }, { accessorKey: 'description', header: 'Name' }, ...levelColumns, diff --git a/client/src/components/dataClassification/exportSegments.ts b/client/src/components/dataClassification/exportSegments.ts index c1b100b..1caed41 100644 --- a/client/src/components/dataClassification/exportSegments.ts +++ b/client/src/components/dataClassification/exportSegments.ts @@ -2,12 +2,12 @@ import type { SegmentTab } from './segments.ts'; import type { CsvColumn } from '@/lib/csv.ts'; import { SFN_CATALOG, - type ChartStringSegment, -} from '@/queries/chartStringSegments.ts'; + type SegmentClassification, +} from '@/queries/segmentClassifications.ts'; type ExportRow = Record; -function classificationLabel(segment: ChartStringSegment): string { +function classificationLabel(segment: SegmentClassification): string { if (segment.includeInReport === null) { return 'Unset'; } @@ -18,7 +18,7 @@ function classificationLabel(segment: ChartStringSegment): string { // search filter). SFN code and description stay separate columns; description // resolves through the catalog. export function buildSegmentExport( - segments: ChartStringSegment[], + segments: SegmentClassification[], tab: SegmentTab ): { columns: CsvColumn[]; filename: string; rows: ExportRow[] } { const levelKeys = [ diff --git a/client/src/components/dataClassification/segments.ts b/client/src/components/dataClassification/segments.ts index 900ed0c..9cd6f94 100644 --- a/client/src/components/dataClassification/segments.ts +++ b/client/src/components/dataClassification/segments.ts @@ -1,7 +1,7 @@ import type { - ChartStringSegment, + SegmentClassification, SegmentType, -} from '@/queries/chartStringSegments.ts'; +} from '@/queries/segmentClassifications.ts'; export interface SegmentTab { classificationHeader: string; @@ -57,14 +57,14 @@ export const SEGMENT_TABS: SegmentTab[] = [ ]; export function segmentsForType( - segments: ChartStringSegment[], + segments: SegmentClassification[], type: SegmentType -): ChartStringSegment[] { +): SegmentClassification[] { return segments.filter((segment) => segment.segmentType === type); } export function unclassifiedCount( - segments: ChartStringSegment[], + segments: SegmentClassification[], type: SegmentType ): number { return segmentsForType(segments, type).filter( @@ -72,6 +72,6 @@ export function unclassifiedCount( ).length; } -export function allClassified(segments: ChartStringSegment[]): boolean { +export function allClassified(segments: SegmentClassification[]): boolean { return segments.every((segment) => segment.includeInReport !== null); } diff --git a/client/src/queries/chartStringSegments.ts b/client/src/queries/segmentClassifications.ts similarity index 86% rename from client/src/queries/chartStringSegments.ts rename to client/src/queries/segmentClassifications.ts index 5ed4a44..bf1c169 100644 --- a/client/src/queries/chartStringSegments.ts +++ b/client/src/queries/segmentClassifications.ts @@ -19,7 +19,7 @@ export interface HierarchyLevel { name: string | null; } -export interface ChartStringSegment { +export interface SegmentClassification { code: string; description: string | null; hierarchy: HierarchyLevel[]; @@ -51,12 +51,12 @@ export const SFN_CATALOG: SfnEntry[] = [ export const FUND_SFNS = SFN_CATALOG.map((entry) => entry.code); export const SFN_MULTIPLE = 'Multiple'; -const SEGMENTS_KEY = ['chartStringSegments'] as const; +const SEGMENTS_KEY = ['segmentClassifications'] as const; -export const chartStringSegmentsQueryOptions = () => +export const segmentClassificationsQueryOptions = () => queryOptions({ queryFn: () => - fetchJson('/api/chartstringsegments'), + fetchJson('/api/segmentclassifications'), queryKey: SEGMENTS_KEY, }); @@ -72,16 +72,16 @@ export const useUpdateSegmentClassification = () => { return useMutation({ mutationFn: (input: UpdateClassificationInput) => - fetchJson('/api/chartstringsegments', { + fetchJson('/api/segmentclassifications', { body: JSON.stringify(input), method: 'PATCH', }), onMutate: async (input) => { await queryClient.cancelQueries({ queryKey: SEGMENTS_KEY }); const previous = - queryClient.getQueryData(SEGMENTS_KEY); + queryClient.getQueryData(SEGMENTS_KEY); - queryClient.setQueryData(SEGMENTS_KEY, (old) => + queryClient.setQueryData(SEGMENTS_KEY, (old) => (old ?? []).map((segment) => segment.segmentType === input.segmentType && segment.code === input.code diff --git a/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx b/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx index 80a4b3a..7b6f480 100644 --- a/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx +++ b/client/src/test/components/dataClassification/SegmentClassificationControl.test.tsx @@ -1,9 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; import { fireEvent, render, screen } from '@testing-library/react'; -import type { ChartStringSegment } from '@/queries/chartStringSegments.ts'; +import type { SegmentClassification } from '@/queries/segmentClassifications.ts'; import { SegmentClassificationControl } from '@/components/dataClassification/SegmentClassificationControl.tsx'; -const unsetAccount: ChartStringSegment = { +const unsetAccount: SegmentClassification = { code: '500000', description: 'Supplies', hierarchy: [], @@ -12,7 +12,7 @@ const unsetAccount: ChartStringSegment = { sfn: null, }; -const unsetFund: ChartStringSegment = { +const unsetFund: SegmentClassification = { code: '70575', description: 'Berry', hierarchy: [], diff --git a/client/src/test/components/dataClassification/SegmentGrid.test.tsx b/client/src/test/components/dataClassification/SegmentGrid.test.tsx index 62d9142..ffa47a9 100644 --- a/client/src/test/components/dataClassification/SegmentGrid.test.tsx +++ b/client/src/test/components/dataClassification/SegmentGrid.test.tsx @@ -1,12 +1,12 @@ import { describe, expect, it, vi } from 'vitest'; import { render, screen } from '@testing-library/react'; -import type { ChartStringSegment } from '@/queries/chartStringSegments.ts'; +import type { SegmentClassification } from '@/queries/segmentClassifications.ts'; import { SegmentGrid } from '@/components/dataClassification/SegmentGrid.tsx'; function account( code: string, includeInReport: boolean | null -): ChartStringSegment { +): SegmentClassification { return { code, description: `Name ${code}`, hierarchy: [], includeInReport, segmentType: 'Account', sfn: null }; } @@ -17,7 +17,7 @@ function isBefore(first: HTMLElement, second: HTMLElement): boolean { ); } -const fundSegments: ChartStringSegment[] = [ +const fundSegments: SegmentClassification[] = [ { code: '45530', description: 'AES State Appropriations', @@ -31,7 +31,7 @@ const fundSegments: ChartStringSegment[] = [ }, ]; -const accountSegments: ChartStringSegment[] = [ +const accountSegments: SegmentClassification[] = [ { code: '500000', description: 'Supplies and Expense', diff --git a/client/src/test/components/dataClassification/exportSegments.test.ts b/client/src/test/components/dataClassification/exportSegments.test.ts index bed5d8e..c08a741 100644 --- a/client/src/test/components/dataClassification/exportSegments.test.ts +++ b/client/src/test/components/dataClassification/exportSegments.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from 'vitest'; import { buildSegmentExport } from '@/components/dataClassification/exportSegments.ts'; import { SEGMENT_TABS } from '@/components/dataClassification/segments.ts'; -import type { ChartStringSegment } from '@/queries/chartStringSegments.ts'; +import type { SegmentClassification } from '@/queries/segmentClassifications.ts'; const fundTab = SEGMENT_TABS.find((tab) => tab.type === 'Fund')!; const deptTab = SEGMENT_TABS.find((tab) => tab.type === 'FinancialDepartment')!; -const fund: ChartStringSegment = { +const fund: SegmentClassification = { code: '45530', description: 'AES State Funds', hierarchy: [ @@ -46,7 +46,7 @@ describe('buildSegmentExport', () => { }); it('omits SFN columns for non-fund types and labels unset rows', () => { - const dept: ChartStringSegment = { + const dept: SegmentClassification = { ...fund, code: 'APLS001', hierarchy: [], diff --git a/client/src/test/components/dataClassification/segments.test.ts b/client/src/test/components/dataClassification/segments.test.ts index aa97c3d..9fccb7a 100644 --- a/client/src/test/components/dataClassification/segments.test.ts +++ b/client/src/test/components/dataClassification/segments.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import type { ChartStringSegment } from '@/queries/chartStringSegments.ts'; +import type { SegmentClassification } from '@/queries/segmentClassifications.ts'; import { allClassified, SEGMENT_TABS, @@ -7,7 +7,7 @@ import { unclassifiedCount, } from '@/components/dataClassification/segments.ts'; -const segments: ChartStringSegment[] = [ +const segments: SegmentClassification[] = [ { code: '45530', description: 'AES', hierarchy: [], includeInReport: true, segmentType: 'Fund', sfn: '220' }, { code: '70575', description: 'Berry', hierarchy: [], includeInReport: null, segmentType: 'Fund', sfn: '219' }, { code: '500000', description: 'S and E', hierarchy: [], includeInReport: null, segmentType: 'Account', sfn: null }, diff --git a/client/src/test/routes/(authenticated)/dataClassification.test.tsx b/client/src/test/routes/(authenticated)/dataClassification.test.tsx index 4371379..2682660 100644 --- a/client/src/test/routes/(authenticated)/dataClassification.test.tsx +++ b/client/src/test/routes/(authenticated)/dataClassification.test.tsx @@ -27,8 +27,8 @@ function mockApi() { let current = [...segments]; server.use( http.get('/api/user/me', () => HttpResponse.json(mockUser)), - http.get('/api/chartstringsegments', () => HttpResponse.json(current)), - http.patch('/api/chartstringsegments', async ({ request }) => { + http.get('/api/segmentclassifications', () => HttpResponse.json(current)), + http.patch('/api/segmentclassifications', async ({ request }) => { const body = await request.json() as { code: string; includeInReport: boolean; segmentType: string; sfn: string | null }; current = current.map((s) => s.code === body.code ? { ...s, includeInReport: body.includeInReport } : s diff --git a/database/data/StoredProcedures/ClassifyTransactions.sql b/database/data/StoredProcedures/ClassifyTransactions.sql index 6cc1b60..0f4ff13 100644 --- a/database/data/StoredProcedures/ClassifyTransactions.sql +++ b/database/data/StoredProcedures/ClassifyTransactions.sql @@ -8,7 +8,7 @@ BEGIN -- Stamps the persisted exclusion columns on the imported transactions: the -- rules step 2 classification does not own. Step 2 based exclusions (fund, -- account, financial dept, activity, ern) are derived at read time from - -- ChartStringSegments and are never stamped here. + -- SegmentClassifications and are never stamped here. IF @cycleStart IS NULL OR @cycleEnd IS NULL THROW 50000, '@cycleStart and @cycleEnd are required.', 1; diff --git a/database/data/StoredProcedures/SeedChartStringSegments.sql b/database/data/StoredProcedures/SeedSegmentClassifications.sql similarity index 90% rename from database/data/StoredProcedures/SeedChartStringSegments.sql rename to database/data/StoredProcedures/SeedSegmentClassifications.sql index 8b282b9..78f584f 100644 --- a/database/data/StoredProcedures/SeedChartStringSegments.sql +++ b/database/data/StoredProcedures/SeedSegmentClassifications.sql @@ -1,10 +1,10 @@ -CREATE PROCEDURE [data].[SeedChartStringSegments] +CREATE PROCEDURE [data].[SeedSegmentClassifications] AS BEGIN SET NOCOUNT ON; -- Insert any segment value present in the imported transactions but missing - -- from ChartStringSegments, as an unclassified row (IncludeInReport NULL). + -- from SegmentClassifications, as an unclassified row (IncludeInReport NULL). -- Existing rows and their classifications are never modified: unclassified -- fails closed downstream until someone classifies the code in step 2. -- @@ -36,7 +36,7 @@ BEGIN UNION SELECT 'Ern', [DosCode] FROM [data].[UcPathTransactions] WHERE [DosCode] IS NOT NULL AND [DosCode] <> 'XXX' ) - INSERT INTO [data].[ChartStringSegments] ([SegmentType], [Code], [Description]) + INSERT INTO [data].[SegmentClassifications] ([SegmentType], [Code], [Description]) OUTPUT inserted.[SegmentType] INTO @inserted ([SegmentType]) SELECT tv.SegmentType, @@ -48,7 +48,7 @@ BEGIN WHERE NOT EXISTS ( SELECT 1 - FROM [data].[ChartStringSegments] existing + FROM [data].[SegmentClassifications] existing WHERE existing.[SegmentType] = tv.SegmentType AND existing.[Code] = tv.Code ); diff --git a/database/data/Tables/ChartStringSegments.sql b/database/data/Tables/SegmentClassifications.sql similarity index 62% rename from database/data/Tables/ChartStringSegments.sql rename to database/data/Tables/SegmentClassifications.sql index cec6865..e7ac733 100644 --- a/database/data/Tables/ChartStringSegments.sql +++ b/database/data/Tables/SegmentClassifications.sql @@ -1,10 +1,10 @@ -CREATE TABLE [data].[ChartStringSegments] +CREATE TABLE [data].[SegmentClassifications] ( [SegmentType] NVARCHAR(20) NOT NULL, -- FinancialDepartment | Account | Fund | Activity | Ern [Code] NVARCHAR(50) NOT NULL, [Description] NVARCHAR(300) NULL, [IncludeInReport] BIT NULL, -- NULL = unclassified, needs review [Sfn] NVARCHAR(10) NULL, -- SFN code (201..223) or 'Multiple'; only for Fund - CONSTRAINT [PK_ChartStringSegments] PRIMARY KEY CLUSTERED ([SegmentType], [Code]), - CONSTRAINT [CK_ChartStringSegments_Sfn] CHECK ([Sfn] IS NULL OR [SegmentType] = 'Fund') + CONSTRAINT [PK_SegmentClassifications] PRIMARY KEY CLUSTERED ([SegmentType], [Code]), + CONSTRAINT [CK_SegmentClassifications_Sfn] CHECK ([Sfn] IS NULL OR [SegmentType] = 'Fund') ); diff --git a/server.core/Data/DataDbContext.cs b/server.core/Data/DataDbContext.cs index e624974..582410c 100644 --- a/server.core/Data/DataDbContext.cs +++ b/server.core/Data/DataDbContext.cs @@ -7,7 +7,7 @@ public class DataDbContext(DbContextOptions options) : DbContext( { public const string DataSchema = "data"; - public DbSet ChartStringSegments => Set(); + public DbSet SegmentClassifications => Set(); public DbSet DepartmentHierarchies => Set(); public DbSet AccountHierarchies => Set(); @@ -21,9 +21,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.HasDefaultSchema(DataSchema); - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => { - entity.ToTable("ChartStringSegments", DataSchema, table => table.ExcludeFromMigrations()); + entity.ToTable("SegmentClassifications", DataSchema, table => table.ExcludeFromMigrations()); entity.HasKey(segment => new { segment.SegmentType, segment.Code }); entity.Property(segment => segment.SegmentType).HasConversion().HasMaxLength(20); entity.Property(segment => segment.Code).HasMaxLength(50); @@ -51,7 +51,7 @@ private static void ConfigureHierarchy(ModelBuilder modelBuilder, string tabl { var maxLength = property.Name switch { - // Match ChartStringSegment.Code so joins/lookups on Code stay aligned. + // Match SegmentClassification.Code so joins/lookups on Code stay aligned. nameof(ISegmentHierarchy.Code) => 50, "Description" => 1000, _ when property.Name.EndsWith("Name") => 1000, diff --git a/server.core/Data/DbInitializer.cs b/server.core/Data/DbInitializer.cs index a57a13e..d3fb22d 100644 --- a/server.core/Data/DbInitializer.cs +++ b/server.core/Data/DbInitializer.cs @@ -40,7 +40,7 @@ private async Task SeedDevelopmentAsync(CancellationToken ct) { // Hierarchy first: chart-string segments are derived from the hierarchy tables. await HierarchySeed.EnsureSeededAsync(_dataDb, ct); - await ChartStringSegmentSeed.EnsureSeededAsync(_dataDb, ct); + await SegmentClassificationSeed.EnsureSeededAsync(_dataDb, ct); } // just a placeholder for any production-safe seeding diff --git a/server.core/Data/ChartStringSegmentSeed.cs b/server.core/Data/SegmentClassificationSeed.cs similarity index 88% rename from server.core/Data/ChartStringSegmentSeed.cs rename to server.core/Data/SegmentClassificationSeed.cs index 44fcca6..d018309 100644 --- a/server.core/Data/ChartStringSegmentSeed.cs +++ b/server.core/Data/SegmentClassificationSeed.cs @@ -9,7 +9,7 @@ namespace Server.Core.Data; /// Segments start unclassified (IncludeInReport = null); a couple per type are /// pre-classified so the grid shows a mix of unset and classified rows. /// -public static class ChartStringSegmentSeed +public static class SegmentClassificationSeed { // Leave roughly this many rows per segment type unclassified so there is still // work to do; the rest are classified (include/exclude) below. @@ -23,12 +23,12 @@ public static class ChartStringSegmentSeed public static async Task EnsureSeededAsync(DataDbContext db, CancellationToken ct = default) { - if (await db.ChartStringSegments.AnyAsync(ct)) + if (await db.SegmentClassifications.AnyAsync(ct)) { return; } - var segments = new List(); + var segments = new List(); segments.AddRange(await BuildAsync(db.AccountHierarchies, SegmentType.Account, ct)); segments.AddRange(await BuildAsync(db.FundHierarchies, SegmentType.Fund, ct)); segments.AddRange(await BuildAsync(db.ActivityHierarchies, SegmentType.Activity, ct)); @@ -36,14 +36,14 @@ public static async Task EnsureSeededAsync(DataDbContext db, CancellationToken c segments.AddRange(BuildErn()); segments.AddRange(await BuildPurposeAsync(db, ct)); - db.ChartStringSegments.AddRange(segments); + db.SegmentClassifications.AddRange(segments); await db.SaveChangesAsync(ct); } // ErnCodes.csv columns: 0 DOS_Code, 1 Description, 2 IncludeInAD419FTE, 3 IsNewInUCP. // IncludeInAD419FTE seeds the classification directly, except the first few (by DOS // code) which stay unset for demo. IsNewInUCP is not consumed yet. - private static List BuildErn() + private static List BuildErn() { var rows = SeedCsv.ReadRows("ErnCodes.csv", fields => ( Code: fields[0].Trim(), @@ -52,7 +52,7 @@ private static List BuildErn() return rows .OrderBy(row => row.Code, StringComparer.Ordinal) - .Select((row, index) => new ChartStringSegment + .Select((row, index) => new SegmentClassification { SegmentType = SegmentType.Ern, Code = row.Code, @@ -71,12 +71,12 @@ private static List BuildErn() private static readonly IReadOnlySet IncludedPurposes = new HashSet { "44", "45" }; - private static async Task> BuildPurposeAsync(DataDbContext db, CancellationToken ct) + private static async Task> BuildPurposeAsync(DataDbContext db, CancellationToken ct) { var rows = await db.PurposeHierarchies.ToListAsync(ct); return rows .OrderBy(row => row.Code, StringComparer.Ordinal) - .Select(row => new ChartStringSegment + .Select(row => new SegmentClassification { SegmentType = SegmentType.Purpose, Code = row.Code, @@ -88,7 +88,7 @@ private static async Task> BuildPurposeAsync(DataDbCont .ToList(); } - private static async Task> BuildAsync( + private static async Task> BuildAsync( DbSet hierarchy, SegmentType type, CancellationToken ct) @@ -109,7 +109,7 @@ private static async Task> BuildAsync( ? FundSfnPool[(StableHash(row.Code) & int.MaxValue) % FundSfnPool.Length] : null; - return new ChartStringSegment + return new SegmentClassification { SegmentType = type, Code = row.Code, diff --git a/server.core/Domain/ChartStringSegment.cs b/server.core/Domain/SegmentClassification.cs similarity index 91% rename from server.core/Domain/ChartStringSegment.cs rename to server.core/Domain/SegmentClassification.cs index 6d964af..928123a 100644 --- a/server.core/Domain/ChartStringSegment.cs +++ b/server.core/Domain/SegmentClassification.cs @@ -10,7 +10,7 @@ public enum SegmentType Purpose, } -public class ChartStringSegment +public class SegmentClassification { public SegmentType SegmentType { get; set; } diff --git a/server/Controllers/ChartStringSegmentsController.cs b/server/Controllers/SegmentClassificationsController.cs similarity index 84% rename from server/Controllers/ChartStringSegmentsController.cs rename to server/Controllers/SegmentClassificationsController.cs index b3b58c0..113726f 100644 --- a/server/Controllers/ChartStringSegmentsController.cs +++ b/server/Controllers/SegmentClassificationsController.cs @@ -2,24 +2,24 @@ using Microsoft.EntityFrameworkCore; using Server.Core.Data; using Server.Core.Domain; -using Server.Models.ChartStringSegments; +using Server.Models.SegmentClassifications; namespace Server.Controllers; -public class ChartStringSegmentsController : ApiControllerBase +public class SegmentClassificationsController : ApiControllerBase { private readonly DataDbContext _db; - public ChartStringSegmentsController(DataDbContext db) + public SegmentClassificationsController(DataDbContext db) { _db = db; } - // GET api/chartstringsegments + // GET api/segmentclassifications [HttpGet] - public async Task>> Get(CancellationToken cancellationToken) + public async Task>> Get(CancellationToken cancellationToken) { - var segments = await _db.ChartStringSegments.ToListAsync(cancellationToken); + var segments = await _db.SegmentClassifications.ToListAsync(cancellationToken); var departments = await _db.DepartmentHierarchies.ToDictionaryAsync(h => h.Code, cancellationToken); var accounts = await _db.AccountHierarchies.ToDictionaryAsync(h => h.Code, cancellationToken); @@ -47,7 +47,7 @@ IReadOnlyList HierarchyFor(SegmentType type, string code) var dtos = segments .OrderBy(segment => segment.SegmentType) .ThenBy(segment => segment.Code) - .Select(segment => new ChartStringSegmentDto( + .Select(segment => new SegmentClassificationDto( segment.SegmentType.ToString(), segment.Code, segment.Description, @@ -59,7 +59,7 @@ IReadOnlyList HierarchyFor(SegmentType type, string code) return Ok(dtos); } - // PATCH api/chartstringsegments + // PATCH api/segmentclassifications [HttpPatch] public async Task UpdateClassification( [FromBody] UpdateClassificationRequest request, @@ -70,7 +70,7 @@ public async Task UpdateClassification( return BadRequest($"Unknown segment type '{request.SegmentType}'."); } - var segment = await _db.ChartStringSegments.FirstOrDefaultAsync( + var segment = await _db.SegmentClassifications.FirstOrDefaultAsync( candidate => candidate.SegmentType == segmentType && candidate.Code == request.Code, cancellationToken); diff --git a/server/Models/ChartStringSegments/FundSfns.cs b/server/Models/SegmentClassifications/FundSfns.cs similarity index 87% rename from server/Models/ChartStringSegments/FundSfns.cs rename to server/Models/SegmentClassifications/FundSfns.cs index 7dd831d..74b6aff 100644 --- a/server/Models/ChartStringSegments/FundSfns.cs +++ b/server/Models/SegmentClassifications/FundSfns.cs @@ -1,4 +1,4 @@ -namespace Server.Models.ChartStringSegments; +namespace Server.Models.SegmentClassifications; public static class FundSfns { diff --git a/server/Models/ChartStringSegments/ChartStringSegmentDtos.cs b/server/Models/SegmentClassifications/SegmentClassificationDtos.cs similarity index 79% rename from server/Models/ChartStringSegments/ChartStringSegmentDtos.cs rename to server/Models/SegmentClassifications/SegmentClassificationDtos.cs index 433bc49..b49664d 100644 --- a/server/Models/ChartStringSegments/ChartStringSegmentDtos.cs +++ b/server/Models/SegmentClassifications/SegmentClassificationDtos.cs @@ -1,6 +1,6 @@ -namespace Server.Models.ChartStringSegments; +namespace Server.Models.SegmentClassifications; -public sealed record ChartStringSegmentDto( +public sealed record SegmentClassificationDto( string SegmentType, string Code, string? Description, diff --git a/server/Models/ChartStringSegments/SfnCatalog.cs b/server/Models/SegmentClassifications/SfnCatalog.cs similarity index 95% rename from server/Models/ChartStringSegments/SfnCatalog.cs rename to server/Models/SegmentClassifications/SfnCatalog.cs index 564d150..fa43add 100644 --- a/server/Models/ChartStringSegments/SfnCatalog.cs +++ b/server/Models/SegmentClassifications/SfnCatalog.cs @@ -1,4 +1,4 @@ -namespace Server.Models.ChartStringSegments; +namespace Server.Models.SegmentClassifications; public sealed record SfnEntry(string Code, string Description); diff --git a/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs b/tests/server.tests/SegmentClassifications/HierarchyMappingTests.cs similarity index 95% rename from tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs rename to tests/server.tests/SegmentClassifications/HierarchyMappingTests.cs index ecf4664..9c80f2b 100644 --- a/tests/server.tests/ChartStringSegments/HierarchyMappingTests.cs +++ b/tests/server.tests/SegmentClassifications/HierarchyMappingTests.cs @@ -2,7 +2,7 @@ using Microsoft.EntityFrameworkCore; using Server.Core.Domain; -namespace Server.Tests.ChartStringSegments; +namespace Server.Tests.SegmentClassifications; public class HierarchyMappingTests { @@ -84,7 +84,7 @@ public void Fund_level_properties_have_max_lengths() entity.FindProperty("ParentLevel0Code")!.GetMaxLength().Should().Be(20); entity.FindProperty("ParentLevel0Name")!.GetMaxLength().Should().Be(1000); entity.FindProperty("Description")!.GetMaxLength().Should().Be(1000); - // Code matches ChartStringSegment.Code (NVARCHAR(50)) so joins on Code align. + // Code matches SegmentClassification.Code (NVARCHAR(50)) so joins on Code align. entity.FindProperty("Code")!.GetMaxLength().Should().Be(50); } } diff --git a/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs b/tests/server.tests/SegmentClassifications/HierarchySeedTests.cs similarity index 98% rename from tests/server.tests/ChartStringSegments/HierarchySeedTests.cs rename to tests/server.tests/SegmentClassifications/HierarchySeedTests.cs index 8b1834d..ee674bf 100644 --- a/tests/server.tests/ChartStringSegments/HierarchySeedTests.cs +++ b/tests/server.tests/SegmentClassifications/HierarchySeedTests.cs @@ -3,7 +3,7 @@ using Server.Core.Data; using Server.Core.Domain; -namespace Server.Tests.ChartStringSegments; +namespace Server.Tests.SegmentClassifications; public class HierarchySeedTests { diff --git a/tests/server.tests/ChartStringSegments/ChartStringSegmentMappingTests.cs b/tests/server.tests/SegmentClassifications/SegmentClassificationMappingTests.cs similarity index 56% rename from tests/server.tests/ChartStringSegments/ChartStringSegmentMappingTests.cs rename to tests/server.tests/SegmentClassifications/SegmentClassificationMappingTests.cs index 45d0ee7..c21ce59 100644 --- a/tests/server.tests/ChartStringSegments/ChartStringSegmentMappingTests.cs +++ b/tests/server.tests/SegmentClassifications/SegmentClassificationMappingTests.cs @@ -3,22 +3,22 @@ using Server.Core.Data; using Server.Core.Domain; -namespace Server.Tests.ChartStringSegments; +namespace Server.Tests.SegmentClassifications; -public class ChartStringSegmentMappingTests +public class SegmentClassificationMappingTests { [Fact] public void Maps_to_data_schema_with_composite_key() { using var db = TestDbContextFactory.CreateDataInMemory(); - var entityType = db.Model.FindEntityType(typeof(ChartStringSegment)); + var entityType = db.Model.FindEntityType(typeof(SegmentClassification)); entityType.Should().NotBeNull(); entityType!.GetSchema().Should().Be("data"); - entityType.GetTableName().Should().Be("ChartStringSegments"); + entityType.GetTableName().Should().Be("SegmentClassifications"); entityType.FindPrimaryKey()!.Properties.Select(p => p.Name) - .Should().Equal(nameof(ChartStringSegment.SegmentType), nameof(ChartStringSegment.Code)); + .Should().Equal(nameof(SegmentClassification.SegmentType), nameof(SegmentClassification.Code)); } [Fact] @@ -26,8 +26,8 @@ public void Stores_segment_type_as_string() { using var db = TestDbContextFactory.CreateDataInMemory(); - var property = db.Model.FindEntityType(typeof(ChartStringSegment))! - .FindProperty(nameof(ChartStringSegment.SegmentType)); + var property = db.Model.FindEntityType(typeof(SegmentClassification))! + .FindProperty(nameof(SegmentClassification.SegmentType)); property!.GetProviderClrType().Should().Be(typeof(string)); } @@ -37,8 +37,8 @@ public void Sfn_column_allows_ten_characters() { using var db = TestDbContextFactory.CreateDataInMemory(); - var property = db.Model.FindEntityType(typeof(ChartStringSegment))! - .FindProperty(nameof(ChartStringSegment.Sfn)); + var property = db.Model.FindEntityType(typeof(SegmentClassification))! + .FindProperty(nameof(SegmentClassification.Sfn)); property!.GetMaxLength().Should().Be(10); } diff --git a/tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs b/tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs similarity index 72% rename from tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs rename to tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs index 124aad1..7eb802b 100644 --- a/tests/server.tests/ChartStringSegments/ChartStringSegmentSeedTests.cs +++ b/tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs @@ -3,9 +3,9 @@ using Server.Core.Data; using Server.Core.Domain; -namespace Server.Tests.ChartStringSegments; +namespace Server.Tests.SegmentClassifications; -public class ChartStringSegmentSeedTests +public class SegmentClassificationSeedTests { [Fact] public async Task EnsureSeeded_derives_one_segment_per_hierarchy_row() @@ -13,7 +13,7 @@ public async Task EnsureSeeded_derives_one_segment_per_hierarchy_row() using var db = TestDbContextFactory.CreateDataInMemory(); await HierarchySeed.EnsureSeededAsync(db); - await ChartStringSegmentSeed.EnsureSeededAsync(db); + await SegmentClassificationSeed.EnsureSeededAsync(db); var expected = await db.AccountHierarchies.CountAsync() + @@ -21,8 +21,8 @@ await db.FundHierarchies.CountAsync() + await db.ActivityHierarchies.CountAsync() + await db.DepartmentHierarchies.CountAsync() + await db.PurposeHierarchies.CountAsync() + - db.ChartStringSegments.Count(segment => segment.SegmentType == SegmentType.Ern); - (await db.ChartStringSegments.CountAsync()).Should().Be(expected); + db.SegmentClassifications.Count(segment => segment.SegmentType == SegmentType.Ern); + (await db.SegmentClassifications.CountAsync()).Should().Be(expected); } [Fact] @@ -31,9 +31,9 @@ public async Task EnsureSeeded_loads_ern_codes_from_csv() using var db = TestDbContextFactory.CreateDataInMemory(); await HierarchySeed.EnsureSeededAsync(db); - await ChartStringSegmentSeed.EnsureSeededAsync(db); + await SegmentClassificationSeed.EnsureSeededAsync(db); - var ern = db.ChartStringSegments + var ern = db.SegmentClassifications .Where(segment => segment.SegmentType == SegmentType.Ern) .ToList(); ern.Count.Should().BeGreaterThan(200); @@ -56,13 +56,13 @@ public async Task EnsureSeeded_maps_segment_types_to_their_hierarchy() using var db = TestDbContextFactory.CreateDataInMemory(); await HierarchySeed.EnsureSeededAsync(db); - await ChartStringSegmentSeed.EnsureSeededAsync(db); + await SegmentClassificationSeed.EnsureSeededAsync(db); // Account 500000 comes from AccountHierarchy, so it is an Account segment. - var account = await db.ChartStringSegments.FindAsync(SegmentType.Account, "500000"); + var account = await db.SegmentClassifications.FindAsync(SegmentType.Account, "500000"); account.Should().NotBeNull(); // Fund 71549 comes from FundHierarchy. - (await db.ChartStringSegments.FindAsync(SegmentType.Fund, "71549")).Should().NotBeNull(); + (await db.SegmentClassifications.FindAsync(SegmentType.Fund, "71549")).Should().NotBeNull(); } [Fact] @@ -71,9 +71,9 @@ public async Task Seeds_purpose_classifications_with_2025_defaults() using var db = TestDbContextFactory.CreateDataInMemory(); await HierarchySeed.EnsureSeededAsync(db); - await ChartStringSegmentSeed.EnsureSeededAsync(db); + await SegmentClassificationSeed.EnsureSeededAsync(db); - var purposes = db.ChartStringSegments + var purposes = db.SegmentClassifications .Where(s => s.SegmentType == SegmentType.Purpose) .ToDictionary(s => s.Code, s => s.IncludeInReport); @@ -92,10 +92,10 @@ public async Task EnsureSeeded_is_idempotent() using var db = TestDbContextFactory.CreateDataInMemory(); await HierarchySeed.EnsureSeededAsync(db); - await ChartStringSegmentSeed.EnsureSeededAsync(db); - var count = await db.ChartStringSegments.CountAsync(); - await ChartStringSegmentSeed.EnsureSeededAsync(db); + await SegmentClassificationSeed.EnsureSeededAsync(db); + var count = await db.SegmentClassifications.CountAsync(); + await SegmentClassificationSeed.EnsureSeededAsync(db); - (await db.ChartStringSegments.CountAsync()).Should().Be(count); + (await db.SegmentClassifications.CountAsync()).Should().Be(count); } } diff --git a/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs b/tests/server.tests/SegmentClassifications/SegmentClassificationsControllerTests.cs similarity index 68% rename from tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs rename to tests/server.tests/SegmentClassifications/SegmentClassificationsControllerTests.cs index 66a7033..bb29649 100644 --- a/tests/server.tests/ChartStringSegments/ChartStringSegmentsControllerTests.cs +++ b/tests/server.tests/SegmentClassifications/SegmentClassificationsControllerTests.cs @@ -2,26 +2,26 @@ using Microsoft.AspNetCore.Mvc; using Server.Controllers; using Server.Core.Domain; -using Server.Models.ChartStringSegments; +using Server.Models.SegmentClassifications; -namespace Server.Tests.ChartStringSegments; +namespace Server.Tests.SegmentClassifications; -public class ChartStringSegmentsControllerTests +public class SegmentClassificationsControllerTests { [Fact] public async Task Get_returns_all_segments() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.AddRange( - new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "45530", Description = "AES", IncludeInReport = true, Sfn = "220" }, - new ChartStringSegment { SegmentType = SegmentType.Account, Code = "500000", Description = "S and E", IncludeInReport = null }); + db.SegmentClassifications.AddRange( + new SegmentClassification { SegmentType = SegmentType.Fund, Code = "45530", Description = "AES", IncludeInReport = true, Sfn = "220" }, + new SegmentClassification { SegmentType = SegmentType.Account, Code = "500000", Description = "S and E", IncludeInReport = null }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.Get(CancellationToken.None); var ok = result.Result.Should().BeOfType().Subject; - ok.Value.Should().BeAssignableTo>() + ok.Value.Should().BeAssignableTo>() .Which.Should().HaveCount(2); } @@ -29,7 +29,7 @@ public async Task Get_returns_all_segments() public async Task Get_includes_hierarchy_for_segment_with_matching_code() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "45530", IncludeInReport = true, Sfn = "220" }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Fund, Code = "45530", IncludeInReport = true, Sfn = "220" }); db.FundHierarchies.Add(new FundHierarchy { Code = "45530", @@ -37,12 +37,12 @@ public async Task Get_includes_hierarchy_for_segment_with_matching_code() ParentLevel1Code = "APPROP", ParentLevel1Name = "Appropriations", }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.Get(CancellationToken.None); var ok = result.Result.Should().BeOfType().Subject; - var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); + var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); dto.Hierarchy.Should().Equal( new HierarchyLevelDto("A", "STATE", "State Funds"), new HierarchyLevelDto("B", "APPROP", "Appropriations")); @@ -59,7 +59,7 @@ public async Task Get_reads_hierarchy_from_the_matching_segment_types_own_table( { using var db = TestDbContextFactory.CreateDataInMemory(); const string code = "12345"; - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = segmentType, Code = code, IncludeInReport = null }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = segmentType, Code = code, IncludeInReport = null }); switch (segmentType) { @@ -96,12 +96,12 @@ public async Task Get_reads_hierarchy_from_the_matching_segment_types_own_table( } await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.Get(CancellationToken.None); var ok = result.Result.Should().BeOfType().Subject; - var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); + var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); dto.Hierarchy.Should().Equal(new HierarchyLevelDto(expectedLevel, "X", "XName")); } @@ -109,14 +109,14 @@ public async Task Get_reads_hierarchy_from_the_matching_segment_types_own_table( public async Task Get_returns_empty_hierarchy_when_no_matching_row() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Account, Code = "500000", IncludeInReport = null }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Account, Code = "500000", IncludeInReport = null }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.Get(CancellationToken.None); var ok = result.Result.Should().BeOfType().Subject; - var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); + var dto = ok.Value.Should().BeAssignableTo>().Subject.Single(); dto.Hierarchy.Should().BeEmpty(); } @@ -124,15 +124,15 @@ public async Task Get_returns_empty_hierarchy_when_no_matching_row() public async Task Patch_updates_include_flag() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null, Sfn = "219" }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null, Sfn = "219" }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Fund", "70575", true, "201"), CancellationToken.None); result.Should().BeOfType(); - var updated = await db.ChartStringSegments.FindAsync(SegmentType.Fund, "70575"); + var updated = await db.SegmentClassifications.FindAsync(SegmentType.Fund, "70575"); updated!.IncludeInReport.Should().BeTrue(); updated.Sfn.Should().Be("201"); } @@ -141,7 +141,7 @@ public async Task Patch_updates_include_flag() public async Task Patch_returns_not_found_for_missing_segment() { using var db = TestDbContextFactory.CreateDataInMemory(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Fund", "00000", false, null), CancellationToken.None); @@ -153,7 +153,7 @@ public async Task Patch_returns_not_found_for_missing_segment() public async Task Patch_returns_bad_request_for_unknown_segment_type() { using var db = TestDbContextFactory.CreateDataInMemory(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Nonsense", "00000", false, null), CancellationToken.None); @@ -165,15 +165,15 @@ public async Task Patch_returns_bad_request_for_unknown_segment_type() public async Task Patch_sets_sfn_for_included_fund() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Fund", "70575", true, "220"), CancellationToken.None); result.Should().BeOfType(); - var updated = await db.ChartStringSegments.FindAsync(SegmentType.Fund, "70575"); + var updated = await db.SegmentClassifications.FindAsync(SegmentType.Fund, "70575"); updated!.IncludeInReport.Should().BeTrue(); updated.Sfn.Should().Be("220"); } @@ -182,15 +182,15 @@ public async Task Patch_sets_sfn_for_included_fund() public async Task Patch_clears_sfn_when_fund_excluded() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "45530", IncludeInReport = true, Sfn = "220" }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Fund, Code = "45530", IncludeInReport = true, Sfn = "220" }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Fund", "45530", false, null), CancellationToken.None); result.Should().BeOfType(); - var updated = await db.ChartStringSegments.FindAsync(SegmentType.Fund, "45530"); + var updated = await db.SegmentClassifications.FindAsync(SegmentType.Fund, "45530"); updated!.IncludeInReport.Should().BeFalse(); updated.Sfn.Should().BeNull(); } @@ -199,9 +199,9 @@ public async Task Patch_clears_sfn_when_fund_excluded() public async Task Patch_rejects_invalid_sfn_for_included_fund() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Fund", "70575", true, "999"), CancellationToken.None); @@ -213,9 +213,9 @@ public async Task Patch_rejects_invalid_sfn_for_included_fund() public async Task Patch_rejects_sfn_on_non_fund_segment() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Account, Code = "500000", IncludeInReport = null }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Account, Code = "500000", IncludeInReport = null }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Account", "500000", true, "201"), CancellationToken.None); @@ -227,15 +227,15 @@ public async Task Patch_rejects_sfn_on_non_fund_segment() public async Task Patch_accepts_multiple_marker_for_included_fund() { using var db = TestDbContextFactory.CreateDataInMemory(); - db.ChartStringSegments.Add(new ChartStringSegment { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null }); + db.SegmentClassifications.Add(new SegmentClassification { SegmentType = SegmentType.Fund, Code = "70575", IncludeInReport = null }); await db.SaveChangesAsync(); - var controller = new ChartStringSegmentsController(db); + var controller = new SegmentClassificationsController(db); var result = await controller.UpdateClassification( new UpdateClassificationRequest("Fund", "70575", true, "Multiple"), CancellationToken.None); result.Should().BeOfType(); - var updated = await db.ChartStringSegments.FindAsync(SegmentType.Fund, "70575"); + var updated = await db.SegmentClassifications.FindAsync(SegmentType.Fund, "70575"); updated!.IncludeInReport.Should().BeTrue(); updated.Sfn.Should().Be("Multiple"); } diff --git a/tests/server.tests/ChartStringSegments/SfnCatalogTests.cs b/tests/server.tests/SegmentClassifications/SfnCatalogTests.cs similarity index 90% rename from tests/server.tests/ChartStringSegments/SfnCatalogTests.cs rename to tests/server.tests/SegmentClassifications/SfnCatalogTests.cs index 90a7bea..0d86977 100644 --- a/tests/server.tests/ChartStringSegments/SfnCatalogTests.cs +++ b/tests/server.tests/SegmentClassifications/SfnCatalogTests.cs @@ -1,7 +1,7 @@ using FluentAssertions; -using Server.Models.ChartStringSegments; +using Server.Models.SegmentClassifications; -namespace Server.Tests.ChartStringSegments; +namespace Server.Tests.SegmentClassifications; public class SfnCatalogTests { From a74312f36df4609bd80ed32cc0db732935a24c4a Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 15:52:36 -0400 Subject: [PATCH 12/15] Drop persisted purpose exclusion from the import pipeline Purpose is a classified segment now, so purpose exclusions derive at read time from SegmentClassifications like the other step 2 types. Removes ExcludedByPurpose columns and the hardcoded 2025 list, the Projects-based 204 guards the carve-out needed, and seeds Purpose codes from transactions alongside the other segment types. Co-Authored-By: Claude Fable 5 --- .../StoredProcedures/ClassifyTransactions.sql | 58 +------------------ .../SeedSegmentClassifications.sql | 4 ++ database/data/Tables/AETransactions.sql | 1 - database/data/Tables/UcPathTransactions.sql | 1 - 4 files changed, 6 insertions(+), 58 deletions(-) diff --git a/database/data/StoredProcedures/ClassifyTransactions.sql b/database/data/StoredProcedures/ClassifyTransactions.sql index 0f4ff13..e0add77 100644 --- a/database/data/StoredProcedures/ClassifyTransactions.sql +++ b/database/data/StoredProcedures/ClassifyTransactions.sql @@ -7,8 +7,8 @@ BEGIN -- Stamps the persisted exclusion columns on the imported transactions: the -- rules step 2 classification does not own. Step 2 based exclusions (fund, - -- account, financial dept, activity, ern) are derived at read time from - -- SegmentClassifications and are never stamped here. + -- account, financial dept, activity, ern, purpose) are derived at read time + -- from SegmentClassifications and are never stamped here. IF @cycleStart IS NULL OR @cycleEnd IS NULL THROW 50000, '@cycleStart and @cycleEnd are required.', 1; @@ -21,34 +21,6 @@ BEGIN IF NOT EXISTS (SELECT 1 FROM [data].[ChartSegments] WHERE [SegmentName] = 'Account') THROW 50000, 'ChartSegments has no Account rows; run the segment reference import first.', 1; - IF NOT EXISTS (SELECT 1 FROM [data].[Projects]) - THROW 50000, 'Projects is empty; run BuildProjects first.', 1; - - -- SFN mismatches on 204 projects are resolved during Project Identification - -- (GetProjectList reports them); none should remain by import time. - IF EXISTS - ( - SELECT 1 FROM [data].[Projects] - WHERE [PgmSfnBucket] IS NOT NULL - AND ( - ([NifaSfn] = '204' AND [PgmSfnBucket] <> '204') - OR ([NifaSfn] <> '204' AND [PgmSfnBucket] = '204') - ) - ) - THROW 50000, 'Projects has 204 SFN disagreements between NIFA and PGM; resolve them in Project Identification first.', 1; - - -- AE projects mapped to a 204 NIFA project. Rows on these projects are - -- reportable regardless of purpose (the 204 carve-out). Any reportable 204 - -- project is mapped through ActiveProjects/AllProjects by the time the - -- import runs, so the materialized project list is the complete source. - SELECT DISTINCT [AEProjectNumber] - INTO #Projects204 - FROM [data].[Projects] - WHERE [NifaSfn] = '204' - AND [AEProjectNumber] IS NOT NULL; - - CREATE UNIQUE CLUSTERED INDEX [IX_Projects204] ON #Projects204 ([AEProjectNumber]); - -- ExcludedByDate, AE: by accounting period membership, consistent with how -- the rows are pulled (period_name list). The cycle months are generated as -- period names ('Oct-24' style, culture pinned) the same way the import @@ -79,32 +51,6 @@ BEGIN ELSE 1 END; - -- ExcludedByPurpose: hardcoded 2025 list, with carve-outs for fund 13U02 - -- (state AES funds, reported regardless of purpose) and 204 projects. - UPDATE t - SET [ExcludedByPurpose] = - CASE - WHEN t.[Purpose] IN ('00', '40', '43', '60', '61', '62', '72', '76', '78', '80') - AND ISNULL(t.[Fund], '') <> '13U02' - AND p204.[AEProjectNumber] IS NULL - THEN 1 - ELSE 0 - END - FROM [data].[AETransactions] t - LEFT JOIN #Projects204 p204 ON p204.[AEProjectNumber] = t.[Project]; - - UPDATE t - SET [ExcludedByPurpose] = - CASE - WHEN t.[Purpose] IN ('00', '40', '43', '60', '61', '62', '72', '76', '78', '80') - AND ISNULL(t.[Fund], '') <> '13U02' - AND p204.[AEProjectNumber] IS NULL - THEN 1 - ELSE 0 - END - FROM [data].[UcPathTransactions] t - LEFT JOIN #Projects204 p204 ON p204.[AEProjectNumber] = t.[Project]; - -- AccountNotInAE: UCPath accounts that do not exist in the AE chart of -- accounts (this should not happen but does). Missing accounts fail closed. UPDATE t diff --git a/database/data/StoredProcedures/SeedSegmentClassifications.sql b/database/data/StoredProcedures/SeedSegmentClassifications.sql index 78f584f..2047bef 100644 --- a/database/data/StoredProcedures/SeedSegmentClassifications.sql +++ b/database/data/StoredProcedures/SeedSegmentClassifications.sql @@ -34,6 +34,10 @@ BEGIN UNION SELECT 'Activity', [Activity] FROM [data].[UcPathTransactions] WHERE [Activity] IS NOT NULL UNION + SELECT 'Purpose', [Purpose] FROM [data].[AETransactions] WHERE [Purpose] IS NOT NULL + UNION + SELECT 'Purpose', [Purpose] FROM [data].[UcPathTransactions] WHERE [Purpose] IS NOT NULL + UNION SELECT 'Ern', [DosCode] FROM [data].[UcPathTransactions] WHERE [DosCode] IS NOT NULL AND [DosCode] <> 'XXX' ) INSERT INTO [data].[SegmentClassifications] ([SegmentType], [Code], [Description]) diff --git a/database/data/Tables/AETransactions.sql b/database/data/Tables/AETransactions.sql index 0e77e36..81965e0 100644 --- a/database/data/Tables/AETransactions.sql +++ b/database/data/Tables/AETransactions.sql @@ -43,7 +43,6 @@ CREATE TABLE [data].[AETransactions] -- Persisted exclusions for rules not owned by step 2 classification. -- NULL = not yet classified (fails closed downstream). [ExcludedByDate] BIT NULL, - [ExcludedByPurpose] BIT NULL, [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_AETransactions_LoadedAt] DEFAULT (SYSUTCDATETIME()), CONSTRAINT [PK_AETransactions] PRIMARY KEY CLUSTERED ([Id]) diff --git a/database/data/Tables/UcPathTransactions.sql b/database/data/Tables/UcPathTransactions.sql index ef819fc..b03a926 100644 --- a/database/data/Tables/UcPathTransactions.sql +++ b/database/data/Tables/UcPathTransactions.sql @@ -39,7 +39,6 @@ CREATE TABLE [data].[UcPathTransactions] -- Persisted exclusions for rules not owned by step 2 classification. -- NULL = not yet classified (fails closed downstream). [ExcludedByDate] BIT NULL, - [ExcludedByPurpose] BIT NULL, [AccountNotInAE] BIT NULL, [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_UcPathTransactions_LoadedAt] DEFAULT (SYSUTCDATETIME()), From bfb77451e0b6ba0afd54ecf28b6c2a87233856aa Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 15:59:22 -0400 Subject: [PATCH 13/15] Require PGM match in Projects, store a single SFN Every included active project has a PGM master data record by build time and SFN agreement is checked during Project Identification, so AEProjectNumber is required and the separate NifaSfn/PgmSfnBucket pair collapses to one Sfn column. BuildProjects fails loudly on unmatched projects instead of inserting NULL AE rows. Co-Authored-By: Claude Fable 5 --- .../data/StoredProcedures/BuildProjects.sql | 36 +++++++++++++------ database/data/Tables/Projects.sql | 9 ++--- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/database/data/StoredProcedures/BuildProjects.sql b/database/data/StoredProcedures/BuildProjects.sql index 865378d..578d07a 100644 --- a/database/data/StoredProcedures/BuildProjects.sql +++ b/database/data/StoredProcedures/BuildProjects.sql @@ -4,15 +4,32 @@ BEGIN SET NOCOUNT ON; -- Materializes the cycle's consolidated project list from ActiveProjects, - -- AllProjects and PGMProjects: one row per NIFA project x AE project pair - -- (a NIFA project with no PGM match is one row with NULL AE fields). + -- AllProjects and PGMProjects: one row per NIFA project x AE project pair. -- Runs as an import stage after step 1 settles the project list; downstream - -- consumers (classify sproc 204 carve-out, expense views, associations) - -- read this table instead of re-deriving the joins. + -- consumers (expense views, associations) read this table instead of + -- re-deriving the joins. SFN comes from the NIFA project number suffix; + -- NIFA/PGM SFN agreement is checked during Project Identification, so a + -- single SFN is stored. IF NOT EXISTS (SELECT 1 FROM [data].[ActiveProjects]) THROW 50000, 'ActiveProjects is empty; complete Project Identification before building the project list.', 1; + -- Every included active project must map to a PGM master data record; an + -- unmatched project means Project Identification is not finished. + IF EXISTS + ( + SELECT 1 + FROM [data].[ActiveProjects] a + LEFT JOIN [data].[AllProjects] ap + ON ap.[ProjectNumber] = a.[ProjectNumber] + LEFT JOIN [data].[v_PgmProjectSfnBuckets] pc + ON ap.[AwardNumber] IS NOT NULL + AND pc.[AwardKey] = REPLACE(ap.[AwardNumber], '-', '') + WHERE ISNULL(a.[ExcludeFromUi], 0) = 0 + AND pc.[ProjectNumber] IS NULL + ) + THROW 50000, 'Active projects exist without a PGM master data match; resolve them in Project Identification first.', 1; + DELETE FROM [data].[Projects]; INSERT INTO [data].[Projects] @@ -26,10 +43,9 @@ BEGIN [ProjectDirector], [UcpEmployeeId], [Is204], - [NifaSfn], + [Sfn], [AEProjectNumber], [SponsorAwardNumber], - [PgmSfnBucket], [PrincipalInvestigatorNames] ) SELECT @@ -51,14 +67,12 @@ BEGIN END, pc.[ProjectNumber], pc.[SponsorAwardNumber], - pc.[PgmSfnBucket], pgm.[PrincipalInvestigatorNames] FROM [data].[ActiveProjects] a - LEFT JOIN [data].[AllProjects] ap + JOIN [data].[AllProjects] ap ON ap.[ProjectNumber] = a.[ProjectNumber] - LEFT JOIN [data].[v_PgmProjectSfnBuckets] pc - ON ap.[AwardNumber] IS NOT NULL - AND pc.[AwardKey] = REPLACE(ap.[AwardNumber], '-', '') + JOIN [data].[v_PgmProjectSfnBuckets] pc + ON pc.[AwardKey] = REPLACE(ap.[AwardNumber], '-', '') LEFT JOIN [data].[PGMProjects] pgm ON pgm.[ProjectId] = pc.[ProjectId] WHERE ISNULL(a.[ExcludeFromUi], 0) = 0; diff --git a/database/data/Tables/Projects.sql b/database/data/Tables/Projects.sql index 8bcfcd6..cc8165d 100644 --- a/database/data/Tables/Projects.sql +++ b/database/data/Tables/Projects.sql @@ -12,12 +12,13 @@ CREATE TABLE [data].[Projects] [ProjectDirector] NVARCHAR(200) NULL, [UcpEmployeeId] NVARCHAR(8) NULL, [Is204] BIT NOT NULL, - [NifaSfn] NVARCHAR(7) NULL, -- from project number suffix: 201 | 202 | 204 | 205 | UNKNOWN + [Sfn] NVARCHAR(7) NOT NULL, -- from project number suffix: 201 | 202 | 204 | 205 | UNKNOWN - -- AE side (from PGMProjects via award number match; NULL when no PGM match) - [AEProjectNumber] NVARCHAR(50) NULL, + -- AE side (from PGMProjects via award number match). Required: every + -- included active project has a PGM master data record by build time, and + -- NIFA/PGM SFN agreement is checked during Project Identification. + [AEProjectNumber] NVARCHAR(50) NOT NULL, [SponsorAwardNumber] NVARCHAR(100) NULL, - [PgmSfnBucket] NVARCHAR(10) NULL, -- CFDA-derived: HATCH | 203 | 204 | 205 | NON-NIFA | NULL [PrincipalInvestigatorNames] NVARCHAR(MAX) NULL, [LoadedAt] DATETIME2(3) NULL CONSTRAINT [DF_Projects_LoadedAt] DEFAULT (SYSUTCDATETIME()), From 870b4d5d3ae345b39ed205236f945bd0c0483e74 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 15:59:22 -0400 Subject: [PATCH 14/15] Tighten UcPathTransactions nullability, rename DosCode to ErnCode Core chart string segments, natural key components, and the fringe/salary marker are required. ERN code replaces the legacy DOS terminology throughout. Co-Authored-By: Claude Fable 5 --- .../SeedSegmentClassifications.sql | 6 ++--- database/data/Tables/UcPathTransactions.sql | 24 +++++++++++-------- server.core/Data/Seed/ErnCodes.csv | 2 +- server.core/Data/SegmentClassificationSeed.cs | 4 ++-- .../SegmentClassificationSeedTests.cs | 2 +- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/database/data/StoredProcedures/SeedSegmentClassifications.sql b/database/data/StoredProcedures/SeedSegmentClassifications.sql index 2047bef..05a5aa0 100644 --- a/database/data/StoredProcedures/SeedSegmentClassifications.sql +++ b/database/data/StoredProcedures/SeedSegmentClassifications.sql @@ -9,9 +9,9 @@ BEGIN -- fails closed downstream until someone classifies the code in step 2. -- -- Descriptions come from the ChartSegments reference data where available. - -- Ern codes (UCPath DOS/earnings codes) have no local description source; + -- ERN codes (UCPath earnings codes) have no local description source; -- the UCPath import fills those in from the PeopleSoft earnings table. - -- 'XXX' is the placeholder DOS code on fringe rows, which carry no FTE, so + -- 'XXX' is the placeholder ERN code on fringe rows, which carry no FTE, so -- classifying it is meaningless and it is not seeded. DECLARE @inserted TABLE ([SegmentType] NVARCHAR(20) NOT NULL); @@ -38,7 +38,7 @@ BEGIN UNION SELECT 'Purpose', [Purpose] FROM [data].[UcPathTransactions] WHERE [Purpose] IS NOT NULL UNION - SELECT 'Ern', [DosCode] FROM [data].[UcPathTransactions] WHERE [DosCode] IS NOT NULL AND [DosCode] <> 'XXX' + SELECT 'Ern', [ErnCode] FROM [data].[UcPathTransactions] WHERE [ErnCode] <> 'XXX' ) INSERT INTO [data].[SegmentClassifications] ([SegmentType], [Code], [Description]) OUTPUT inserted.[SegmentType] INTO @inserted ([SegmentType]) diff --git a/database/data/Tables/UcPathTransactions.sql b/database/data/Tables/UcPathTransactions.sql index b03a926..d0f367b 100644 --- a/database/data/Tables/UcPathTransactions.sql +++ b/database/data/Tables/UcPathTransactions.sql @@ -4,20 +4,24 @@ CREATE TABLE [data].[UcPathTransactions] -- emplid, empl_rcd, erncd ('XXX' for fringe rows), run id. [LaborTransactionId] NVARCHAR(125) NOT NULL, - -- Chart string segments (column names aligned with AETransactions for a future union) - [Entity] NVARCHAR(50) NULL, - [Fund] NVARCHAR(50) NULL, - [FinancialDepartment] NVARCHAR(50) NULL, + -- Chart string segments (column names aligned with AETransactions for a future + -- union). The core chart string is required on every posted labor ledger row; + -- the optional COA segments (parent dept rollup, program, project, activity) + -- stay nullable. + [Entity] NVARCHAR(50) NOT NULL, + [Fund] NVARCHAR(50) NOT NULL, + [FinancialDepartment] NVARCHAR(50) NOT NULL, [ParentDepartment] NVARCHAR(50) NULL, - [Account] NVARCHAR(50) NULL, - [Purpose] NVARCHAR(50) NULL, + [Account] NVARCHAR(50) NOT NULL, + [Purpose] NVARCHAR(50) NOT NULL, [Program] NVARCHAR(50) NULL, [Project] NVARCHAR(50) NULL, [Activity] NVARCHAR(50) NULL, [FinanceDocTypeCd] NVARCHAR(4) NULL, - [DosCode] NVARCHAR(3) NULL, -- ERNCD for salary rows, 'XXX' for fringe rows - [EmployeeId] NVARCHAR(10) NULL, + -- Components of the natural key are required. + [ErnCode] NVARCHAR(3) NOT NULL, -- ERNCD for salary rows, 'XXX' for fringe rows + [EmployeeId] NVARCHAR(10) NOT NULL, [EmployeeName] NVARCHAR(100) NULL, [PositionNumber] NVARCHAR(8) NULL, [EffDt] DATETIME2(7) NULL, @@ -28,12 +32,12 @@ CREATE TABLE [data].[UcPathTransactions] [PayRate] DECIMAL(17, 4) NULL, [CalculatedFte] DECIMAL(9, 6) NULL, -- hours / hours in federal fiscal year (2088 or 2096) [PayPeriodEndDate] DATETIME2(7) NULL, -- authoritative date for cycle-window logic (fiscal year/period unreliable on corrections) - [FringeBenefitSalaryCd] NVARCHAR(1) NULL, -- 'S' salary, 'F' fringe + [FringeBenefitSalaryCd] NVARCHAR(1) NOT NULL, -- 'S' salary, 'F' fringe (assigned by the import per source view) [PaidPercent] DECIMAL(7, 4) NULL, [ErnDerivedPercent] DECIMAL(7, 4) NULL, [FiscalYear] INT NULL, -- PeopleSoft fiscal bookkeeping, kept for QA totals by period [Period] NVARCHAR(2) NULL, - [EmpRcd] SMALLINT NULL, + [EmpRcd] SMALLINT NOT NULL, [EffSeq] SMALLINT NULL, -- Persisted exclusions for rules not owned by step 2 classification. diff --git a/server.core/Data/Seed/ErnCodes.csv b/server.core/Data/Seed/ErnCodes.csv index 8ba0505..9124700 100644 --- a/server.core/Data/Seed/ErnCodes.csv +++ b/server.core/Data/Seed/ErnCodes.csv @@ -1,4 +1,4 @@ -DOS_Code,Description,IncludeInAD419FTE,IsNewInUCP +ErnCode,Description,IncludeInAD419FTE,IsNewInUCP 8EF,Retro-EPSL-8EF,true,1 901,Retro Regular Pay-Non Base,false,1 92L,Retro Holiday-Premium Hourly,false, diff --git a/server.core/Data/SegmentClassificationSeed.cs b/server.core/Data/SegmentClassificationSeed.cs index d018309..e22d227 100644 --- a/server.core/Data/SegmentClassificationSeed.cs +++ b/server.core/Data/SegmentClassificationSeed.cs @@ -40,8 +40,8 @@ public static async Task EnsureSeededAsync(DataDbContext db, CancellationToken c await db.SaveChangesAsync(ct); } - // ErnCodes.csv columns: 0 DOS_Code, 1 Description, 2 IncludeInAD419FTE, 3 IsNewInUCP. - // IncludeInAD419FTE seeds the classification directly, except the first few (by DOS + // ErnCodes.csv columns: 0 ErnCode, 1 Description, 2 IncludeInAD419FTE, 3 IsNewInUCP. + // IncludeInAD419FTE seeds the classification directly, except the first few (by ERN // code) which stay unset for demo. IsNewInUCP is not consumed yet. private static List BuildErn() { diff --git a/tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs b/tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs index 7eb802b..7a2ebaa 100644 --- a/tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs +++ b/tests/server.tests/SegmentClassifications/SegmentClassificationSeedTests.cs @@ -43,7 +43,7 @@ public async Task EnsureSeeded_loads_ern_codes_from_csv() reg.IncludeInReport.Should().BeTrue(); reg.Description.Should().Be("Regular Pay"); - // The first 10 by ordinal DOS code are left unset for demo. + // The first 10 by ordinal ERN code are left unset for demo. ern.OrderBy(segment => segment.Code, StringComparer.Ordinal) .Take(10) .All(segment => segment.IncludeInReport == null) From 480ac409b415a1d428d0bd97e75b341ad65231c8 Mon Sep 17 00:00:00 2001 From: Rob Martinsen Date: Wed, 8 Jul 2026 16:39:57 -0400 Subject: [PATCH 15/15] Require profiled-complete UcPathTransactions columns Warehouse profiling of both labor ledger views over the filtered population shows every mapped column except Program and JobCode fully populated, so parent department, project, activity, position number, amounts, pay period end date, and fiscal fields are now required. Import-supplied values cover fringe gaps (hours 0, FTE 0, ERN 'XXX'); rows without a position number are excluded at import. The classify sproc drops its dead null-date branch. Co-Authored-By: Claude Fable 5 --- .../StoredProcedures/ClassifyTransactions.sql | 4 +- database/data/Tables/UcPathTransactions.sql | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/database/data/StoredProcedures/ClassifyTransactions.sql b/database/data/StoredProcedures/ClassifyTransactions.sql index e0add77..a6caae9 100644 --- a/database/data/StoredProcedures/ClassifyTransactions.sql +++ b/database/data/StoredProcedures/ClassifyTransactions.sql @@ -41,12 +41,10 @@ BEGIN LEFT JOIN @cyclePeriods cp ON cp.[PeriodName] = t.[PeriodName]; -- ExcludedByDate, UCPath: by pay period end date (authoritative; fiscal - -- year/period are unreliable on payroll corrections). Missing dates fail - -- closed as excluded. + -- year/period are unreliable on payroll corrections). UPDATE [data].[UcPathTransactions] SET [ExcludedByDate] = CASE - WHEN [PayPeriodEndDate] IS NULL THEN 1 WHEN CAST([PayPeriodEndDate] AS DATE) BETWEEN @cycleStart AND @cycleEnd THEN 0 ELSE 1 END; diff --git a/database/data/Tables/UcPathTransactions.sql b/database/data/Tables/UcPathTransactions.sql index d0f367b..da9db73 100644 --- a/database/data/Tables/UcPathTransactions.sql +++ b/database/data/Tables/UcPathTransactions.sql @@ -5,38 +5,40 @@ CREATE TABLE [data].[UcPathTransactions] [LaborTransactionId] NVARCHAR(125) NOT NULL, -- Chart string segments (column names aligned with AETransactions for a future - -- union). The core chart string is required on every posted labor ledger row; - -- the optional COA segments (parent dept rollup, program, project, activity) - -- stay nullable. + -- union). Warehouse profiling (salary and fringe views, 2026-07) shows every + -- segment except Program fully populated on the filtered population, so only + -- Program stays nullable. The source encodes empty as a blank string, so the + -- import must trim and blank-to-NULL on the way in for the constraints to mean + -- anything. [Entity] NVARCHAR(50) NOT NULL, [Fund] NVARCHAR(50) NOT NULL, [FinancialDepartment] NVARCHAR(50) NOT NULL, - [ParentDepartment] NVARCHAR(50) NULL, + [ParentDepartment] NVARCHAR(50) NOT NULL, [Account] NVARCHAR(50) NOT NULL, [Purpose] NVARCHAR(50) NOT NULL, [Program] NVARCHAR(50) NULL, - [Project] NVARCHAR(50) NULL, - [Activity] NVARCHAR(50) NULL, + [Project] NVARCHAR(50) NOT NULL, + [Activity] NVARCHAR(50) NOT NULL, [FinanceDocTypeCd] NVARCHAR(4) NULL, -- Components of the natural key are required. - [ErnCode] NVARCHAR(3) NOT NULL, -- ERNCD for salary rows, 'XXX' for fringe rows + [ErnCode] NVARCHAR(3) NOT NULL, -- ERNCD for salary rows, 'XXX' for fringe rows (source ERNCD is blank on most fringe) [EmployeeId] NVARCHAR(10) NOT NULL, [EmployeeName] NVARCHAR(100) NULL, - [PositionNumber] NVARCHAR(8) NULL, + [PositionNumber] NVARCHAR(8) NOT NULL, -- rare source rows without one are excluded at import [EffDt] DATETIME2(7) NULL, - [JobCode] NVARCHAR(4) NULL, + [JobCode] NVARCHAR(4) NULL, -- blank at source for many rows; backfilled by the title-code step [RateTypeCd] NVARCHAR(1) NULL, - [Hours] DECIMAL(18, 6) NULL, - [Amount] DECIMAL(19, 4) NULL, - [PayRate] DECIMAL(17, 4) NULL, - [CalculatedFte] DECIMAL(9, 6) NULL, -- hours / hours in federal fiscal year (2088 or 2096) - [PayPeriodEndDate] DATETIME2(7) NULL, -- authoritative date for cycle-window logic (fiscal year/period unreliable on corrections) + [Hours] DECIMAL(18, 6) NOT NULL, -- 0 on fringe rows (source fringe view has no hours) + [Amount] DECIMAL(19, 4) NOT NULL, + [PayRate] DECIMAL(17, 4) NULL, -- amount/hours; undefined on fringe rows + [CalculatedFte] DECIMAL(9, 6) NOT NULL, -- hours / hours in federal fiscal year (2088 or 2096); 0 on fringe rows + [PayPeriodEndDate] DATETIME2(7) NOT NULL, -- authoritative date for cycle-window logic (fiscal year/period unreliable on corrections) [FringeBenefitSalaryCd] NVARCHAR(1) NOT NULL, -- 'S' salary, 'F' fringe (assigned by the import per source view) - [PaidPercent] DECIMAL(7, 4) NULL, - [ErnDerivedPercent] DECIMAL(7, 4) NULL, - [FiscalYear] INT NULL, -- PeopleSoft fiscal bookkeeping, kept for QA totals by period - [Period] NVARCHAR(2) NULL, + [PaidPercent] DECIMAL(7, 4) NULL, -- salary rows only; source fringe view has no percent columns + [ErnDerivedPercent] DECIMAL(7, 4) NULL, -- salary rows only + [FiscalYear] INT NOT NULL, -- PeopleSoft fiscal bookkeeping, kept for QA totals by period + [Period] NVARCHAR(2) NOT NULL, [EmpRcd] SMALLINT NOT NULL, [EffSeq] SMALLINT NULL,