From d5ef2d845ed3da67d9e9b83a863280b03aac24db Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sun, 15 Feb 2026 16:01:27 -0500 Subject: [PATCH 1/8] Add route to submit data through file upload This work is based on the function that had been written and removed in commit: e24f7d6 This leaves the data provider with the existing submit functionality described as submitting single entity data. The new submit endpoint allows submitting files with filenames matching teh entity name. --- packages/data-provider/package.json | 11 +- packages/data-provider/src/config/config.ts | 6 + .../src/controllers/categoryController.ts | 4 +- .../src/controllers/submissionController.ts | 173 +++-- .../controllers/submittedDataController.ts | 4 +- packages/data-provider/src/core/provider.ts | 3 +- .../repository/activeSubmissionRepository.ts | 42 +- .../src/routers/submissionRouter.ts | 35 +- .../src/services/submission/submission.ts | 203 ++++-- .../{processor.ts => submissionProcessor.ts} | 276 +++++--- .../services/submission/submissionService.ts | 631 ++++++++++++++++++ .../services/submittedData/submmittedData.ts | 10 +- packages/data-provider/src/utils/fileUtils.ts | 237 +++++++ .../src/utils/requestValidation.ts | 8 +- packages/data-provider/src/utils/schemas.ts | 137 ++-- .../src/utils/submissionUtils.ts | 130 +++- packages/data-provider/src/utils/types.ts | 33 +- .../test/utils/fileUtils.spec.ts | 66 ++ .../parseSubmissionSummaryResponse.spec.ts | 17 +- 19 files changed, 1712 insertions(+), 314 deletions(-) rename packages/data-provider/src/services/submission/{processor.ts => submissionProcessor.ts} (89%) create mode 100644 packages/data-provider/src/services/submission/submissionService.ts create mode 100644 packages/data-provider/src/utils/fileUtils.ts create mode 100644 packages/data-provider/test/utils/fileUtils.spec.ts diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index 86593134..d8920552 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -37,11 +37,15 @@ "@overture-stack/lectern-client": "2.0.0-beta.6", "@overture-stack/lyric-data-model": "workspace:^", "@overture-stack/sqon-builder": "^1.1.0", + "bytes": "^3.1.2", + "csv-parse": "^6.1.0", "dotenv": "^16.4.5", "drizzle-orm": "^0.29.5", "express": "^4.19.2", - "lodash-es": "^4.17.21", + "firstline": "^2.0.2", "jszip": "^3.10.1", + "lodash-es": "^4.17.21", + "multer": "^2.0.2", "nanoid": "^5.0.7", "pg": "^8.12.0", "plur": "^5.1.0", @@ -49,13 +53,16 @@ "zod": "^3.23.8" }, "devDependencies": { + "@types/bytes": "^3.1.5", "@types/chai-as-promised": "^8.0.1", "@types/deep-freeze": "^0.1.5", "@types/express": "^4.17.21", "@types/express-serve-static-core": "^4.19.5", + "@types/firstline": "^2.0.4", + "@types/jszip": "^3.4.1", "@types/lodash": "^4.17.7", "@types/lodash-es": "^4.17.12", - "@types/jszip": "^3.4.1", + "@types/multer": "^2.0.0", "@types/pg": "^8.11.6", "@types/qs": "^6.9.15", "chai-as-promised": "^8.0.0", diff --git a/packages/data-provider/src/config/config.ts b/packages/data-provider/src/config/config.ts index 8e5dbace..49da312f 100644 --- a/packages/data-provider/src/config/config.ts +++ b/packages/data-provider/src/config/config.ts @@ -24,6 +24,10 @@ export type SchemaServiceConfig = { url: string; }; +export type SubmissionServiceConfig = { + maxFileSize?: number; +}; + export type LoggerConfig = { level?: string; file?: boolean; @@ -55,6 +59,7 @@ export type AppConfig = { logger: LoggerConfig; onFinishCommit?: (resultOnCommit: ResultOnCommit) => void; schemaService: SchemaServiceConfig; + submissionService: SubmissionServiceConfig; validator: ValidatorConfig; }; @@ -68,4 +73,5 @@ export interface BaseDependencies { logger: Logger; onFinishCommit?: (resultOnCommit: ResultOnCommit) => void; schemaService: SchemaServiceConfig; + submissionService?: SubmissionServiceConfig; } diff --git a/packages/data-provider/src/controllers/categoryController.ts b/packages/data-provider/src/controllers/categoryController.ts index 9487d21a..84a3c41e 100644 --- a/packages/data-provider/src/controllers/categoryController.ts +++ b/packages/data-provider/src/controllers/categoryController.ts @@ -4,14 +4,14 @@ import { BaseDependencies } from '../config/config.js'; import categorySvc from '../services/categoryService.js'; import { BadRequest } from '../utils/errors.js'; import { validateRequest } from '../utils/requestValidation.js'; -import { cagegoryDetailsRequestSchema } from '../utils/schemas.js'; +import { categoryDetailsRequestSchema } from '../utils/schemas.js'; const controller = (dependencies: BaseDependencies) => { const categoryService = categorySvc(dependencies); const { logger } = dependencies; const LOG_MODULE = 'CATEGORY_CONTROLLER'; return { - getDetails: validateRequest(cagegoryDetailsRequestSchema, async (req, res, next) => { + getDetails: validateRequest(categoryDetailsRequestSchema, async (req, res, next) => { try { const categoryId = Number(req.params.categoryId); diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts index 076c71cd..44c87c25 100644 --- a/packages/data-provider/src/controllers/submissionController.ts +++ b/packages/data-provider/src/controllers/submissionController.ts @@ -1,16 +1,18 @@ +import type { Response } from 'express'; import { isEmpty } from 'lodash-es'; import { BaseDependencies } from '../config/config.js'; import { type AuthConfig, shouldBypassAuth } from '../middleware/auth.js'; -import submissionService from '../services/submission/submission.js'; +import submissionService from '../services/submission/submissionService.js'; import submittedDataService from '../services/submittedData/submmittedData.js'; import { hasUserWriteAccess } from '../utils/authUtils.js'; import { BadRequest, Forbidden, NotFound } from '../utils/errors.js'; +import { extractFileExtension, SUPPORTED_FILE_EXTENSIONS } from '../utils/fileUtils.js'; import { asArray } from '../utils/formatUtils.js'; import { validateRequest } from '../utils/requestValidation.js'; import { dataDeleteBySystemIdRequestSchema, - dataEditRequestSchema, + editSingleEntityRequestSchema, submissionActiveByOrganizationRequestSchema, submissionByIdRequestSchema, submissionCommitRequestSchema, @@ -18,10 +20,17 @@ import { submissionDeleteRequestSchema, submissionDetailsRequestSchema, submissionsByCategoryRequestSchema, + uploadSingleEntitySubmissionDataRequestSchema, uploadSubmissionRequestSchema, } from '../utils/schemas.js'; import { parseSubmissionActionTypes } from '../utils/submissionUtils.js'; -import { SUBMISSION_ACTION_TYPE } from '../utils/types.js'; +import { + BATCH_ERROR_TYPE, + BatchError, + type PaginatedResponse, + SUBMISSION_ACTION_TYPE, + type SubmissionSummary, +} from '../utils/types.js'; const controller = ({ baseDependencies, @@ -164,11 +173,11 @@ const controller = ({ } }), - editSubmittedData: validateRequest(dataEditRequestSchema, async (req, res, next) => { + editSubmittedData: validateRequest(editSingleEntityRequestSchema, async (req, res, next) => { try { const categoryId = Number(req.params.categoryId); - const entityName = req.query.entityName; - const organization = req.query.organization; + const entityName = req.params.entityName; + const organization = req.params.organizationId; const payload = req.body; const user = req.user; @@ -200,44 +209,47 @@ const controller = ({ next(error); } }), - getSubmissionsByCategory: validateRequest(submissionsByCategoryRequestSchema, async (req, res, next) => { - try { - const categoryId = Number(req.params.categoryId); - const onlyActive = req.query.onlyActive?.toLowerCase() === 'true'; - const organization = req.query.organization; - const page = parseInt(String(req.query.page)) || defaultPage; - const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; - const username = req.query.username; - - logger.info( - LOG_MODULE, - `Request Submission categoryId '${categoryId}'`, - `pagination params: page '${page}' pageSize '${pageSize}'`, - `onlyActive '${onlyActive}'`, - `organization '${organization}'`, - ); - - const submissionsResult = await service.getSubmissionsByCategory( - categoryId, - { page, pageSize }, - { onlyActive, username, organization }, - ); + getSubmissionsByCategory: validateRequest( + submissionsByCategoryRequestSchema, + async (req, res: Response>, next) => { + try { + const categoryId = Number(req.params.categoryId); + const onlyActive = req.query.onlyActive?.toLowerCase() === 'true'; + const organization = req.query.organization; + const page = parseInt(String(req.query.page)) || defaultPage; + const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize; + const username = req.query.username; + + logger.info( + LOG_MODULE, + `Request Submission categoryId '${categoryId}'`, + `pagination params: page '${page}' pageSize '${pageSize}'`, + `onlyActive '${onlyActive}'`, + `organization '${organization}'`, + ); - const response = { - pagination: { - currentPage: page, - pageSize: pageSize, - totalPages: Math.ceil(submissionsResult.metadata.totalRecords / pageSize), - totalRecords: submissionsResult.metadata.totalRecords, - }, - records: submissionsResult.result, - }; + const submissionsResult = await service.getSubmissionsByCategory( + categoryId, + { page, pageSize }, + { onlyActive, username, organization }, + ); - return res.status(200).send(response); - } catch (error) { - next(error); - } - }), + const response: PaginatedResponse = { + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: Math.ceil(submissionsResult.metadata.totalRecords / pageSize), + totalRecords: submissionsResult.metadata.totalRecords, + }, + records: submissionsResult.result, + }; + + return res.status(200).send(response); + } catch (error) { + next(error); + } + }, + ), getSubmissionById: validateRequest(submissionByIdRequestSchema, async (req, res, next) => { try { const submissionId = Number(req.params.submissionId); @@ -311,11 +323,11 @@ const controller = ({ next(error); } }), - submit: validateRequest(uploadSubmissionRequestSchema, async (req, res, next) => { + submitSingleEntityData: validateRequest(uploadSingleEntitySubmissionDataRequestSchema, async (req, res, next) => { try { const categoryId = Number(req.params.categoryId); - const entityName = req.query.entityName; - const organization = req.query.organization; + const entityName = req.params.entityName; + const organization = req.params.organizationId; const payload = req.body; const user = req.user; @@ -326,7 +338,9 @@ const controller = ({ ` entityName '${entityName}'`, ); - if (!payload || payload.length == 0) { + // TODO: parse body payload + + if (!payload || !Array.isArray(payload) || payload.length == 0) { throw new BadRequest( 'The "payload" parameter is missing or empty. Please include the records in the request for processing.', ); @@ -338,7 +352,7 @@ const controller = ({ const username = user?.username || ''; - const resultSubmission = await service.submit({ + const resultSubmission = await service.submitJson({ data: { [entityName]: payload }, categoryId, organization, @@ -351,6 +365,71 @@ const controller = ({ next(error); } }), + + submitFiles: validateRequest(uploadSubmissionRequestSchema, async (req, res, next) => { + try { + const categoryId = Number(req.params.categoryId); + const files = Array.isArray(req.files) ? req.files : []; + const organization = req.params.organizationId; + + // Get username from auth + const username = req.user?.username || ''; + + logger.info( + LOG_MODULE, + `Upload Submission Request: categoryId '${categoryId}'`, + ` organization '${organization}'`, + ` files '${files?.map((f) => f.originalname)}'`, + ); + + if (!files || files.length == 0) { + throw new BadRequest( + 'The "files" parameter is missing or empty. Please include files in the request for processing.', + ); + } + + // sort files into validFiles and fileErrors based on correct file extension + const { validFiles, fileErrors } = files.reduce<{ + validFiles: Express.Multer.File[]; + fileErrors: BatchError[]; + }>( + (acc, file) => { + if (extractFileExtension(file.originalname)) { + acc.validFiles.push(file); + } else { + const batchError: BatchError = { + type: BATCH_ERROR_TYPE.INVALID_FILE_EXTENSION, + message: `File '${file.originalname}' has invalid file extension. File extension must be '${SUPPORTED_FILE_EXTENSIONS.options}'.`, + batchName: file.originalname, + }; + acc.fileErrors.push(batchError); + } + return acc; + }, + { validFiles: [], fileErrors: [] }, + ); + + const resultSubmission = await service.submitFiles({ + files: validFiles, + categoryId, + organization, + username, + }); + + if (fileErrors.length == 0 && resultSubmission.batchErrors.length == 0) { + logger.info(LOG_MODULE, `Submission uploaded successfully`); + } else { + logger.info(LOG_MODULE, 'Found some errors processing this request'); + } + + // This response provides the details of file Submission + return res + .status(200) + .send({ ...resultSubmission, batchErrors: [...fileErrors, ...resultSubmission.batchErrors] }); + } catch (error) { + next(error); + } + }), }; }; diff --git a/packages/data-provider/src/controllers/submittedDataController.ts b/packages/data-provider/src/controllers/submittedDataController.ts index fffe52d7..1fd76130 100644 --- a/packages/data-provider/src/controllers/submittedDataController.ts +++ b/packages/data-provider/src/controllers/submittedDataController.ts @@ -12,7 +12,7 @@ import { dataGetByCategoryRequestSchema, dataGetByOrganizationRequestSchema, dataGetByQueryRequestSchema, - dataGetBySystemIdRequestSchema, + DataGetBySystemIdRequestSchema, } from '../utils/schemas.js'; import { convertToViewType } from '../utils/submittedDataUtils.js'; import { SubmittedDataPaginatedResponse, VIEW_TYPE } from '../utils/types.js'; @@ -185,7 +185,7 @@ const controller = ({ next(error); } }), - getSubmittedDataBySystemId: validateRequest(dataGetBySystemIdRequestSchema, async (req, res, next) => { + getSubmittedDataBySystemId: validateRequest(DataGetBySystemIdRequestSchema, async (req, res, next) => { try { const categoryId = Number(req.params.categoryId); const systemId = req.params.systemId; diff --git a/packages/data-provider/src/core/provider.ts b/packages/data-provider/src/core/provider.ts index 54fd260d..cb66c60f 100644 --- a/packages/data-provider/src/core/provider.ts +++ b/packages/data-provider/src/core/provider.ts @@ -21,7 +21,7 @@ import validationRouter from '../routers/validationRouter.js'; import auditService from '../services/auditService.js'; import categoryService from '../services/categoryService.js'; import dictionaryService from '../services/dictionaryService.js'; -import submissionService from '../services/submission/submission.js'; +import submissionService from '../services/submission/submissionService.js'; import submittedDataService from '../services/submittedData/submmittedData.js'; import validationService from '../services/validationService.js'; import * as auditUtils from '../utils/auditUtils.js'; @@ -46,6 +46,7 @@ const provider = (configData: AppConfig) => { idService: configData.idService, logger: getLogger(configData.logger), schemaService: configData.schemaService, + submissionService: configData.submissionService, onFinishCommit: configData.onFinishCommit, }; diff --git a/packages/data-provider/src/repository/activeSubmissionRepository.ts b/packages/data-provider/src/repository/activeSubmissionRepository.ts index b96f0380..a541af75 100644 --- a/packages/data-provider/src/repository/activeSubmissionRepository.ts +++ b/packages/data-provider/src/repository/activeSubmissionRepository.ts @@ -17,7 +17,7 @@ import type { SubmissionErrorsSummary, } from '../utils/types.js'; -const repository = (dependencies: BaseDependencies) => { +const activeSubmissionRepository = (dependencies: BaseDependencies) => { const LOG_MODULE = 'ACTIVE_SUBMISSION_REPOSITORY'; const { db, logger } = dependencies; @@ -30,14 +30,14 @@ const repository = (dependencies: BaseDependencies) => { createdBy: true, updatedAt: true, updatedBy: true, - } as const satisfies BooleanTrueObject; + }; // Submission columns for full detail queries including `data` and `errors` columns const submissionColumnsWithData: BooleanTrueObject = { ...submissionColumns, data: true, errors: true, - } as const satisfies BooleanTrueObject; + }; const submissionDictionaryRelationColumns = { dictionary: { @@ -193,6 +193,36 @@ jsonb_build_object( } }, + /** + * Returns the entire active submission, including all data. + */ + getActiveSubmissionDetails: async ({ + categoryId, + organization, + username, + }: { + categoryId: number; + username: string; + organization: string; + }): Promise | undefined> => { + try { + const dbResponse = await db.query.submissions.findFirst({ + where: and( + eq(submissions.dictionaryCategoryId, categoryId), + eq(submissions.createdBy, username), + eq(submissions.organization, organization), + activeStatusesCondition, + ), + columns: submissionColumnsWithData, + with: submissionDictionaryRelationColumns, + }); + return dbResponse; + } catch (error) { + logger.error(LOG_MODULE, `Failed getting active submission data`, error); + throw new ServiceUnavailable(); + } + }, + /** * Finds the current Active Submission by parameters * @param {Object} params @@ -201,7 +231,7 @@ jsonb_build_object( * @param {string} params.organization Organization name * @returns */ - getActiveSubmission: async ({ + getActiveSubmissionSummary: async ({ categoryId, username, organization, @@ -223,7 +253,7 @@ jsonb_build_object( extras: { data: dataSummaryQuery, errors: errorsSummaryQuery }, }); } catch (error) { - logger.error(LOG_MODULE, `Failed getting active Submission`, error); + logger.error(LOG_MODULE, `Failed getting active submission summary`, error); throw new ServiceUnavailable(); } }, @@ -374,4 +404,4 @@ jsonb_build_object( }; }; -export default repository; +export default activeSubmissionRepository; diff --git a/packages/data-provider/src/routers/submissionRouter.ts b/packages/data-provider/src/routers/submissionRouter.ts index 2d44fd48..0a5e96d9 100644 --- a/packages/data-provider/src/routers/submissionRouter.ts +++ b/packages/data-provider/src/routers/submissionRouter.ts @@ -1,4 +1,6 @@ +import bytes from 'bytes'; import { json, Router, urlencoded } from 'express'; +import multer from 'multer'; import { BaseDependencies } from '../config/config.js'; import submissionController from '../controllers/submissionController.js'; @@ -11,9 +13,18 @@ const router = ({ baseDependencies: BaseDependencies; authConfig: AuthConfig; }): Router => { + const upload = multer({ dest: '/tmp', limits: { fileSize: baseDependencies.submissionService?.maxFileSize } }); + + // Handles null edgecase and values of 0 as no limit. + const bytesFileLimit = bytes.format(baseDependencies.submissionService?.maxFileSize ?? 0) || undefined; + const router = Router(); router.use(urlencoded({ extended: false })); - router.use(json()); + router.use( + json({ + limit: bytesFileLimit, + }), + ); router.use(authMiddleware(authConfig)); @@ -65,12 +76,28 @@ const router = ({ }).getActiveByOrganization, ); + /* =============================================================== + * Submit Data + * - Submit files for multiple entities + * - Submit data for single entity (Files or request body text) + * =============================================================== */ + + router.post( + '/category/:categoryId/organization/:organizationId', + upload.array('files'), + submissionController({ + baseDependencies, + authConfig, + }).submitFiles, + ); + router.post( - '/category/:categoryId/data', + '/category/:categoryId/organization/:organizationId/entity/:entityName', + upload.array('files'), submissionController({ baseDependencies, authConfig, - }).submit, + }).submitSingleEntityData, ); router.delete( @@ -82,7 +109,7 @@ const router = ({ ); router.put( - `/category/:categoryId/data`, + `/category/:categoryId/organization/:organizationId/entity/:entityName`, submissionController({ baseDependencies, authConfig, diff --git a/packages/data-provider/src/services/submission/submission.ts b/packages/data-provider/src/services/submission/submission.ts index 60d44fc2..18b43b9d 100644 --- a/packages/data-provider/src/services/submission/submission.ts +++ b/packages/data-provider/src/services/submission/submission.ts @@ -9,13 +9,15 @@ import { import { BaseDependencies } from '../../config/config.js'; import systemIdGenerator from '../../external/systemIdGenerator.js'; -import submissionRepository from '../../repository/activeSubmissionRepository.js'; -import categoryRepository from '../../repository/categoryRepository.js'; -import submittedRepository from '../../repository/submittedRepository.js'; +import createSubmissionRepository from '../../repository/activeSubmissionRepository.js'; +import createCategoryRepository from '../../repository/categoryRepository.js'; +import createSubmittedDataRepository from '../../repository/submittedRepository.js'; import { getSchemaByName } from '../../utils/dictionaryUtils.js'; import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; import { filterAndPaginateSubmissionData, type FlattenedSubmissionData } from '../../utils/submissionResponseParser.js'; import { + checkEntityFieldNames, + checkFileNames, createSubmissionSummaryResponse, isSubmissionActive, removeItemsFromSubmission, @@ -23,7 +25,8 @@ import { import { CommitSubmissionResult, CREATE_SUBMISSION_STATUS, - type CreateSubmissionResult, + type CreateSubmissionJsonResult, + type SubmitFileResult, type DeleteSubmissionResult, type EntityData, type PaginationOptions, @@ -32,12 +35,18 @@ import { type SubmissionActionType, SubmissionSummary, } from '../../utils/types.js'; -import processor from './processor.js'; +import { default as createSubmissionProcessor } from './processor.js'; -const service = (dependencies: BaseDependencies) => { +const submissionService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'SUBMISSION_SERVICE'; const { logger, onFinishCommit } = dependencies; - const { performCommitSubmissionAsync, performDataValidation } = processor(dependencies); + + const categoryRepository = createCategoryRepository(dependencies); + const submissionProcessor = createSubmissionProcessor(dependencies); + const submissionRepository = createSubmissionRepository(dependencies); + const submittedDataRepository = createSubmittedDataRepository(dependencies); + + const { generateIdentifier } = systemIdGenerator(dependencies); /** * Runs Schema validation asynchronously and moves the Active Submission to Submitted Data @@ -50,12 +59,10 @@ const service = (dependencies: BaseDependencies) => { submissionId: number, username: string, ): Promise => { - const { getSubmissionDetailsById } = submissionRepository(dependencies); - const { getSubmittedDataByCategoryIdAndOrganization } = submittedRepository(dependencies); - const { getActiveDictionaryByCategory } = categoryRepository(dependencies); - const { generateIdentifier } = systemIdGenerator(dependencies); + const { getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; + const { getActiveDictionaryByCategory } = categoryRepository; - const submission = await getSubmissionDetailsById(submissionId); + const submission = await submissionRepository.getSubmissionDetailsById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); } @@ -120,7 +127,7 @@ const service = (dependencies: BaseDependencies) => { ); // To Commit Active Submission we need to validate SubmittedData + Active Submission - performCommitSubmissionAsync({ + submissionProcessor.performCommitSubmissionAsync({ dataToValidate: { inserts: insertsToValidate, submittedData: submittedDataToValidate, @@ -155,9 +162,7 @@ const service = (dependencies: BaseDependencies) => { submissionId: number, username: string, ): Promise => { - const { getSubmissionById, update } = submissionRepository(dependencies); - - const submission = await getSubmissionById(submissionId); + const submission = await submissionRepository.getSubmissionById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); } @@ -166,7 +171,7 @@ const service = (dependencies: BaseDependencies) => { throw new StatusConflict('Submission is not active. Only Active Submission can be deleted'); } - const updatedRecordId = await update(submission.id, { + const updatedRecordId = await submissionRepository.update(submission.id, { status: SUBMISSION_STATUS.CLOSED, updatedBy: username, }); @@ -187,7 +192,7 @@ const service = (dependencies: BaseDependencies) => { * @param {number} submissionId * @param {string} entityName * @param {string} username - * @returns { Promise} + * @returns { Promise} */ const deleteActiveSubmissionEntity = async ( submissionId: number, @@ -197,10 +202,8 @@ const service = (dependencies: BaseDependencies) => { entityName: string; index: number | null; }, - ): Promise => { - const { getSubmissionDetailsById } = submissionRepository(dependencies); - - const submission = await getSubmissionDetailsById(submissionId); + ): Promise => { + const submission = await submissionRepository.getSubmissionDetailsById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); } @@ -236,7 +239,7 @@ const service = (dependencies: BaseDependencies) => { }); // Validate and update Active Submission after removing the entity - performDataValidation({ + submissionProcessor.performDataValidation({ submissionId: submission.id, submissionData: updatedActiveSubmissionData, username, @@ -275,9 +278,11 @@ const service = (dependencies: BaseDependencies) => { result: SubmissionSummary[]; metadata: { totalRecords: number; errorMessage?: string }; }> => { - const { getSubmissionsByCategory, getTotalSubmissionsByCategory } = submissionRepository(dependencies); - - const recordsPaginated = await getSubmissionsByCategory(categoryId, paginationOptions, filterOptions); + const recordsPaginated = await submissionRepository.getSubmissionsByCategory( + categoryId, + paginationOptions, + filterOptions, + ); if (!recordsPaginated || recordsPaginated.length === 0) { return { result: [], @@ -287,7 +292,7 @@ const service = (dependencies: BaseDependencies) => { }; } - const totalRecords = await getTotalSubmissionsByCategory(categoryId, filterOptions); + const totalRecords = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions); return { metadata: { totalRecords, @@ -302,9 +307,7 @@ const service = (dependencies: BaseDependencies) => { * @returns One Submission */ const getSubmissionById = async (submissionId: number) => { - const { getSubmissionById } = submissionRepository(dependencies); - - const submission = await getSubmissionById(submissionId); + const submission = await submissionRepository.getSubmissionById(submissionId); if (_.isEmpty(submission)) { return; } @@ -332,9 +335,7 @@ const service = (dependencies: BaseDependencies) => { paginationOptions: PaginationOptions; filterOptions: { entityNames: string[]; actionTypes: SubmissionActionType[] }; }): Promise<{ data: FlattenedSubmissionData[]; errors?: SubmissionRecordErrorDetails[] }> => { - const { getSubmissionDetailsById } = submissionRepository(dependencies); - - const submission = await getSubmissionDetailsById(submissionId); + const submission = await submissionRepository.getSubmissionDetailsById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); } @@ -378,9 +379,7 @@ const service = (dependencies: BaseDependencies) => { username: string; organization: string; }): Promise => { - const { getActiveSubmission } = submissionRepository(dependencies); - - const submission = await getActiveSubmission({ + const submission = await submissionRepository.getActiveSubmissionSummary({ organization, username, categoryId, @@ -406,15 +405,18 @@ const service = (dependencies: BaseDependencies) => { organization: string; }): Promise => { const { categoryId, username, organization } = params; - const submissionRepo = submissionRepository(dependencies); - const categoryRepo = categoryRepository(dependencies); + const { getActiveDictionaryByCategory } = categoryRepository; - const activeSubmission = await submissionRepo.getActiveSubmission({ categoryId, username, organization }); + const activeSubmission = await submissionRepository.getActiveSubmissionSummary({ + categoryId, + username, + organization, + }); if (activeSubmission) { return activeSubmission.id; } - const currentDictionary = await categoryRepo.getActiveDictionaryByCategory(categoryId); + const currentDictionary = await getActiveDictionaryByCategory(categoryId); if (!currentDictionary) { throw new InternalServerError(`Dictionary in category '${categoryId}' not found`); @@ -430,7 +432,7 @@ const service = (dependencies: BaseDependencies) => { status: SUBMISSION_STATUS.OPEN, }; - return submissionRepo.save(newSubmissionInput); + return submissionRepository.save(newSubmissionInput); }; /** @@ -443,7 +445,7 @@ const service = (dependencies: BaseDependencies) => { * @param {string} params.username User name creating the Submission * @returns The Active Submission created or Updated */ - const submit = async ({ + const submitJson = async ({ data, categoryId, organization, @@ -453,15 +455,12 @@ const service = (dependencies: BaseDependencies) => { categoryId: number; organization: string; username: string; - }): Promise => { + }): Promise => { const entityNames = Object.keys(data); logger.info( LOG_MODULE, `Processing '${entityNames.length}' entities on category id '${categoryId}' organization '${organization}'`, ); - const { getActiveDictionaryByCategory } = categoryRepository(dependencies); - const { processInsertRecordsAsync } = processor(dependencies); - if (entityNames.length === 0) { return { status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, @@ -469,7 +468,7 @@ const service = (dependencies: BaseDependencies) => { }; } - const currentDictionary = await getActiveDictionaryByCategory(categoryId); + const currentDictionary = await categoryRepository.getActiveDictionaryByCategory(categoryId); if (_.isEmpty(currentDictionary)) { return { @@ -498,7 +497,7 @@ const service = (dependencies: BaseDependencies) => { // Schema validation runs asynchronously and does not block execution. // The results will be saved to the database. - processInsertRecordsAsync({ + submissionProcessor.processInsertRecordsAsync({ records: data, submissionId: activeSubmissionId, schemasDictionary, @@ -512,6 +511,109 @@ const service = (dependencies: BaseDependencies) => { }; }; + /** + * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database + * @param {object} params + * @param {Express.Multer.File[]} params.files An array of files + * @param {number} params.categoryId Category ID of the Submission + * @param {string} params.organization Organization name + * @param {string} params.userName User name creating the Submission + * @returns The Active Submission created or Updated + */ + const submitFiles = async ({ + files, + categoryId, + organization, + username, + }: { + files: Express.Multer.File[]; + categoryId: number; + organization: string; + username: string; + }): Promise => { + logger.info(LOG_MODULE, `Processing '${files.length}' files on category id '${categoryId}'`); + + if (files.length === 0) { + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: 'No valid files for submission', + batchErrors: [], + inProcessEntities: [], + }; + } + + const currentDictionary = await categoryRepository.getActiveDictionaryByCategory(categoryId); + + if (_.isEmpty(currentDictionary)) { + throw new BadRequest(`Dictionary in category '${categoryId}' not found`); + } + + const schemasDictionary: SchemasDictionary = { + name: currentDictionary.name, + version: currentDictionary.version, + schemas: currentDictionary.schemas, + }; + + // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates) + // TODO: Use filename map to identify files, concatenate records if multiple files map to same entity + const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name); + const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames); + + if (_.isEmpty(validFileEntity)) { + logger.info(LOG_MODULE, `No valid files for submission`); + } + + // step 2 Validation. Validate fieldNames (missing required fields based on schema) + const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity); + + const batchErrors = [...fileNamesErrors, ...fieldNameErrors]; + const entitiesToProcess = Object.keys(checkedEntities); + + if (_.isEmpty(checkedEntities)) { + logger.info(LOG_MODULE, 'Found errors on Submission files.', JSON.stringify(batchErrors)); + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: 'No valid entities in submission', + batchErrors, + inProcessEntities: entitiesToProcess, + }; + } + + // Get Active Submission or Open a new one + const activeSubmissionId = await getOrCreateActiveSubmission({ categoryId, username, organization }); + + // TODO: Add files to submission, then run validation separately. Currently these processes are both + // done by the function that adds the files to the submission. + + // Start background process of adding files to submission + // Running Schema validation in the background do not need to wait + // Result of validations will be stored in database + submissionProcessor.addFilesToSubmissionAsync(checkedEntities, { + schemasDictionary, + categoryId, + organization, + username, + }); + + if (batchErrors.length === 0) { + return { + status: CREATE_SUBMISSION_STATUS.PROCESSING, + description: 'Submission files are being processed', + submissionId: activeSubmissionId, + batchErrors, + inProcessEntities: entitiesToProcess, + }; + } + + return { + status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION, + description: 'Some Submission files are being processed while others were unable to process', + submissionId: activeSubmissionId, + batchErrors, + inProcessEntities: entitiesToProcess, + }; + }; + return { commitSubmission, deleteActiveSubmissionById, @@ -521,8 +623,9 @@ const service = (dependencies: BaseDependencies) => { getSubmissionDetailsById, getActiveSubmissionByOrganization, getOrCreateActiveSubmission, - submit, + submitJson, + submitFiles, }; }; -export default service; +export default submissionService; diff --git a/packages/data-provider/src/services/submission/processor.ts b/packages/data-provider/src/services/submission/submissionProcessor.ts similarity index 89% rename from packages/data-provider/src/services/submission/processor.ts rename to packages/data-provider/src/services/submission/submissionProcessor.ts index e539da44..e51852b1 100644 --- a/packages/data-provider/src/services/submission/processor.ts +++ b/packages/data-provider/src/services/submission/submissionProcessor.ts @@ -1,3 +1,4 @@ +import bytes from 'bytes'; import * as _ from 'lodash-es'; import type { DataRecord, Schema } from '@overture-stack/lectern-client'; @@ -14,7 +15,7 @@ import type { } from '@overture-stack/lyric-data-model/models'; import { BaseDependencies } from '../../config/config.js'; -import submissionRepository from '../../repository/activeSubmissionRepository.js'; +import createSubmissionRepository from '../../repository/activeSubmissionRepository.js'; import categoryRepository from '../../repository/categoryRepository.js'; import dictionaryRepository from '../../repository/dictionaryRepository.js'; import submittedRepository from '../../repository/submittedRepository.js'; @@ -37,6 +38,7 @@ import { mergeUpdatesBySystemId, parseToSchema, segregateFieldChangeRecords, + submissionInsertDataFromFiles, validateSchemas, } from '../../utils/submissionUtils.js'; import { @@ -54,13 +56,72 @@ import { type SchemasDictionary, SUBMISSION_STATUS, type SubmittedDataResponse, + type ValidateFilesParams, } from '../../utils/types.js'; import searchDataRelations from '../submittedData/searchDataRelations.js'; -const processor = (dependencies: BaseDependencies) => { +const submissionProcessor = (dependencies: BaseDependencies) => { const LOG_MODULE = 'SUBMISSION_PROCESSOR_SERVICE'; const { logger } = dependencies; + /** + * Processes a list of data records and compares them with previously submitted data. + * @param {DataRecord[]} records An array of data records to be processed + * @param {string} schemaName The name of the schema associated with the records + * @returns {Promise} An array of `SubmissionUpdateData` objects. Each object + * contains the `systemId`, `old` data, and `new` data representing the differences + * between the previously submitted data and the updated record. + */ + const compareUpdatedData = async (records: DataRecord[], schemaName: string): Promise => { + const { getSubmittedDataBySystemId } = submittedRepository(dependencies); + const results: SubmissionUpdateData[] = []; + + const promises = records.map(async (record) => { + const systemId = record['systemId']?.toString(); + if (!systemId) { + return; + } + + const foundSubmittedData = await getSubmittedDataBySystemId(systemId); + if (foundSubmittedData?.data) { + if (foundSubmittedData.entityName !== schemaName) { + logger.error( + LOG_MODULE, + `Entity name mismatch for system ID '${systemId}': expected '${schemaName}', found '${foundSubmittedData.entityName}'`, + ); + results.push({ + systemId: systemId, + old: {}, + new: {}, + }); + return; + } + const changeData = _.omit(record, 'systemId'); + const diffData = computeDataDiff(foundSubmittedData.data, changeData); + if (!_.isEmpty(diffData.old) && !_.isEmpty(diffData.new)) { + results.push({ + systemId: systemId, + old: diffData.old, + new: diffData.new, + }); + } + } else { + logger.error(LOG_MODULE, `No submitted data found for system ID '${systemId}'`); + results.push({ + systemId: systemId, + old: {}, + new: {}, + }); + } + return; + }); + + // Wait for all records to be processed + await Promise.all(promises); + + return results; + }; + /** * Finds and returns the dependent updates based on the provided submission update data. * @@ -155,6 +216,7 @@ const processor = (dependencies: BaseDependencies) => { */ const handleIdFieldChanges = async (idFieldChangeRecord: Record) => { const { getSubmittedDataBySystemId } = submittedRepository(dependencies); + return Object.entries(idFieldChangeRecord).reduce< Promise<{ inserts: Record; @@ -219,7 +281,7 @@ const processor = (dependencies: BaseDependencies) => { * @returns void */ const performCommitSubmissionAsync = async (params: CommitSubmissionParams): Promise => { - const submissionRepo = submissionRepository(dependencies); + const submissionRepo = createSubmissionRepository(dependencies); const dataSubmittedRepo = submittedRepository(dependencies); const { dictionary, dataToValidate, submissionId, username } = params; @@ -416,7 +478,7 @@ const processor = (dependencies: BaseDependencies) => { const { getActiveDictionaryByCategory } = categoryRepository(dependencies); const { getSubmittedDataByCategoryIdAndOrganization } = submittedRepository(dependencies); - const { getSubmissionById } = submissionRepository(dependencies); + const { getSubmissionById } = createSubmissionRepository(dependencies); // Get Active Submission from database const originalSubmission = await getSubmissionById(submissionId); @@ -537,7 +599,7 @@ const processor = (dependencies: BaseDependencies) => { }, ): Promise => { const { getDictionary } = dictionaryRepository(dependencies); - const { getSubmissionDetailsById } = submissionRepository(dependencies); + const { getSubmissionDetailsById } = createSubmissionRepository(dependencies); try { // Parse file data @@ -546,7 +608,6 @@ const processor = (dependencies: BaseDependencies) => { const filesDataProcessed = await compareUpdatedData(recordsParsed, schema.name); const submission = await getSubmissionDetailsById(submissionId); - if (!submission) { throw new Error(`Submission '${submissionId}' not found`); } @@ -634,101 +695,6 @@ const processor = (dependencies: BaseDependencies) => { logger.info(LOG_MODULE, `Finished validating files`); }; - /** - * Processes a list of data records and compares them with previously submitted data. - * @param {DataRecord[]} records An array of data records to be processed - * @param {string} schemaName The name of the schema associated with the records - * @returns {Promise} An array of `SubmissionUpdateData` objects. Each object - * contains the `systemId`, `old` data, and `new` data representing the differences - * between the previously submitted data and the updated record. - */ - const compareUpdatedData = async (records: DataRecord[], schemaName: string): Promise => { - const { getSubmittedDataBySystemId } = submittedRepository(dependencies); - const results: SubmissionUpdateData[] = []; - - const promises = records.map(async (record) => { - const systemId = record['systemId']?.toString(); - if (!systemId) { - return; - } - - const foundSubmittedData = await getSubmittedDataBySystemId(systemId); - if (foundSubmittedData?.data) { - if (foundSubmittedData.entityName !== schemaName) { - logger.error( - LOG_MODULE, - `Entity name mismatch for system ID '${systemId}': expected '${schemaName}', found '${foundSubmittedData.entityName}'`, - ); - results.push({ - systemId: systemId, - old: {}, - new: {}, - }); - return; - } - const changeData = _.omit(record, 'systemId'); - const diffData = computeDataDiff(foundSubmittedData.data, changeData); - if (!_.isEmpty(diffData.old) && !_.isEmpty(diffData.new)) { - results.push({ - systemId: systemId, - old: diffData.old, - new: diffData.new, - }); - } - } else { - logger.error(LOG_MODULE, `No submitted data found for system ID '${systemId}'`); - results.push({ - systemId: systemId, - old: {}, - new: {}, - }); - } - return; - }); - - // Wait for all records to be processed - await Promise.all(promises); - - return results; - }; - - /** - * Update Active Submission in database - * @param {Object} input - * @param {number} input.dictionaryId The Dictionary ID of the Submission - * @param {SubmissionData} input.submissionData Data to be submitted grouped on inserts, updates and deletes - * @param {number} input.idActiveSubmission ID of the Active Submission - * @param {SubmissionErrors} input.schemaErrors Array of schemaErrors - * @param {string} input.username User updating the active submission - * @returns {Promise} An Active Submission updated - */ - const updateActiveSubmission = async (input: { - dictionaryId: number; - submissionData: SubmissionData; - idActiveSubmission: number; - schemaErrors: SubmissionErrors; - username: string; - }): Promise => { - const { dictionaryId, submissionData, idActiveSubmission, schemaErrors, username } = input; - const { update } = submissionRepository(dependencies); - const newStatusSubmission = - Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID; - // Update with new data - const updatedActiveSubmissionId = await update(idActiveSubmission, { - data: submissionData, - status: newStatusSubmission, - dictionaryId: dictionaryId, - updatedBy: username, - errors: schemaErrors, - }); - - logger.info( - LOG_MODULE, - `Updated Active submission '${updatedActiveSubmissionId}' with status '${newStatusSubmission}'`, - ); - return updatedActiveSubmissionId; - }; - /** * Processes and validates a batch of incoming records for an active submission. * This function updates the submission merging the new records with existing submission data. @@ -751,7 +717,7 @@ const processor = (dependencies: BaseDependencies) => { submissionId: number; username: string; }) => { - const { getSubmissionDetailsById, update } = submissionRepository(dependencies); + const { getSubmissionDetailsById, update } = createSubmissionRepository(dependencies); try { // Get Active Submission from database @@ -794,16 +760,114 @@ const processor = (dependencies: BaseDependencies) => { JSON.stringify(error), ); } - logger.info(LOG_MODULE, `Finished validating files`); + logger.info(LOG_MODULE, `Finished processInsertRecordsAsync for submission ${submissionId}`); + }; + + /** + * Update Active Submission in database + * @param {Object} input + * @param {number} input.dictionaryId The Dictionary ID of the Submission + * @param {SubmissionData} input.submissionData Data to be submitted grouped on inserts, updates and deletes + * @param {number} input.idActiveSubmission ID of the Active Submission + * @param {SubmissionErrors} input.schemaErrors Array of schemaErrors + * @param {string} input.username User updating the active submission + * @returns {Promise} An Active Submission updated + */ + const updateActiveSubmission = async (input: { + dictionaryId: number; + submissionData: SubmissionData; + idActiveSubmission: number; + schemaErrors: SubmissionErrors; + username: string; + }): Promise => { + const { dictionaryId, submissionData, idActiveSubmission, schemaErrors, username } = input; + const { update } = createSubmissionRepository(dependencies); + const newStatusSubmission = + Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID; + // Update with new data + const updatedActiveSubmissionId = await update(idActiveSubmission, { + data: submissionData, + status: newStatusSubmission, + dictionaryId: dictionaryId, + updatedBy: username, + errors: schemaErrors, + }); + + logger.info( + LOG_MODULE, + `Updated Active submission '${updatedActiveSubmissionId}' with status '${newStatusSubmission}'`, + ); + return updatedActiveSubmissionId; + }; + + /** + * Void function to process and validate uploaded files on an Active Submission. + * Performs the schema data validation combined with all Submitted Data. + * @param {Record} files Uploaded files to be processed + * @param {Object} params + * @param {number} params.categoryId Category Identifier + * @param {string} params.organization Organization name + * @param {SchemasDictionary} params.schemasDictionary Dictionary to parse files with + * @param {string} params.username User who performs the action + * @returns {void} + */ + const addFilesToSubmissionAsync = async (files: Record, params: ValidateFilesParams) => { + for (const [fileName, fileInfo] of Object.entries(files)) { + const sizeString = bytes.format(fileInfo.size, { decimalPlaces: 2 }); + logger.info(`Processing file '${fileName}' size '${sizeString}'`); + } + + // TODO: This only gets a summary, we need to insert data into an active submission so we need all the insert statements. + const submissionRepository = createSubmissionRepository(dependencies); + + const { categoryId, organization, username, schemasDictionary } = params; + + try { + // Parse file data + const filesDataProcessed = await submissionInsertDataFromFiles(files, schemasDictionary); + + // Get Active Submission from database + const activeSubmission = await submissionRepository.getActiveSubmissionDetails({ + categoryId, + username, + organization, + }); + if (!activeSubmission) { + throw new BadRequest(`Submission '${activeSubmission}' not found`); + } + + // Merge Active Submission data with incoming TSV file data processed + const insertActiveSubmissionData = mergeInsertsRecords(activeSubmission.data.inserts ?? {}, filesDataProcessed); + + // Perform Schema Data validation Async. + await performDataValidation({ + username, + submissionId: activeSubmission.id, + submissionData: { + inserts: insertActiveSubmissionData, + deletes: activeSubmission.data.deletes, + updates: activeSubmission.data.updates, + }, + }); + } catch (error) { + logger.error( + `There was an error processing files: ${Object.entries(files).map(([entityName]) => entityName)}`, + JSON.stringify(error), + ); + } + logger.info( + `Finished addFilesToSubmissionAsync for active submission in category "${params.categoryId}" for organization "${params.organization}" submitted by user "${params.username}"`, + ); }; return { - processEditRecordsAsync, performCommitSubmissionAsync, performDataValidation, - updateActiveSubmission, + processEditRecordsAsync, processInsertRecordsAsync, + updateActiveSubmission, + addFilesToSubmissionAsync, }; }; -export default processor; +export default submissionProcessor; diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts new file mode 100644 index 00000000..e595d2af --- /dev/null +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -0,0 +1,631 @@ +import * as _ from 'lodash-es'; + +import { Dictionary as SchemasDictionary } from '@overture-stack/lectern-client'; +import { + type NewSubmission, + type SubmissionRecordErrorDetails, + type SubmissionUpdateData, +} from '@overture-stack/lyric-data-model/models'; + +import { BaseDependencies } from '../../config/config.js'; +import systemIdGenerator from '../../external/systemIdGenerator.js'; +import createSubmissionRepository from '../../repository/activeSubmissionRepository.js'; +import createCategoryRepository from '../../repository/categoryRepository.js'; +import createSubmittedDataRepository from '../../repository/submittedRepository.js'; +import { getSchemaByName } from '../../utils/dictionaryUtils.js'; +import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; +import { filterAndPaginateSubmissionData, type FlattenedSubmissionData } from '../../utils/submissionResponseParser.js'; +import { + checkEntityFieldNames, + checkFileNames, + createSubmissionSummaryResponse, + isSubmissionActive, + removeItemsFromSubmission, +} from '../../utils/submissionUtils.js'; +import { + CommitSubmissionResult, + CREATE_SUBMISSION_STATUS, + type CreateSubmissionJsonResult, + type SubmitFileResult, + type DeleteSubmissionResult, + type EntityData, + type PaginationOptions, + SUBMISSION_ACTION_TYPE, + SUBMISSION_STATUS, + type SubmissionActionType, + SubmissionSummary, +} from '../../utils/types.js'; +import { default as createSubmissionProcessor } from './submissionProcessor.js'; + +const submissionService = (dependencies: BaseDependencies) => { + const LOG_MODULE = 'SUBMISSION_SERVICE'; + const { logger, onFinishCommit } = dependencies; + + const categoryRepository = createCategoryRepository(dependencies); + const submissionProcessor = createSubmissionProcessor(dependencies); + const submissionRepository = createSubmissionRepository(dependencies); + const submittedDataRepository = createSubmittedDataRepository(dependencies); + + const { generateIdentifier } = systemIdGenerator(dependencies); + + /** + * Runs Schema validation asynchronously and moves the Active Submission to Submitted Data + * @param {number} categoryId + * @param {number} submissionId + * @returns {Promise} + */ + const commitSubmission = async ( + categoryId: number, + submissionId: number, + username: string, + ): Promise => { + const { getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; + const { getActiveDictionaryByCategory } = categoryRepository; + + const submission = await submissionRepository.getSubmissionDetailsById(submissionId); + if (!submission) { + throw new BadRequest(`Submission '${submissionId}' not found`); + } + + if (submission.dictionaryCategory.id !== categoryId) { + throw new BadRequest(`Category ID provided does not match the category for the Submission`); + } + + if (submission.status !== SUBMISSION_STATUS.VALID) { + throw new StatusConflict('Submission does not have status VALID and cannot be committed'); + } + + const currentDictionary = await getActiveDictionaryByCategory(categoryId); + if (_.isEmpty(currentDictionary)) { + throw new BadRequest(`Dictionary in category '${categoryId}' not found`); + } + + const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization( + categoryId, + submission?.organization, + ); + + const entitiesToProcess = new Set(); + + submittedDataToValidate?.forEach((data) => entitiesToProcess.add(data.entityName)); + + const insertsToValidate = submission.data?.inserts + ? Object.entries(submission.data.inserts).flatMap(([entityName, submissionData]) => { + entitiesToProcess.add(entityName); + + return submissionData.records.map((record) => ({ + data: record, + dictionaryCategoryId: categoryId, + entityName, + isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise + organization: submission.organization, + originalSchemaId: currentDictionary.id, + systemId: generateIdentifier(entityName, record), + createdBy: username, + })); + }) + : []; + + const deleteDataArray = submission.data?.deletes + ? Object.entries(submission.data.deletes).flatMap(([entityName, submissionDeleteData]) => { + entitiesToProcess.add(entityName); + return submissionDeleteData; + }) + : []; + + const updateDataArray = + submission.data?.updates && + Object.entries(submission.data.updates).reduce>( + (acc, [entityName, submissionUpdateData]) => { + entitiesToProcess.add(entityName); + submissionUpdateData.forEach((record) => { + acc[record.systemId] = record; + }); + return acc; + }, + {}, + ); + + // To Commit Active Submission we need to validate SubmittedData + Active Submission + submissionProcessor.performCommitSubmissionAsync({ + dataToValidate: { + inserts: insertsToValidate, + submittedData: submittedDataToValidate, + deletes: deleteDataArray, + updates: updateDataArray, + }, + submissionId: submission.id, + dictionary: currentDictionary, + username: username, + onFinishCommit, + }); + + return { + status: CREATE_SUBMISSION_STATUS.PROCESSING, + dictionary: { + name: currentDictionary.name, + version: currentDictionary.version, + }, + processedEntities: Array.from(entitiesToProcess.values()), + }; + }; + + /** + * Updates Submission status to CLOSED + * This action is allowed only if current Submission Status as OPEN, VALID or INVALID + * Returns the resulting ID of the Submission + * @param {number} submissionId + * @param {string} username + * @returns {Promise} + */ + const deleteActiveSubmissionById = async ( + submissionId: number, + username: string, + ): Promise => { + const submission = await submissionRepository.getSubmissionById(submissionId); + if (!submission) { + throw new BadRequest(`Submission '${submissionId}' not found`); + } + + if (!isSubmissionActive(submission.status)) { + throw new StatusConflict('Submission is not active. Only Active Submission can be deleted'); + } + + const updatedRecordId = await submissionRepository.update(submission.id, { + status: SUBMISSION_STATUS.CLOSED, + updatedBy: username, + }); + + logger.info(LOG_MODULE, `Submission '${submissionId}' updated with new status '${SUBMISSION_STATUS.CLOSED}'`); + + return { + status: SUBMISSION_STATUS.CLOSED, + description: 'Submission closed successfully', + submissionId: updatedRecordId, + }; + }; + + /** + * Function to remove an entity from an Active Submission by given Submission ID + * It validates resulting Active Submission running cross schema validation along with the existing Submitted Data + * Returns the resulting ID of the Active Submission + * @param {number} submissionId + * @param {string} entityName + * @param {string} username + * @returns { Promise} + */ + const deleteActiveSubmissionEntity = async ( + submissionId: number, + username: string, + filter: { + actionType: SubmissionActionType; + entityName: string; + index: number | null; + }, + ): Promise => { + const submission = await submissionRepository.getSubmissionDetailsById(submissionId); + if (!submission) { + throw new BadRequest(`Submission '${submissionId}' not found`); + } + + if (!isSubmissionActive(submission.status)) { + throw new StatusConflict('Submission is not active. Only Active Submission can be modified'); + } + + if ( + SUBMISSION_ACTION_TYPE.Values.INSERTS.includes(filter.actionType) && + !_.has(submission.data.inserts, filter.entityName) + ) { + throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); + } + + if ( + SUBMISSION_ACTION_TYPE.Values.UPDATES.includes(filter.actionType) && + !_.has(submission.data.updates, filter.entityName) + ) { + throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); + } + + if ( + SUBMISSION_ACTION_TYPE.Values.DELETES.includes(filter.actionType) && + !_.has(submission.data.deletes, filter.entityName) + ) { + throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); + } + + // Remove entity from the Submission + const updatedActiveSubmissionData = removeItemsFromSubmission(submission.data, { + ...filter, + }); + + // Validate and update Active Submission after removing the entity + submissionProcessor.performDataValidation({ + submissionId: submission.id, + submissionData: updatedActiveSubmissionData, + username, + }); + + logger.info(LOG_MODULE, `Submission '${submission.id}' updated after removing entity '${filter.entityName}'`); + + return { + status: CREATE_SUBMISSION_STATUS.PROCESSING, + description: 'Submission records are being processed', + submissionId: submission.id, + }; + }; + + /** + * Get Submissions by Category + * @param {number} categoryId - The ID of the category for which data is being fetched. + * @param {Object} paginationOptions - Pagination properties + * @param {number} paginationOptions.page - Page number + * @param {number} paginationOptions.pageSize - Items per page + * @param {Object} filterOptions + * @param {boolean} filterOptions.onlyActive - Filter by Active status + * @param {string} filterOptions.username - User Name + * @returns an array of Submission + */ + + const getSubmissionsByCategory = async ( + categoryId: number, + paginationOptions: PaginationOptions, + filterOptions: { + onlyActive: boolean; + username?: string; + organization?: string; + }, + ): Promise<{ + result: SubmissionSummary[]; + metadata: { totalRecords: number; errorMessage?: string }; + }> => { + const recordsPaginated = await submissionRepository.getSubmissionsByCategory( + categoryId, + paginationOptions, + filterOptions, + ); + if (!recordsPaginated || recordsPaginated.length === 0) { + return { + result: [], + metadata: { + totalRecords: 0, + }, + }; + } + + const totalRecords = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions); + return { + metadata: { + totalRecords, + }, + result: recordsPaginated.map((response) => createSubmissionSummaryResponse(response)), + }; + }; + + /** + * Get Submission by Submission ID + * @param {number} submissionId A Submission ID + * @returns One Submission + */ + const getSubmissionById = async (submissionId: number) => { + const submission = await submissionRepository.getSubmissionById(submissionId); + if (_.isEmpty(submission)) { + return; + } + + return createSubmissionSummaryResponse(submission); + }; + + /** + * Get Submission Records paginated + * @param {number} submissionId A Submission ID + * @param {Object} paginationOptions - Pagination properties + * @param {number} paginationOptions.page - Page number + * @param {number} paginationOptions.pageSize - Items per page + * @param {Object} filterOptions + * @param {string} filterOptions.entityName - Filter by Entity name + * @param {string} filterOptions.actionType - Filter by Action type + * @returns One Submission + */ + const getSubmissionDetailsById = async ({ + submissionId, + paginationOptions, + filterOptions, + }: { + submissionId: number; + paginationOptions: PaginationOptions; + filterOptions: { entityNames: string[]; actionTypes: SubmissionActionType[] }; + }): Promise<{ data: FlattenedSubmissionData[]; errors?: SubmissionRecordErrorDetails[] }> => { + const submission = await submissionRepository.getSubmissionDetailsById(submissionId); + if (!submission) { + throw new BadRequest(`Submission '${submissionId}' not found`); + } + + const submissionEntityNames = [ + ...Object.keys(submission.data.inserts ?? {}), + ...Object.keys(submission.data.updates ?? {}), + ...Object.keys(submission.data.deletes ?? {}), + ]; + + const missingEntityNames = filterOptions.entityNames.filter((name) => !submissionEntityNames.includes(name)); + + if (filterOptions.entityNames.length > 0 && missingEntityNames.length > 0) { + throw new BadRequest( + `Invalid entity name(s) '${missingEntityNames.join(', ')}' for Submission '${submissionId}'`, + ); + } + + return filterAndPaginateSubmissionData({ + data: submission.data, + errors: submission.errors || {}, + filterOptions, + paginationOptions, + }); + }; + + /** + * Get an active Submission by Organization + * @param {Object} params + * @param {number} params.categoryId + * @param {string} params.username + * @param {string} params.organization + * @returns One Active Submission + */ + const getActiveSubmissionByOrganization = async ({ + categoryId, + username, + organization, + }: { + categoryId: number; + username: string; + organization: string; + }): Promise => { + const submission = await submissionRepository.getActiveSubmissionSummary({ + organization, + username, + categoryId, + }); + if (_.isEmpty(submission)) { + return; + } + + return createSubmissionSummaryResponse(submission); + }; + + /** + * Find the current Active Submission or Create an Open Active Submission with initial values and no schema data. + * @param {object} params + * @param {string} params.username Owner of the Submission + * @param {number} params.categoryId Category ID of the Submission + * @param {string} params.organization Organization name + * @returns number ID of the Active Submission + */ + const getOrCreateActiveSubmission = async (params: { + username: string; + categoryId: number; + organization: string; + }): Promise => { + const { categoryId, username, organization } = params; + const { getActiveDictionaryByCategory } = categoryRepository; + + const activeSubmission = await submissionRepository.getActiveSubmissionSummary({ + categoryId, + username, + organization, + }); + if (activeSubmission) { + return activeSubmission.id; + } + + const currentDictionary = await getActiveDictionaryByCategory(categoryId); + + if (!currentDictionary) { + throw new InternalServerError(`Dictionary in category '${categoryId}' not found`); + } + + const newSubmissionInput: NewSubmission = { + createdBy: username, + data: {}, + dictionaryCategoryId: categoryId, + dictionaryId: currentDictionary.id, + errors: {}, + organization: organization, + status: SUBMISSION_STATUS.OPEN, + }; + + return submissionRepository.save(newSubmissionInput); + }; + + /** + * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database + * @param {object} params + * @param {Record[]} params.records An array of records + * @param {string} params.entityName Entity Name of the Records + * @param {number} params.categoryId Category ID of the Submission + * @param {string} params.organization Organization name + * @param {string} params.username User name creating the Submission + * @returns The Active Submission created or Updated + */ + const submit = async ({ + data, + categoryId, + organization, + username, + }: { + data: EntityData; + categoryId: number; + organization: string; + username: string; + }): Promise => { + const entityNames = Object.keys(data); + logger.info( + LOG_MODULE, + `Processing '${entityNames.length}' entities on category id '${categoryId}' organization '${organization}'`, + ); + if (entityNames.length === 0) { + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: 'No valid data for submission', + }; + } + + const currentDictionary = await categoryRepository.getActiveDictionaryByCategory(categoryId); + + if (_.isEmpty(currentDictionary)) { + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: `Dictionary in category '${categoryId}' not found`, + }; + } + + const schemasDictionary: SchemasDictionary = { + name: currentDictionary.name, + version: currentDictionary.version, + schemas: currentDictionary.schemas, + }; + + // Validate entity name + const invalidEntities = entityNames.filter((name) => !getSchemaByName(name, schemasDictionary)); + if (invalidEntities.length) { + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: `Invalid entity name '${invalidEntities}' for submission`, + }; + } + + // Get Active Submission or Open a new one + const activeSubmissionId = await getOrCreateActiveSubmission({ categoryId, username, organization }); + + // Schema validation runs asynchronously and does not block execution. + // The results will be saved to the database. + submissionProcessor.processInsertRecordsAsync({ + records: data, + submissionId: activeSubmissionId, + schemasDictionary, + username, + }); + + return { + status: CREATE_SUBMISSION_STATUS.PROCESSING, + description: 'Submission records are being processed', + submissionId: activeSubmissionId, + }; + }; + + /** + * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database + * @param {object} params + * @param {Express.Multer.File[]} params.files An array of files + * @param {number} params.categoryId Category ID of the Submission + * @param {string} params.organization Organization name + * @param {string} params.username User name creating the Submission + * @returns The Active Submission created or Updated + */ + const submitFiles = async ({ + files, + categoryId, + organization, + username, + }: { + files: Express.Multer.File[]; + categoryId: number; + organization: string; + username: string; + }): Promise => { + logger.info(LOG_MODULE, `Processing '${files.length}' files on category id '${categoryId}'`); + + if (files.length === 0) { + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: 'No valid files for submission', + batchErrors: [], + inProcessEntities: [], + }; + } + + const currentDictionary = await categoryRepository.getActiveDictionaryByCategory(categoryId); + + if (_.isEmpty(currentDictionary)) { + throw new BadRequest(`Dictionary in category '${categoryId}' not found`); + } + + const schemasDictionary: SchemasDictionary = { + name: currentDictionary.name, + version: currentDictionary.version, + schemas: currentDictionary.schemas, + }; + + // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates) + // TODO: Use filename map to identify files, concatenate records if multiple files map to same entity + const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name); + const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames); + + if (_.isEmpty(validFileEntity)) { + logger.info(LOG_MODULE, `No valid files for submission`); + } + + // step 2 Validation. Validate fieldNames (missing required fields based on schema) + const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity); + + const batchErrors = [...fileNamesErrors, ...fieldNameErrors]; + const entitiesToProcess = Object.keys(checkedEntities); + + if (_.isEmpty(checkedEntities)) { + logger.info(LOG_MODULE, 'Found errors on Submission files.', JSON.stringify(batchErrors)); + return { + status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, + description: 'No valid entities in submission', + batchErrors, + inProcessEntities: entitiesToProcess, + }; + } + + // Get Active Submission or Open a new one + const activeSubmissionId = await getOrCreateActiveSubmission({ categoryId, username, organization }); + + // TODO: Add files to submission, then run validation separately. Currently these processes are both + // done by the function that adds the files to the submission. + + // Start background process of adding files to submission + // Running Schema validation in the background do not need to wait + // Result of validations will be stored in database + submissionProcessor.addFilesToSubmissionAsync(checkedEntities, { + schemasDictionary, + categoryId, + organization, + username, + }); + + if (batchErrors.length === 0) { + return { + status: CREATE_SUBMISSION_STATUS.PROCESSING, + description: 'Submission files are being processed', + submissionId: activeSubmissionId, + batchErrors, + inProcessEntities: entitiesToProcess, + }; + } + + return { + status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION, + description: 'Some Submission files are being processed while others were unable to process', + submissionId: activeSubmissionId, + batchErrors, + inProcessEntities: entitiesToProcess, + }; + }; + + return { + commitSubmission, + deleteActiveSubmissionById, + deleteActiveSubmissionEntity, + getSubmissionsByCategory, + getSubmissionById, + getSubmissionDetailsById, + getActiveSubmissionByOrganization, + getOrCreateActiveSubmission, + submitJson: submit, + submitFiles, + }; +}; + +export default submissionService; diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts index 2147fe6f..13089ed0 100644 --- a/packages/data-provider/src/services/submittedData/submmittedData.ts +++ b/packages/data-provider/src/services/submittedData/submmittedData.ts @@ -23,8 +23,8 @@ import { VIEW_TYPE, type ViewType, } from '../../utils/types.js'; -import processor from '../submission/processor.js'; -import submissionService from '../submission/submission.js'; +import submissionProcessor from '../submission/submissionProcessor.js'; +import submissionService from '../submission/submissionService.js'; import searchDataRelations from './searchDataRelations.js'; import viewMode from './viewMode.js'; @@ -54,7 +54,7 @@ const submittedData = (dependencies: BaseDependencies) => { const { getActiveDictionaryByCategory } = categoryRepository(dependencies); const { getSubmissionDetailsById } = submissionRepository(dependencies); const { getOrCreateActiveSubmission } = submissionService(dependencies); - const { performDataValidation } = processor(dependencies); + const { performDataValidation } = submissionProcessor(dependencies); // get SubmittedData by SystemId const foundRecordToDelete = await getSubmittedDataBySystemId(systemId); @@ -171,12 +171,12 @@ const submittedData = (dependencies: BaseDependencies) => { ); const { getActiveDictionaryByCategory } = categoryRepository(dependencies); const { getOrCreateActiveSubmission } = submissionService(dependencies); - const { processEditRecordsAsync } = processor(dependencies); + const { processEditRecordsAsync } = submissionProcessor(dependencies); if (records.length === 0) { return { status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: 'No valid records for submission', + description: 'No valid records provided.', }; } diff --git a/packages/data-provider/src/utils/fileUtils.ts b/packages/data-provider/src/utils/fileUtils.ts new file mode 100644 index 00000000..f62cd3b4 --- /dev/null +++ b/packages/data-provider/src/utils/fileUtils.ts @@ -0,0 +1,237 @@ +import bytes, { type Unit } from 'bytes'; +import { parse as csvParse } from 'csv-parse'; +import firstline from 'firstline'; +import fs from 'fs'; +import { z } from 'zod'; + +import { + type DataRecord, + parse, + type ParseSchemaError, + type Schema, + type UnprocessedDataRecord, +} from '@overture-stack/lectern-client'; + +import { BATCH_ERROR_TYPE, type BatchError } from './types.js'; + +export const SUPPORTED_FILE_EXTENSIONS = z.enum(['tsv', 'csv']); +export type SupportedFileExtensions = z.infer; + +export const columnSeparatorValue = { + tsv: '\t', + csv: ',', +} as const satisfies Record; + +/** + * Extracts the extension from the filename and returns it if it's supported. + * Otherwise it returns undefined. + * @param {string} fileName + * @returns {SupportedFileExtensions | undefined} + */ +export const extractFileExtension = (fileName: string): SupportedFileExtensions | undefined => { + // Extract the file extension + const fileExtension = fileName.split('.').pop()?.toLowerCase(); + + try { + // Parse to validate the extension against the Zod enum + return SUPPORTED_FILE_EXTENSIONS.parse(fileExtension); + } catch (error) { + return; + } +}; + +/** + * Determines the separator character for a given file based on its extension. + * @param fileName The name of the file whose extension determines the separator character. + * @returns The separator character associated with the file extension, or `undefined` if + * the file extension is invalid or unrecognized. + */ +export const getSeparatorCharacter = (fileName: string): string | undefined => { + const fileExtension = extractFileExtension(fileName); + if (fileExtension) { + return columnSeparatorValue[fileExtension]; + } + return; +}; + +/** + * Maps a record array to an object with keys from headers, formatting each value for compatibility. + * @param headers An array of header names, used as keys for the returned object. + * @param record An array of values corresponding to each header, to be formatted and mapped. + * @returns An `UnprocessedDataRecord` object where each header in `headers` is a key, + * and each value is the corresponding entry in `record` formatted for compatibility. + */ +export const mapRecordToHeaders = (headers: string[], record: string[]) => { + return headers.reduce((obj: UnprocessedDataRecord, nextKey, index) => { + const dataStr = record[index] || ''; + const formattedData = formatForExcelCompatibility(dataStr); + obj[nextKey] = formattedData; + return obj; + }, {}); +}; + +/** + * Reads only first line of the file + * Usefull when file is too large and we're only interested in column names + * @param file A file we want to read + * @returns a string with the content of the first line of the file + */ +export const readHeaders = async (file: Express.Multer.File) => { + return firstline(file.path); +}; + +/** + * Reads a text file and parse it to a JSON format. + * Records are parsed to match schema field types. + * Supported files: .tsv and .csv + * @param {Express.Multer.File} file A file to read + * @param {Schema} schema Schema to parse data with + * @returns a JSON format objet + */ +export const readTextFile = async ( + file: Express.Multer.File, + schema: Schema, +): Promise<{ records: DataRecord[]; errors?: ParseSchemaError[] }> => { + const returnRecords: DataRecord[] = []; + const returnErrors: ParseSchemaError[] = []; + const separatorCharacter = getSeparatorCharacter(file.originalname); + if (!separatorCharacter) { + throw new Error('Invalid file Extension'); + } + + let headers: string[] = []; + let lineNumber = 0; + + return new Promise((resolve, reject) => { + const stream = fs.createReadStream(file.path).pipe(csvParse({ delimiter: separatorCharacter })); + + stream.on('data', (record: string[]) => { + lineNumber++; + if (!headers.length) { + headers = Object.values(record); + } else { + const mappedRecord = mapRecordToHeaders(headers, record); + + try { + const parseSchemaResult = parse.parseRecordValues(mappedRecord, schema); + if (parseSchemaResult.success) { + returnRecords.push(parseSchemaResult.data.record); + } else { + returnRecords.push(parseSchemaResult.data.record); + returnErrors.push({ + recordErrors: parseSchemaResult.data.errors, + recordIndex: lineNumber, + }); + } + + if (lineNumber % 1000 === 0) { + // TODO: Add batch processing logic here (e.g., write to database or process as needed) + // returnRecords = []; // Clear the array after processing the batch + // returnErrors = []; // Clear the array after processing the batch + } + } catch (error) { + console.error(`Catching error parsing data: ${error}`); + } + } + }); + + stream.on('end', () => { + resolve({ records: returnRecords, errors: returnErrors }); + }); + + stream.on('close', () => { + stream.destroy(); + fs.unlink(file.path, () => {}); + }); + + stream.on('error', () => { + reject({ records: returnRecords, errors: returnErrors }); + }); + }); +}; + +function formatForExcelCompatibility(data: string) { + // tsv exported from excel might add double quotations to indicate string and escape double quotes + // this function removes those extra double quatations from a given string + + return data + .trim() + .replace(/^"/, '') // excel might add a beginning double quotes to indicate string + .replace(/"$/, '') // excel might add a trailing double quote to indicate string + .replace(/""/g, '"') // excel might've used a second double quote to escape a double quote in a string + .trim(); +} + +export function getSizeInBytes(size: string | number): number { + // Parse the string value into an integer in bytes. + // If value is a number it is assumed is in bytes. + return bytes.parse(size) || 0; +} + +/** + * Formats a file size from bytes to a specified unit with a defined precision. + * + * @param sizeInBytes - The file size in bytes to be formatted. + * @param unit - The unit to which the size should be converted (e.g., 'MB', 'GB'). + * @param precision - The number of decimal places to include in the formatted output. + * @returns The file size formatted as a string in the specified unit with the given precision. Returns null if sizeInBytes is not a Finite number. + * + */ +export const formatByteSize = (sizeInBytes: number, unit: Unit, precision: number): string | null => { + // Returns null if sizeInBytes is not Finite + return bytes.format(sizeInBytes, { unit, decimalPlaces: precision }); +}; + +type FileProcessingResult = { + validFiles: Express.Multer.File[]; + fileErrors: BatchError[]; +}; + +/** + * Processes an array of uploaded files, filtering valid `.tsv` files and checking for required headers + * + * @param {Express.Multer.File[]} files An array of `Express.Multer.File` objects representing the uploaded files. + * @returns A `Promise` that resolves to an object containing two arrays: + * - `validFiles`: Files that have a `.tsv` extension and contain the `systemId` header. + * - `fileErrors`: Files that either have an invalid extension or are missing the required `systemId` header. + */ +export async function processFiles(files: Express.Multer.File[]): Promise { + const result: FileProcessingResult = { + validFiles: [], + fileErrors: [], + }; + + for (const file of files) { + try { + if (extractFileExtension(file.originalname)) { + const fileHeaders = await readHeaders(file); // Wait for the async operation + if (fileHeaders.includes('systemId')) { + result.validFiles.push(file); + } else { + const batchError: BatchError = { + type: BATCH_ERROR_TYPE.MISSING_REQUIRED_HEADER, + message: `File '${file.originalname}' is missing the column 'systemId'`, + batchName: file.originalname, + }; + result.fileErrors.push(batchError); + } + } else { + const batchError: BatchError = { + type: BATCH_ERROR_TYPE.INVALID_FILE_EXTENSION, + message: `File '${file.originalname}' has invalid file extension. File extension must be '${SUPPORTED_FILE_EXTENSIONS.options}'`, + batchName: file.originalname, + }; + result.fileErrors.push(batchError); + } + } catch (error) { + const batchError: BatchError = { + type: BATCH_ERROR_TYPE.FILE_READ_ERROR, + message: `Error reading file '${file.originalname}'`, + batchName: file.originalname, + }; + result.fileErrors.push(batchError); + } + } + + return result; +} diff --git a/packages/data-provider/src/utils/requestValidation.ts b/packages/data-provider/src/utils/requestValidation.ts index 68eb1337..7cf1631f 100644 --- a/packages/data-provider/src/utils/requestValidation.ts +++ b/packages/data-provider/src/utils/requestValidation.ts @@ -1,15 +1,15 @@ import { NextFunction, Request, Response } from 'express'; import type { ParamsDictionary, RequestHandler } from 'express-serve-static-core'; import type { ParsedQs } from 'qs'; -import { ZodError, ZodSchema } from 'zod'; +import { ZodError, ZodSchema, type ZodType } from 'zod'; import type { UserSession } from '../middleware/auth.js'; import { BadRequest, InternalServerError } from './errors.js'; export declare type RequestValidation = { - body?: ZodSchema; - query?: ZodSchema; - pathParams?: ZodSchema; + body?: ZodType; + query?: ZodType; + pathParams?: ZodType; }; type RequestWithUser< diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts index c7b7b1c7..bfa0aa92 100644 --- a/packages/data-provider/src/utils/schemas.ts +++ b/packages/data-provider/src/utils/schemas.ts @@ -127,7 +127,7 @@ const submissionIdSchema = z const stringNotEmpty = z.string().trim().min(1); // Common Category Path Params -export interface categoryPathParams extends ParamsDictionary { +export interface CategoryPathParams extends ParamsDictionary { categoryId: string; } @@ -136,16 +136,19 @@ export const categoryPathParamsSchema = z.object({ }); // Common Category and Organization Path Params - -export interface categoryOrganizationPathParams extends ParamsDictionary { - categoryId: string; - organization: string; -} - export const categoryOrganizationPathParamsSchema = z.object({ categoryId: categoryIdSchema, organization: organizationSchema, }); +export type CategoryOrganizationPathParams = z.infer; + +// Common Category, Organization, and EntityName Path Params +export const categoryOrganizationEntityPathParamsSchema = z.object({ + categoryId: categoryIdSchema, + organizationId: organizationSchema, + entityName: entityNameSchema, +}); +export type CategoryOrganizationEntityPathParams = z.infer; // Common Submission Path Params @@ -159,7 +162,7 @@ const submissionIdPathParamSchema = z.object({ // Common Pagination Query Params -export interface paginationQueryParams extends ParsedQs { +export interface PaginationQueryParams extends ParsedQs { page?: string; pageSize?: string; } @@ -171,7 +174,7 @@ const paginationQuerySchema = z.object({ // Audit Request -export interface auditQueryParams extends ParsedQs { +export interface AuditQueryParams extends ParsedQs { entityName?: string; eventType?: string; systemId?: string; @@ -191,8 +194,8 @@ const auditQuerySchema = z export const auditByCatAndOrgRequestSchema: RequestValidation< object, - paginationQueryParams & auditQueryParams, - categoryOrganizationPathParams + PaginationQueryParams & AuditQueryParams, + CategoryOrganizationPathParams > = { query: auditQuerySchema, pathParams: categoryOrganizationPathParamsSchema, @@ -200,13 +203,13 @@ export const auditByCatAndOrgRequestSchema: RequestValidation< // Category Request -export const cagegoryDetailsRequestSchema: RequestValidation = { +export const categoryDetailsRequestSchema: RequestValidation = { pathParams: categoryPathParamsSchema, }; // Dictionary Request -export interface dictionaryRegisterBodyParams { +export interface DictionaryRegisterBodyParams { categoryName: string; dictionaryName: string; dictionaryVersion: string; @@ -214,7 +217,7 @@ export interface dictionaryRegisterBodyParams { } export const dictionaryRegisterRequestSchema: RequestValidation< - dictionaryRegisterBodyParams, + DictionaryRegisterBodyParams, ParsedQs, ParamsDictionary > = { @@ -228,7 +231,7 @@ export const dictionaryRegisterRequestSchema: RequestValidation< // Submission Requests -export interface submissionsByCategoryQueryParams extends paginationQueryParams { +export interface SubmissionsByCategoryQueryParams extends PaginationQueryParams { onlyActive?: string; organization?: string; username?: string; @@ -236,8 +239,8 @@ export interface submissionsByCategoryQueryParams extends paginationQueryParams export const submissionsByCategoryRequestSchema: RequestValidation< object, - submissionsByCategoryQueryParams, - categoryPathParams + SubmissionsByCategoryQueryParams, + CategoryPathParams > = { query: z.object({ onlyActive: booleanSchema.default('false'), @@ -250,14 +253,14 @@ export const submissionsByCategoryRequestSchema: RequestValidation< export const submissionByIdRequestSchema: RequestValidation = { pathParams: submissionIdPathParamSchema, }; -export interface submissionsDetailsQueryParams extends paginationQueryParams { +export interface SubmissionsDetailsQueryParams extends PaginationQueryParams { entityNames?: string | string[]; actionTypes?: string | string[]; } export const submissionDetailsRequestSchema: RequestValidation< object, - submissionsDetailsQueryParams, + SubmissionsDetailsQueryParams, submissionIdPathParam > = { query: z @@ -272,7 +275,7 @@ export const submissionDetailsRequestSchema: RequestValidation< export const submissionActiveByOrganizationRequestSchema: RequestValidation< object, ParsedQs, - categoryOrganizationPathParams + CategoryOrganizationPathParams > = { pathParams: categoryOrganizationPathParamsSchema, }; @@ -293,20 +296,20 @@ export const submissionDeleteRequestSchema: RequestValidation = { query: z.object({ entityName: entityNameSchema, @@ -318,10 +321,17 @@ export const submissionDeleteEntityNameRequestSchema: RequestValidation< }), }; -export interface uploadSubmissionRequestQueryParams extends ParsedQs { - entityName: string; - organization: string; -} +export type UploadSubmissionPathParams = { + organizationId: string; + categoryId: string; +}; + +export const filenameEntityPair = z.object({ + filename: z.string(), + entity: z.string(), +}); + +export type FilenameEntityPair = z.infer; const dataRecordValueSchema = z.union([ z.string(), @@ -333,61 +343,68 @@ const dataRecordValueSchema = z.union([ z.undefined(), ]); +const dataRecordSchema = z.record(dataRecordValueSchema); + export const uploadSubmissionRequestSchema: RequestValidation< - Array, - uploadSubmissionRequestQueryParams, - categoryPathParams + unknown, + Record, + UploadSubmissionPathParams > = { - body: z.record(dataRecordValueSchema).array(), - pathParams: categoryPathParamsSchema, - query: z.object({ - entityName: entityNameSchema, - organization: organizationSchema, + pathParams: z.object({ + categoryId: categoryIdSchema, + organizationId: organizationSchema, }), + // body: z.array(filenameEntityPair).optional(), +}; + +export const uploadSingleEntitySubmissionDataRequestSchema: RequestValidation< + Array, + Record, + CategoryOrganizationEntityPathParams +> = { + body: dataRecordSchema.array(), + pathParams: categoryOrganizationEntityPathParamsSchema, }; // Submitted Data -export interface dataDeleteBySystemIdPathParams extends ParamsDictionary { +export interface DataDeleteBySystemIdPathParams extends ParamsDictionary { systemId: string; categoryId: string; } -export const dataDeleteBySystemIdRequestSchema: RequestValidation = { +export const dataDeleteBySystemIdRequestSchema: RequestValidation = { pathParams: z.object({ systemId: stringNotEmpty, categoryId: categoryIdSchema, }), }; -export interface dataEditRequestSchemaQueryParams extends ParsedQs { +export interface DataEditRequestSchemaQueryParams extends ParsedQs { entityName: string; organization: string; } -export const dataEditRequestSchema: RequestValidation< +// TODO: Need type validation for the edit request schema +export const editSingleEntityRequestSchema: RequestValidation< Array>, - dataEditRequestSchemaQueryParams, - categoryPathParams + Record, + CategoryOrganizationEntityPathParams > = { body: z.record(z.unknown()).array(), - pathParams: categoryPathParamsSchema, - query: z.object({ - entityName: entityNameSchema, - organization: organizationSchema, - }), + pathParams: categoryOrganizationEntityPathParamsSchema, }; -export interface dataQueryParams extends paginationQueryParams { +export interface DataQueryParams extends PaginationQueryParams { entityName?: string | string[]; view?: string; } -export interface getDataQueryParams extends ParsedQs { +export interface GetDataQueryParams extends ParsedQs { view?: string; } -export const dataGetByCategoryRequestSchema: RequestValidation = { +export const dataGetByCategoryRequestSchema: RequestValidation = { query: z .object({ entityName: z.union([entityNameSchema, entityNameSchema.array()]).optional(), @@ -408,8 +425,8 @@ export const dataGetByCategoryRequestSchema: RequestValidation = { query: z .object({ @@ -429,7 +446,7 @@ export const dataGetByOrganizationRequestSchema: RequestValidation< pathParams: categoryOrganizationPathParamsSchema, }; -export const dataGetByQueryRequestSchema: RequestValidation = { +export const dataGetByQueryRequestSchema: RequestValidation = { body: sqonSchema, query: z .object({ @@ -439,15 +456,15 @@ export const dataGetByQueryRequestSchema: RequestValidation = { query: z.object({ view: viewSchema.optional(), @@ -471,7 +488,7 @@ export const validationPathParamsSchema = z.object({ entityName: entityNameSchema, }); -export interface validationPathParams extends ParamsDictionary { +export interface ValidationPathParams extends ParamsDictionary { categoryId: string; entityName: string; } @@ -480,12 +497,12 @@ const validationQuerySchema = z.object({ organization: organizationSchema, value: stringNotEmpty, }); -export interface validationQueryParam extends ParsedQs { +export interface ValidationQueryParam extends ParsedQs { organization: string; value: string; } -export const validationRequestSchema: RequestValidation = { +export const validationRequestSchema: RequestValidation = { query: validationQuerySchema, pathParams: validationPathParamsSchema, }; diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts index 4f656d1e..d8890c0c 100644 --- a/packages/data-provider/src/utils/submissionUtils.ts +++ b/packages/data-provider/src/utils/submissionUtils.ts @@ -24,6 +24,8 @@ import type { SchemaChildNode } from './dictionarySchemaRelations.js'; import { asArray, deepCompare } from './formatUtils.js'; import { groupErrorsByIndex, mapAndMergeSubmittedDataToRecordReferences } from './submittedDataUtils.js'; import { + BATCH_ERROR_TYPE, + type BatchError, type DataRecordReference, type EditSubmittedDataReference, MERGE_REFERENCE_TYPE, @@ -40,6 +42,8 @@ import { type SubmissionSummary, SubmittedDataReference, } from './types.js'; +import { readHeaders, readTextFile } from './fileUtils.js'; +import { getSchemaFieldNames } from './dictionaryUtils.js'; // Only "open", "valid", and "invalid" statuses are considered Active Submission export const openSubmissionStatus = [ @@ -58,6 +62,102 @@ export const isSubmissionActive = (status: SubmissionStatus): status is OpenSubm return openStatuses.includes(status); }; +/** + * Checks if file contains required fields based on schema + * @param {SchemasDictionary} dictionary A dictionary to validate with + * @param {Record} entityFileMap A Record to map a file with a entityName as a key + * @returns a list of valid files and a list of errors + */ +export const checkEntityFieldNames = async ( + dictionary: SchemasDictionary, + entityFileMap: Record, +) => { + const checkedEntities: Record = {}; + const fieldNameErrors: BatchError[] = []; + + for (const [entityName, file] of Object.entries(entityFileMap)) { + try { + const fileHeaders = await readHeaders(file); + const entitySchema = dictionary.schemas.find((schema) => schema.name === entityName); + + if (!entitySchema) { + fieldNameErrors.push({ + type: BATCH_ERROR_TYPE.INVALID_FILE_NAME, + message: `No matching schema for entity ${entityName}.`, + batchName: file.originalname, + }); + continue; + } + + const schemaFieldNames = getSchemaFieldNames(entitySchema); + + const missingRequiredFields = schemaFieldNames.required.filter( + (requiredField) => !fileHeaders.includes(requiredField), + ); + if (missingRequiredFields.length > 0) { + fieldNameErrors.push({ + type: BATCH_ERROR_TYPE.MISSING_REQUIRED_HEADER, + message: `Missing required fields '${JSON.stringify(missingRequiredFields)}'`, + batchName: file.originalname, + }); + } else { + checkedEntities[entityName] = file; + } + } catch (error) { + fieldNameErrors.push({ + type: BATCH_ERROR_TYPE.FILE_READ_ERROR, + message: `Error reading file '${file.originalname}'`, + batchName: file.originalname, + }); + } + } + return { + checkedEntities, + fieldNameErrors, + }; +}; + +/** + * Removes invalid/duplicated files + * @param {Express.Multer.File[]} files An array of files + * @param {string[]} dictionarySchemaNames Schema names in the dictionary + * @returns A list of valid files mapped by schema/entity names + */ +export const checkFileNames = async ( + files: Express.Multer.File[], + dictionarySchemaNames: string[], +): Promise<{ validFileEntity: Record; batchErrors: BatchError[] }> => { + const validFileEntity: Record = {}; + const batchErrors: BatchError[] = []; + + for (const file of files) { + const matchingName = dictionarySchemaNames.filter( + (schemaName) => schemaName.toLowerCase() == file.originalname.split('.')[0].toLowerCase(), + ); + + if (matchingName.length > 1) { + batchErrors.push({ + type: BATCH_ERROR_TYPE.MULTIPLE_TYPED_FILES, + message: 'Multiple schemas matches this file', + batchName: file.originalname, + }); + } else if (matchingName.length === 1) { + validFileEntity[matchingName[0]] = file; + } else { + batchErrors.push({ + type: BATCH_ERROR_TYPE.INVALID_FILE_NAME, + message: 'Filename does not relate any schema name', + batchName: file.originalname, + }); + } + } + + return { + validFileEntity, + batchErrors, + }; +}; + /** * Checks if object is a Submission or a SubmittedData * @param {SubmittedDataReference | NewSubmittedDataReference | EditSubmittedDataReference} toBeDetermined @@ -573,7 +673,7 @@ export const createSubmissionSummaryResponse = ( dictionaryCategory: submission.dictionaryCategory, errors: { ...submission.errors, - total: sumRecordsCount(submission.errors), + total: sumRecordsCount(submission.errors ?? {}), }, organization: submission.organization, status: submission.status, @@ -730,6 +830,34 @@ export const segregateFieldChangeRecords = ( ); }; +/** + * Construct a SubmissionInsertData object per each file returning a Record type based on entityName + * @param {Record} files + * @param {SchemasDictionary} schemasDictionary + * @returns {Promise>} + */ +export const submissionInsertDataFromFiles = async ( + files: Record, + schemasDictionary: SchemasDictionary, +): Promise> => { + return await Object.entries(files).reduce>>( + async (accPromise, [entityName, file]) => { + const acc = await accPromise; + const schema = schemasDictionary.schemas.find((schema) => schema.name === entityName); + if (!schema) { + throw new Error(`No schema found for : '${entityName}'`); + } + const parsedFileData = await readTextFile(file, schema); + acc[entityName] = { + batchName: file.originalname, + records: parsedFileData.records, + }; + return Promise.resolve(acc); + }, + Promise.resolve({}), + ); +}; + /** * Validate a full set of Schema Data using a Dictionary * @param {SchemasDictionary & {id: number }} dictionary diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts index 5520a9aa..0381fe8d 100644 --- a/packages/data-provider/src/utils/types.ts +++ b/packages/data-provider/src/utils/types.ts @@ -74,10 +74,7 @@ export type AuditDataResponse = { * Include an array of the filtered records and a summary of the pagination * Response type used to query submitted data endpoint */ -export type AuditPaginatedResponse = { - pagination: PaginationMetadata; - records: AuditDataResponse[]; -}; +export type AuditPaginatedResponse = PaginatedResponse; /** * Type that describes the options used as a filter on Audit Table @@ -94,19 +91,27 @@ export type AuditFilterOptions = PaginationOptions & { * Enum used in the Reponse on Create new Submissions */ export const CREATE_SUBMISSION_STATUS = { + PARTIAL_SUBMISSION: 'PARTIAL_SUBMISSION', PROCESSING: 'PROCESSING', INVALID_SUBMISSION: 'INVALID_SUBMISSION', } as const; export type CreateSubmissionStatus = ObjectValues; /** - * Used as a Response type on a Create new Active Submission (Upload endpoint) + * Used as a Response type for submitting data */ -export type CreateSubmissionResult = { +export interface SubmitDataResult { submissionId?: number; status: CreateSubmissionStatus; description: string; -}; +} +/** + * Used as a Response type for submitting data via file upload + */ +export interface SubmitFileResult extends SubmitDataResult { + batchErrors: BatchError[]; + inProcessEntities: string[]; +} /** * Response type on Commit Active Submission (Commit endpoint) @@ -171,7 +176,7 @@ export type BatchError = { export interface ValidateFilesParams { categoryId: number; organization: string; - schema: Schema; + schemasDictionary: SchemasDictionary; username: string; } @@ -271,7 +276,7 @@ export type SubmissionDataSummaryRepositoryRecord = { data: SubmissionDataSummary; dictionary: DictionarySummary; dictionaryCategory: CategorySummary; - errors: SubmissionErrorsSummary; + errors: SubmissionErrorsSummary | null; organization: string; status: SubmissionStatus; createdAt: Date | null; @@ -356,6 +361,11 @@ export type PaginationMetadata = { totalRecords: number; }; +export type PaginatedResponse = { + pagination: PaginationMetadata; + records: T[]; +}; + /** * Type that describes the options used as a filter on Submitted Data */ @@ -367,10 +377,7 @@ export type SubmittedDataFilterOptions = PaginationOptions & { * Include an array of the filtered records and a summary of the pagination * Response type used to query submitted data endpoint */ -export type SubmittedDataPaginatedResponse = { - pagination: PaginationMetadata; - records: SubmittedDataResponse[]; -}; +export type SubmittedDataPaginatedResponse = PaginatedResponse; /** * Enum used to merge SubmittedData and Submissions diff --git a/packages/data-provider/test/utils/fileUtils.spec.ts b/packages/data-provider/test/utils/fileUtils.spec.ts new file mode 100644 index 00000000..815590bb --- /dev/null +++ b/packages/data-provider/test/utils/fileUtils.spec.ts @@ -0,0 +1,66 @@ +import { expect, use } from 'chai'; +import chaiAsPromised from 'chai-as-promised'; +import { describe, it } from 'mocha'; + +import { extractFileExtension, getSeparatorCharacter, mapRecordToHeaders } from '../../src/utils/fileUtils.js'; + +use(chaiAsPromised); + +describe('File Utils', () => { + describe('Map records to headers', () => { + it('should return an unprocessed record object', () => { + const headers: string[] = ['id', 'name', 'description']; + const record: string[] = ['100', 'Cat', 'Feline animal']; + const response = mapRecordToHeaders(headers, record); + expect(response).to.eql({ id: '100', name: 'Cat', description: 'Feline animal' }); + }); + it('should return empty object when no headers are passed', () => { + const headers: string[] = []; + const record: string[] = ['100', 'Cat', 'Feline animal']; + const response = mapRecordToHeaders(headers, record); + expect(response).to.eql({}); + }); + it('should return object with empty string values when no records are passed', () => { + const headers: string[] = ['id', 'name', 'description']; + const record: string[] = []; + const response = mapRecordToHeaders(headers, record); + expect(response).to.eql({ id: '', name: '', description: '' }); + }); + }); + + describe('Validate file Extension', () => { + it('should return invalid file extension', () => { + const response = extractFileExtension('archive.xls'); + expect(response).to.be.undefined; + }); + + it('should return invalid file extension when there is no extension', () => { + const response = extractFileExtension('noextension'); + expect(response).to.be.undefined; + }); + + it('should return tsv file extension', () => { + const response = extractFileExtension('archive.tsv'); + expect(response).to.eql('tsv'); + }); + it('should return csv file extension', () => { + const response = extractFileExtension('archive.csv'); + expect(response).to.eql('csv'); + }); + }); + + describe('Get delimiter character from file extension', () => { + it('should identify the delimiter character for a .csv file', () => { + const response = getSeparatorCharacter('myFile.csv'); + expect(response).to.eql(','); + }); + it('should identify the delimiter character for a .tsv file', () => { + const response = getSeparatorCharacter('myFile.tsv'); + expect(response).to.eql('\t'); + }); + it('should return undefined when file extension is invalid', () => { + const response = getSeparatorCharacter('myFile.xyz'); + expect(response).to.be.undefined; + }); + }); +}); diff --git a/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts b/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts index 6bafb950..b05bd06e 100644 --- a/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts +++ b/packages/data-provider/test/utils/submission/parseSubmissionSummaryResponse.spec.ts @@ -9,14 +9,10 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ it('should return a Summary without any data ', () => { const submissionDataSummaryRepositoryRecord: SubmissionDataSummaryRepositoryRecord = { id: 4, - data: { - inserts: undefined, - updates: undefined, - deletes: undefined, - }, + data: {}, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, + errors: null, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate, @@ -28,13 +24,11 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ expect(response).to.eql({ id: 4, data: { - inserts: undefined, - updates: undefined, - deletes: undefined, + total: 0, }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, + errors: { total: 0 }, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate.toISOString(), @@ -78,6 +72,7 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ expect(response).to.eql({ id: 3, data: { + total: 3, inserts: { books: { batchName: 'books.tsv', @@ -97,7 +92,7 @@ describe('Submission Utils - Parse a Submission object to a Summary of the Activ }, dictionary: { name: 'books', version: '1' }, dictionaryCategory: { name: 'favorite books', id: 1 }, - errors: {}, + errors: { total: 0 }, organization: 'oicr', status: SUBMISSION_STATUS.VALID, createdAt: todaysDate.toISOString(), From ce52bb4bdb9f7556915275fb1d09601f244b9006 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sun, 15 Feb 2026 16:03:57 -0500 Subject: [PATCH 2/8] Add max file size for file uploads to env config --- apps/server/.env.schema | 44 ++++++++++++++++++++++------ apps/server/src/config/app.ts | 40 +++++--------------------- apps/server/src/config/envUtils.ts | 46 ++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 42 deletions(-) create mode 100644 apps/server/src/config/envUtils.ts diff --git a/apps/server/.env.schema b/apps/server/.env.schema index 6f9e51e1..d36d6ae5 100644 --- a/apps/server/.env.schema +++ b/apps/server/.env.schema @@ -1,9 +1,35 @@ -PORT=3030 -DB_HOST= -DB_PORT= -DB_NAME= -DB_USER= -DB_PASSWORD= -LECTERN_URL= -LOG_LEVEL= -ID_USELOCAL= \ No newline at end of file +## Copy this file into `server/.env` to run the server locally. +## Variables that are not commented require values before the app can run. They have been provided default values +## that will work with the docker-compose setup. +## Commented values are optional configs, showing their default values. + +## Server +# PORT=3030 +# LOG_LEVEL=debug + +# ALLOWED_ORIGINS= +# CORS_ENABLED=false + +## DB Connection +DB_HOST=localhost +DB_NAME=lyric +DB_PASSWORD=secret +DB_PORT=5432 +DB_USER=postgres + +## Dictionary +LECTERN_URL=http://localhost:3000 + + +## Submission +# AUDIT_ENABLED=true +# SUBMISSION_MAX_SIZE=10485760 # Default 10Mb, value in bytes + +## Data +# ID_USELOCAL=true +# ID_CUSTOM_ALPHABET=0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ +# ID_CUSTOM_SIZE=20 +# PLURALIZE_SCHEMAS_ENABLED=true + +## Validator +# VALIDATOR_CONFIG=[] \ No newline at end of file diff --git a/apps/server/src/config/app.ts b/apps/server/src/config/app.ts index cf79eb28..371850e8 100644 --- a/apps/server/src/config/app.ts +++ b/apps/server/src/config/app.ts @@ -2,6 +2,10 @@ import 'dotenv/config'; import { type AppConfig, type ValidatorEntry } from '@overture-stack/lyric'; +import { getBoolean, getJSONConfig, getOptionalIntegerConfig, getRequiredConfig } from './envUtils.js'; + +const DEFAULT_SUBMISSION_MAX_SIZE = 10 * 1024 * 1024; // 10Mb + export const getServerConfig = () => { return { port: process.env.PORT || 3030, @@ -10,39 +14,6 @@ export const getServerConfig = () => { }; }; -export const getBoolean = (env: string | undefined, defaultValue: boolean): boolean => { - switch ((env ?? '').toLocaleLowerCase()) { - case 'true': - return true; - case 'false': - return false; - default: - return defaultValue; - } -}; - -const getRequiredConfig = (name: string) => { - const value = process.env[name]; - if (!value) { - throw new Error(`No Environment Variable provided for required configuration parameter '${name}'`); - } - return value; -}; - -const getJSONConfig = (name: string) => { - const value = process.env[name]; - - if (!value) { - return; - } - - try { - return JSON.parse(value); - } catch (error) { - throw new Error(`Environment variable '${name}' must be a valid JSON.`); - } -}; - const isValidValidatorEntry = (obj: unknown): obj is ValidatorEntry => { return typeof obj === 'object' && obj !== null && 'categoryId' in obj && 'entityName' in obj && 'fieldName' in obj; }; @@ -92,5 +63,8 @@ export const appConfig: AppConfig = { schemaService: { url: getRequiredConfig('LECTERN_URL'), }, + submissionService: { + maxFileSize: getOptionalIntegerConfig('SUBMISSION_MAX_SIZE') || DEFAULT_SUBMISSION_MAX_SIZE, + }, validator: getValidatorConfig('VALIDATOR_CONFIG'), }; diff --git a/apps/server/src/config/envUtils.ts b/apps/server/src/config/envUtils.ts new file mode 100644 index 00000000..260ad50b --- /dev/null +++ b/apps/server/src/config/envUtils.ts @@ -0,0 +1,46 @@ +export const getBoolean = (env: string | undefined, defaultValue: boolean): boolean => { + switch ((env ?? '').toLocaleLowerCase()) { + case 'true': + return true; + case 'false': + return false; + default: + return defaultValue; + } +}; + +export const getRequiredConfig = (name: string) => { + const value = process.env[name]; + if (!value) { + throw new Error(`No Environment Variable provided for required configuration parameter '${name}'`); + } + return value; +}; + +export const getJSONConfig = (name: string) => { + const value = process.env[name]; + + if (!value) { + return; + } + + try { + return JSON.parse(value); + } catch (error) { + throw new Error(`Environment variable '${name}' must be a valid JSON.`); + } +}; + +export const getOptionalIntegerConfig = (name: string): number | undefined => { + const value = process.env[name]; + + if (value === undefined || value === '') { + return undefined; + } + const number = Number(value); + + if (!(Number.isFinite(number) && Number.isInteger(number))) { + throw new Error(`Environment variable '${name}' must be an integer. Recieved invalid value ${value}`); + } + return number; +}; From b09af64fce0a4515b62db1a9f52fb1fcae060c4e Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sun, 15 Feb 2026 16:04:51 -0500 Subject: [PATCH 3/8] Update swagger documentation to include submissionRouter chagnes for file upload --- apps/server/swagger/audit-api.yml | 14 ++----- apps/server/swagger/categories-api.yml | 7 +--- apps/server/swagger/dictionary-api.yml | 14 +------ apps/server/swagger/parameters.yml | 9 ++++- apps/server/swagger/schemas.yml | 4 +- apps/server/swagger/submission-api.yml | 53 ++++++++++++++++++++++---- 6 files changed, 61 insertions(+), 40 deletions(-) diff --git a/apps/server/swagger/audit-api.yml b/apps/server/swagger/audit-api.yml index 6d0450a1..a5e148d3 100644 --- a/apps/server/swagger/audit-api.yml +++ b/apps/server/swagger/audit-api.yml @@ -1,19 +1,11 @@ /audit/category/{categoryId}/organization/{organization}: get: - summary: Retrive change history of stored data + summary: Retrieve change history of stored data tags: - Audit parameters: - - name: categoryId - in: path - description: ID of the Category - type: string - required: true - - name: organization - in: path - description: Name of the Organization - type: string - required: true + - $ref: '#/components/parameters/path/CategoryId' + - $ref: '#/components/parameters/path/Organization' - name: entityName description: Filter events based on entity name in: query diff --git a/apps/server/swagger/categories-api.yml b/apps/server/swagger/categories-api.yml index ebfda13a..ab64a7d3 100644 --- a/apps/server/swagger/categories-api.yml +++ b/apps/server/swagger/categories-api.yml @@ -27,12 +27,7 @@ tags: - Category parameters: - - name: categoryId - in: path - required: true - schema: - type: string - description: ID of the category + - $ref: '#/components/parameters/path/CategoryId' responses: 200: description: Category Details diff --git a/apps/server/swagger/dictionary-api.yml b/apps/server/swagger/dictionary-api.yml index 535e5071..a77c0294 100644 --- a/apps/server/swagger/dictionary-api.yml +++ b/apps/server/swagger/dictionary-api.yml @@ -50,12 +50,7 @@ tags: - Dictionary parameters: - - name: categoryId - in: path - required: true - schema: - type: integer - description: The ID of the category containing the dictionary + - $ref: '#/components/parameters/path/CategoryId' responses: 200: description: Dictionary schema JSON @@ -77,12 +72,7 @@ tags: - Dictionary parameters: - - name: categoryId - in: path - required: true - schema: - type: integer - description: The ID of the category containing the dictionary + - $ref: '#/components/parameters/path/CategoryId' - name: fileType in: query required: false diff --git a/apps/server/swagger/parameters.yml b/apps/server/swagger/parameters.yml index 4fbcf474..1e4ec9f7 100644 --- a/apps/server/swagger/parameters.yml +++ b/apps/server/swagger/parameters.yml @@ -6,7 +6,7 @@ components: in: path required: true schema: - type: string + type: integer description: ID of the category to which the data belongs EntityName: description: The name of the Entity @@ -22,6 +22,13 @@ components: schema: type: string description: Organization name + OrganizationId: + name: organizationId + in: path + required: true + schema: + type: string + description: ID of the organization SubmissionId: name: submissionId in: path diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml index 00b989de..a4d160be 100644 --- a/apps/server/swagger/schemas.yml +++ b/apps/server/swagger/schemas.yml @@ -200,7 +200,7 @@ components: createdBy: type: string description: Name of user who created the submission - udpatedAt: + updatedAt: type: string description: Date and time of latest update updatedBy: @@ -258,7 +258,7 @@ components: createdBy: type: string description: User name who created the submission - udpatedAt: + updatedAt: type: string description: Date and time of latest update updatedBy: diff --git a/apps/server/swagger/submission-api.yml b/apps/server/swagger/submission-api.yml index e733c9bf..012d5069 100644 --- a/apps/server/swagger/submission-api.yml +++ b/apps/server/swagger/submission-api.yml @@ -207,15 +207,52 @@ 503: $ref: '#/components/responses/ServiceUnavailableError' -/submission/category/{categoryId}/data: +/submission/category/{categoryId}/organization/{organizationId}: post: - summary: Add new data to a submission for the specified category. Returns an Active Submission containing the newly created records + summary: Upload and process submission files for the specified category and organization. tags: - Submission parameters: - $ref: '#/components/parameters/path/CategoryId' - - $ref: '#/components/parameters/query/EntityName' - - $ref: '#/components/parameters/query/Organization' + - $ref: '#/components/parameters/path/OrganizationId' + requestBody: + description: File(s) to be added to the submission + required: true + content: + multipart/form-data: + schema: + type: object + properties: + files: + type: array + items: + type: string + format: binary + responses: + 200: + description: Submission accepted + content: + application/json: + schema: + $ref: '#/components/schemas/CreateSubmissionResult' + 400: + $ref: '#/components/responses/BadRequest' + 401: + $ref: '#/components/responses/UnauthorizedError' + 500: + $ref: '#/components/responses/ServerError' + 503: + $ref: '#/components/responses/ServiceUnavailableError' + +/submission/category/{categoryId}/organization/{organizationId}/entity/{entityName}: + post: + summary: Add new data for a single entity to a submission for the specified category and organization. + tags: + - Submission + parameters: + - $ref: '#/components/parameters/path/CategoryId' + - $ref: '#/components/parameters/path/OrganizationId' + - $ref: '#/components/parameters/path/EntityName' requestBody: description: The JSON payload containing the data to be added to the submission required: true @@ -241,15 +278,15 @@ 503: $ref: '#/components/responses/ServiceUnavailableError' put: - summary: Modifies existing data for a submission. Returns an Active Submission containing the records that will be updated + summary: Modifies existing data for a signle entity in the active submission of the specified category and organization. tags: - Submission parameters: - $ref: '#/components/parameters/path/CategoryId' - - $ref: '#/components/parameters/query/EntityName' - - $ref: '#/components/parameters/query/Organization' + - $ref: '#/components/parameters/path/OrganizationId' + - $ref: '#/components/parameters/path/EntityName' requestBody: - description: The JSON payload containing the data to be added to the submission + description: The JSON payload containing the data to be updated in the submission required: true content: application/json: From d74c6b69b37eae32de796783d9345ad9a4693cba Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sun, 15 Feb 2026 16:05:29 -0500 Subject: [PATCH 4/8] package lock for data-provider dependencies for file processing --- pnpm-lock.yaml | 115 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 84d8eb28..dbfcf882 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,6 +160,12 @@ importers: '@overture-stack/sqon-builder': specifier: ^1.1.0 version: 1.1.0 + bytes: + specifier: ^3.1.2 + version: 3.1.2 + csv-parse: + specifier: ^6.1.0 + version: 6.1.0 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -169,12 +175,18 @@ importers: express: specifier: ^4.19.2 version: 4.19.2 + firstline: + specifier: ^2.0.2 + version: 2.0.2 jszip: specifier: ^3.10.1 version: 3.10.1 lodash-es: specifier: ^4.17.21 version: 4.17.21 + multer: + specifier: ^2.0.2 + version: 2.0.2 nanoid: specifier: ^5.0.7 version: 5.0.7 @@ -191,6 +203,9 @@ importers: specifier: ^3.23.8 version: 3.23.8 devDependencies: + '@types/bytes': + specifier: ^3.1.5 + version: 3.1.5 '@types/chai-as-promised': specifier: ^8.0.1 version: 8.0.2 @@ -203,6 +218,9 @@ importers: '@types/express-serve-static-core': specifier: ^4.19.5 version: 4.19.5 + '@types/firstline': + specifier: ^2.0.4 + version: 2.0.4 '@types/jszip': specifier: ^3.4.1 version: 3.4.1 @@ -212,6 +230,9 @@ importers: '@types/lodash-es': specifier: ^4.17.12 version: 4.17.12 + '@types/multer': + specifier: ^2.0.0 + version: 2.0.0 '@types/pg': specifier: ^8.11.6 version: 8.11.6 @@ -786,6 +807,9 @@ packages: '@types/body-parser@1.19.5': resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + '@types/bytes@3.1.5': + resolution: {integrity: sha512-VgZkrJckypj85YxEsEavcMmmSOIzkUHqWmM4CCyia5dc54YwsXzJ5uT4fYxBQNEXx+oF1krlhgCbvfubXqZYsQ==} + '@types/chai-as-promised@8.0.2': resolution: {integrity: sha512-meQ1wDr1K5KRCSvG2lX7n7/5wf70BeptTKst0axGvnN6zqaVpRqegoIbugiAPSqOW9K9aL8gDVrm7a2LXOtn2Q==} @@ -807,6 +831,9 @@ packages: '@types/express@4.17.21': resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} + '@types/firstline@2.0.4': + resolution: {integrity: sha512-EYoMzk783ncj3soLGADXD/rklDMw1PAO5Hc3lRZa5G21vkfacwkdTlIdhTJ39omqDLezTSmxjDG1psd4A/mUHg==} + '@types/http-errors@2.0.4': resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} @@ -832,6 +859,9 @@ packages: '@types/mocha@10.0.7': resolution: {integrity: sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==} + '@types/multer@2.0.0': + resolution: {integrity: sha512-C3Z9v9Evij2yST3RSBktxP9STm6OdMc5uR1xF1SGr98uv8dUlAL2hqwrZ3GVB3uyMyiegnscEK6PGtYvNrjTjw==} + '@types/node@20.14.10': resolution: {integrity: sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==} @@ -964,6 +994,9 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1014,6 +1047,10 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -1127,6 +1164,10 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + concat-stream@2.0.0: + resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} + engines: {'0': node >= 6.0} + content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -1164,6 +1205,9 @@ packages: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} + csv-parse@6.1.0: + resolution: {integrity: sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==} + d@1.0.2: resolution: {integrity: sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==} engines: {node: '>=0.12'} @@ -1527,6 +1571,10 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} + firstline@2.0.2: + resolution: {integrity: sha512-8KcmfI0jgSECnzdhucm0i7vrwef3BWwgjimW2YkRC5eSFwjb5DibVoA0YvgkYwwxuJi9c+7M7X3b3lX8o9B6wg==} + engines: {node: '>=6.4.0'} + flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -1605,20 +1653,21 @@ packages: glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@7.1.6: resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} engines: {node: '>=12'} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me globals@14.0.0: resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} @@ -1947,6 +1996,10 @@ packages: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} + mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + mkdirp@1.0.4: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} @@ -1966,6 +2019,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + multer@2.0.2: + resolution: {integrity: sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==} + engines: {node: '>= 10.16.0'} + nanoid@5.0.7: resolution: {integrity: sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==} engines: {node: ^18 || >=20} @@ -2341,6 +2398,10 @@ packages: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -2470,6 +2531,9 @@ packages: type@2.7.3: resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==} + typedarray@0.0.6: + resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + typescript-eslint@7.16.1: resolution: {integrity: sha512-889oE5qELj65q/tGeOSvlreNKhimitFwZqQ0o7PcWC7/lgRkAMknznsCsV8J8mZGTP/Z+cIbX8accf2DE33hrA==} engines: {node: ^18.18.0 || >=20.0.0} @@ -2985,6 +3049,8 @@ snapshots: '@types/connect': 3.4.38 '@types/node': 20.14.10 + '@types/bytes@3.1.5': {} + '@types/chai-as-promised@8.0.2': dependencies: '@types/chai': 4.3.16 @@ -3015,6 +3081,10 @@ snapshots: '@types/qs': 6.9.15 '@types/serve-static': 1.15.7 + '@types/firstline@2.0.4': + dependencies: + '@types/node': 20.14.10 + '@types/http-errors@2.0.4': {} '@types/istanbul-lib-coverage@2.0.6': {} @@ -3035,6 +3105,10 @@ snapshots: '@types/mocha@10.0.7': {} + '@types/multer@2.0.0': + dependencies: + '@types/express': 4.17.21 + '@types/node@20.14.10': dependencies: undici-types: 5.26.5 @@ -3191,6 +3265,8 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + append-field@1.0.0: {} + argparse@2.0.1: {} array-flatten@1.1.1: {} @@ -3249,6 +3325,10 @@ snapshots: buffer-from@1.1.2: {} + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + bytes@3.1.2: {} c8@10.1.2: @@ -3379,6 +3459,13 @@ snapshots: concat-map@0.0.1: {} + concat-stream@2.0.0: + dependencies: + buffer-from: 1.1.2 + inherits: 2.0.4 + readable-stream: 3.6.2 + typedarray: 0.0.6 + content-disposition@0.5.4: dependencies: safe-buffer: 5.2.1 @@ -3418,6 +3505,8 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + csv-parse@6.1.0: {} + d@1.0.2: dependencies: es5-ext: 0.10.64 @@ -3878,6 +3967,8 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 + firstline@2.0.2: {} + flat-cache@4.0.1: dependencies: flatted: 3.3.1 @@ -4268,6 +4359,10 @@ snapshots: minipass@7.1.2: {} + mkdirp@0.5.6: + dependencies: + minimist: 1.2.8 + mkdirp@1.0.4: {} mocha@10.6.0: @@ -4299,6 +4394,16 @@ snapshots: ms@2.1.3: {} + multer@2.0.2: + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 2.0.0 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + nanoid@5.0.7: {} natural-compare@1.4.0: {} @@ -4666,6 +4771,8 @@ snapshots: statuses@2.0.1: {} + streamsearch@1.1.0: {} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -4799,6 +4906,8 @@ snapshots: type@2.7.3: {} + typedarray@0.0.6: {} + typescript-eslint@7.16.1(eslint@9.7.0)(typescript@5.5.3): dependencies: '@typescript-eslint/eslint-plugin': 7.16.1(@typescript-eslint/parser@7.16.1(eslint@9.7.0)(typescript@5.5.3))(eslint@9.7.0)(typescript@5.5.3) From b249bd419c4bbdb8b2bb38a421737affa1671167 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sun, 15 Feb 2026 16:32:34 -0500 Subject: [PATCH 5/8] Fix type errors from incomplete type rename --- .../src/services/submission/submission.ts | 12 ++++++------ .../src/services/submission/submissionService.ts | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/data-provider/src/services/submission/submission.ts b/packages/data-provider/src/services/submission/submission.ts index 18b43b9d..45c9d7c9 100644 --- a/packages/data-provider/src/services/submission/submission.ts +++ b/packages/data-provider/src/services/submission/submission.ts @@ -25,8 +25,6 @@ import { import { CommitSubmissionResult, CREATE_SUBMISSION_STATUS, - type CreateSubmissionJsonResult, - type SubmitFileResult, type DeleteSubmissionResult, type EntityData, type PaginationOptions, @@ -34,8 +32,10 @@ import { SUBMISSION_STATUS, type SubmissionActionType, SubmissionSummary, + type SubmitDataResult, + type SubmitFileResult, } from '../../utils/types.js'; -import { default as createSubmissionProcessor } from './processor.js'; +import { default as createSubmissionProcessor } from './submissionProcessor.js'; const submissionService = (dependencies: BaseDependencies) => { const LOG_MODULE = 'SUBMISSION_SERVICE'; @@ -192,7 +192,7 @@ const submissionService = (dependencies: BaseDependencies) => { * @param {number} submissionId * @param {string} entityName * @param {string} username - * @returns { Promise} + * @returns { Promise} */ const deleteActiveSubmissionEntity = async ( submissionId: number, @@ -202,7 +202,7 @@ const submissionService = (dependencies: BaseDependencies) => { entityName: string; index: number | null; }, - ): Promise => { + ): Promise => { const submission = await submissionRepository.getSubmissionDetailsById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); @@ -455,7 +455,7 @@ const submissionService = (dependencies: BaseDependencies) => { categoryId: number; organization: string; username: string; - }): Promise => { + }): Promise => { const entityNames = Object.keys(data); logger.info( LOG_MODULE, diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index e595d2af..9bfe1dc5 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -25,7 +25,7 @@ import { import { CommitSubmissionResult, CREATE_SUBMISSION_STATUS, - type CreateSubmissionJsonResult, + type SubmitDataResult, type SubmitFileResult, type DeleteSubmissionResult, type EntityData, @@ -192,7 +192,7 @@ const submissionService = (dependencies: BaseDependencies) => { * @param {number} submissionId * @param {string} entityName * @param {string} username - * @returns { Promise} + * @returns { Promise} */ const deleteActiveSubmissionEntity = async ( submissionId: number, @@ -202,7 +202,7 @@ const submissionService = (dependencies: BaseDependencies) => { entityName: string; index: number | null; }, - ): Promise => { + ): Promise => { const submission = await submissionRepository.getSubmissionDetailsById(submissionId); if (!submission) { throw new BadRequest(`Submission '${submissionId}' not found`); @@ -455,7 +455,7 @@ const submissionService = (dependencies: BaseDependencies) => { categoryId: number; organization: string; username: string; - }): Promise => { + }): Promise => { const entityNames = Object.keys(data); logger.info( LOG_MODULE, From 6814ea222ce98fe09be6eb79951332a0c086dbb5 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 16 Feb 2026 08:16:21 -0500 Subject: [PATCH 6/8] Undo rename of submissionService.submit --- .../data-provider/src/controllers/submissionController.ts | 8 ++++---- .../src/services/submission/submissionService.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts index 44c87c25..8927806d 100644 --- a/packages/data-provider/src/controllers/submissionController.ts +++ b/packages/data-provider/src/controllers/submissionController.ts @@ -352,7 +352,7 @@ const controller = ({ const username = user?.username || ''; - const resultSubmission = await service.submitJson({ + const resultSubmission = await service.submit({ data: { [entityName]: payload }, categoryId, organization, @@ -409,14 +409,14 @@ const controller = ({ { validFiles: [], fileErrors: [] }, ); - const resultSubmission = await service.submitFiles({ + const submitFilesResult = await service.submitFiles({ files: validFiles, categoryId, organization, username, }); - if (fileErrors.length == 0 && resultSubmission.batchErrors.length == 0) { + if (fileErrors.length == 0 && submitFilesResult.batchErrors.length == 0) { logger.info(LOG_MODULE, `Submission uploaded successfully`); } else { logger.info(LOG_MODULE, 'Found some errors processing this request'); @@ -425,7 +425,7 @@ const controller = ({ // This response provides the details of file Submission return res .status(200) - .send({ ...resultSubmission, batchErrors: [...fileErrors, ...resultSubmission.batchErrors] }); + .send({ ...submitFilesResult, batchErrors: [...fileErrors, ...submitFilesResult.batchErrors] }); } catch (error) { next(error); } diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index 9bfe1dc5..858ea1ea 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -623,7 +623,7 @@ const submissionService = (dependencies: BaseDependencies) => { getSubmissionDetailsById, getActiveSubmissionByOrganization, getOrCreateActiveSubmission, - submitJson: submit, + submit, submitFiles, }; }; From 2bc045860d09e4f30304b1555f8dec1a30b5243a Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 16 Feb 2026 08:19:34 -0500 Subject: [PATCH 7/8] Use singular noun for SupportedFileExtension type name --- packages/data-provider/src/utils/fileUtils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/data-provider/src/utils/fileUtils.ts b/packages/data-provider/src/utils/fileUtils.ts index f62cd3b4..917c0275 100644 --- a/packages/data-provider/src/utils/fileUtils.ts +++ b/packages/data-provider/src/utils/fileUtils.ts @@ -15,20 +15,20 @@ import { import { BATCH_ERROR_TYPE, type BatchError } from './types.js'; export const SUPPORTED_FILE_EXTENSIONS = z.enum(['tsv', 'csv']); -export type SupportedFileExtensions = z.infer; +export type SupportedFileExtension = z.infer; export const columnSeparatorValue = { tsv: '\t', csv: ',', -} as const satisfies Record; +} as const satisfies Record; /** * Extracts the extension from the filename and returns it if it's supported. * Otherwise it returns undefined. * @param {string} fileName - * @returns {SupportedFileExtensions | undefined} + * @returns {SupportedFileExtension | undefined} */ -export const extractFileExtension = (fileName: string): SupportedFileExtensions | undefined => { +export const extractFileExtension = (fileName: string): SupportedFileExtension | undefined => { // Extract the file extension const fileExtension = fileName.split('.').pop()?.toLowerCase(); From f21683a0bf9e955d7742c037cc17da1bb2ecc2ec Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 16 Feb 2026 09:08:27 -0500 Subject: [PATCH 8/8] Rename submission.ts to submissionService.ts --- .../src/services/submission/submission.ts | 631 ------------------ .../services/submission/submissionService.ts | 4 +- 2 files changed, 2 insertions(+), 633 deletions(-) delete mode 100644 packages/data-provider/src/services/submission/submission.ts diff --git a/packages/data-provider/src/services/submission/submission.ts b/packages/data-provider/src/services/submission/submission.ts deleted file mode 100644 index 45c9d7c9..00000000 --- a/packages/data-provider/src/services/submission/submission.ts +++ /dev/null @@ -1,631 +0,0 @@ -import * as _ from 'lodash-es'; - -import { Dictionary as SchemasDictionary } from '@overture-stack/lectern-client'; -import { - type NewSubmission, - type SubmissionRecordErrorDetails, - type SubmissionUpdateData, -} from '@overture-stack/lyric-data-model/models'; - -import { BaseDependencies } from '../../config/config.js'; -import systemIdGenerator from '../../external/systemIdGenerator.js'; -import createSubmissionRepository from '../../repository/activeSubmissionRepository.js'; -import createCategoryRepository from '../../repository/categoryRepository.js'; -import createSubmittedDataRepository from '../../repository/submittedRepository.js'; -import { getSchemaByName } from '../../utils/dictionaryUtils.js'; -import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js'; -import { filterAndPaginateSubmissionData, type FlattenedSubmissionData } from '../../utils/submissionResponseParser.js'; -import { - checkEntityFieldNames, - checkFileNames, - createSubmissionSummaryResponse, - isSubmissionActive, - removeItemsFromSubmission, -} from '../../utils/submissionUtils.js'; -import { - CommitSubmissionResult, - CREATE_SUBMISSION_STATUS, - type DeleteSubmissionResult, - type EntityData, - type PaginationOptions, - SUBMISSION_ACTION_TYPE, - SUBMISSION_STATUS, - type SubmissionActionType, - SubmissionSummary, - type SubmitDataResult, - type SubmitFileResult, -} from '../../utils/types.js'; -import { default as createSubmissionProcessor } from './submissionProcessor.js'; - -const submissionService = (dependencies: BaseDependencies) => { - const LOG_MODULE = 'SUBMISSION_SERVICE'; - const { logger, onFinishCommit } = dependencies; - - const categoryRepository = createCategoryRepository(dependencies); - const submissionProcessor = createSubmissionProcessor(dependencies); - const submissionRepository = createSubmissionRepository(dependencies); - const submittedDataRepository = createSubmittedDataRepository(dependencies); - - const { generateIdentifier } = systemIdGenerator(dependencies); - - /** - * Runs Schema validation asynchronously and moves the Active Submission to Submitted Data - * @param {number} categoryId - * @param {number} submissionId - * @returns {Promise} - */ - const commitSubmission = async ( - categoryId: number, - submissionId: number, - username: string, - ): Promise => { - const { getSubmittedDataByCategoryIdAndOrganization } = submittedDataRepository; - const { getActiveDictionaryByCategory } = categoryRepository; - - const submission = await submissionRepository.getSubmissionDetailsById(submissionId); - if (!submission) { - throw new BadRequest(`Submission '${submissionId}' not found`); - } - - if (submission.dictionaryCategory.id !== categoryId) { - throw new BadRequest(`Category ID provided does not match the category for the Submission`); - } - - if (submission.status !== SUBMISSION_STATUS.VALID) { - throw new StatusConflict('Submission does not have status VALID and cannot be committed'); - } - - const currentDictionary = await getActiveDictionaryByCategory(categoryId); - if (_.isEmpty(currentDictionary)) { - throw new BadRequest(`Dictionary in category '${categoryId}' not found`); - } - - const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization( - categoryId, - submission?.organization, - ); - - const entitiesToProcess = new Set(); - - submittedDataToValidate?.forEach((data) => entitiesToProcess.add(data.entityName)); - - const insertsToValidate = submission.data?.inserts - ? Object.entries(submission.data.inserts).flatMap(([entityName, submissionData]) => { - entitiesToProcess.add(entityName); - - return submissionData.records.map((record) => ({ - data: record, - dictionaryCategoryId: categoryId, - entityName, - isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise - organization: submission.organization, - originalSchemaId: currentDictionary.id, - systemId: generateIdentifier(entityName, record), - createdBy: username, - })); - }) - : []; - - const deleteDataArray = submission.data?.deletes - ? Object.entries(submission.data.deletes).flatMap(([entityName, submissionDeleteData]) => { - entitiesToProcess.add(entityName); - return submissionDeleteData; - }) - : []; - - const updateDataArray = - submission.data?.updates && - Object.entries(submission.data.updates).reduce>( - (acc, [entityName, submissionUpdateData]) => { - entitiesToProcess.add(entityName); - submissionUpdateData.forEach((record) => { - acc[record.systemId] = record; - }); - return acc; - }, - {}, - ); - - // To Commit Active Submission we need to validate SubmittedData + Active Submission - submissionProcessor.performCommitSubmissionAsync({ - dataToValidate: { - inserts: insertsToValidate, - submittedData: submittedDataToValidate, - deletes: deleteDataArray, - updates: updateDataArray, - }, - submissionId: submission.id, - dictionary: currentDictionary, - username: username, - onFinishCommit, - }); - - return { - status: CREATE_SUBMISSION_STATUS.PROCESSING, - dictionary: { - name: currentDictionary.name, - version: currentDictionary.version, - }, - processedEntities: Array.from(entitiesToProcess.values()), - }; - }; - - /** - * Updates Submission status to CLOSED - * This action is allowed only if current Submission Status as OPEN, VALID or INVALID - * Returns the resulting ID of the Submission - * @param {number} submissionId - * @param {string} username - * @returns {Promise} - */ - const deleteActiveSubmissionById = async ( - submissionId: number, - username: string, - ): Promise => { - const submission = await submissionRepository.getSubmissionById(submissionId); - if (!submission) { - throw new BadRequest(`Submission '${submissionId}' not found`); - } - - if (!isSubmissionActive(submission.status)) { - throw new StatusConflict('Submission is not active. Only Active Submission can be deleted'); - } - - const updatedRecordId = await submissionRepository.update(submission.id, { - status: SUBMISSION_STATUS.CLOSED, - updatedBy: username, - }); - - logger.info(LOG_MODULE, `Submission '${submissionId}' updated with new status '${SUBMISSION_STATUS.CLOSED}'`); - - return { - status: SUBMISSION_STATUS.CLOSED, - description: 'Submission closed successfully', - submissionId: updatedRecordId, - }; - }; - - /** - * Function to remove an entity from an Active Submission by given Submission ID - * It validates resulting Active Submission running cross schema validation along with the existing Submitted Data - * Returns the resulting ID of the Active Submission - * @param {number} submissionId - * @param {string} entityName - * @param {string} username - * @returns { Promise} - */ - const deleteActiveSubmissionEntity = async ( - submissionId: number, - username: string, - filter: { - actionType: SubmissionActionType; - entityName: string; - index: number | null; - }, - ): Promise => { - const submission = await submissionRepository.getSubmissionDetailsById(submissionId); - if (!submission) { - throw new BadRequest(`Submission '${submissionId}' not found`); - } - - if (!isSubmissionActive(submission.status)) { - throw new StatusConflict('Submission is not active. Only Active Submission can be modified'); - } - - if ( - SUBMISSION_ACTION_TYPE.Values.INSERTS.includes(filter.actionType) && - !_.has(submission.data.inserts, filter.entityName) - ) { - throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); - } - - if ( - SUBMISSION_ACTION_TYPE.Values.UPDATES.includes(filter.actionType) && - !_.has(submission.data.updates, filter.entityName) - ) { - throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); - } - - if ( - SUBMISSION_ACTION_TYPE.Values.DELETES.includes(filter.actionType) && - !_.has(submission.data.deletes, filter.entityName) - ) { - throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`); - } - - // Remove entity from the Submission - const updatedActiveSubmissionData = removeItemsFromSubmission(submission.data, { - ...filter, - }); - - // Validate and update Active Submission after removing the entity - submissionProcessor.performDataValidation({ - submissionId: submission.id, - submissionData: updatedActiveSubmissionData, - username, - }); - - logger.info(LOG_MODULE, `Submission '${submission.id}' updated after removing entity '${filter.entityName}'`); - - return { - status: CREATE_SUBMISSION_STATUS.PROCESSING, - description: 'Submission records are being processed', - submissionId: submission.id, - }; - }; - - /** - * Get Submissions by Category - * @param {number} categoryId - The ID of the category for which data is being fetched. - * @param {Object} paginationOptions - Pagination properties - * @param {number} paginationOptions.page - Page number - * @param {number} paginationOptions.pageSize - Items per page - * @param {Object} filterOptions - * @param {boolean} filterOptions.onlyActive - Filter by Active status - * @param {string} filterOptions.username - User Name - * @returns an array of Submission - */ - - const getSubmissionsByCategory = async ( - categoryId: number, - paginationOptions: PaginationOptions, - filterOptions: { - onlyActive: boolean; - username?: string; - organization?: string; - }, - ): Promise<{ - result: SubmissionSummary[]; - metadata: { totalRecords: number; errorMessage?: string }; - }> => { - const recordsPaginated = await submissionRepository.getSubmissionsByCategory( - categoryId, - paginationOptions, - filterOptions, - ); - if (!recordsPaginated || recordsPaginated.length === 0) { - return { - result: [], - metadata: { - totalRecords: 0, - }, - }; - } - - const totalRecords = await submissionRepository.getTotalSubmissionsByCategory(categoryId, filterOptions); - return { - metadata: { - totalRecords, - }, - result: recordsPaginated.map((response) => createSubmissionSummaryResponse(response)), - }; - }; - - /** - * Get Submission by Submission ID - * @param {number} submissionId A Submission ID - * @returns One Submission - */ - const getSubmissionById = async (submissionId: number) => { - const submission = await submissionRepository.getSubmissionById(submissionId); - if (_.isEmpty(submission)) { - return; - } - - return createSubmissionSummaryResponse(submission); - }; - - /** - * Get Submission Records paginated - * @param {number} submissionId A Submission ID - * @param {Object} paginationOptions - Pagination properties - * @param {number} paginationOptions.page - Page number - * @param {number} paginationOptions.pageSize - Items per page - * @param {Object} filterOptions - * @param {string} filterOptions.entityName - Filter by Entity name - * @param {string} filterOptions.actionType - Filter by Action type - * @returns One Submission - */ - const getSubmissionDetailsById = async ({ - submissionId, - paginationOptions, - filterOptions, - }: { - submissionId: number; - paginationOptions: PaginationOptions; - filterOptions: { entityNames: string[]; actionTypes: SubmissionActionType[] }; - }): Promise<{ data: FlattenedSubmissionData[]; errors?: SubmissionRecordErrorDetails[] }> => { - const submission = await submissionRepository.getSubmissionDetailsById(submissionId); - if (!submission) { - throw new BadRequest(`Submission '${submissionId}' not found`); - } - - const submissionEntityNames = [ - ...Object.keys(submission.data.inserts ?? {}), - ...Object.keys(submission.data.updates ?? {}), - ...Object.keys(submission.data.deletes ?? {}), - ]; - - const missingEntityNames = filterOptions.entityNames.filter((name) => !submissionEntityNames.includes(name)); - - if (filterOptions.entityNames.length > 0 && missingEntityNames.length > 0) { - throw new BadRequest( - `Invalid entity name(s) '${missingEntityNames.join(', ')}' for Submission '${submissionId}'`, - ); - } - - return filterAndPaginateSubmissionData({ - data: submission.data, - errors: submission.errors || {}, - filterOptions, - paginationOptions, - }); - }; - - /** - * Get an active Submission by Organization - * @param {Object} params - * @param {number} params.categoryId - * @param {string} params.username - * @param {string} params.organization - * @returns One Active Submission - */ - const getActiveSubmissionByOrganization = async ({ - categoryId, - username, - organization, - }: { - categoryId: number; - username: string; - organization: string; - }): Promise => { - const submission = await submissionRepository.getActiveSubmissionSummary({ - organization, - username, - categoryId, - }); - if (_.isEmpty(submission)) { - return; - } - - return createSubmissionSummaryResponse(submission); - }; - - /** - * Find the current Active Submission or Create an Open Active Submission with initial values and no schema data. - * @param {object} params - * @param {string} params.username Owner of the Submission - * @param {number} params.categoryId Category ID of the Submission - * @param {string} params.organization Organization name - * @returns number ID of the Active Submission - */ - const getOrCreateActiveSubmission = async (params: { - username: string; - categoryId: number; - organization: string; - }): Promise => { - const { categoryId, username, organization } = params; - const { getActiveDictionaryByCategory } = categoryRepository; - - const activeSubmission = await submissionRepository.getActiveSubmissionSummary({ - categoryId, - username, - organization, - }); - if (activeSubmission) { - return activeSubmission.id; - } - - const currentDictionary = await getActiveDictionaryByCategory(categoryId); - - if (!currentDictionary) { - throw new InternalServerError(`Dictionary in category '${categoryId}' not found`); - } - - const newSubmissionInput: NewSubmission = { - createdBy: username, - data: {}, - dictionaryCategoryId: categoryId, - dictionaryId: currentDictionary.id, - errors: {}, - organization: organization, - status: SUBMISSION_STATUS.OPEN, - }; - - return submissionRepository.save(newSubmissionInput); - }; - - /** - * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database - * @param {object} params - * @param {Record[]} params.records An array of records - * @param {string} params.entityName Entity Name of the Records - * @param {number} params.categoryId Category ID of the Submission - * @param {string} params.organization Organization name - * @param {string} params.username User name creating the Submission - * @returns The Active Submission created or Updated - */ - const submitJson = async ({ - data, - categoryId, - organization, - username, - }: { - data: EntityData; - categoryId: number; - organization: string; - username: string; - }): Promise => { - const entityNames = Object.keys(data); - logger.info( - LOG_MODULE, - `Processing '${entityNames.length}' entities on category id '${categoryId}' organization '${organization}'`, - ); - if (entityNames.length === 0) { - return { - status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: 'No valid data for submission', - }; - } - - const currentDictionary = await categoryRepository.getActiveDictionaryByCategory(categoryId); - - if (_.isEmpty(currentDictionary)) { - return { - status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: `Dictionary in category '${categoryId}' not found`, - }; - } - - const schemasDictionary: SchemasDictionary = { - name: currentDictionary.name, - version: currentDictionary.version, - schemas: currentDictionary.schemas, - }; - - // Validate entity name - const invalidEntities = entityNames.filter((name) => !getSchemaByName(name, schemasDictionary)); - if (invalidEntities.length) { - return { - status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: `Invalid entity name '${invalidEntities}' for submission`, - }; - } - - // Get Active Submission or Open a new one - const activeSubmissionId = await getOrCreateActiveSubmission({ categoryId, username, organization }); - - // Schema validation runs asynchronously and does not block execution. - // The results will be saved to the database. - submissionProcessor.processInsertRecordsAsync({ - records: data, - submissionId: activeSubmissionId, - schemasDictionary, - username, - }); - - return { - status: CREATE_SUBMISSION_STATUS.PROCESSING, - description: 'Submission records are being processed', - submissionId: activeSubmissionId, - }; - }; - - /** - * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database - * @param {object} params - * @param {Express.Multer.File[]} params.files An array of files - * @param {number} params.categoryId Category ID of the Submission - * @param {string} params.organization Organization name - * @param {string} params.userName User name creating the Submission - * @returns The Active Submission created or Updated - */ - const submitFiles = async ({ - files, - categoryId, - organization, - username, - }: { - files: Express.Multer.File[]; - categoryId: number; - organization: string; - username: string; - }): Promise => { - logger.info(LOG_MODULE, `Processing '${files.length}' files on category id '${categoryId}'`); - - if (files.length === 0) { - return { - status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: 'No valid files for submission', - batchErrors: [], - inProcessEntities: [], - }; - } - - const currentDictionary = await categoryRepository.getActiveDictionaryByCategory(categoryId); - - if (_.isEmpty(currentDictionary)) { - throw new BadRequest(`Dictionary in category '${categoryId}' not found`); - } - - const schemasDictionary: SchemasDictionary = { - name: currentDictionary.name, - version: currentDictionary.version, - schemas: currentDictionary.schemas, - }; - - // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates) - // TODO: Use filename map to identify files, concatenate records if multiple files map to same entity - const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name); - const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames); - - if (_.isEmpty(validFileEntity)) { - logger.info(LOG_MODULE, `No valid files for submission`); - } - - // step 2 Validation. Validate fieldNames (missing required fields based on schema) - const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity); - - const batchErrors = [...fileNamesErrors, ...fieldNameErrors]; - const entitiesToProcess = Object.keys(checkedEntities); - - if (_.isEmpty(checkedEntities)) { - logger.info(LOG_MODULE, 'Found errors on Submission files.', JSON.stringify(batchErrors)); - return { - status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION, - description: 'No valid entities in submission', - batchErrors, - inProcessEntities: entitiesToProcess, - }; - } - - // Get Active Submission or Open a new one - const activeSubmissionId = await getOrCreateActiveSubmission({ categoryId, username, organization }); - - // TODO: Add files to submission, then run validation separately. Currently these processes are both - // done by the function that adds the files to the submission. - - // Start background process of adding files to submission - // Running Schema validation in the background do not need to wait - // Result of validations will be stored in database - submissionProcessor.addFilesToSubmissionAsync(checkedEntities, { - schemasDictionary, - categoryId, - organization, - username, - }); - - if (batchErrors.length === 0) { - return { - status: CREATE_SUBMISSION_STATUS.PROCESSING, - description: 'Submission files are being processed', - submissionId: activeSubmissionId, - batchErrors, - inProcessEntities: entitiesToProcess, - }; - } - - return { - status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION, - description: 'Some Submission files are being processed while others were unable to process', - submissionId: activeSubmissionId, - batchErrors, - inProcessEntities: entitiesToProcess, - }; - }; - - return { - commitSubmission, - deleteActiveSubmissionById, - deleteActiveSubmissionEntity, - getSubmissionsByCategory, - getSubmissionById, - getSubmissionDetailsById, - getActiveSubmissionByOrganization, - getOrCreateActiveSubmission, - submitJson, - submitFiles, - }; -}; - -export default submissionService; diff --git a/packages/data-provider/src/services/submission/submissionService.ts b/packages/data-provider/src/services/submission/submissionService.ts index 858ea1ea..0d967356 100644 --- a/packages/data-provider/src/services/submission/submissionService.ts +++ b/packages/data-provider/src/services/submission/submissionService.ts @@ -25,8 +25,6 @@ import { import { CommitSubmissionResult, CREATE_SUBMISSION_STATUS, - type SubmitDataResult, - type SubmitFileResult, type DeleteSubmissionResult, type EntityData, type PaginationOptions, @@ -34,6 +32,8 @@ import { SUBMISSION_STATUS, type SubmissionActionType, SubmissionSummary, + type SubmitDataResult, + type SubmitFileResult, } from '../../utils/types.js'; import { default as createSubmissionProcessor } from './submissionProcessor.js';