Restore Data Submission by File Upload

apps/server/.env.schema

@@ -1,9 +1,35 @@
-PORT=3030
-DB_HOST=
-DB_PORT=
-DB_NAME=
-DB_USER=
-DB_PASSWORD=
-LECTERN_URL=
-LOG_LEVEL=
-ID_USELOCAL=
+## Copy this file into `server/.env` to run the server locally.
+## Variables that are not commented require values before the app can run. They have been provided default values
+## that will work with the docker-compose setup.
+## Commented values are optional configs, showing their default values.
+
+## Server
+# PORT=3030
+# LOG_LEVEL=debug
+
+# ALLOWED_ORIGINS=
+# CORS_ENABLED=false
+
+## DB Connection
+DB_HOST=localhost
+DB_NAME=lyric
+DB_PASSWORD=secret
+DB_PORT=5432
+DB_USER=postgres
+
+## Dictionary 
+LECTERN_URL=http://localhost:3000
+
+
+## Submission
+# AUDIT_ENABLED=true
+# SUBMISSION_MAX_SIZE=10485760 # Default 10Mb, value in bytes
+
+## Data
+# ID_USELOCAL=true
+# ID_CUSTOM_ALPHABET=0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ
+# ID_CUSTOM_SIZE=20
+# PLURALIZE_SCHEMAS_ENABLED=true
+
+## Validator
+# VALIDATOR_CONFIG=[]

apps/server/src/config/app.ts

@@ -2,6 +2,10 @@ import 'dotenv/config';
 
 import { type AppConfig, type ValidatorEntry } from '@overture-stack/lyric';
 
+import { getBoolean, getJSONConfig, getOptionalIntegerConfig, getRequiredConfig } from './envUtils.js';
+
+const DEFAULT_SUBMISSION_MAX_SIZE = 10 * 1024 * 1024; // 10Mb
+
 export const getServerConfig = () => {
 	return {
 		port: process.env.PORT || 3030,
@@ -10,39 +14,6 @@ export const getServerConfig = () => {
 	};
 };
 
-export const getBoolean = (env: string | undefined, defaultValue: boolean): boolean => {
-	switch ((env ?? '').toLocaleLowerCase()) {
-		case 'true':
-			return true;
-		case 'false':
-			return false;
-		default:
-			return defaultValue;
-	}
-};
-
-const getRequiredConfig = (name: string) => {
-	const value = process.env[name];
-	if (!value) {
-		throw new Error(`No Environment Variable provided for required configuration parameter '${name}'`);
-	}
-	return value;
-};
-
-const getJSONConfig = (name: string) => {
-	const value = process.env[name];
-
-	if (!value) {
-		return;
-	}
-
-	try {
-		return JSON.parse(value);
-	} catch (error) {
-		throw new Error(`Environment variable '${name}' must be a valid JSON.`);
-	}
-};
-
 const isValidValidatorEntry = (obj: unknown): obj is ValidatorEntry => {
 	return typeof obj === 'object' && obj !== null && 'categoryId' in obj && 'entityName' in obj && 'fieldName' in obj;
 };
@@ -92,5 +63,8 @@ export const appConfig: AppConfig = {
 	schemaService: {
 		url: getRequiredConfig('LECTERN_URL'),
 	},
+	submissionService: {
+		maxFileSize: getOptionalIntegerConfig('SUBMISSION_MAX_SIZE') || DEFAULT_SUBMISSION_MAX_SIZE,
+	},
 	validator: getValidatorConfig('VALIDATOR_CONFIG'),
 };

apps/server/src/config/envUtils.ts

@@ -0,0 +1,46 @@
+export const getBoolean = (env: string | undefined, defaultValue: boolean): boolean => {
+	switch ((env ?? '').toLocaleLowerCase()) {
+		case 'true':
+			return true;
+		case 'false':
+			return false;
+		default:
+			return defaultValue;
+	}
+};
+
+export const getRequiredConfig = (name: string) => {
+	const value = process.env[name];
+	if (!value) {
+		throw new Error(`No Environment Variable provided for required configuration parameter '${name}'`);
+	}
+	return value;
+};
+
+export const getJSONConfig = (name: string) => {
+	const value = process.env[name];
+
+	if (!value) {
+		return;
+	}
+
+	try {
+		return JSON.parse(value);
+	} catch (error) {
+		throw new Error(`Environment variable '${name}' must be a valid JSON.`);
+	}
+};
+
+export const getOptionalIntegerConfig = (name: string): number | undefined => {
+	const value = process.env[name];
+
+	if (value === undefined || value === '') {
+		return undefined;
+	}
+	const number = Number(value);
+
+	if (!(Number.isFinite(number) && Number.isInteger(number))) {
+		throw new Error(`Environment variable '${name}' must be an integer. Recieved invalid value ${value}`);
+	}
+	return number;
+};

apps/server/swagger/audit-api.yml

@@ -1,19 +1,11 @@
 /audit/category/{categoryId}/organization/{organization}:
   get:
-    summary: Retrive change history of stored data
+    summary: Retrieve change history of stored data
     tags:
       - Audit
     parameters:
-      - name: categoryId
-        in: path
-        description: ID of the Category
-        type: string
-        required: true
-      - name: organization
-        in: path
-        description: Name of the Organization
-        type: string
-        required: true
+      - $ref: '#/components/parameters/path/CategoryId'
+      - $ref: '#/components/parameters/path/Organization'
       - name: entityName
         description: Filter events based on entity name
         in: query

apps/server/swagger/categories-api.yml

@@ -27,12 +27,7 @@
     tags:
       - Category
     parameters:
-      - name: categoryId
-        in: path
-        required: true
-        schema:
-          type: string
-        description: ID of the category
+      - $ref: '#/components/parameters/path/CategoryId'
     responses:
       200:
         description: Category Details

apps/server/swagger/dictionary-api.yml

@@ -50,12 +50,7 @@
     tags:
       - Dictionary
     parameters:
-      - name: categoryId
-        in: path
-        required: true
-        schema:
-          type: integer
-        description: The ID of the category containing the dictionary
+      - $ref: '#/components/parameters/path/CategoryId'
     responses:
       200:
         description: Dictionary schema JSON
@@ -77,12 +72,7 @@
     tags:
       - Dictionary
     parameters:
-      - name: categoryId
-        in: path
-        required: true
-        schema:
-          type: integer
-        description: The ID of the category containing the dictionary
+      - $ref: '#/components/parameters/path/CategoryId'
       - name: fileType
         in: query
         required: false

apps/server/swagger/parameters.yml

@@ -6,7 +6,7 @@ components:
         in: path
         required: true
         schema:
-          type: string
+          type: integer
         description: ID of the category to which the data belongs
       EntityName:
         description: The name of the Entity
@@ -22,6 +22,13 @@ components:
         schema:
           type: string
         description: Organization name
+      OrganizationId:
+        name: organizationId
+        in: path
+        required: true
+        schema:
+          type: string
+        description: ID of the organization
       SubmissionId:
         name: submissionId
         in: path

apps/server/swagger/schemas.yml

@@ -200,7 +200,7 @@ components:
         createdBy:
           type: string
           description: Name of user who created the submission
-        udpatedAt:
+        updatedAt:
           type: string
           description: Date and time of latest update
         updatedBy:
@@ -258,7 +258,7 @@ components:
         createdBy:
           type: string
           description: User name who created the submission
-        udpatedAt:
+        updatedAt:
           type: string
           description: Date and time of latest update
         updatedBy:

apps/server/swagger/submission-api.yml

@@ -207,15 +207,52 @@
       503:
         $ref: '#/components/responses/ServiceUnavailableError'
 
-/submission/category/{categoryId}/data:
+/submission/category/{categoryId}/organization/{organizationId}:
   post:
-    summary: Add new data to a submission for the specified category. Returns an Active Submission containing the newly created records
+    summary: Upload and process submission files for the specified category and organization.
     tags:
       - Submission
     parameters:
       - $ref: '#/components/parameters/path/CategoryId'
-      - $ref: '#/components/parameters/query/EntityName'
-      - $ref: '#/components/parameters/query/Organization'
+      - $ref: '#/components/parameters/path/OrganizationId'
+    requestBody:
+      description: File(s) to be added to the submission
+      required: true
+      content:
+        multipart/form-data:
+          schema:
+            type: object
+            properties:
+              files:
+                type: array
+                items:
+                  type: string
+                  format: binary
+    responses:
+      200:
+        description: Submission accepted
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateSubmissionResult'
+      400:
+        $ref: '#/components/responses/BadRequest'
+      401:
+        $ref: '#/components/responses/UnauthorizedError'
+      500:
+        $ref: '#/components/responses/ServerError'
+      503:
+        $ref: '#/components/responses/ServiceUnavailableError'
+
+/submission/category/{categoryId}/organization/{organizationId}/entity/{entityName}:
+  post:
+    summary: Add new data for a single entity to a submission for the specified category and organization.
+    tags:
+      - Submission
+    parameters:
+      - $ref: '#/components/parameters/path/CategoryId'
+      - $ref: '#/components/parameters/path/OrganizationId'
+      - $ref: '#/components/parameters/path/EntityName'
     requestBody:
       description: The JSON payload containing the data to be added to the submission
       required: true
@@ -241,15 +278,15 @@
       503:
         $ref: '#/components/responses/ServiceUnavailableError'
   put:
-    summary: Modifies existing data for a submission. Returns an Active Submission containing the records that will be updated
+    summary: Modifies existing data for a signle entity in the active submission of the specified category and organization.
     tags:
       - Submission
     parameters:
       - $ref: '#/components/parameters/path/CategoryId'
-      - $ref: '#/components/parameters/query/EntityName'
-      - $ref: '#/components/parameters/query/Organization'
+      - $ref: '#/components/parameters/path/OrganizationId'
+      - $ref: '#/components/parameters/path/EntityName'
     requestBody:
-      description: The JSON payload containing the data to be added to the submission
+      description: The JSON payload containing the data to be updated in the submission
       required: true
       content:
         application/json:

packages/data-provider/package.json

@@ -37,25 +37,32 @@
 		"@overture-stack/lectern-client": "2.0.0-beta.6",
 		"@overture-stack/lyric-data-model": "workspace:^",
 		"@overture-stack/sqon-builder": "^1.1.0",
+		"bytes": "^3.1.2",
+		"csv-parse": "^6.1.0",
 		"dotenv": "^16.4.5",
 		"drizzle-orm": "^0.29.5",
 		"express": "^4.19.2",
-		"lodash-es": "^4.17.21",
+		"firstline": "^2.0.2",
 		"jszip": "^3.10.1",
+		"lodash-es": "^4.17.21",
+		"multer": "^2.0.2",
 		"nanoid": "^5.0.7",
 		"pg": "^8.12.0",
 		"plur": "^5.1.0",
 		"winston": "^3.13.1",
 		"zod": "^3.23.8"
 	},
 	"devDependencies": {
+		"@types/bytes": "^3.1.5",
 		"@types/chai-as-promised": "^8.0.1",
 		"@types/deep-freeze": "^0.1.5",
 		"@types/express": "^4.17.21",
 		"@types/express-serve-static-core": "^4.19.5",
+		"@types/firstline": "^2.0.4",
+		"@types/jszip": "^3.4.1",
 		"@types/lodash": "^4.17.7",
 		"@types/lodash-es": "^4.17.12",
-		"@types/jszip": "^3.4.1",
+		"@types/multer": "^2.0.0",
 		"@types/pg": "^8.11.6",
 		"@types/qs": "^6.9.15",
 		"chai-as-promised": "^8.0.0",

packages/data-provider/src/config/config.ts

@@ -24,6 +24,10 @@ export type SchemaServiceConfig = {
 	url: string;
 };
 
+export type SubmissionServiceConfig = {
+	maxFileSize?: number;
+};
+
 export type LoggerConfig = {
 	level?: string;
 	file?: boolean;
@@ -55,6 +59,7 @@ export type AppConfig = {
 	logger: LoggerConfig;
 	onFinishCommit?: (resultOnCommit: ResultOnCommit) => void;
 	schemaService: SchemaServiceConfig;
+	submissionService: SubmissionServiceConfig;
 	validator: ValidatorConfig;
 };
 
@@ -68,4 +73,5 @@ export interface BaseDependencies {
 	logger: Logger;
 	onFinishCommit?: (resultOnCommit: ResultOnCommit) => void;
 	schemaService: SchemaServiceConfig;
+	submissionService?: SubmissionServiceConfig;
 }