Media upload feature

.github/workflows/api-deploy-staging.yml

@@ -45,6 +45,12 @@ jobs:
           echo "S3_ACCESS_KEY='$( echo '${{ secrets.S3_ACCESS_KEY }}' )'" >> .env
           echo "S3_SECRET_KEY='$( echo '${{ secrets.S3_SECRET_KEY }}' )'" >> .env
           echo "S3_IMG_BUCKET='$( echo '${{ secrets.S3_IMG_BUCKET }}' )'" >> .env
+          echo "S3_MEDIA_BUCKET='$( echo '${{ secrets.S3_MEDIA_BUCKET }}' )'" >> .env
+          echo "S3_MEDIA_PORT='$( echo '${{ secrets.S3_MEDIA_PORT }}' )'" >> .env
+          echo "S3_MEDIA_USE_SSL='$( echo '${{ secrets.S3_MEDIA_USE_SSL }}' )'" >> .env
+          echo "S3_MEDIA_ACCESS_KEY='$( echo '${{ secrets.S3_MEDIA_ACCESS_KEY }}' )'" >> .env
+          echo "S3_MEDIA_SECRET_KEY='$( echo '${{ secrets.S3_MEDIA_SECRET_KEY }}' )'" >> .env
+          echo "S3_MEDIA_ENDPOINT='$( echo '${{ secrets.S3_MEDIA_ENDPOINT }}' )'" >> .env
           echo "CORS_ORIGIN='$( echo '${{ vars.CORS_ORIGIN }}' )'" >> .env
           echo "JWT_MAPPINGS='$( echo '${{ vars.JWT_MAPPINGS }}' )'" >> .env
           echo "MAX_HTTP_BUFFER_SIZE='$( echo '${{ vars.MAX_HTTP_BUFFER_SIZE }}' )'" >> .env

api/.env.example

@@ -31,6 +31,17 @@ S3_SECRET_KEY="minio123"
 S3_IMG_BUCKET="images"
 S3_IMG_QUALITY=80
 
+# Local development S3 settings
+S3_MEDIA_ENDPOINT="localhost"
+S3_MEDIA_PORT=9000
+S3_MEDIA_USE_SSL=false
+S3_MEDIA_ACCESS_KEY="minio"
+S3_MEDIA_SECRET_KEY="minio123"
+S3_MEDIA_BUCKET="audios"
+
 # Socket.io settings
 # Optional maximum upload file size in bytes (default 10MB)
 MAX_HTTP_BUFFER_SIZE=10000000
+
+# Maximum file size for media uploads in bytes
+MAX_MEDIA_UPLOAD_FILE_SIZE=15000000 # 15MB

api/.env.test.example

@@ -14,3 +14,10 @@ S3_PORT=9000
 S3_USE_SSL=false
 S3_ACCESS_KEY="minio"
 S3_SECRET_KEY="minio123"
+
+# S3 Media storage access details (defaults to same as S3 if not set)
+S3_MEDIA_ENDPOINT="127.0.0.1"
+S3_MEDIA_PORT=9000
+S3_MEDIA_USE_SSL=false
+S3_MEDIA_ACCESS_KEY="minio"
+S3_MEDIA_SECRET_KEY="minio123"

api/package.json

@@ -32,6 +32,7 @@
         "class-validator": "^0.14.1",
         "jsonwebtoken": "^9.0.2",
         "minio": "^8.0.0",
+        "music-metadata": "^11.9.0",
         "nano": "^10.1.4",
         "nest-winston": "^1.9.4",
         "reflect-metadata": "^0.1.13",

api/src/app.module.ts

@@ -12,6 +12,7 @@ import { SearchService } from "./endpoints/search.service";
 import { ChangeRequestService } from "./endpoints/changeRequest.service";
 import { ChangeRequestController } from "./endpoints/changeRequest.controller";
 import * as winston from "winston";
+import { S3MediaService } from "./s3-media/media.service";
 
 let winstonTransport: winston.transport;
 if (!process.env.NODE_ENV || process.env.NODE_ENV === "development") {
@@ -47,6 +48,13 @@ if (!process.env.NODE_ENV || process.env.NODE_ENV === "development") {
         }),
     ],
     controllers: [AppController, SearchController, ChangeRequestController],
-    providers: [DbService, Socketio, S3Service, SearchService, ChangeRequestService],
+    providers: [
+        DbService,
+        Socketio,
+        S3Service,
+        S3MediaService,
+        SearchService,
+        ChangeRequestService,
+    ],
 })
 export class AppModule {}

api/src/changeRequests/documentProcessing/processContentDto.spec.ts

@@ -9,15 +9,18 @@ import { ChangeReqDto } from "../../dto/ChangeReqDto";
 import { PostDto } from "../../dto/PostDto";
 import { PublishStatus } from "../../enums";
 import { TagDto } from "../../dto/TagDto";
+import { S3MediaService } from "../../s3-media/media.service";
 
 describe("processContentDto", () => {
     let db: DbService;
     let s3: S3Service;
+    let s3Media: S3MediaService;
 
     beforeAll(async () => {
         const testingModule = await createTestingModule("process-content-dto");
         db = testingModule.dbService;
         s3 = testingModule.s3Service;
+        s3Media = testingModule.s3MediaService;
         PermissionSystem.upsertGroups((await db.getGroups()).docs);
     });
 
@@ -27,7 +30,14 @@ describe("processContentDto", () => {
         changeRequest.doc._id = "test-slug-1";
         changeRequest.doc.slug = "this-is-a-test-slug";
 
-        const res = await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3);
+        const res = await processChangeRequest(
+            "",
+            changeRequest,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
         const dbDoc = await db.getDoc(changeRequest.doc._id);
 
         expect(res.result.ok).toBe(true);
@@ -40,8 +50,15 @@ describe("processContentDto", () => {
         changeRequest.doc._id = "test-slug-1";
         changeRequest.doc.slug = "this-is-a-test-slug";
 
-        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3); // ensure that the slug is already in use
-        const res = await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3);
+        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3, s3Media); // ensure that the slug is already in use
+        const res = await processChangeRequest(
+            "",
+            changeRequest,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
         const dbDoc = await db.getDoc(changeRequest.doc._id);
 
         expect(res.result.ok).toBe(true);
@@ -54,15 +71,22 @@ describe("processContentDto", () => {
         changeRequest1.doc.parentId = "post-blog1";
         changeRequest1.doc._id = "test-slug-1";
         changeRequest1.doc.slug = "this-is-a-test-slug";
-        await processChangeRequest("", changeRequest1, ["group-super-admins"], db, s3);
+        await processChangeRequest("", changeRequest1, ["group-super-admins"], db, s3, s3Media);
 
         // Create a new change request with the same slug
         const changeRequest2 = changeRequest_content();
         changeRequest2.doc.parentId = "post-blog1";
         changeRequest2.doc._id = "test-slug-2";
         changeRequest2.doc.slug = "this-is-a-test-slug";
 
-        const res = await processChangeRequest("", changeRequest2, ["group-super-admins"], db, s3);
+        const res = await processChangeRequest(
+            "",
+            changeRequest2,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
         const dbDoc = await db.getDoc(changeRequest2.doc._id);
 
         expect(res.result.ok).toBe(true);
@@ -75,7 +99,14 @@ describe("processContentDto", () => {
         changeRequest.doc._id = "test-slug-1";
         changeRequest.doc.slug = 'Invalid Slug! 123 無效的 Bør ikke være tilladt "#$%&/()=?`';
 
-        const res = await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3);
+        const res = await processChangeRequest(
+            "",
+            changeRequest,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
         const dbDoc = await db.getDoc(changeRequest.doc._id);
 
         expect(res.result.ok).toBe(true);
@@ -91,7 +122,7 @@ describe("processContentDto", () => {
         delete changeRequest.doc.parentPostType;
         delete changeRequest.doc.parentPublishDateVisible;
 
-        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3);
+        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3, s3Media);
         const dbDoc = await db.getDoc(changeRequest.doc._id);
 
         expect(dbDoc.docs[0].memberOf).toEqual(["group-public-content"]);
@@ -121,7 +152,7 @@ describe("processContentDto", () => {
         delete changeRequest.doc.parentPublishDateVisible;
         delete changeRequest.doc.parentTaggedDocs;
 
-        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3);
+        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3, s3Media);
         const dbDoc = await db.getDoc(changeRequest.doc._id);
 
         expect(dbDoc.docs[0].memberOf).toEqual(["group-public-content"]);
@@ -147,7 +178,7 @@ describe("processContentDto", () => {
         };
 
         PermissionSystem.upsertGroups((await db.getGroups()).docs);
-        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3);
+        await processChangeRequest("", changeRequest, ["group-super-admins"], db, s3, s3Media);
 
         const res = await db.getContentByParentId(changeRequest.doc._id);
         const docsCount = res.docs.length;
@@ -178,7 +209,14 @@ describe("processContentDto", () => {
         changeRequest1.doc.parentId = "post-blog1";
         changeRequest1.doc._id = "content-en";
         changeRequest1.doc.language = "lang-eng";
-        await processChangeRequest("test-user", changeRequest1, ["group-super-admins"], db, s3);
+        await processChangeRequest(
+            "test-user",
+            changeRequest1,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
 
         // Add a new translation for the same parent
         const changeRequest2 = changeRequest_content();
@@ -191,6 +229,7 @@ describe("processContentDto", () => {
             ["group-super-admins"],
             db,
             s3,
+            s3Media,
         );
 
         // Fetch the documents from the database
@@ -214,7 +253,14 @@ describe("processContentDto", () => {
         changeRequest1.doc._id = "content-en";
         changeRequest1.doc.language = "lang-eng";
         changeRequest1.doc.status = PublishStatus.Published;
-        await processChangeRequest("test-user", changeRequest1, ["group-super-admins"], db, s3);
+        await processChangeRequest(
+            "test-user",
+            changeRequest1,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
 
         // Add a new translation for the same parent
         const changeRequest2 = changeRequest_content();
@@ -228,6 +274,7 @@ describe("processContentDto", () => {
             ["group-super-admins"],
             db,
             s3,
+            s3Media,
         );
 
         // Fetch the documents from the database
@@ -245,7 +292,14 @@ describe("processContentDto", () => {
 
         // Update the English content document to draft
         changeRequest1.doc.status = PublishStatus.Draft;
-        await processChangeRequest("test-user", changeRequest1, ["group-super-admins"], db, s3);
+        await processChangeRequest(
+            "test-user",
+            changeRequest1,
+            ["group-super-admins"],
+            db,
+            s3,
+            s3Media,
+        );
 
         // Fetch the documents from the database
         const dbDocEn2 = await db.getDoc("content-en");

api/src/changeRequests/documentProcessing/processContentDto.ts

@@ -22,6 +22,7 @@ export default async function processContentDto(doc: ContentDto, db: DbService)
         doc.memberOf = parentDoc.memberOf;
         doc.parentTags = parentDoc.tags;
         doc.parentImageData = parentDoc.imageData;
+        doc.parentMedia = parentDoc.media;
 
         if (parentDoc.type == DocType.Post) {
             doc.parentPostType = (parentDoc as PostDto).postType;