export: export all static files to the same s3 bucket

localstack/init/ready.d/init.sh

@@ -1,9 +1,7 @@
 #!/bin/sh
 set -eu
 
-bucket_prefix="${EXPORT_BUCKET_PREFIX:-gbt-exports}"
-bucket_name="${bucket_prefix}-local"
-
+bucket_name="${STATIC_ASSET_BUCKET:gbt-static-assets}"
 if ! awslocal s3api head-bucket --bucket "$bucket_name" >/dev/null 2>&1; then
   awslocal s3 mb "s3://${bucket_name}" >/dev/null
 fi

src/modules/export/data-access/exportStorageRepository.ts

@@ -0,0 +1,47 @@
+import { Upload } from "@aws-sdk/lib-storage";
+import { Readable } from "stream";
+import { createLogger } from "@/logging";
+import { getS3Client } from "@/shared/s3";
+
+const EXPORT_BUCKET = process.env.STATIC_ASSET_BUCKET ?? "gbt-static-assets";
+
+const s3Client = getS3Client();
+
+export const exportStorageRepository = {
+  async upload({
+    key,
+    source,
+    type,
+  }: {
+    key: string;
+    source: Readable | Buffer;
+    type: string;
+  }): Promise<string> {
+    const logger = createLogger({ bucket: EXPORT_BUCKET, key });
+
+    const upload = new Upload({
+      client: s3Client,
+      params: {
+        Bucket: EXPORT_BUCKET,
+        Key: key,
+        Body: source,
+        ContentType: type,
+      },
+    });
+
+    await upload.done();
+
+    const location = `s3://${EXPORT_BUCKET}/${key}`;
+    logger.info(`Export PDF uploaded to ${location}`);
+
+    return location;
+  },
+
+  publicUrl({ key }: { key: string }): string {
+    if (process.env.NODE_ENV === "production") {
+      return `https://assets.globalbibletools.com/${key}`;
+    } else {
+      return `${process.env.EXPORT_PUBLIC_S3_ENDPOINT}/${EXPORT_BUCKET}/${key}`;
+    }
+  },
+};

src/modules/export/jobs/exportInterlinearPdfHandler.ts

@@ -1,7 +1,6 @@
 import { logger } from "@/logging";
 import jobRepo from "@/shared/jobs/data-access/jobRepository";
-import { getStorageEnvironment } from "@/shared/storageEnvironment";
-import exportStorageRepository from "../data-access/ExportStorageRepository";
+import { exportStorageRepository } from "../data-access/exportStorageRepository";
 import { detectScript } from "@/shared/scriptDetection";
 import interlinearQueryService from "../data-access/InterlinearQueryService";
 import {
@@ -15,11 +14,9 @@ export async function exportInterlinearPdfHandler(
 ) {
   const jobLogger = logger.child({ jobId: job.id, jobType: job.type });
 
-  const environment = getStorageEnvironment();
-
   const { languageCode, languageId } = job.payload;
 
-  const exportKey = `interlinear/${languageCode}/${job.id}.pdf`;
+  const exportKey = `interlinear-pdf/${languageCode}.pdf`;
 
   try {
     const books =
@@ -61,14 +58,13 @@ export async function exportInterlinearPdfHandler(
       },
     });
 
-    await exportStorageRepository.uploadPdf({
-      environment,
+    await exportStorageRepository.upload({
       key: exportKey,
-      stream,
+      source: stream,
+      type: "application/pdf",
     });
 
-    const downloadUrl = exportStorageRepository.publicPdfUrl({
-      environment,
+    const downloadUrl = exportStorageRepository.publicUrl({
       key: exportKey,
     });
 
@@ -114,4 +110,4 @@ function formatChapterLabel(chapters: number[]): string {
 
 function formatChapterRange(start: number, end: number): string {
   return start === end ? `${start}` : `${start}-${end}`;
-}
+}

src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts

@@ -7,14 +7,14 @@ import jobRepository from "@/shared/jobs/data-access/jobRepository";
 
 const {
   mockFetchBooksWithApprovedGlossChapters,
-  mockUploadPdf,
-  mockPublicPdfUrl,
+  mockUpload,
+  mockPublicUrl,
   mockGenerateInterlinearPdfDocument,
 } = vi.hoisted(() => {
   return {
     mockFetchBooksWithApprovedGlossChapters: vi.fn(),
-    mockUploadPdf: vi.fn(),
-    mockPublicPdfUrl: vi.fn(),
+    mockUpload: vi.fn(),
+    mockPublicUrl: vi.fn(),
     mockGenerateInterlinearPdfDocument: vi.fn(),
   };
 });
@@ -29,12 +29,12 @@ vi.mock("@/modules/export/data-access/InterlinearQueryService", () => {
     },
   };
 });
-vi.mock("@/modules/export/data-access/ExportStorageRepository", () => {
+vi.mock("@/modules/export/data-access/exportStorageRepository", () => {
   const repo = {
-    uploadPdf: mockUploadPdf,
-    publicPdfUrl: mockPublicPdfUrl,
+    upload: mockUpload,
+    publicUrl: mockPublicUrl,
   };
-  return { __esModule: true, exportStorageRepository: repo, default: repo };
+  return { __esModule: true, exportStorageRepository: repo };
 });
 vi.mock("@/modules/export/pdf/InterlinearPdfGenerator", () => ({
   generateInterlinearPdfDocument: mockGenerateInterlinearPdfDocument,
@@ -59,8 +59,8 @@ describe("exportInterlinearPdfHandler", () => {
   beforeEach(() => {
     vi.useFakeTimers();
     mockFetchBooksWithApprovedGlossChapters.mockReset();
-    mockUploadPdf.mockReset();
-    mockPublicPdfUrl.mockReset();
+    mockUpload.mockReset();
+    mockPublicUrl.mockReset();
     mockGenerateInterlinearPdfDocument.mockReset();
     mockJobRepoCommit.mockReset();
 
@@ -116,7 +116,7 @@ describe("exportInterlinearPdfHandler", () => {
       stream: Readable.from(["pdf"]),
       pageCount: 3,
     }));
-    mockPublicPdfUrl.mockReturnValue("https://exports.example.com/final.pdf");
+    mockPublicUrl.mockReturnValue("https://exports.example.com/final.pdf");
   });
 
   afterEach(() => {
@@ -149,21 +149,19 @@ describe("exportInterlinearPdfHandler", () => {
         }),
       }),
     );
-    expect(mockUploadPdf).toHaveBeenCalledExactlyOnceWith(
-      expect.objectContaining({
-        environment: "local",
-        key: "interlinear/spa/job-1.pdf",
-      }),
-    );
-    expect(mockPublicPdfUrl).toHaveBeenCalledExactlyOnceWith({
-      environment: "local",
-      key: "interlinear/spa/job-1.pdf",
+    expect(mockUpload).toHaveBeenCalledExactlyOnceWith({
+      key: "interlinear-pdf/spa.pdf",
+      source: expect.anything(),
+      type: "application/pdf",
+    });
+    expect(mockPublicUrl).toHaveBeenCalledExactlyOnceWith({
+      key: "interlinear-pdf/spa.pdf",
     });
     expect(mockJobRepoCommit).toHaveBeenCalledExactlyOnceWith(
       expect.objectContaining({
         id: "job-1",
         data: {
-          exportKey: "interlinear/spa/job-1.pdf",
+          exportKey: "interlinear-pdf/spa.pdf",
           downloadUrl: "https://exports.example.com/final.pdf",
           pages: 3,
         },
@@ -216,20 +214,19 @@ describe("exportInterlinearPdfHandler", () => {
   });
 
   it("does not record job data when final URL generation fails", async () => {
-    mockPublicPdfUrl.mockImplementationOnce(() => {
+    mockPublicUrl.mockImplementationOnce(() => {
       throw new Error("public URL failed");
     });
 
     await expect(exportInterlinearPdfHandler(baseJob)).rejects.toThrow(
       /public URL failed/,
     );
 
-    expect(mockUploadPdf).toHaveBeenCalledExactlyOnceWith(
-      expect.objectContaining({
-        environment: "local",
-        key: "interlinear/spa/job-1.pdf",
-      }),
-    );
+    expect(mockUpload).toHaveBeenCalledExactlyOnceWith({
+      key: "interlinear-pdf/spa.pdf",
+      source: expect.anything(),
+      type: "application/pdf",
+    });
     expect(mockJobRepoCommit).not.toHaveBeenCalled();
   });
 
@@ -241,6 +238,6 @@ describe("exportInterlinearPdfHandler", () => {
     );
 
     expect(mockGenerateInterlinearPdfDocument).not.toHaveBeenCalled();
-    expect(mockUploadPdf).not.toHaveBeenCalled();
+    expect(mockUpload).not.toHaveBeenCalled();
   });
 });

src/shared/jobs/model.ts

@@ -78,7 +78,7 @@ export function createJobModel<
         id: ulid(),
         parentJobId: options?.parentJobId,
         status: JobStatus.Pending,
-        payload: payloadSchema.parse(payload ?? {}),
+        payload: payloadSchema.parse(payload),
         data: undefined as Data | undefined,
         createdAt: now,
         updatedAt: now,
@@ -90,7 +90,7 @@ export function createJobModel<
         id: raw.id,
         parentJobId: raw.parentJobId,
         status: raw.status,
-        payload: payloadSchema.parse(raw.payload ?? {}),
+        payload: payloadSchema.parse(raw.payload),
         data: raw.data != null ? resolvedDataSchema.parse(raw.data) : undefined,
         createdAt: raw.createdAt,
         updatedAt: raw.updatedAt,