From b479f92dd1364fe9b6ab97a8baa671effcae46ae Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 17:01:37 -0500 Subject: [PATCH 1/2] export: make upload repository work for any file type --- localstack/init/ready.d/init.sh | 4 +- .../data-access/ExportStorageRepository.ts | 81 ------------------- .../data-access/exportStorageRepository.ts | 47 +++++++++++ .../jobs/exportInterlinearPdfHandler.ts | 16 ++-- .../jobs/exportInterlinearPdfHandler.unit.ts | 51 ++++++------ src/shared/jobs/model.ts | 4 +- src/shared/storageEnvironment.ts | 5 -- 7 files changed, 80 insertions(+), 128 deletions(-) delete mode 100644 src/modules/export/data-access/ExportStorageRepository.ts create mode 100644 src/modules/export/data-access/exportStorageRepository.ts delete mode 100644 src/shared/storageEnvironment.ts diff --git a/localstack/init/ready.d/init.sh b/localstack/init/ready.d/init.sh index 14097702..be4d35da 100755 --- a/localstack/init/ready.d/init.sh +++ b/localstack/init/ready.d/init.sh @@ -1,9 +1,7 @@ #!/bin/sh set -eu -bucket_prefix="${EXPORT_BUCKET_PREFIX:-gbt-exports}" -bucket_name="${bucket_prefix}-local" - +bucket_name="${STATIC_ASSET_BUCKET:gbt-static-assets}" if ! awslocal s3api head-bucket --bucket "$bucket_name" >/dev/null 2>&1; then awslocal s3 mb "s3://${bucket_name}" >/dev/null fi diff --git a/src/modules/export/data-access/ExportStorageRepository.ts b/src/modules/export/data-access/ExportStorageRepository.ts deleted file mode 100644 index 93284afb..00000000 --- a/src/modules/export/data-access/ExportStorageRepository.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Upload } from "@aws-sdk/lib-storage"; -import { Readable } from "stream"; -import { createLogger } from "@/logging"; -import { getS3Client } from "@/shared/s3"; - -const EXPORT_BUCKET_PREFIX = process.env.EXPORT_BUCKET_PREFIX ?? "gbt-exports"; - -const s3Client = getS3Client(); - -export interface ExportStorageOptions { - environment: "prod" | "local"; -} - -function exportBucketName(environment: "prod" | "local"): string { - return `${EXPORT_BUCKET_PREFIX}-${environment}`; -} - -function encodeObjectKey(key: string): string { - return key.split("/").map(encodeURIComponent).join("/"); -} - -function joinUrl(baseUrl: string, path: string): string { - const normalizedBase = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; - return new URL(path, normalizedBase).toString(); -} - -export const exportStorageRepository = { - async uploadPdf({ - environment, - key, - stream, - }: ExportStorageOptions & { - key: string; - stream: Readable; - }): Promise { - const bucket = exportBucketName(environment); - const logger = createLogger({ bucket, key }); - - const upload = new Upload({ - client: s3Client, - params: { - Bucket: bucket, - Key: key, - Body: stream, - ContentType: "application/pdf", - }, - }); - - await upload.done(); - logger.info("Export PDF uploaded"); - - return `s3://${bucket}/${key}`; - }, - - publicPdfUrl({ - environment, - key, - }: ExportStorageOptions & { key: string }): string { - const bucket = exportBucketName(environment); - const encodedKey = encodeObjectKey(key); - - const publicBaseUrl = process.env.EXPORT_PUBLIC_BASE_URL; - if (publicBaseUrl) { - return joinUrl(publicBaseUrl, encodedKey); - } - - const publicS3Endpoint = process.env.EXPORT_PUBLIC_S3_ENDPOINT; - if (publicS3Endpoint) { - return joinUrl(publicS3Endpoint, `${bucket}/${encodedKey}`); - } - - const region = process.env.AWS_REGION ?? "us-east-1"; - if (region === "us-east-1") { - return `https://${bucket}.s3.amazonaws.com/${encodedKey}`; - } - - return `https://${bucket}.s3.${region}.amazonaws.com/${encodedKey}`; - }, -}; - -export default exportStorageRepository; diff --git a/src/modules/export/data-access/exportStorageRepository.ts b/src/modules/export/data-access/exportStorageRepository.ts new file mode 100644 index 00000000..457d7ba8 --- /dev/null +++ b/src/modules/export/data-access/exportStorageRepository.ts @@ -0,0 +1,47 @@ +import { Upload } from "@aws-sdk/lib-storage"; +import { Readable } from "stream"; +import { createLogger } from "@/logging"; +import { getS3Client } from "@/shared/s3"; + +const EXPORT_BUCKET = process.env.STATIC_ASSET_BUCKET ?? "gbt-static-assets"; + +const s3Client = getS3Client(); + +export const exportStorageRepository = { + async upload({ + key, + source, + type, + }: { + key: string; + source: Readable | Buffer; + type: string; + }): Promise { + const logger = createLogger({ bucket: EXPORT_BUCKET, key }); + + const upload = new Upload({ + client: s3Client, + params: { + Bucket: EXPORT_BUCKET, + Key: key, + Body: source, + ContentType: type, + }, + }); + + await upload.done(); + + const location = `s3://${EXPORT_BUCKET}/${key}`; + logger.info(`Export PDF uploaded to ${location}`); + + return location; + }, + + publicUrl({ key }: { key: string }): string { + if (process.env.NODE_ENV === "production") { + return `https://assets.globalbibletools.com/${key}`; + } else { + return `${process.env.EXPORT_PUBLIC_S3_ENDPOINT}/${key}`; + } + }, +}; diff --git a/src/modules/export/jobs/exportInterlinearPdfHandler.ts b/src/modules/export/jobs/exportInterlinearPdfHandler.ts index 5c3cc953..8372d079 100644 --- a/src/modules/export/jobs/exportInterlinearPdfHandler.ts +++ b/src/modules/export/jobs/exportInterlinearPdfHandler.ts @@ -1,7 +1,6 @@ import { logger } from "@/logging"; import jobRepo from "@/shared/jobs/data-access/jobRepository"; -import { getStorageEnvironment } from "@/shared/storageEnvironment"; -import exportStorageRepository from "../data-access/ExportStorageRepository"; +import { exportStorageRepository } from "../data-access/exportStorageRepository"; import { detectScript } from "@/shared/scriptDetection"; import interlinearQueryService from "../data-access/InterlinearQueryService"; import { @@ -15,8 +14,6 @@ export async function exportInterlinearPdfHandler( ) { const jobLogger = logger.child({ jobId: job.id, jobType: job.type }); - const environment = getStorageEnvironment(); - const { languageCode, languageId } = job.payload; const exportKey = `interlinear/${languageCode}/${job.id}.pdf`; @@ -61,14 +58,13 @@ export async function exportInterlinearPdfHandler( }, }); - await exportStorageRepository.uploadPdf({ - environment, + await exportStorageRepository.upload({ key: exportKey, - stream, + source: stream, + type: "application/pdf", }); - const downloadUrl = exportStorageRepository.publicPdfUrl({ - environment, + const downloadUrl = exportStorageRepository.publicUrl({ key: exportKey, }); @@ -114,4 +110,4 @@ function formatChapterLabel(chapters: number[]): string { function formatChapterRange(start: number, end: number): string { return start === end ? `${start}` : `${start}-${end}`; -} \ No newline at end of file +} diff --git a/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts b/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts index 15aaea2c..38802087 100644 --- a/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts +++ b/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts @@ -7,14 +7,14 @@ import jobRepository from "@/shared/jobs/data-access/jobRepository"; const { mockFetchBooksWithApprovedGlossChapters, - mockUploadPdf, - mockPublicPdfUrl, + mockUpload, + mockPublicUrl, mockGenerateInterlinearPdfDocument, } = vi.hoisted(() => { return { mockFetchBooksWithApprovedGlossChapters: vi.fn(), - mockUploadPdf: vi.fn(), - mockPublicPdfUrl: vi.fn(), + mockUpload: vi.fn(), + mockPublicUrl: vi.fn(), mockGenerateInterlinearPdfDocument: vi.fn(), }; }); @@ -29,12 +29,12 @@ vi.mock("@/modules/export/data-access/InterlinearQueryService", () => { }, }; }); -vi.mock("@/modules/export/data-access/ExportStorageRepository", () => { +vi.mock("@/modules/export/data-access/exportStorageRepository", () => { const repo = { - uploadPdf: mockUploadPdf, - publicPdfUrl: mockPublicPdfUrl, + upload: mockUpload, + publicUrl: mockPublicUrl, }; - return { __esModule: true, exportStorageRepository: repo, default: repo }; + return { __esModule: true, exportStorageRepository: repo }; }); vi.mock("@/modules/export/pdf/InterlinearPdfGenerator", () => ({ generateInterlinearPdfDocument: mockGenerateInterlinearPdfDocument, @@ -59,8 +59,8 @@ describe("exportInterlinearPdfHandler", () => { beforeEach(() => { vi.useFakeTimers(); mockFetchBooksWithApprovedGlossChapters.mockReset(); - mockUploadPdf.mockReset(); - mockPublicPdfUrl.mockReset(); + mockUpload.mockReset(); + mockPublicUrl.mockReset(); mockGenerateInterlinearPdfDocument.mockReset(); mockJobRepoCommit.mockReset(); @@ -116,7 +116,7 @@ describe("exportInterlinearPdfHandler", () => { stream: Readable.from(["pdf"]), pageCount: 3, })); - mockPublicPdfUrl.mockReturnValue("https://exports.example.com/final.pdf"); + mockPublicUrl.mockReturnValue("https://exports.example.com/final.pdf"); }); afterEach(() => { @@ -149,14 +149,12 @@ describe("exportInterlinearPdfHandler", () => { }), }), ); - expect(mockUploadPdf).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - environment: "local", - key: "interlinear/spa/job-1.pdf", - }), - ); - expect(mockPublicPdfUrl).toHaveBeenCalledExactlyOnceWith({ - environment: "local", + expect(mockUpload).toHaveBeenCalledExactlyOnceWith({ + key: "interlinear/spa/job-1.pdf", + source: expect.anything(), + type: "application/pdf", + }); + expect(mockPublicUrl).toHaveBeenCalledExactlyOnceWith({ key: "interlinear/spa/job-1.pdf", }); expect(mockJobRepoCommit).toHaveBeenCalledExactlyOnceWith( @@ -216,7 +214,7 @@ describe("exportInterlinearPdfHandler", () => { }); it("does not record job data when final URL generation fails", async () => { - mockPublicPdfUrl.mockImplementationOnce(() => { + mockPublicUrl.mockImplementationOnce(() => { throw new Error("public URL failed"); }); @@ -224,12 +222,11 @@ describe("exportInterlinearPdfHandler", () => { /public URL failed/, ); - expect(mockUploadPdf).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - environment: "local", - key: "interlinear/spa/job-1.pdf", - }), - ); + expect(mockUpload).toHaveBeenCalledExactlyOnceWith({ + key: "interlinear/spa/job-1.pdf", + source: expect.anything(), + type: "application/pdf", + }); expect(mockJobRepoCommit).not.toHaveBeenCalled(); }); @@ -241,6 +238,6 @@ describe("exportInterlinearPdfHandler", () => { ); expect(mockGenerateInterlinearPdfDocument).not.toHaveBeenCalled(); - expect(mockUploadPdf).not.toHaveBeenCalled(); + expect(mockUpload).not.toHaveBeenCalled(); }); }); diff --git a/src/shared/jobs/model.ts b/src/shared/jobs/model.ts index bd8fe0c6..c5132753 100644 --- a/src/shared/jobs/model.ts +++ b/src/shared/jobs/model.ts @@ -78,7 +78,7 @@ export function createJobModel< id: ulid(), parentJobId: options?.parentJobId, status: JobStatus.Pending, - payload: payloadSchema.parse(payload ?? {}), + payload: payloadSchema.parse(payload), data: undefined as Data | undefined, createdAt: now, updatedAt: now, @@ -90,7 +90,7 @@ export function createJobModel< id: raw.id, parentJobId: raw.parentJobId, status: raw.status, - payload: payloadSchema.parse(raw.payload ?? {}), + payload: payloadSchema.parse(raw.payload), data: raw.data != null ? resolvedDataSchema.parse(raw.data) : undefined, createdAt: raw.createdAt, updatedAt: raw.updatedAt, diff --git a/src/shared/storageEnvironment.ts b/src/shared/storageEnvironment.ts deleted file mode 100644 index 87590364..00000000 --- a/src/shared/storageEnvironment.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type StorageEnvironment = "prod" | "local"; - -export function getStorageEnvironment(): StorageEnvironment { - return process.env.NODE_ENV === "production" ? "prod" : "local"; -} From 709e66cd099c0f1e1b2ffc24d160695b818f122b Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 17:05:52 -0500 Subject: [PATCH 2/2] fix upload directory --- src/modules/export/data-access/exportStorageRepository.ts | 2 +- src/modules/export/jobs/exportInterlinearPdfHandler.ts | 2 +- .../export/jobs/exportInterlinearPdfHandler.unit.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/modules/export/data-access/exportStorageRepository.ts b/src/modules/export/data-access/exportStorageRepository.ts index 457d7ba8..00e63d46 100644 --- a/src/modules/export/data-access/exportStorageRepository.ts +++ b/src/modules/export/data-access/exportStorageRepository.ts @@ -41,7 +41,7 @@ export const exportStorageRepository = { if (process.env.NODE_ENV === "production") { return `https://assets.globalbibletools.com/${key}`; } else { - return `${process.env.EXPORT_PUBLIC_S3_ENDPOINT}/${key}`; + return `${process.env.EXPORT_PUBLIC_S3_ENDPOINT}/${EXPORT_BUCKET}/${key}`; } }, }; diff --git a/src/modules/export/jobs/exportInterlinearPdfHandler.ts b/src/modules/export/jobs/exportInterlinearPdfHandler.ts index 8372d079..f02bb6f8 100644 --- a/src/modules/export/jobs/exportInterlinearPdfHandler.ts +++ b/src/modules/export/jobs/exportInterlinearPdfHandler.ts @@ -16,7 +16,7 @@ export async function exportInterlinearPdfHandler( const { languageCode, languageId } = job.payload; - const exportKey = `interlinear/${languageCode}/${job.id}.pdf`; + const exportKey = `interlinear-pdf/${languageCode}.pdf`; try { const books = diff --git a/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts b/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts index 38802087..4ab1b4df 100644 --- a/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts +++ b/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts @@ -150,18 +150,18 @@ describe("exportInterlinearPdfHandler", () => { }), ); expect(mockUpload).toHaveBeenCalledExactlyOnceWith({ - key: "interlinear/spa/job-1.pdf", + key: "interlinear-pdf/spa.pdf", source: expect.anything(), type: "application/pdf", }); expect(mockPublicUrl).toHaveBeenCalledExactlyOnceWith({ - key: "interlinear/spa/job-1.pdf", + key: "interlinear-pdf/spa.pdf", }); expect(mockJobRepoCommit).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ id: "job-1", data: { - exportKey: "interlinear/spa/job-1.pdf", + exportKey: "interlinear-pdf/spa.pdf", downloadUrl: "https://exports.example.com/final.pdf", pages: 3, }, @@ -223,7 +223,7 @@ describe("exportInterlinearPdfHandler", () => { ); expect(mockUpload).toHaveBeenCalledExactlyOnceWith({ - key: "interlinear/spa/job-1.pdf", + key: "interlinear-pdf/spa.pdf", source: expect.anything(), type: "application/pdf", });