diff --git a/localstack/init/ready.d/init.sh b/localstack/init/ready.d/init.sh index 14097702..be4d35da 100755 --- a/localstack/init/ready.d/init.sh +++ b/localstack/init/ready.d/init.sh @@ -1,9 +1,7 @@ #!/bin/sh set -eu -bucket_prefix="${EXPORT_BUCKET_PREFIX:-gbt-exports}" -bucket_name="${bucket_prefix}-local" - +bucket_name="${STATIC_ASSET_BUCKET:gbt-static-assets}" if ! awslocal s3api head-bucket --bucket "$bucket_name" >/dev/null 2>&1; then awslocal s3 mb "s3://${bucket_name}" >/dev/null fi diff --git a/src/modules/export/data-access/ExportStorageRepository.ts b/src/modules/export/data-access/ExportStorageRepository.ts deleted file mode 100644 index 93284afb..00000000 --- a/src/modules/export/data-access/ExportStorageRepository.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Upload } from "@aws-sdk/lib-storage"; -import { Readable } from "stream"; -import { createLogger } from "@/logging"; -import { getS3Client } from "@/shared/s3"; - -const EXPORT_BUCKET_PREFIX = process.env.EXPORT_BUCKET_PREFIX ?? "gbt-exports"; - -const s3Client = getS3Client(); - -export interface ExportStorageOptions { - environment: "prod" | "local"; -} - -function exportBucketName(environment: "prod" | "local"): string { - return `${EXPORT_BUCKET_PREFIX}-${environment}`; -} - -function encodeObjectKey(key: string): string { - return key.split("/").map(encodeURIComponent).join("/"); -} - -function joinUrl(baseUrl: string, path: string): string { - const normalizedBase = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; - return new URL(path, normalizedBase).toString(); -} - -export const exportStorageRepository = { - async uploadPdf({ - environment, - key, - stream, - }: ExportStorageOptions & { - key: string; - stream: Readable; - }): Promise { - const bucket = exportBucketName(environment); - const logger = createLogger({ bucket, key }); - - const upload = new Upload({ - client: s3Client, - params: { - Bucket: bucket, - Key: key, - Body: stream, - ContentType: "application/pdf", - }, - }); - - await upload.done(); - logger.info("Export PDF uploaded"); - - return `s3://${bucket}/${key}`; - }, - - publicPdfUrl({ - environment, - key, - }: ExportStorageOptions & { key: string }): string { - const bucket = exportBucketName(environment); - const encodedKey = encodeObjectKey(key); - - const publicBaseUrl = process.env.EXPORT_PUBLIC_BASE_URL; - if (publicBaseUrl) { - return joinUrl(publicBaseUrl, encodedKey); - } - - const publicS3Endpoint = process.env.EXPORT_PUBLIC_S3_ENDPOINT; - if (publicS3Endpoint) { - return joinUrl(publicS3Endpoint, `${bucket}/${encodedKey}`); - } - - const region = process.env.AWS_REGION ?? "us-east-1"; - if (region === "us-east-1") { - return `https://${bucket}.s3.amazonaws.com/${encodedKey}`; - } - - return `https://${bucket}.s3.${region}.amazonaws.com/${encodedKey}`; - }, -}; - -export default exportStorageRepository; diff --git a/src/modules/export/data-access/exportStorageRepository.ts b/src/modules/export/data-access/exportStorageRepository.ts new file mode 100644 index 00000000..00e63d46 --- /dev/null +++ b/src/modules/export/data-access/exportStorageRepository.ts @@ -0,0 +1,47 @@ +import { Upload } from "@aws-sdk/lib-storage"; +import { Readable } from "stream"; +import { createLogger } from "@/logging"; +import { getS3Client } from "@/shared/s3"; + +const EXPORT_BUCKET = process.env.STATIC_ASSET_BUCKET ?? "gbt-static-assets"; + +const s3Client = getS3Client(); + +export const exportStorageRepository = { + async upload({ + key, + source, + type, + }: { + key: string; + source: Readable | Buffer; + type: string; + }): Promise { + const logger = createLogger({ bucket: EXPORT_BUCKET, key }); + + const upload = new Upload({ + client: s3Client, + params: { + Bucket: EXPORT_BUCKET, + Key: key, + Body: source, + ContentType: type, + }, + }); + + await upload.done(); + + const location = `s3://${EXPORT_BUCKET}/${key}`; + logger.info(`Export PDF uploaded to ${location}`); + + return location; + }, + + publicUrl({ key }: { key: string }): string { + if (process.env.NODE_ENV === "production") { + return `https://assets.globalbibletools.com/${key}`; + } else { + return `${process.env.EXPORT_PUBLIC_S3_ENDPOINT}/${EXPORT_BUCKET}/${key}`; + } + }, +}; diff --git a/src/modules/export/jobs/exportInterlinearPdfHandler.ts b/src/modules/export/jobs/exportInterlinearPdfHandler.ts index 5c3cc953..f02bb6f8 100644 --- a/src/modules/export/jobs/exportInterlinearPdfHandler.ts +++ b/src/modules/export/jobs/exportInterlinearPdfHandler.ts @@ -1,7 +1,6 @@ import { logger } from "@/logging"; import jobRepo from "@/shared/jobs/data-access/jobRepository"; -import { getStorageEnvironment } from "@/shared/storageEnvironment"; -import exportStorageRepository from "../data-access/ExportStorageRepository"; +import { exportStorageRepository } from "../data-access/exportStorageRepository"; import { detectScript } from "@/shared/scriptDetection"; import interlinearQueryService from "../data-access/InterlinearQueryService"; import { @@ -15,11 +14,9 @@ export async function exportInterlinearPdfHandler( ) { const jobLogger = logger.child({ jobId: job.id, jobType: job.type }); - const environment = getStorageEnvironment(); - const { languageCode, languageId } = job.payload; - const exportKey = `interlinear/${languageCode}/${job.id}.pdf`; + const exportKey = `interlinear-pdf/${languageCode}.pdf`; try { const books = @@ -61,14 +58,13 @@ export async function exportInterlinearPdfHandler( }, }); - await exportStorageRepository.uploadPdf({ - environment, + await exportStorageRepository.upload({ key: exportKey, - stream, + source: stream, + type: "application/pdf", }); - const downloadUrl = exportStorageRepository.publicPdfUrl({ - environment, + const downloadUrl = exportStorageRepository.publicUrl({ key: exportKey, }); @@ -114,4 +110,4 @@ function formatChapterLabel(chapters: number[]): string { function formatChapterRange(start: number, end: number): string { return start === end ? `${start}` : `${start}-${end}`; -} \ No newline at end of file +} diff --git a/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts b/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts index 15aaea2c..4ab1b4df 100644 --- a/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts +++ b/src/modules/export/jobs/exportInterlinearPdfHandler.unit.ts @@ -7,14 +7,14 @@ import jobRepository from "@/shared/jobs/data-access/jobRepository"; const { mockFetchBooksWithApprovedGlossChapters, - mockUploadPdf, - mockPublicPdfUrl, + mockUpload, + mockPublicUrl, mockGenerateInterlinearPdfDocument, } = vi.hoisted(() => { return { mockFetchBooksWithApprovedGlossChapters: vi.fn(), - mockUploadPdf: vi.fn(), - mockPublicPdfUrl: vi.fn(), + mockUpload: vi.fn(), + mockPublicUrl: vi.fn(), mockGenerateInterlinearPdfDocument: vi.fn(), }; }); @@ -29,12 +29,12 @@ vi.mock("@/modules/export/data-access/InterlinearQueryService", () => { }, }; }); -vi.mock("@/modules/export/data-access/ExportStorageRepository", () => { +vi.mock("@/modules/export/data-access/exportStorageRepository", () => { const repo = { - uploadPdf: mockUploadPdf, - publicPdfUrl: mockPublicPdfUrl, + upload: mockUpload, + publicUrl: mockPublicUrl, }; - return { __esModule: true, exportStorageRepository: repo, default: repo }; + return { __esModule: true, exportStorageRepository: repo }; }); vi.mock("@/modules/export/pdf/InterlinearPdfGenerator", () => ({ generateInterlinearPdfDocument: mockGenerateInterlinearPdfDocument, @@ -59,8 +59,8 @@ describe("exportInterlinearPdfHandler", () => { beforeEach(() => { vi.useFakeTimers(); mockFetchBooksWithApprovedGlossChapters.mockReset(); - mockUploadPdf.mockReset(); - mockPublicPdfUrl.mockReset(); + mockUpload.mockReset(); + mockPublicUrl.mockReset(); mockGenerateInterlinearPdfDocument.mockReset(); mockJobRepoCommit.mockReset(); @@ -116,7 +116,7 @@ describe("exportInterlinearPdfHandler", () => { stream: Readable.from(["pdf"]), pageCount: 3, })); - mockPublicPdfUrl.mockReturnValue("https://exports.example.com/final.pdf"); + mockPublicUrl.mockReturnValue("https://exports.example.com/final.pdf"); }); afterEach(() => { @@ -149,21 +149,19 @@ describe("exportInterlinearPdfHandler", () => { }), }), ); - expect(mockUploadPdf).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - environment: "local", - key: "interlinear/spa/job-1.pdf", - }), - ); - expect(mockPublicPdfUrl).toHaveBeenCalledExactlyOnceWith({ - environment: "local", - key: "interlinear/spa/job-1.pdf", + expect(mockUpload).toHaveBeenCalledExactlyOnceWith({ + key: "interlinear-pdf/spa.pdf", + source: expect.anything(), + type: "application/pdf", + }); + expect(mockPublicUrl).toHaveBeenCalledExactlyOnceWith({ + key: "interlinear-pdf/spa.pdf", }); expect(mockJobRepoCommit).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ id: "job-1", data: { - exportKey: "interlinear/spa/job-1.pdf", + exportKey: "interlinear-pdf/spa.pdf", downloadUrl: "https://exports.example.com/final.pdf", pages: 3, }, @@ -216,7 +214,7 @@ describe("exportInterlinearPdfHandler", () => { }); it("does not record job data when final URL generation fails", async () => { - mockPublicPdfUrl.mockImplementationOnce(() => { + mockPublicUrl.mockImplementationOnce(() => { throw new Error("public URL failed"); }); @@ -224,12 +222,11 @@ describe("exportInterlinearPdfHandler", () => { /public URL failed/, ); - expect(mockUploadPdf).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - environment: "local", - key: "interlinear/spa/job-1.pdf", - }), - ); + expect(mockUpload).toHaveBeenCalledExactlyOnceWith({ + key: "interlinear-pdf/spa.pdf", + source: expect.anything(), + type: "application/pdf", + }); expect(mockJobRepoCommit).not.toHaveBeenCalled(); }); @@ -241,6 +238,6 @@ describe("exportInterlinearPdfHandler", () => { ); expect(mockGenerateInterlinearPdfDocument).not.toHaveBeenCalled(); - expect(mockUploadPdf).not.toHaveBeenCalled(); + expect(mockUpload).not.toHaveBeenCalled(); }); }); diff --git a/src/shared/jobs/model.ts b/src/shared/jobs/model.ts index bd8fe0c6..c5132753 100644 --- a/src/shared/jobs/model.ts +++ b/src/shared/jobs/model.ts @@ -78,7 +78,7 @@ export function createJobModel< id: ulid(), parentJobId: options?.parentJobId, status: JobStatus.Pending, - payload: payloadSchema.parse(payload ?? {}), + payload: payloadSchema.parse(payload), data: undefined as Data | undefined, createdAt: now, updatedAt: now, @@ -90,7 +90,7 @@ export function createJobModel< id: raw.id, parentJobId: raw.parentJobId, status: raw.status, - payload: payloadSchema.parse(raw.payload ?? {}), + payload: payloadSchema.parse(raw.payload), data: raw.data != null ? resolvedDataSchema.parse(raw.data) : undefined, createdAt: raw.createdAt, updatedAt: raw.updatedAt, diff --git a/src/shared/storageEnvironment.ts b/src/shared/storageEnvironment.ts deleted file mode 100644 index 87590364..00000000 --- a/src/shared/storageEnvironment.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type StorageEnvironment = "prod" | "local"; - -export function getStorageEnvironment(): StorageEnvironment { - return process.env.NODE_ENV === "production" ? "prod" : "local"; -}