From c0b8306ad61202c274a7874764a3af36c5e63f20 Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 20:43:32 -0500 Subject: [PATCH 1/7] scaffold sqlite job # Conflicts: # src/shared/jobs/jobHandlerRegistry.ts # src/shared/jobs/jobRegistry.ts # Conflicts: # src/shared/jobs/jobHandlerRegistry.ts # src/shared/jobs/jobRegistry.ts --- .../export/jobs/ExportGlossesSqliteJob.ts | 11 ++++++++++ .../export/jobs/exportGlossesSqliteHandler.ts | 22 +++++++++++++++++++ src/shared/jobs/jobHandlerRegistry.ts | 5 +++++ src/shared/jobs/jobRegistry.ts | 2 ++ 4 files changed, 40 insertions(+) create mode 100644 src/modules/export/jobs/ExportGlossesSqliteJob.ts create mode 100644 src/modules/export/jobs/exportGlossesSqliteHandler.ts diff --git a/src/modules/export/jobs/ExportGlossesSqliteJob.ts b/src/modules/export/jobs/ExportGlossesSqliteJob.ts new file mode 100644 index 00000000..9147c6c2 --- /dev/null +++ b/src/modules/export/jobs/ExportGlossesSqliteJob.ts @@ -0,0 +1,11 @@ +import { createJobModel } from "@/shared/jobs/model"; +import * as z from "zod"; + +const ExportGlossesSqlitePayloadSchema = z.object({ + languageCodes: z.array(z.string()), +}); + +export class ExportGlossesSqliteJob extends createJobModel({ + type: "export_glosses_sqlite", + payloadSchema: ExportGlossesSqlitePayloadSchema, +}) {} diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.ts new file mode 100644 index 00000000..1fba3c08 --- /dev/null +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.ts @@ -0,0 +1,22 @@ +import { logger } from "@/logging"; +import { ExportGlossesSqliteJob } from "./ExportGlossesSqliteJob"; + +export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { + const jobLogger = logger.child({ + job: { + id: job.id, + type: job.type, + }, + }); + + const { languageCodes } = job.payload; + + for (const languageCode of languageCodes) { + jobLogger.info({ languageCode }, "Exporting glosses SQLite for language"); + } + + jobLogger.info( + { languageCount: languageCodes.length }, + "Glosses SQLite export complete", + ); +} diff --git a/src/shared/jobs/jobHandlerRegistry.ts b/src/shared/jobs/jobHandlerRegistry.ts index 56911850..852388f6 100644 --- a/src/shared/jobs/jobHandlerRegistry.ts +++ b/src/shared/jobs/jobHandlerRegistry.ts @@ -8,6 +8,7 @@ import { updateBookCompletionProgressHandler } from "@/modules/reporting/jobs/up import { importAIGlossesHandler } from "@/modules/translation/jobs/importAIGlossesHandler"; import { syncAIGlossLanguagesHandler } from "@/modules/translation/jobs/syncAIGlossLanguagesHandler"; import { sendEmailHandler } from "@/shared/email/jobs/sendEmailHandler"; +import { exportGlossesSqliteHandler } from "@/modules/export/jobs/exportGlossesSqliteHandler"; export type JobHandlerRegistry = { [Type in JobType]: { @@ -53,4 +54,8 @@ export const jobHandlerRegistry: JobHandlerRegistry = { handler: syncAIGlossLanguagesHandler, timeout: 60 * 5, }, + export_glosses_sqlite: { + handler: exportGlossesSqliteHandler, + timeout: 60 * 15, + }, }; diff --git a/src/shared/jobs/jobRegistry.ts b/src/shared/jobs/jobRegistry.ts index bc9eb92c..48ff7f36 100644 --- a/src/shared/jobs/jobRegistry.ts +++ b/src/shared/jobs/jobRegistry.ts @@ -8,6 +8,7 @@ import { ExportGlossesChildJob } from "@/modules/export/jobs/ExportGlossesChildJ import { ExportGlossesFinalizeJob } from "@/modules/export/jobs/ExportGlossesFinalizeJob"; import { ImportAIGlossesJob } from "@/modules/translation/jobs/ImportAIGlossesJob"; import { SyncAIGlossLanguagesJob } from "@/modules/translation/jobs/SyncAIGlossLanguagesJob"; +import { ExportGlossesSqliteJob } from "@/modules/export/jobs/ExportGlossesSqliteJob"; export const jobRegistry = { [SendEmailJob.type]: SendEmailJob, @@ -19,6 +20,7 @@ export const jobRegistry = { [ExportGlossesFinalizeJob.type]: ExportGlossesFinalizeJob, [ImportAIGlossesJob.type]: ImportAIGlossesJob, [SyncAIGlossLanguagesJob.type]: SyncAIGlossLanguagesJob, + [ExportGlossesSqliteJob.type]: ExportGlossesSqliteJob, } satisfies Record< string, ReturnType> From 1137fbdd7399a97ea3688aa768e90eef6e1cc728 Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Mon, 8 Jun 2026 22:54:25 -0500 Subject: [PATCH 2/7] export ui --- .../components/AppDatabaseExportPanel.tsx | 104 ++++++++++++++++++ src/ui/admin/routes/jobs.tsx | 5 + .../serverFns/getGlossesSqliteExportData.ts | 55 +++++++++ 3 files changed, 164 insertions(+) create mode 100644 src/ui/admin/components/AppDatabaseExportPanel.tsx create mode 100644 src/ui/admin/serverFns/getGlossesSqliteExportData.ts diff --git a/src/ui/admin/components/AppDatabaseExportPanel.tsx b/src/ui/admin/components/AppDatabaseExportPanel.tsx new file mode 100644 index 00000000..d457d410 --- /dev/null +++ b/src/ui/admin/components/AppDatabaseExportPanel.tsx @@ -0,0 +1,104 @@ +"use client"; + +import { format } from "date-fns"; +import Button from "@/components/Button"; +import ComboboxInput, { type ComboboxItem } from "@/components/ComboboxInput"; +import { JobStatus } from "@/shared/jobs/types"; +import { queueJobAction } from "@/shared/jobs/queueJobAction"; +import { getGlossesSqliteExportData } from "@/ui/admin/serverFns/getGlossesSqliteExportData"; +import { useSuspenseQuery } from "@tanstack/react-query"; +import { useMemo, useState } from "react"; + +export default function AppDatabaseExportPanel() { + const { data: sqliteData, refetch: refetchSqliteJobs } = useSuspenseQuery({ + queryKey: ["glossesSqliteExport"], + queryFn: () => getGlossesSqliteExportData(), + refetchInterval: ({ state }) => { + const job = state.data?.latestJob; + if ( + job && + job.status !== JobStatus.Complete && + job.status !== JobStatus.Failed + ) { + return 3000; + } + return false; + }, + }); + + const [selectedLanguage, setSelectedLanguage] = useState(""); + + const languageItems: ComboboxItem[] = useMemo( + () => + sqliteData.languages.map((lang) => ({ + label: `${lang.englishName} (${lang.code})`, + value: lang.code, + })), + [sqliteData.languages], + ); + + const latestJob = sqliteData.latestJob; + const isSubmitting = + !!latestJob && + latestJob.status !== JobStatus.Complete && + latestJob.status !== JobStatus.Failed; + + return ( +
+

App Database Export

+ +
+ + +
+ + {latestJob && ( +
+
+

Last Run:

+ {format(latestJob.createdAt, "MMM dd, yyy hh:mm aaa")} + + {latestJob.status.replace("-", " ")} + +
+
+ )} +
+ ); +} + +function getStatusClassName(status: JobStatus): string { + switch (status) { + case JobStatus.Complete: + return "bg-green-200 text-gray-900"; + case JobStatus.Failed: + return "bg-red-300 text-gray-900"; + case JobStatus.InProgress: + return "bg-brown-100 text-gray-900"; + case JobStatus.Pending: + default: + return "bg-gray-200 text-gray-900"; + } +} diff --git a/src/ui/admin/routes/jobs.tsx b/src/ui/admin/routes/jobs.tsx index 6018b80c..4f7ee795 100644 --- a/src/ui/admin/routes/jobs.tsx +++ b/src/ui/admin/routes/jobs.tsx @@ -17,6 +17,7 @@ import { withDocumentTitle } from "@/documentTitle"; import { getActiveJobs } from "@/ui/admin/serverFns/getActiveJobs"; import { Policy } from "@/modules/access"; import { routerGuard } from "@/modules/access/routerGuard"; +import AppDatabaseExportPanel from "@/ui/admin/components/AppDatabaseExportPanel"; const policy = new Policy({ systemRoles: [Policy.SystemRole.Admin] }); @@ -72,6 +73,10 @@ function AdminJobsView() { Sync AI Gloss Languages +
+ +
+

GitHub Export

diff --git a/src/ui/admin/serverFns/getGlossesSqliteExportData.ts b/src/ui/admin/serverFns/getGlossesSqliteExportData.ts new file mode 100644 index 00000000..8cef038b --- /dev/null +++ b/src/ui/admin/serverFns/getGlossesSqliteExportData.ts @@ -0,0 +1,55 @@ +import { sql } from "kysely"; +import { createPolicyMiddleware, Policy } from "@/modules/access"; +import { getDb } from "@/db"; +import { createServerFn } from "@tanstack/react-start"; +import { getAllLanguagesReadModel } from "@/ui/admin/readModels/getAllLanguagesReadModel"; + +const policy = new Policy({ + systemRoles: [Policy.SystemRole.Admin], +}); + +export interface GlossesSqliteExportJob { + id: string; + status: string; + createdAt: Date; + updatedAt: Date; + languageCodes: string[]; +} + +export const getGlossesSqliteExportData = createServerFn() + .middleware([ + createPolicyMiddleware({ + policy, + }), + ]) + .handler(async () => { + const [latestJob, languages] = await Promise.all([ + getLatestGlossesSqliteExportJob(), + getAllLanguagesReadModel(), + ]); + + return { latestJob, languages }; + }); + +async function getLatestGlossesSqliteExportJob(): Promise< + GlossesSqliteExportJob | undefined +> { + const row = await getDb() + .selectFrom("job") + .where("type", "=", "export_glosses_sqlite") + .orderBy("created_at", "desc") + .select([ + "id", + "status", + "created_at as createdAt", + "updated_at as updatedAt", + (eb) => + sql`${eb.ref("job.payload")}->'languageCodes'`.as( + "languageCodes", + ), + ]) + .limit(1) + .executeTakeFirst(); + + return row; +} From 573f9727872788e46a980149402116b6cbd63f30 Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 10:29:49 -0500 Subject: [PATCH 3/7] generate sqlite file for a language export --- Dockerfile.job-worker | 4 + package-lock.json | 318 +++++++++++++++++- package.json | 4 +- .../export/data-access/AppGlossRepository.ts | 100 ++++++ .../export/jobs/exportGlossesSqliteHandler.ts | 102 +++++- 5 files changed, 523 insertions(+), 5 deletions(-) create mode 100644 src/modules/export/data-access/AppGlossRepository.ts diff --git a/Dockerfile.job-worker b/Dockerfile.job-worker index 8c6bc59f..454c9c17 100644 --- a/Dockerfile.job-worker +++ b/Dockerfile.job-worker @@ -10,6 +10,10 @@ COPY src/assets/fonts dist/fonts/ RUN npm run build:job-worker FROM public.ecr.aws/lambda/nodejs:22 AS runner + +COPY package*.json . +RUN ["npm", "install", "better-sqlite3"] + ENV NODE_ENV=production ENV PDFKIT_DATA_DIR=/var/task/data COPY --from=build /app/dist/job-worker.js ${LAMBDA_TASK_ROOT} diff --git a/package-lock.json b/package-lock.json index 1984615e..e16fa30d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@tiptap/react": "3.18.0", "@tiptap/starter-kit": "3.18.0", "aws-lambda": "1.0.7", + "better-sqlite3": "12.10.0", "chart.js": "4.4.4", "chartjs-adapter-date-fns": "3.0.0", "d3": "7.9.0", @@ -57,6 +58,7 @@ "@tanstack/eslint-plugin-router": "1.161.6", "@testing-library/react": "16.3.1", "@types/aws-lambda": "8.10.147", + "@types/better-sqlite3": "7.6.13", "@types/d3": "7.4.3", "@types/dompurify": "3.0.5", "@types/lodash": "4.17.7", @@ -6599,6 +6601,16 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "dev": true, @@ -7655,6 +7667,20 @@ "integrity": "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==", "license": "Apache-2.0" }, + "node_modules/better-sqlite3": { + "version": "12.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.10.0.tgz", + "integrity": "sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x || 26.x" + } + }, "node_modules/bidi-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", @@ -7686,6 +7712,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/birecord": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/birecord/-/birecord-0.1.1.tgz", @@ -7693,6 +7728,41 @@ "dev": true, "license": "(MIT OR Apache-2.0)" }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -8034,6 +8104,12 @@ "node": ">= 6" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -8768,6 +8844,21 @@ "optional": true, "peer": true }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/deep-equal": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", @@ -8800,6 +8891,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "dev": true, @@ -8865,7 +8965,6 @@ }, "node_modules/detect-libc": { "version": "2.1.2", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -10102,6 +10201,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/expect-type": { "version": "1.3.0", "dev": true, @@ -10279,6 +10387,12 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/fill-range": { "version": "7.1.1", "license": "MIT", @@ -10398,6 +10512,12 @@ "node": ">=12.20.0" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -10572,6 +10692,12 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -11273,6 +11399,12 @@ "version": "2.0.4", "license": "ISC" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, "node_modules/internal-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", @@ -12935,6 +13067,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", @@ -12955,7 +13099,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -12970,6 +13113,12 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "license": "MIT" @@ -12990,6 +13139,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "dev": true, @@ -13212,6 +13367,18 @@ } } }, + "node_modules/node-abi": { + "version": "3.92.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.92.0.tgz", + "integrity": "sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -14086,6 +14253,33 @@ "dev": true, "license": "MIT" }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "dev": true, @@ -14825,6 +15019,16 @@ "node": ">=12.0.0" } }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "dev": true, @@ -14884,6 +15088,30 @@ "version": "4.0.4", "license": "MIT" }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react": { "version": "19.2.4", "license": "MIT", @@ -15372,7 +15600,6 @@ }, "node_modules/semver": { "version": "7.7.3", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -15539,6 +15766,51 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -15835,6 +16107,34 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/teeny-request": { "version": "10.1.2", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", @@ -16189,6 +16489,18 @@ "@esbuild/win32-x64": "0.27.4" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "dev": true, diff --git a/package.json b/package.json index d27b36ac..3e6051fc 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "dev": "vite dev", "dev:job-worker": "node src/shared/jobs/bin/workerDevServer.js", "build": "vite build", - "build:job-worker": "esbuild src/shared/jobs/bin/worker.ts --bundle --outfile=dist/job-worker.js --format=cjs --platform=node --target=node20", + "build:job-worker": "esbuild src/shared/jobs/bin/worker.ts --bundle --outfile=dist/job-worker.js --format=cjs --platform=node --target=node20 --external:better-sqlite3", "start": "node ./output/server/index.mjs", "lint": "eslint ./src", "format": "prettier . --write", @@ -33,6 +33,7 @@ "@tiptap/react": "3.18.0", "@tiptap/starter-kit": "3.18.0", "aws-lambda": "1.0.7", + "better-sqlite3": "12.10.0", "chart.js": "4.4.4", "chartjs-adapter-date-fns": "3.0.0", "d3": "7.9.0", @@ -66,6 +67,7 @@ "@tanstack/eslint-plugin-router": "1.161.6", "@testing-library/react": "16.3.1", "@types/aws-lambda": "8.10.147", + "@types/better-sqlite3": "7.6.13", "@types/d3": "7.4.3", "@types/dompurify": "3.0.5", "@types/lodash": "4.17.7", diff --git a/src/modules/export/data-access/AppGlossRepository.ts b/src/modules/export/data-access/AppGlossRepository.ts new file mode 100644 index 00000000..3c244da0 --- /dev/null +++ b/src/modules/export/data-access/AppGlossRepository.ts @@ -0,0 +1,100 @@ +import SQLite, { Database, Statement } from "better-sqlite3"; +import { Writable } from "stream"; + +interface TextTable { + _id: number; + text: string; +} + +interface VersesTable { + _id: number; + text: number; +} + +export class AppGlossRepository { + private db: Database; + + constructor() { + this.db = new SQLite(":memory:", { + nativeBinding: + "node_modules/better-sqlite3/build/Release/better_sqlite3.node", + }); + + this.db.pragma("journal_mode = WAL"); + + this.db + .prepare( + `create table text ( + _id integer primary key, + text text not null + )`, + ) + .run(); + this.db + .prepare( + `create table verses ( + _id integer primary key, + text integer + )`, + ) + .run(); + } + + getVerseWritableStream() { + return new SqliteWritableStream( + this.db, + "insert into verses (_id, text) values (?, ?)", + (stmt, row: VersesTable) => stmt.run(row._id, row.text), + ); + } + + getTextWritableStream() { + return new SqliteWritableStream( + this.db, + "insert into text (_id, text) values (?, ?)", + (stmt, row: TextTable) => stmt.run(row._id, row.text), + ); + } + + serialize() { + return this.db.serialize(); + } +} + +class SqliteWritableStream extends Writable { + readonly insertStmt: Statement; + + constructor( + readonly database: Database, + readonly statement: string, + readonly run: (statement: Statement, row: T) => void, + ) { + super({ objectMode: true }); + + this.insertStmt = database.prepare(statement); + + database.prepare("begin").run(); + } + + override _write( + row: T, + _encoding: BufferEncoding, + callback: (error?: Error | null) => void, + ) { + try { + this.run(this.insertStmt, row); + callback(); + } catch (error) { + callback(error instanceof Error ? error : new Error(String(error))); + } + } + + override _final(callback: (error?: Error | null) => void) { + try { + this.database.prepare("commit").run(); + callback(); + } catch (error) { + callback(error instanceof Error ? error : new Error(String(error))); + } + } +} diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.ts index 1fba3c08..021b1afe 100644 --- a/src/modules/export/jobs/exportGlossesSqliteHandler.ts +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.ts @@ -1,5 +1,12 @@ import { logger } from "@/logging"; import { ExportGlossesSqliteJob } from "./ExportGlossesSqliteJob"; +import { getDb } from "@/db"; +import { GlossStateRaw } from "@/modules/translation/types"; +import { resolveLanguageByCode } from "@/modules/languages"; +import { Logger } from "pino"; +import { pipeline } from "stream/promises"; +import { writeFileSync } from "fs"; +import { AppGlossRepository } from "../data-access/AppGlossRepository"; export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { const jobLogger = logger.child({ @@ -12,7 +19,16 @@ export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { const { languageCodes } = job.payload; for (const languageCode of languageCodes) { - jobLogger.info({ languageCode }, "Exporting glosses SQLite for language"); + const buffer = await createSqliteDb(languageCode, jobLogger); + + if (buffer) { + writeFileSync(`${languageCode}.sqlite`, buffer); + } + + jobLogger.info( + { languageCode }, + `Finished exporting glosses SQLite for language ${languageCode}`, + ); } jobLogger.info( @@ -20,3 +36,87 @@ export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { "Glosses SQLite export complete", ); } + +async function createSqliteDb( + languageCode: string, + logger: Logger, +): Promise { + const language = await resolveLanguageByCode(languageCode); + + if (!language) { + logger.error({ languageCode }, "Could not find language to export"); + return; + } + + const glossStream = streamGlossesForLanguage(language.id); + + let nextTextId = 1; + const textIdMap = new Map(); + + const appGlossRepository = new AppGlossRepository(); + + await pipeline( + glossStream, + async function* (stream) { + for await (const row of stream) { + if (!row.gloss) continue; + + let textId = textIdMap.get(row.gloss); + if (!textId) { + textId = nextTextId; + nextTextId += 1; + textIdMap.set(row.gloss, textId); + } + + yield { + _id: row.wordId, + text: textId, + }; + } + }, + appGlossRepository.getVerseWritableStream(), + ); + + await pipeline(async function* () { + for (const [text, _id] of textIdMap.entries()) { + yield { _id, text }; + } + }, appGlossRepository.getTextWritableStream()); + + return appGlossRepository.serialize(); +} + +interface GlossExportRow { + wordId: string; + gloss: string | null; +} + +function streamGlossesForLanguage( + languageId: string, +): AsyncIterableIterator { + return getDb() + .with("completed_books", (db) => + db + .selectFrom("book_completion") + .where("language_id", "=", languageId) + .where("completed_at", "is not", null) + .select("book_id"), + ) + .with("gloss_word", (db) => + db + .selectFrom("phrase_word as pw") + .innerJoin("phrase as ph", "ph.id", "pw.phrase_id") + .innerJoin("gloss as g", "g.phrase_id", "ph.id") + .innerJoin("book_word_map as w", "w.word_id", "pw.word_id") + .innerJoin("completed_books as b", "b.book_id", "w.book_id") + .where("ph.language_id", "=", languageId) + .where("ph.deleted_at", "is", null) + .where("g.state", "=", GlossStateRaw.Approved) + .select(["pw.word_id", "g.gloss"]), + ) + .selectFrom("word") + .leftJoin("gloss_word", "gloss_word.word_id", "word.id") + .select(["word.id as wordId", "gloss_word.gloss"]) + .orderBy("word.id") + .stream(); +} From 45288c3d1e2f5149d09979a4a5c7783e4aaf4d16 Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 20:40:30 -0500 Subject: [PATCH 4/7] sqlite export working # Conflicts: # src/modules/export/data-access/ExportStorageRepository.ts # src/modules/export/jobs/exportInterlinearPdfHandler.ts --- .../export/data-access/AppGlossRepository.ts | 3 +++ .../export/jobs/exportGlossesSqliteHandler.ts | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/modules/export/data-access/AppGlossRepository.ts b/src/modules/export/data-access/AppGlossRepository.ts index 3c244da0..f3477725 100644 --- a/src/modules/export/data-access/AppGlossRepository.ts +++ b/src/modules/export/data-access/AppGlossRepository.ts @@ -1,3 +1,4 @@ +import { logger } from "@/logging"; import SQLite, { Database, Statement } from "better-sqlite3"; import { Writable } from "stream"; @@ -85,6 +86,7 @@ class SqliteWritableStream extends Writable { this.run(this.insertStmt, row); callback(); } catch (error) { + logger.error({ err: error, row }, "Failed to insert sqlite rows"); callback(error instanceof Error ? error : new Error(String(error))); } } @@ -94,6 +96,7 @@ class SqliteWritableStream extends Writable { this.database.prepare("commit").run(); callback(); } catch (error) { + logger.error({ err: error }, "Failed to commit sqlite transaction"); callback(error instanceof Error ? error : new Error(String(error))); } } diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.ts index 021b1afe..75dc8786 100644 --- a/src/modules/export/jobs/exportGlossesSqliteHandler.ts +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.ts @@ -5,8 +5,9 @@ import { GlossStateRaw } from "@/modules/translation/types"; import { resolveLanguageByCode } from "@/modules/languages"; import { Logger } from "pino"; import { pipeline } from "stream/promises"; -import { writeFileSync } from "fs"; import { AppGlossRepository } from "../data-access/AppGlossRepository"; +import { getStorageEnvironment } from "@/shared/storageEnvironment"; +import exportStorageRepository from "../data-access/ExportStorageRepository"; export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { const jobLogger = logger.child({ @@ -18,11 +19,17 @@ export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { const { languageCodes } = job.payload; + const environment = getStorageEnvironment(); + for (const languageCode of languageCodes) { const buffer = await createSqliteDb(languageCode, jobLogger); - if (buffer) { - writeFileSync(`${languageCode}.sqlite`, buffer); + await exportStorageRepository.upload({ + environment, + key: `glosses/v1/${languageCode}.db`, + stream: buffer, + type: "unknown", + }); } jobLogger.info( @@ -61,6 +68,9 @@ async function createSqliteDb( for await (const row of stream) { if (!row.gloss) continue; + // The database format does not support word ids with hyphens yet, so we skip these for now. + if (row.wordId.includes("-")) continue; + let textId = textIdMap.get(row.gloss); if (!textId) { textId = nextTextId; From 01768c022c2c8c16d0786fa75e527ccd90cd5a40 Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 20:52:05 -0500 Subject: [PATCH 5/7] fix merge issues --- .../export/jobs/exportGlossesSqliteHandler.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.ts index 75dc8786..473fac90 100644 --- a/src/modules/export/jobs/exportGlossesSqliteHandler.ts +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.ts @@ -6,8 +6,7 @@ import { resolveLanguageByCode } from "@/modules/languages"; import { Logger } from "pino"; import { pipeline } from "stream/promises"; import { AppGlossRepository } from "../data-access/AppGlossRepository"; -import { getStorageEnvironment } from "@/shared/storageEnvironment"; -import exportStorageRepository from "../data-access/ExportStorageRepository"; +import { exportStorageRepository } from "../data-access/exportStorageRepository"; export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { const jobLogger = logger.child({ @@ -19,16 +18,13 @@ export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { const { languageCodes } = job.payload; - const environment = getStorageEnvironment(); - for (const languageCode of languageCodes) { const buffer = await createSqliteDb(languageCode, jobLogger); if (buffer) { await exportStorageRepository.upload({ - environment, key: `glosses/v1/${languageCode}.db`, - stream: buffer, - type: "unknown", + source: buffer, + type: "application/vnd.sqlite3", }); } @@ -55,20 +51,21 @@ async function createSqliteDb( return; } + const appGlossRepository = new AppGlossRepository(); + const glossStream = streamGlossesForLanguage(language.id); let nextTextId = 1; const textIdMap = new Map(); - const appGlossRepository = new AppGlossRepository(); - await pipeline( glossStream, async function* (stream) { for await (const row of stream) { if (!row.gloss) continue; - // The database format does not support word ids with hyphens yet, so we skip these for now. + // The database format does not support word ids with hyphens yet, + // so we skip these for now. if (row.wordId.includes("-")) continue; let textId = textIdMap.get(row.gloss); From f7dfa1d384e334a00153ea9520d38cbbc60e4ab0 Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Sat, 13 Jun 2026 21:13:19 -0500 Subject: [PATCH 6/7] test for new job --- .../jobs/exportGlossesSqliteHandler.test.ts | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 src/modules/export/jobs/exportGlossesSqliteHandler.test.ts diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts new file mode 100644 index 00000000..6799d07e --- /dev/null +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts @@ -0,0 +1,249 @@ +import { initializeDatabase } from "@/tests/vitest/dbUtils"; +import { beforeEach, expect, test, vitest } from "vitest"; +import { languageFactory } from "@/modules/languages/test-utils/languageFactory"; +import { phraseFactory } from "@/modules/translation/test-utils/phraseFactory"; +import { GlossStateRaw } from "@/modules/translation/types"; +import { + HAGGAI_BOOK_ID, + bibleFactory, +} from "@/modules/bible-core/test-utils/bibleFactory"; +import { getDb } from "@/db"; +import { exportGlossesSqliteHandler } from "./exportGlossesSqliteHandler"; +import { ExportGlossesSqliteJob } from "./ExportGlossesSqliteJob"; +import { exportStorageRepository } from "../data-access/exportStorageRepository"; +import Database from "better-sqlite3"; + +vitest.mock("../data-access/exportStorageRepository", () => ({ + exportStorageRepository: { + upload: vitest.fn(), + publicUrl: vitest.fn(), + }, +})); + +initializeDatabase(); + +const mockedUpload = vitest.mocked(exportStorageRepository.upload); + +beforeEach(() => { + mockedUpload.mockReset(); +}); + +function querySqliteTables(buffer: Buffer) { + const db = new Database(buffer); + const verses = db.prepare("select * from verses order by _id").all() as { + _id: number; + text: number; + }[]; + const texts = db.prepare("select * from text order by _id").all() as { + _id: number; + text: string; + }[]; + db.close(); + return { verses, texts }; +} + +test("exports approved glosses for a language as a SQLite database", async () => { + const { language } = await languageFactory.build({ code: "spa" }); + + await getDb() + .insertInto("book_completion") + .values({ + language_id: language.id, + book_id: HAGGAI_BOOK_ID, + refreshed_at: new Date(), + updated_at: new Date(), + completed_at: new Date(), + }) + .execute(); + + const word = await bibleFactory.word(); + + await phraseFactory.build({ + languageId: language.id, + wordIds: [word.id], + events: true, + gloss: { + state: GlossStateRaw.Approved, + gloss: "test gloss", + }, + }); + + const job = ExportGlossesSqliteJob.create({ + languageCodes: [language.code], + }); + + await exportGlossesSqliteHandler(job); + + expect(mockedUpload).toHaveBeenCalledExactlyOnceWith({ + key: `glosses/v1/${language.code}.db`, + source: expect.any(Buffer), + type: "application/vnd.sqlite3", + }); + + const buffer = mockedUpload.mock.calls[0][0].source as Buffer; + const { verses, texts } = querySqliteTables(buffer); + + expect(verses).toEqual([{ _id: Number(word.id), text: 1 }]); + expect(texts).toEqual([{ _id: 1, text: "test gloss" }]); +}); + +test("skips words with null glosses", async () => { + const { language } = await languageFactory.build({ code: "hin" }); + + await getDb() + .insertInto("book_completion") + .values({ + language_id: language.id, + book_id: HAGGAI_BOOK_ID, + refreshed_at: new Date(), + updated_at: new Date(), + completed_at: new Date(), + }) + .execute(); + + // Create a phrase with unapproved gloss — should not appear in export + await phraseFactory.build({ + languageId: language.id, + events: true, + gloss: "unapproved", + }); + + const job = ExportGlossesSqliteJob.create({ + languageCodes: [language.code], + }); + + await exportGlossesSqliteHandler(job); + + expect(mockedUpload).toHaveBeenCalledExactlyOnceWith({ + key: `glosses/v1/${language.code}.db`, + source: expect.any(Buffer), + type: "application/vnd.sqlite3", + }); + + const buffer = mockedUpload.mock.calls[0][0].source as Buffer; + const { verses, texts } = querySqliteTables(buffer); + + expect(verses).toEqual([]); + expect(texts).toEqual([]); +}); + +test("skips a language code that does not exist", async () => { + const job = ExportGlossesSqliteJob.create({ + languageCodes: ["nonexistent"], + }); + + await exportGlossesSqliteHandler(job); + + expect(mockedUpload).not.toHaveBeenCalled(); +}); + +test("exports multiple languages in separate databases", async () => { + const { language: language1 } = await languageFactory.build({ code: "spa" }); + const { language: language2 } = await languageFactory.build({ code: "hin" }); + + for (const language of [language1, language2]) { + await getDb() + .insertInto("book_completion") + .values({ + language_id: language.id, + book_id: HAGGAI_BOOK_ID, + refreshed_at: new Date(), + updated_at: new Date(), + completed_at: new Date(), + }) + .execute(); + } + + const word = await bibleFactory.word(); + + await phraseFactory.build({ + languageId: language1.id, + wordIds: [word.id], + events: true, + gloss: { + state: GlossStateRaw.Approved, + gloss: "hello", + }, + }); + + await phraseFactory.build({ + languageId: language2.id, + wordIds: [word.id], + events: true, + gloss: { + state: GlossStateRaw.Approved, + gloss: "namaste", + }, + }); + + const job = ExportGlossesSqliteJob.create({ + languageCodes: [language1.code, language2.code], + }); + + await exportGlossesSqliteHandler(job); + + expect(mockedUpload).toHaveBeenCalledTimes(2); + + const spaBuffer = mockedUpload.mock.calls[0][0].source as Buffer; + const { verses: spaVerses, texts: spaTexts } = querySqliteTables(spaBuffer); + expect(spaVerses).toEqual([{ _id: Number(word.id), text: 1 }]); + expect(spaTexts).toEqual([{ _id: 1, text: "hello" }]); + + const hinBuffer = mockedUpload.mock.calls[1][0].source as Buffer; + const { verses: hinVerses, texts: hinTexts } = querySqliteTables(hinBuffer); + expect(hinVerses).toEqual([{ _id: Number(word.id), text: 1 }]); + expect(hinTexts).toEqual([{ _id: 1, text: "namaste" }]); +}); + +test("deduplicates gloss text entries", async () => { + const { language } = await languageFactory.build({ code: "arb" }); + + await getDb() + .insertInto("book_completion") + .values({ + language_id: language.id, + book_id: HAGGAI_BOOK_ID, + refreshed_at: new Date(), + updated_at: new Date(), + completed_at: new Date(), + }) + .execute(); + + // Create two phrases with the same gloss text on different words + const words = await bibleFactory.words({ count: 2 }); + + await phraseFactory.build({ + languageId: language.id, + wordIds: [words[0].id], + events: true, + gloss: { + state: GlossStateRaw.Approved, + gloss: "same gloss", + }, + }); + + await phraseFactory.build({ + languageId: language.id, + wordIds: [words[1].id], + events: true, + gloss: { + state: GlossStateRaw.Approved, + gloss: "same gloss", + }, + }); + + const job = ExportGlossesSqliteJob.create({ + languageCodes: [language.code], + }); + + await exportGlossesSqliteHandler(job); + + const buffer = mockedUpload.mock.calls[0][0].source as Buffer; + const { verses, texts } = querySqliteTables(buffer); + + expect(verses).toEqual([ + { _id: Number(words[0].id), text: 1 }, + { _id: Number(words[1].id), text: 1 }, + ]); + expect(texts).toEqual([{ _id: 1, text: "same gloss" }]); +}); From 21116ecf05a58bd7cb1257f075c5d799fae640fd Mon Sep 17 00:00:00 2001 From: Adrian Rocke Date: Tue, 16 Jun 2026 20:45:22 -0500 Subject: [PATCH 7/7] zip up sqlite db --- package-lock.json | 683 +++++++++++++++++- package.json | 2 + .../data-access/exportStorageRepository.ts | 25 +- .../jobs/exportGlossesSqliteHandler.test.ts | 31 +- .../export/jobs/exportGlossesSqliteHandler.ts | 6 +- 5 files changed, 724 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index e16fa30d..0459949a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@tiptap/pm": "3.18.0", "@tiptap/react": "3.18.0", "@tiptap/starter-kit": "3.18.0", + "archiver": "8.0.0", "aws-lambda": "1.0.7", "better-sqlite3": "12.10.0", "chart.js": "4.4.4", @@ -57,6 +58,7 @@ "@tanstack/eslint-plugin-query": "5.91.5", "@tanstack/eslint-plugin-router": "1.161.6", "@testing-library/react": "16.3.1", + "@types/archiver": "8.0.0", "@types/aws-lambda": "8.10.147", "@types/better-sqlite3": "7.6.13", "@types/d3": "7.4.3", @@ -6545,6 +6547,17 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/archiver": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-8.0.0.tgz", + "integrity": "sha512-YpXPbEuv9+eUIPPQWUPahj3cvs9isWRuF+J4z+KbdYVDO3rWorWQFxUVHnwPu2AgKwvgpki5F2VMX0Xx+mX45A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/readdir-glob": "*" + } + }, "node_modules/@types/aria-query": { "version": "5.0.4", "dev": true, @@ -6972,6 +6985,16 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/readdir-glob": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", + "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "devOptional": true, @@ -7363,6 +7386,18 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -7468,6 +7503,119 @@ "node": ">= 8" } }, + "node_modules/archiver": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-8.0.0.tgz", + "integrity": "sha512-fV1orZfsnPn9BaSByR/qE67rJCLJEy2Ox5bq7nJh+jquWaNh6Sfec75kJ2T6PtdGUbPQlrVoSVCEOa5SdiTQ1g==", + "license": "MIT", + "dependencies": { + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "is-stream": "^4.0.0", + "lazystream": "^1.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^3.0.0", + "tar-stream": "^3.0.0", + "zip-stream": "^7.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/archiver/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/archiver/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/archiver/node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/archiver/node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver/node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/argparse": { "version": "2.0.1", "license": "Python-2.0" @@ -7527,6 +7675,12 @@ "node": ">=4" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, "node_modules/atomic-sleep": { "version": "1.0.0", "license": "MIT", @@ -7609,6 +7763,20 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/babel-dead-code-elimination": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/babel-dead-code-elimination/-/babel-dead-code-elimination-1.0.12.tgz", @@ -7625,12 +7793,103 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, "license": "MIT", "engines": { "node": "18 || 20 || >=22" } }, + "node_modules/bare-events": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.1.tgz", + "integrity": "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.7.2", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.2.tgz", + "integrity": "sha512-aTvMFUWkBmjzKtEQMDGGDNF8bkfpD5N1b/FCwt7A3wrU4t1o/e/85Wzkluh6JlODCjqVESYCkQCdTXqZ9G7VFg==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.1.tgz", + "integrity": "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ==", + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.1.tgz", + "integrity": "sha512-ghj2DSK/2e99a1anTVPCV4m4YIYtrbXhfM7V3D7XZLOTsybnYyaJloymGqssQc8l/or0UoDyRtNQkmkEF/ysgQ==", + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.13.3", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.3.tgz", + "integrity": "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.5.tgz", + "integrity": "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "funding": [ @@ -7777,7 +8036,6 @@ "version": "5.0.4", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -7856,6 +8114,15 @@ "isarray": "^1.0.0" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "license": "BSD-3-Clause" @@ -8287,6 +8554,103 @@ "dev": true, "license": "MIT" }, + "node_modules/compress-commons": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-7.0.1.tgz", + "integrity": "sha512-g0S8KAD8qf4+V//pr3BfB1aBnARLXNz2Gx+jmHU0LEriUuoQUOPOulVquHKTJ8+EAIIO7fhseNDr9wK5Q9FKBQ==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^7.0.1", + "is-stream": "^4.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/compress-commons/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/compress-commons/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/compress-commons/node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/compress-commons/node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -8316,6 +8680,106 @@ "integrity": "sha512-RAj4E421UYRgqokKUmotqAwuplYw15qtdXfY+hGzgCJ/MBjCVZcSoHK/kH9kocfjRjcDME7IiDWR/1WX1TM2Pg==", "license": "MIT" }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-7.0.1.tgz", + "integrity": "sha512-IBWsY8xznyQrcHn8h4bC8/4ErNke5elzgG8GcqF4RFPw6aHkWWRc7Tgw6upjaTX/CT/yQgqYENkxYsTYN+hW2g==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/crc32-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/crc32-stream/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/crc32-stream/node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/crelt": { "version": "1.0.6", "license": "MIT" @@ -10156,6 +10620,15 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "5.0.1", "dev": true, @@ -10168,6 +10641,15 @@ "node": ">=0.4.x" } }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/execa": { "version": "8.0.1", "dev": true, @@ -10270,6 +10752,12 @@ "node": ">=6.0.0" } }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -12143,6 +12631,54 @@ "node": ">=18.0.0" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/levn": { "version": "0.4.1", "dev": true, @@ -13083,7 +13619,6 @@ "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.2" @@ -14810,6 +15345,21 @@ "license": "MIT", "peer": true }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/process-warning": { "version": "4.0.1", "funding": [ @@ -15151,6 +15701,21 @@ "node": ">= 6" } }, + "node_modules/readdir-glob": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-3.0.0.tgz", + "integrity": "sha512-AhNB2KgKeVJr16nK9LLZbJNWnYoT23ZrumNKFDebHBdkC8KHSqWo871JAUhoWC/RtjEVdqNMFpM6qrwRbaUqpw==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^10.2.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/yqnn" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -15957,6 +16522,17 @@ "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", "license": "MIT" }, + "node_modules/streamx": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz", + "integrity": "sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "license": "MIT", @@ -16168,6 +16744,24 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -17839,6 +18433,89 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zip-stream": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-7.0.5.tgz", + "integrity": "sha512-dSvYKdvLsAHCDqPOhIwk/q5CvuWtTB3Dgpoe0uVEFjTzIOAmsQpprX25InCvrvJsirEbu1OHyy67n/kAj1Sw/w==", + "license": "MIT", + "dependencies": { + "compress-commons": "^7.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/zip-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/zip-stream/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/zip-stream/node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/zod": { "version": "3.23.8", "license": "MIT", diff --git a/package.json b/package.json index 3e6051fc..f202c0a9 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@tiptap/pm": "3.18.0", "@tiptap/react": "3.18.0", "@tiptap/starter-kit": "3.18.0", + "archiver": "8.0.0", "aws-lambda": "1.0.7", "better-sqlite3": "12.10.0", "chart.js": "4.4.4", @@ -66,6 +67,7 @@ "@tanstack/eslint-plugin-query": "5.91.5", "@tanstack/eslint-plugin-router": "1.161.6", "@testing-library/react": "16.3.1", + "@types/archiver": "8.0.0", "@types/aws-lambda": "8.10.147", "@types/better-sqlite3": "7.6.13", "@types/d3": "7.4.3", diff --git a/src/modules/export/data-access/exportStorageRepository.ts b/src/modules/export/data-access/exportStorageRepository.ts index 00e63d46..ea144f3b 100644 --- a/src/modules/export/data-access/exportStorageRepository.ts +++ b/src/modules/export/data-access/exportStorageRepository.ts @@ -1,5 +1,6 @@ import { Upload } from "@aws-sdk/lib-storage"; -import { Readable } from "stream"; +import { PassThrough, Readable } from "stream"; +import { ZipArchive } from "archiver"; import { createLogger } from "@/logging"; import { getS3Client } from "@/shared/s3"; @@ -31,12 +32,32 @@ export const exportStorageRepository = { await upload.done(); - const location = `s3://${EXPORT_BUCKET}/${key}`; + const location = this.publicUrl({ key }); logger.info(`Export PDF uploaded to ${location}`); return location; }, + async uploadZip({ + key, + source, + fileName, + }: { + key: string; + source: Buffer; + fileName: string; + }): Promise { + const archive = new ZipArchive(); + archive.append(source, { name: fileName }); + archive.finalize(); + + return this.upload({ + key, + source: archive.pipe(new PassThrough()), + type: "application/zip", + }); + }, + publicUrl({ key }: { key: string }): string { if (process.env.NODE_ENV === "production") { return `https://assets.globalbibletools.com/${key}`; diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts index 6799d07e..22e2245f 100644 --- a/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.test.ts @@ -16,16 +16,17 @@ import Database from "better-sqlite3"; vitest.mock("../data-access/exportStorageRepository", () => ({ exportStorageRepository: { upload: vitest.fn(), + uploadZip: vitest.fn(), publicUrl: vitest.fn(), }, })); initializeDatabase(); -const mockedUpload = vitest.mocked(exportStorageRepository.upload); +const mockedUploadZip = vitest.mocked(exportStorageRepository.uploadZip); beforeEach(() => { - mockedUpload.mockReset(); + mockedUploadZip.mockReset(); }); function querySqliteTables(buffer: Buffer) { @@ -74,13 +75,13 @@ test("exports approved glosses for a language as a SQLite database", async () => await exportGlossesSqliteHandler(job); - expect(mockedUpload).toHaveBeenCalledExactlyOnceWith({ - key: `glosses/v1/${language.code}.db`, + expect(mockedUploadZip).toHaveBeenCalledExactlyOnceWith({ + key: `glosses/v1/${language.code}.db.zip`, source: expect.any(Buffer), - type: "application/vnd.sqlite3", + fileName: `${language.code}.db`, }); - const buffer = mockedUpload.mock.calls[0][0].source as Buffer; + const buffer = mockedUploadZip.mock.calls[0][0].source as Buffer; const { verses, texts } = querySqliteTables(buffer); expect(verses).toEqual([{ _id: Number(word.id), text: 1 }]); @@ -114,13 +115,13 @@ test("skips words with null glosses", async () => { await exportGlossesSqliteHandler(job); - expect(mockedUpload).toHaveBeenCalledExactlyOnceWith({ - key: `glosses/v1/${language.code}.db`, + expect(mockedUploadZip).toHaveBeenCalledExactlyOnceWith({ + key: `glosses/v1/${language.code}.db.zip`, source: expect.any(Buffer), - type: "application/vnd.sqlite3", + fileName: `${language.code}.db`, }); - const buffer = mockedUpload.mock.calls[0][0].source as Buffer; + const buffer = mockedUploadZip.mock.calls[0][0].source as Buffer; const { verses, texts } = querySqliteTables(buffer); expect(verses).toEqual([]); @@ -134,7 +135,7 @@ test("skips a language code that does not exist", async () => { await exportGlossesSqliteHandler(job); - expect(mockedUpload).not.toHaveBeenCalled(); + expect(mockedUploadZip).not.toHaveBeenCalled(); }); test("exports multiple languages in separate databases", async () => { @@ -182,14 +183,14 @@ test("exports multiple languages in separate databases", async () => { await exportGlossesSqliteHandler(job); - expect(mockedUpload).toHaveBeenCalledTimes(2); + expect(mockedUploadZip).toHaveBeenCalledTimes(2); - const spaBuffer = mockedUpload.mock.calls[0][0].source as Buffer; + const spaBuffer = mockedUploadZip.mock.calls[0][0].source as Buffer; const { verses: spaVerses, texts: spaTexts } = querySqliteTables(spaBuffer); expect(spaVerses).toEqual([{ _id: Number(word.id), text: 1 }]); expect(spaTexts).toEqual([{ _id: 1, text: "hello" }]); - const hinBuffer = mockedUpload.mock.calls[1][0].source as Buffer; + const hinBuffer = mockedUploadZip.mock.calls[1][0].source as Buffer; const { verses: hinVerses, texts: hinTexts } = querySqliteTables(hinBuffer); expect(hinVerses).toEqual([{ _id: Number(word.id), text: 1 }]); expect(hinTexts).toEqual([{ _id: 1, text: "namaste" }]); @@ -238,7 +239,7 @@ test("deduplicates gloss text entries", async () => { await exportGlossesSqliteHandler(job); - const buffer = mockedUpload.mock.calls[0][0].source as Buffer; + const buffer = mockedUploadZip.mock.calls[0][0].source as Buffer; const { verses, texts } = querySqliteTables(buffer); expect(verses).toEqual([ diff --git a/src/modules/export/jobs/exportGlossesSqliteHandler.ts b/src/modules/export/jobs/exportGlossesSqliteHandler.ts index 473fac90..4d878fd1 100644 --- a/src/modules/export/jobs/exportGlossesSqliteHandler.ts +++ b/src/modules/export/jobs/exportGlossesSqliteHandler.ts @@ -21,10 +21,10 @@ export async function exportGlossesSqliteHandler(job: ExportGlossesSqliteJob) { for (const languageCode of languageCodes) { const buffer = await createSqliteDb(languageCode, jobLogger); if (buffer) { - await exportStorageRepository.upload({ - key: `glosses/v1/${languageCode}.db`, + await exportStorageRepository.uploadZip({ + key: `glosses/v1/${languageCode}.db.zip`, source: buffer, - type: "application/vnd.sqlite3", + fileName: `${languageCode}.db`, }); }