From 080b6a195a3382dda19bdaed0d7a2c375a64520d Mon Sep 17 00:00:00 2001 From: Tom Taylor Date: Mon, 23 Mar 2026 20:52:10 +0000 Subject: [PATCH 01/27] Fix: Map duplicate assets during import instead of warning (#172) --- lib/AdaptFrameworkImport.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index ae87e31..5ab9771 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -600,7 +600,12 @@ class AdaptFrameworkImport { const resolved = path.relative(`${this.coursePath}/..`, filepath) this.assetMap[resolved] = asset._id.toString() } catch (e) { - this.statusReport.warn.push({ code: 'ASSET_IMPORT_FAILED', data: { filepath } }) + if (e.code === 'DUPLICATE_ASSET') { + const resolved = path.relative(`${this.coursePath}/..`, filepath) + this.assetMap[resolved] = e.data.assetId + } else { + this.statusReport.warn.push({ code: 'ASSET_IMPORT_FAILED', data: { filepath } }) + } } imagesImported++ })) From 74276ab3605c704e2d154b4cb4818bc71fa0c314 Mon Sep 17 00:00:00 2001 From: Tom Taylor Date: Mon, 23 Mar 2026 20:56:41 +0000 Subject: [PATCH 02/27] New: Run adapt-migrations directly for all trigger points (#174) --- errors/errors.json | 7 + index.js | 2 +- lib/AdaptFrameworkImport.js | 76 ++++---- lib/AdaptFrameworkModule.js | 21 ++- lib/handlers.js | 5 +- lib/utils.js | 4 + lib/utils/collectMigrationScripts.js | 15 ++ lib/utils/migrateExistingCourses.js | 78 +++++++++ lib/utils/readFrameworkPluginVersions.js | 21 +++ lib/utils/runContentMigration.js | 32 ++++ package.json | 1 + tests/utils-collectMigrationScripts.spec.js | 45 +++++ tests/utils-migrateExistingCourses.spec.js | 163 ++++++++++++++++++ .../utils-readFrameworkPluginVersions.spec.js | 48 ++++++ tests/utils-runContentMigration.spec.js | 92 ++++++++++ 15 files changed, 573 insertions(+), 37 deletions(-) create mode 100644 lib/utils/collectMigrationScripts.js create mode 100644 lib/utils/migrateExistingCourses.js create mode 100644 lib/utils/readFrameworkPluginVersions.js create mode 100644 lib/utils/runContentMigration.js create mode 100644 tests/utils-collectMigrationScripts.spec.js create mode 100644 tests/utils-migrateExistingCourses.spec.js create mode 100644 tests/utils-readFrameworkPluginVersions.spec.js create mode 100644 tests/utils-runContentMigration.spec.js diff --git a/errors/errors.json b/errors/errors.json index 4273c2a..17d8c63 100644 --- a/errors/errors.json +++ b/errors/errors.json @@ -112,6 +112,13 @@ "description": "Update of the framework failed", "statusCode": 500 }, + "FW_UPDATE_MIGRATION_FAILED": { + "data": { + "errors": "Array of per-course migration errors" + }, + "description": "Content migration after framework or plugin update encountered errors", + "statusCode": 500 + }, "FW_INVALID_VERSION": { "data": { "name": "Incompatible plugin name", diff --git a/index.js b/index.js index 5fb33a0..05c162e 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,6 @@ * @namespace adaptframework */ export { default } from './lib/AdaptFrameworkModule.js' -export { copyFrameworkSource } from './lib/utils.js' +export { copyFrameworkSource, readFrameworkPluginVersions } from './lib/utils.js' export { default as AdaptFrameworkBuild } from './lib/AdaptFrameworkBuild.js' export { default as AdaptFrameworkImport } from './lib/AdaptFrameworkImport.js' diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index 5ab9771..73c5bb8 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -1,13 +1,12 @@ -import { App, Hook, spawn, readJson, writeJson } from 'adapt-authoring-core' +import { App, Hook, readJson, writeJson } from 'adapt-authoring-core' import { parseObjectId } from 'adapt-authoring-mongodb' import fs from 'node:fs/promises' import { glob } from 'glob' import octopus from 'adapt-octopus' import path from 'upath' -import { randomBytes } from 'node:crypto' import semver from 'semver' import { unzip } from 'zipper' -import { log, logDir, getImportSummary, getImportContentCounts } from './utils.js' +import { log, logDir, getImportSummary, getImportContentCounts, readFrameworkPluginVersions, collectMigrationScripts, runContentMigration } from './utils.js' import ComponentTransform from './migrations/component.js' import ConfigTransform from './migrations/config.js' @@ -269,9 +268,8 @@ class AdaptFrameworkImport { [this.importCourseAssets, importContent], [this.importCoursePlugins, isDryRun && importPlugins], [this.importCoursePlugins, !isDryRun && importContent], - [this.loadCourseData, isDryRun && importContent], + [this.loadCourseData, importContent], [this.migrateCourseData, !isDryRun && migrateContent], - [this.loadCourseData, !isDryRun && importContent], [this.importCourseData, !isDryRun && importContent], [this.generateSummary] ] @@ -492,47 +490,61 @@ class AdaptFrameworkImport { } /** - * Run grunt task - * @return {Promise} - */ - async runGruntMigration (subTask, { outputDir, captureDir, outputFilePath }) { - const output = await spawn({ - cmd: 'npx', - args: ['grunt', `migration:${subTask}`, `--outputdir=${outputDir}`, `--capturedir=${captureDir}`], - cwd: this.frameworkPath ?? this.framework.path - }) - if (outputFilePath) await fs.writeFile(outputFilePath, output) - } - - /** - * Handle migrate course data, installs adapt-migrations/capture data/adds updated scripts/migrates data + * Migrates course data in-memory using adapt-migrations */ async migrateCourseData () { try { await this.patchThemeName() await this.patchCustomStyle() - const migrationId = `${this.userId}-${randomBytes(4).toString('hex')}` - - const opts = { - outputDir: path.relative(this.framework.path, path.resolve(this.coursePath, '..')), - captureDir: path.join(`./${migrationId}-migrations`), - outputFilePath: path.join(this.framework.path, 'migrations', `${migrationId}.txt`) - } - log('debug', 'MIGRATION_ID', migrationId) - logDir('captureDir', opts.captureDir) - logDir('outputDir', opts.outputDir) + const content = this.flattenContentJson() + const fromPlugins = Object.values(this.usedContentPlugins).map(p => ({ + name: p.name, + version: p.version + })) + const toPlugins = await readFrameworkPluginVersions(this.framework.path) + const scripts = await collectMigrationScripts(this.framework.path) - await this.runGruntMigration('capture', opts) - await this.runGruntMigration('migrate', opts) + const migrated = await runContentMigration({ content, fromPlugins, toPlugins, scripts }) - await fs.rm(path.join(this.framework.path, opts.captureDir), { recursive: true }) + this.unflattenContentJson(migrated) + log('info', 'in-memory content migration completed') } catch (error) { log('error', 'Migration process failed', error) throw App.instance.errors.FW_IMPORT_MIGRATION_FAILED.setData({ reason: error.message }) } } + /** + * Flattens this.contentJson into a flat array for adapt-migrations + * @returns {Array} + */ + flattenContentJson () { + const content = [] + if (this.contentJson.course?._id) content.push(this.contentJson.course) + if (this.contentJson.config?._id) content.push(this.contentJson.config) + for (const item of Object.values(this.contentJson.contentObjects)) { + if (item?._id) content.push(item) + } + return content + } + + /** + * Writes migrated content back into the contentJson structure + * @param {Array} migrated The migrated content array + */ + unflattenContentJson (migrated) { + for (const item of migrated) { + if (item._type === 'course') { + this.contentJson.course = item + } else if (item._type === 'config') { + this.contentJson.config = item + } else { + this.contentJson.contentObjects[item._id] = item + } + } + } + /** * Imports any specified tags * @return {Promise} diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index 3fd8c98..7f55c3a 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -4,7 +4,7 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js' import fs from 'node:fs/promises' import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js' import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server' -import { runCliCommand } from './utils.js' +import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses } from './utils.js' import path from 'node:path' import semver from 'semver' @@ -206,17 +206,22 @@ class AdaptFrameworkModule extends AbstractModule { * @return {Promise} */ async updateFramework (version) { + let migrationResult try { if (version) { this.checkVersionCompatibility(version) } + const fromPlugins = await readFrameworkPluginVersions(this.path) await this.runCliCommand('updateFramework', { version: version ?? this.targetVersionRange }) this._version = await this.runCliCommand('getCurrentFrameworkVersion') + const toPlugins = await readFrameworkPluginVersions(this.path) + migrationResult = await migrateExistingCourses({ fromPlugins, toPlugins, frameworkDir: this.path }) } catch (e) { this.log('error', `failed to update framework, ${e.message}`) throw e.statusCode ? e : this.app.errors.FW_UPDATE_FAILED.setData({ reason: e.message }) } - this.postUpdateHook.invoke() + await this.postUpdateHook.invoke() + return migrationResult } /** @@ -301,6 +306,18 @@ class AdaptFrameworkModule extends AbstractModule { this.contentMigrations.push(migration) } + /** + * Migrates content for specific courses. Called by contentplugin on plugin update. + * @param {Object} options + * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before the update + * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after the update + * @param {String[]} options.courseIds Course IDs to migrate + * @returns {Promise<{migrated: Number, failed: Number, errors: Array}>} + */ + async migrateCourses ({ fromPlugins, toPlugins, courseIds }) { + return migrateExistingCourses({ fromPlugins, toPlugins, frameworkDir: this.path, courseIds }) + } + /** * Builds a single Adapt framework course * @param {AdaptFrameworkBuildOptions} options diff --git a/lib/handlers.js b/lib/handlers.js index 7db26da..029682f 100644 --- a/lib/handlers.js +++ b/lib/handlers.js @@ -133,11 +133,12 @@ export async function postUpdateHandler (req, res, next) { } log('info', 'running framework update') const previousVersion = framework.version - await framework.updateFramework(req.body.version) + const migrationResult = await framework.updateFramework(req.body.version) const currentVersion = framework.version !== previousVersion ? framework.version : undefined res.json({ from: previousVersion, - to: currentVersion + to: currentVersion, + migration: migrationResult }) } catch (e) { return next(e) diff --git a/lib/utils.js b/lib/utils.js index 6b4efdf..3171e32 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -7,3 +7,7 @@ export { retrieveBuildData } from './utils/retrieveBuildData.js' export { getImportSummary } from './utils/getImportSummary.js' export { slugifyTitle } from './utils/slugifyTitle.js' export { copyFrameworkSource } from './utils/copyFrameworkSource.js' +export { readFrameworkPluginVersions } from './utils/readFrameworkPluginVersions.js' +export { collectMigrationScripts } from './utils/collectMigrationScripts.js' +export { runContentMigration } from './utils/runContentMigration.js' +export { migrateExistingCourses } from './utils/migrateExistingCourses.js' diff --git a/lib/utils/collectMigrationScripts.js b/lib/utils/collectMigrationScripts.js new file mode 100644 index 0000000..3210a90 --- /dev/null +++ b/lib/utils/collectMigrationScripts.js @@ -0,0 +1,15 @@ +import { glob } from 'glob' +import path from 'node:path' + +/** + * Collects all migration script paths from the framework's src directory + * @param {String} frameworkDir Absolute path to the framework directory + * @returns {Promise} Absolute paths to migration scripts + */ +export async function collectMigrationScripts (frameworkDir) { + const srcDir = path.join(frameworkDir, 'src') + return glob([ + 'core/migrations/**/*.js', + '*/*/migrations/**/*.js' + ], { cwd: srcDir, absolute: true }) +} diff --git a/lib/utils/migrateExistingCourses.js b/lib/utils/migrateExistingCourses.js new file mode 100644 index 0000000..91f63e3 --- /dev/null +++ b/lib/utils/migrateExistingCourses.js @@ -0,0 +1,78 @@ +import { App } from 'adapt-authoring-core' +import { isDeepStrictEqual } from 'node:util' +import { collectMigrationScripts } from './collectMigrationScripts.js' +import { runContentMigration } from './runContentMigration.js' +import { log } from './log.js' + +/** + * Migrates content for a set of courses by courseId + * @param {Object} options + * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before update + * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after update + * @param {String} options.frameworkDir Absolute path to the framework directory + * @param {String[]} [options.courseIds] Specific course IDs to migrate (if omitted, migrates all) + * @returns {Promise<{migrated: Number, failed: Number, errors: Array}>} + */ +export async function migrateExistingCourses ({ fromPlugins, toPlugins, frameworkDir, courseIds }) { + const content = await App.instance.waitForModule('content') + const scripts = await collectMigrationScripts(frameworkDir) + + if (!scripts.length) { + log('debug', 'no migration scripts found, skipping') + return { migrated: 0, failed: 0, errors: [] } + } + + const courses = courseIds + ? await Promise.all(courseIds.map(async _id => content.findOne({ _id }))) + : await content.find({ _type: 'course' }) + + let migrated = 0 + let failed = 0 + const errors = [] + + for (const course of courses) { + try { + const courseId = course._id.toString() + log('debug', `migrating course ${courseId}`) + + const courseContent = await fetchCourseContent(content, course) + const originals = JSON.parse(JSON.stringify(courseContent)) + + const migratedContent = await runContentMigration({ + content: courseContent, + fromPlugins: JSON.parse(JSON.stringify(fromPlugins)), + toPlugins, + scripts + }) + + let updatedCount = 0 + for (let i = 0; i < migratedContent.length; i++) { + if (!isDeepStrictEqual(originals[i], migratedContent[i])) { + await content.update({ _id: migratedContent[i]._id }, migratedContent[i]) + updatedCount++ + } + } + if (updatedCount > 0) { + log('info', `migrated ${updatedCount} items in course ${courseId}`) + } + migrated++ + } catch (e) { + const courseId = course._id?.toString() ?? 'unknown' + log('error', `migration failed for course ${courseId}`, e.message) + errors.push({ courseId, error: e.message }) + failed++ + } + } + + log('info', `migration complete: ${migrated} succeeded, ${failed} failed`) + return { migrated, failed, errors } +} + +async function fetchCourseContent (content, course) { + const config = await content.findOne({ _courseId: course._id, _type: 'config' }, { strict: false }) + const items = await content.find({ _courseId: course._id, _type: { $nin: ['course', 'config'] } }) + const result = [course] + if (config) result.push(config) + result.push(...items) + return result +} diff --git a/lib/utils/readFrameworkPluginVersions.js b/lib/utils/readFrameworkPluginVersions.js new file mode 100644 index 0000000..cbacb45 --- /dev/null +++ b/lib/utils/readFrameworkPluginVersions.js @@ -0,0 +1,21 @@ +import { readJson } from 'adapt-authoring-core' +import { glob } from 'glob' +import path from 'node:path' + +/** + * Reads bower.json files from the framework's src directory to build a list of plugin names and versions + * @param {String} frameworkDir Absolute path to the framework directory + * @returns {Promise>} + */ +export async function readFrameworkPluginVersions (frameworkDir) { + const srcDir = path.join(frameworkDir, 'src') + const bowerPaths = await glob([ + 'core/bower.json', + '{components,extensions,menu,theme}/*/bower.json' + ], { cwd: srcDir, absolute: true }) + const plugins = await Promise.all(bowerPaths.map(async p => { + const { name, version } = await readJson(p) + return { name, version } + })) + return plugins +} diff --git a/lib/utils/runContentMigration.js b/lib/utils/runContentMigration.js new file mode 100644 index 0000000..d9d80ef --- /dev/null +++ b/lib/utils/runContentMigration.js @@ -0,0 +1,32 @@ +import { load, migrate, Journal, Logger } from 'adapt-migrations' + +/** + * Runs adapt-migrations on a content array. Shared by framework update, course import, and plugin update. + * @param {Object} options + * @param {Array} options.content Flat array of content objects (course, config, contentObjects, etc.) + * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before the update + * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after the update + * @param {String[]} options.scripts Absolute paths to migration scripts + * @param {String} [options.cachePath] Optional cache path for adapt-migrations + * @returns {Promise} The migrated content array + */ +export async function runContentMigration ({ content, fromPlugins, toPlugins, scripts, cachePath }) { + const logger = Logger.getInstance() + + await load({ scripts, cachePath, logger }) + + const originalFromPlugins = JSON.parse(JSON.stringify(fromPlugins)) + const journal = new Journal({ + logger, + data: { + content, + fromPlugins, + originalFromPlugins, + toPlugins + } + }) + + await migrate({ journal, logger }) + + return journal.data.content +} diff --git a/package.json b/package.json index 6aee035..52eb2d4 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "adapt-authoring-mongodb": "^3.0.0", "adapt-authoring-spoortracking": "^1.0.2", "adapt-cli": "^3.3.3", + "adapt-migrations": "^1.4.0", "adapt-octopus": "^0.1.2", "bytes": "^3.1.2", "fs-extra": "11.3.3", diff --git a/tests/utils-collectMigrationScripts.spec.js b/tests/utils-collectMigrationScripts.spec.js new file mode 100644 index 0000000..4944ab3 --- /dev/null +++ b/tests/utils-collectMigrationScripts.spec.js @@ -0,0 +1,45 @@ +import { describe, it, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import os from 'node:os' + +import { collectMigrationScripts } from '../lib/utils/collectMigrationScripts.js' + +describe('collectMigrationScripts()', () => { + let tmpDir + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fw-mig-')) + const srcDir = path.join(tmpDir, 'src') + await fs.mkdir(path.join(srcDir, 'core', 'migrations'), { recursive: true }) + await fs.mkdir(path.join(srcDir, 'components', 'adapt-contrib-text', 'migrations'), { recursive: true }) + + await fs.writeFile(path.join(srcDir, 'core', 'migrations', '6.24.2.js'), '// core migration') + await fs.writeFile(path.join(srcDir, 'components', 'adapt-contrib-text', 'migrations', '5.0.1.js'), '// text migration') + }) + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true }) + }) + + it('should find core and plugin migration scripts', async () => { + const scripts = await collectMigrationScripts(tmpDir) + assert.equal(scripts.length, 2) + assert.ok(scripts.some(s => s.includes('core/migrations/6.24.2.js'))) + assert.ok(scripts.some(s => s.includes('adapt-contrib-text/migrations/5.0.1.js'))) + }) + + it('should return absolute paths', async () => { + const scripts = await collectMigrationScripts(tmpDir) + scripts.forEach(s => assert.ok(path.isAbsolute(s))) + }) + + it('should return empty array when no migration scripts exist', async () => { + const emptyDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fw-nomig-')) + await fs.mkdir(path.join(emptyDir, 'src'), { recursive: true }) + const scripts = await collectMigrationScripts(emptyDir) + assert.deepEqual(scripts, []) + await fs.rm(emptyDir, { recursive: true }) + }) +}) diff --git a/tests/utils-migrateExistingCourses.spec.js b/tests/utils-migrateExistingCourses.spec.js new file mode 100644 index 0000000..9ca0ee7 --- /dev/null +++ b/tests/utils-migrateExistingCourses.spec.js @@ -0,0 +1,163 @@ +import { describe, it, mock } from 'node:test' +import assert from 'node:assert/strict' + +const mockContentModule = { + find: mock.fn(async () => [ + { _id: 'course1', _type: 'course', title: 'Course 1' } + ]), + findOne: mock.fn(async ({ _id, _type, _courseId }) => { + if (_type === 'config') return { _id: 'cfg1', _type: 'config', _courseId } + return { _id, _type: 'course', title: 'Course 1' } + }), + update: mock.fn(async () => {}) +} + +mock.module('adapt-authoring-core', { + namedExports: { + App: { + instance: { + waitForModule: mock.fn(async () => mockContentModule) + } + } + } +}) + +const mockCollectMigrationScripts = mock.fn(async () => ['/path/to/script.js']) +mock.module('../lib/utils/collectMigrationScripts.js', { + namedExports: { + collectMigrationScripts: mockCollectMigrationScripts + } +}) + +const mockRunContentMigration = mock.fn(async ({ content }) => { + return content.map(item => ({ + ...item, + title: item.title ? item.title + ' (migrated)' : item.title + })) +}) +mock.module('../lib/utils/runContentMigration.js', { + namedExports: { + runContentMigration: mockRunContentMigration + } +}) + +mock.module('../lib/utils/log.js', { + namedExports: { + log: () => {} + } +}) + +const { migrateExistingCourses } = await import('../lib/utils/migrateExistingCourses.js') + +describe('migrateExistingCourses()', () => { + it('should collect migration scripts from frameworkDir', async () => { + mockCollectMigrationScripts.mock.resetCalls() + await migrateExistingCourses({ + fromPlugins: [{ name: 'core', version: '1.0.0' }], + toPlugins: [{ name: 'core', version: '2.0.0' }], + frameworkDir: '/fw' + }) + assert.equal(mockCollectMigrationScripts.mock.calls.length, 1) + assert.equal(mockCollectMigrationScripts.mock.calls[0].arguments[0], '/fw') + }) + + it('should query all courses when no courseIds provided', async () => { + mockContentModule.find.mock.resetCalls() + await migrateExistingCourses({ + fromPlugins: [], + toPlugins: [], + frameworkDir: '/fw' + }) + const findCalls = mockContentModule.find.mock.calls + assert.ok(findCalls.some(c => + JSON.stringify(c.arguments[0]) === JSON.stringify({ _type: 'course' }) + )) + }) + + it('should return migration result counts', async () => { + mockContentModule.find.mock.resetCalls() + mockContentModule.update.mock.resetCalls() + mockRunContentMigration.mock.resetCalls() + mockRunContentMigration.mock.mockImplementation(async ({ content }) => { + return content.map(item => ({ + ...item, + title: item.title ? item.title + ' (migrated)' : item.title + })) + }) + const result = await migrateExistingCourses({ + fromPlugins: [{ name: 'core', version: '1.0.0' }], + toPlugins: [{ name: 'core', version: '2.0.0' }], + frameworkDir: '/fw' + }) + assert.equal(result.migrated, 1) + assert.equal(result.failed, 0) + assert.deepEqual(result.errors, []) + }) + + it('should only update changed items in DB', async () => { + mockContentModule.update.mock.resetCalls() + mockRunContentMigration.mock.mockImplementation(async ({ content }) => { + return content.map(item => ({ + ...item, + title: item.title ? item.title + ' (migrated)' : item.title + })) + }) + await migrateExistingCourses({ + fromPlugins: [], + toPlugins: [], + frameworkDir: '/fw' + }) + assert.ok(mockContentModule.update.mock.calls.length > 0) + }) + + it('should skip DB writes when content is unchanged', async () => { + mockContentModule.update.mock.resetCalls() + mockRunContentMigration.mock.mockImplementation(async ({ content }) => content) + await migrateExistingCourses({ + fromPlugins: [], + toPlugins: [], + frameworkDir: '/fw' + }) + assert.equal(mockContentModule.update.mock.calls.length, 0) + }) + + it('should return early with zero counts when no scripts found', async () => { + mockCollectMigrationScripts.mock.mockImplementation(async () => []) + const result = await migrateExistingCourses({ + fromPlugins: [], + toPlugins: [], + frameworkDir: '/fw' + }) + assert.equal(result.migrated, 0) + assert.equal(result.failed, 0) + // restore + mockCollectMigrationScripts.mock.mockImplementation(async () => ['/path/to/script.js']) + }) + + it('should isolate per-course errors and continue', async () => { + mockContentModule.find.mock.mockImplementation(async (query) => { + if (query._type === 'course') { + return [ + { _id: 'course1', _type: 'course', title: 'OK' }, + { _id: 'course2', _type: 'course', title: 'Fails' } + ] + } + return [] + }) + let callCount = 0 + mockRunContentMigration.mock.mockImplementation(async ({ content }) => { + callCount++ + if (callCount === 2) throw new Error('migration error') + return content.map(item => ({ ...item, title: item.title + ' (migrated)' })) + }) + const result = await migrateExistingCourses({ + fromPlugins: [], + toPlugins: [], + frameworkDir: '/fw' + }) + assert.equal(result.migrated, 1) + assert.equal(result.failed, 1) + assert.equal(result.errors.length, 1) + assert.equal(result.errors[0].courseId, 'course2') + }) +}) diff --git a/tests/utils-readFrameworkPluginVersions.spec.js b/tests/utils-readFrameworkPluginVersions.spec.js new file mode 100644 index 0000000..9393b23 --- /dev/null +++ b/tests/utils-readFrameworkPluginVersions.spec.js @@ -0,0 +1,48 @@ +import { describe, it, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import os from 'node:os' + +import { readFrameworkPluginVersions } from '../lib/utils/readFrameworkPluginVersions.js' + +describe('readFrameworkPluginVersions()', () => { + let tmpDir + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fw-test-')) + const srcDir = path.join(tmpDir, 'src') + await fs.mkdir(path.join(srcDir, 'core'), { recursive: true }) + await fs.mkdir(path.join(srcDir, 'components', 'adapt-contrib-text'), { recursive: true }) + await fs.mkdir(path.join(srcDir, 'extensions', 'adapt-contrib-trickle'), { recursive: true }) + + await fs.writeFile(path.join(srcDir, 'core', 'bower.json'), JSON.stringify({ name: 'adapt-contrib-core', version: '6.24.1' })) + await fs.writeFile(path.join(srcDir, 'components', 'adapt-contrib-text', 'bower.json'), JSON.stringify({ name: 'adapt-contrib-text', version: '5.0.0' })) + await fs.writeFile(path.join(srcDir, 'extensions', 'adapt-contrib-trickle', 'bower.json'), JSON.stringify({ name: 'adapt-contrib-trickle', version: '4.2.1' })) + }) + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true }) + }) + + it('should return plugin names and versions from bower.json files', async () => { + const plugins = await readFrameworkPluginVersions(tmpDir) + assert.equal(plugins.length, 3) + const names = plugins.map(p => p.name).sort() + assert.deepEqual(names, ['adapt-contrib-core', 'adapt-contrib-text', 'adapt-contrib-trickle']) + }) + + it('should return name and version for each plugin', async () => { + const plugins = await readFrameworkPluginVersions(tmpDir) + const core = plugins.find(p => p.name === 'adapt-contrib-core') + assert.equal(core.version, '6.24.1') + }) + + it('should return empty array when src dir has no bower files', async () => { + const emptyDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fw-empty-')) + await fs.mkdir(path.join(emptyDir, 'src'), { recursive: true }) + const plugins = await readFrameworkPluginVersions(emptyDir) + assert.deepEqual(plugins, []) + await fs.rm(emptyDir, { recursive: true }) + }) +}) diff --git a/tests/utils-runContentMigration.spec.js b/tests/utils-runContentMigration.spec.js new file mode 100644 index 0000000..6173754 --- /dev/null +++ b/tests/utils-runContentMigration.spec.js @@ -0,0 +1,92 @@ +import { describe, it, mock } from 'node:test' +import assert from 'node:assert/strict' + +const mockLoad = mock.fn(async () => {}) +const mockMigrate = mock.fn(async ({ journal }) => { + journal.data.content[0].title = 'migrated' +}) + +class FakeJournal { + constructor ({ data }) { + this.data = data + } +} + +class FakeLogger { + info () {} + error () {} + warn () {} + debug () {} + log () {} +} + +mock.module('adapt-migrations', { + namedExports: { + load: mockLoad, + migrate: mockMigrate, + Journal: FakeJournal, + Logger: { getInstance: () => new FakeLogger() } + } +}) + +const { runContentMigration } = await import('../lib/utils/runContentMigration.js') + +describe('runContentMigration()', () => { + it('should call load with scripts and logger', async () => { + mockLoad.mock.resetCalls() + const scripts = ['/path/to/migration.js'] + await runContentMigration({ + content: [{ _id: 'c1', title: 'old' }], + fromPlugins: [{ name: 'core', version: '1.0.0' }], + toPlugins: [{ name: 'core', version: '2.0.0' }], + scripts + }) + assert.equal(mockLoad.mock.calls.length, 1) + assert.deepEqual(mockLoad.mock.calls[0].arguments[0].scripts, scripts) + }) + + it('should create a Journal with correct data shape', async () => { + mockMigrate.mock.resetCalls() + await runContentMigration({ + content: [{ _id: 'c1', title: 'old' }], + fromPlugins: [{ name: 'core', version: '1.0.0' }], + toPlugins: [{ name: 'core', version: '2.0.0' }], + scripts: [] + }) + assert.equal(mockMigrate.mock.calls.length, 1) + const journal = mockMigrate.mock.calls[0].arguments[0].journal + assert.ok(journal.data.content) + assert.ok(journal.data.fromPlugins) + assert.ok(journal.data.originalFromPlugins) + assert.ok(journal.data.toPlugins) + }) + + it('should return mutated content', async () => { + mockMigrate.mock.resetCalls() + mockMigrate.mock.mockImplementation(async ({ journal }) => { + journal.data.content[0].title = 'migrated' + }) + const result = await runContentMigration({ + content: [{ _id: 'c1', title: 'old' }], + fromPlugins: [{ name: 'core', version: '1.0.0' }], + toPlugins: [{ name: 'core', version: '2.0.0' }], + scripts: [] + }) + assert.equal(result[0].title, 'migrated') + }) + + it('should deep-clone fromPlugins into originalFromPlugins', async () => { + mockMigrate.mock.resetCalls() + mockMigrate.mock.mockImplementation(async () => {}) + const fromPlugins = [{ name: 'core', version: '1.0.0' }] + await runContentMigration({ + content: [{ _id: 'c1' }], + fromPlugins, + toPlugins: [{ name: 'core', version: '2.0.0' }], + scripts: [] + }) + const journal = mockMigrate.mock.calls[0].arguments[0].journal + assert.deepEqual(journal.data.originalFromPlugins, fromPlugins) + assert.notEqual(journal.data.originalFromPlugins, journal.data.fromPlugins) + }) +}) From 491081f99b33050ffcbc02c950ce172a977bd0ec Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 18 Mar 2026 15:22:19 +0000 Subject: [PATCH 03/27] New: Pre-built compilation cache for preview builds (refs #176) Cache shared grunt output (JS bundle, templates, CSS, plugin assets) per theme+menu combo so subsequent preview builds skip compilation entirely, reducing build time from ~60s to ~2-5s. --- lib/AdaptFrameworkBuild.js | 55 ++++++- lib/AdaptFrameworkModule.js | 28 +++- lib/utils.js | 4 + lib/utils/applyBuildReplacements.js | 23 +++ lib/utils/computePluginHash.js | 14 ++ lib/utils/generateLanguageManifest.js | 9 ++ lib/utils/prebuiltCache.js | 159 +++++++++++++++++++ tests/utils-applyBuildReplacements.spec.js | 57 +++++++ tests/utils-computePluginHash.spec.js | 40 +++++ tests/utils-generateLanguageManifest.spec.js | 27 ++++ tests/utils-prebuiltCache.spec.js | 113 +++++++++++++ 11 files changed, 527 insertions(+), 2 deletions(-) create mode 100644 lib/utils/applyBuildReplacements.js create mode 100644 lib/utils/computePluginHash.js create mode 100644 lib/utils/generateLanguageManifest.js create mode 100644 lib/utils/prebuiltCache.js create mode 100644 tests/utils-applyBuildReplacements.spec.js create mode 100644 tests/utils-computePluginHash.spec.js create mode 100644 tests/utils-generateLanguageManifest.spec.js create mode 100644 tests/utils-prebuiltCache.spec.js diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index 67b2c00..0a9a00f 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -3,7 +3,7 @@ import { App, Hook, ensureDir, writeJson } from 'adapt-authoring-core' import { parseObjectId } from 'adapt-authoring-mongodb' import { createWriteStream } from 'node:fs' import AdaptCli from 'adapt-cli' -import { log, logDir, logMemory, copyFrameworkSource } from './utils.js' +import { log, logDir, logMemory, copyFrameworkSource, hasCachedBuild, populateCache, restoreFromCache, generateLanguageManifest, applyBuildReplacements } from './utils.js' import fs from 'node:fs/promises' import path from 'upath' import semver from 'semver' @@ -199,6 +199,32 @@ class AdaptFrameworkBuild { await this.loadCourseData() + // Check for cached preview build + if (this.isPreview && !contentOnly) { + const prebuiltCacheRoot = path.join(framework.getConfig('buildDir'), 'prebuilt-cache') + const pluginHash = await framework.getPluginHash() + const theme = this.courseData.config.data._theme + const menu = this.courseData.config.data._menu + + if (await hasCachedBuild(prebuiltCacheRoot, pluginHash, theme, menu)) { + await restoreFromCache(prebuiltCacheRoot, pluginHash, theme, menu, this.buildDir) + await this.copyAssets() + await this.preBuildHook.invoke(this) + await this.writeContentJson() + await this.writeLanguageManifest() + await applyBuildReplacements(this.buildDir, { + defaultLanguage: this.courseData.config.data._defaultLanguage ?? 'en', + defaultDirection: this.courseData.config.data._defaultDirection ?? 'ltr', + buildType: 'development', + timestamp: Date.now() + }) + this.location = path.join(this.dir, 'build') + await this.postBuildHook.invoke(this) + this.buildData = await this.recordBuildAttempt() + return this + } + } + const tasks = [this.copyAssets()] if (!contentOnly) { tasks.push(copyFrameworkSource({ @@ -233,6 +259,18 @@ class AdaptFrameworkBuild { .setData(e) } } + // Populate prebuilt cache after successful grunt build for preview + if (this.isPreview && !contentOnly) { + const prebuiltCacheRoot = path.join(framework.getConfig('buildDir'), 'prebuilt-cache') + const pluginHash = await framework.getPluginHash() + const theme = this.courseData.config.data._theme + const menu = this.courseData.config.data._menu + try { + await populateCache(this.buildDir, prebuiltCacheRoot, pluginHash, theme, menu) + } catch (e) { + log('warn', 'CACHE', `failed to populate prebuilt cache: ${e.message}`) + } + } if (this.compress) { this.location = await this.prepareZip() } else { @@ -506,6 +544,21 @@ class AdaptFrameworkBuild { })) } + /** + * Writes the language_data_manifest.js for each language dir. + * Only needed on cache-hit builds where grunt is skipped. + * @return {Promise} + */ + async writeLanguageManifest () { + const langDir = this.courseData.course.dir + const fileNames = Object.values(this.courseData) + .filter(d => d.dir === langDir) + .map(d => d.fileName) + const manifest = generateLanguageManifest(fileNames) + await ensureDir(langDir) + await writeJson(path.join(langDir, 'language_data_manifest.js'), manifest) + } + /** * Creates a zip file containing all files relevant to the type of build being performed * @return {Promise} diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index 7f55c3a..d165da2 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -4,7 +4,7 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js' import fs from 'node:fs/promises' import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js' import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server' -import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses } from './utils.js' +import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, invalidateCache } from './utils.js' import path from 'node:path' import semver from 'semver' @@ -83,6 +83,12 @@ class AdaptFrameworkModule extends AbstractModule { await this.installFramework() + const contentplugin = await this.app.waitForModule('contentplugin') + contentplugin.postInsertHook.tap(() => this.invalidatePrebuiltCache()) + contentplugin.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) + contentplugin.postDeleteHook.tap(() => this.invalidatePrebuiltCache()) + this.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) + process.env.BROWSERSLIST_IGNORE_OLD_DATA = '1' if (this.app.args['update-framework'] === true) { @@ -224,6 +230,26 @@ class AdaptFrameworkModule extends AbstractModule { return migrationResult } + /** + * Returns a cached plugin hash, computing it on first call + * @return {Promise} + */ + async getPluginHash () { + if (!this._pluginHash) { + this._pluginHash = await computePluginHash(this.path) + } + return this._pluginHash + } + + /** + * Invalidates the prebuilt compilation cache + */ + invalidatePrebuiltCache () { + const cacheRoot = path.join(this.getConfig('buildDir'), 'prebuilt-cache') + invalidateCache(cacheRoot) + this._pluginHash = null + } + /** * Logs relevant framework status messages */ diff --git a/lib/utils.js b/lib/utils.js index 3171e32..6faab23 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -11,3 +11,7 @@ export { readFrameworkPluginVersions } from './utils/readFrameworkPluginVersions export { collectMigrationScripts } from './utils/collectMigrationScripts.js' export { runContentMigration } from './utils/runContentMigration.js' export { migrateExistingCourses } from './utils/migrateExistingCourses.js' +export { computePluginHash } from './utils/computePluginHash.js' +export { hasCachedBuild, populateCache, restoreFromCache, invalidateCache } from './utils/prebuiltCache.js' +export { generateLanguageManifest } from './utils/generateLanguageManifest.js' +export { applyBuildReplacements } from './utils/applyBuildReplacements.js' diff --git a/lib/utils/applyBuildReplacements.js b/lib/utils/applyBuildReplacements.js new file mode 100644 index 0000000..a8d0539 --- /dev/null +++ b/lib/utils/applyBuildReplacements.js @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises' +import path from 'upath' + +/** + * Applies @@placeholder substitutions in index.html + * @param {String} buildDir The build output directory + * @param {Object} data Replacement values + * @param {String} data.defaultLanguage The default language code + * @param {String} data.defaultDirection The default text direction + * @param {String} data.buildType The build type (e.g. 'development') + * @param {Number} data.timestamp The build timestamp + * @return {Promise} + */ +export async function applyBuildReplacements (buildDir, { defaultLanguage, defaultDirection, buildType, timestamp }) { + const indexPath = path.join(buildDir, 'index.html') + let html = await fs.readFile(indexPath, 'utf8') + html = html + .replace(/@@config\._defaultLanguage/g, defaultLanguage) + .replace(/@@config\._defaultDirection/g, defaultDirection) + .replace(/@@build\.type/g, buildType) + .replace(/@@build\.timestamp/g, String(timestamp)) + await fs.writeFile(indexPath, html) +} diff --git a/lib/utils/computePluginHash.js b/lib/utils/computePluginHash.js new file mode 100644 index 0000000..c8420a1 --- /dev/null +++ b/lib/utils/computePluginHash.js @@ -0,0 +1,14 @@ +import { createHash } from 'node:crypto' +import Project from 'adapt-cli/lib/integration/Project.js' + +/** + * Computes a deterministic hash from the installed plugin set + * @param {String} frameworkDir Path to the local framework installation + * @return {Promise} 16-char hex hash + */ +export async function computePluginHash (frameworkDir) { + const project = new Project({ cwd: frameworkDir }) + const deps = await project.getInstalledDependencies() + const sorted = Object.entries(deps).sort(([a], [b]) => a.localeCompare(b)) + return createHash('sha256').update(JSON.stringify(sorted)).digest('hex').slice(0, 16) +} diff --git a/lib/utils/generateLanguageManifest.js b/lib/utils/generateLanguageManifest.js new file mode 100644 index 0000000..ce94a10 --- /dev/null +++ b/lib/utils/generateLanguageManifest.js @@ -0,0 +1,9 @@ +/** + * Returns the list of JSON filenames that belong in a language manifest. + * The framework runtime reads this to know which data files to fetch. + * @param {Array} jsonFileNames All JSON filenames written to the language dir + * @return {Array} Filtered list excluding the manifest itself and assets.json + */ +export function generateLanguageManifest (jsonFileNames) { + return jsonFileNames.filter(f => f !== 'language_data_manifest.js' && f !== 'assets.json') +} diff --git a/lib/utils/prebuiltCache.js b/lib/utils/prebuiltCache.js new file mode 100644 index 0000000..815d7d2 --- /dev/null +++ b/lib/utils/prebuiltCache.js @@ -0,0 +1,159 @@ +import fs from 'node:fs/promises' +import path from 'upath' +import { log } from './log.js' + +/** Shared artifact paths relative to build dir (JS bundle, templates, plugin assets) */ +const SHARED_GLOBS = [ + 'adapt/js', + 'templates.js', + 'assets', + 'libraries', + 'index.html' +] + +/** CSS artifact paths relative to build dir */ +const CSS_PATHS = ['adapt/css'] + +/** + * Returns the cache directory paths for a given plugin hash, theme, and menu + * @param {String} cacheRoot Root cache directory + * @param {String} pluginHash Hash of installed plugins + * @param {String} theme Theme name + * @param {String} menu Menu name + * @return {{ sharedDir: String, cssDir: String }} + */ +export function getCachePaths (cacheRoot, pluginHash, theme, menu) { + return { + sharedDir: path.join(cacheRoot, pluginHash), + cssDir: path.join(cacheRoot, `${pluginHash}_${theme}_${menu}`) + } +} + +/** + * Checks whether a cached build exists for the given parameters + * @param {String} cacheRoot Root cache directory + * @param {String} pluginHash Hash of installed plugins + * @param {String} theme Theme name + * @param {String} menu Menu name + * @return {Promise} + */ +export async function hasCachedBuild (cacheRoot, pluginHash, theme, menu) { + const { sharedDir, cssDir } = getCachePaths(cacheRoot, pluginHash, theme, menu) + try { + await Promise.all([ + fs.access(sharedDir), + fs.access(cssDir) + ]) + return true + } catch { + return false + } +} + +/** + * Extracts shared artifacts from a completed grunt build into the cache. + * Uses atomic rename for parallel safety. + * @param {String} buildOutputDir The build output directory (contains adapt/, index.html, etc.) + * @param {String} cacheRoot Root cache directory + * @param {String} pluginHash Hash of installed plugins + * @param {String} theme Theme name + * @param {String} menu Menu name + * @return {Promise} + */ +export async function populateCache (buildOutputDir, cacheRoot, pluginHash, theme, menu) { + const { sharedDir, cssDir } = getCachePaths(cacheRoot, pluginHash, theme, menu) + await fs.mkdir(cacheRoot, { recursive: true }) + + // Write to temp dirs first, then rename atomically + const tmpShared = `${sharedDir}_tmp_${Date.now()}` + const tmpCss = `${cssDir}_tmp_${Date.now()}` + + try { + await fs.mkdir(tmpShared, { recursive: true }) + await fs.mkdir(tmpCss, { recursive: true }) + + // Copy shared artifacts + for (const rel of SHARED_GLOBS) { + const src = path.join(buildOutputDir, rel) + const dest = path.join(tmpShared, rel) + try { + const stat = await fs.stat(src) + if (stat.isDirectory()) { + await fs.cp(src, dest, { recursive: true }) + } else { + await fs.mkdir(path.dirname(dest), { recursive: true }) + await fs.copyFile(src, dest) + } + } catch (e) { + if (e.code !== 'ENOENT') throw e + // Skip missing optional artifacts + } + } + + // Copy CSS artifacts + for (const rel of CSS_PATHS) { + const src = path.join(buildOutputDir, rel) + const dest = path.join(tmpCss, rel) + try { + await fs.cp(src, dest, { recursive: true }) + } catch (e) { + if (e.code !== 'ENOENT') throw e + } + } + + // Atomic rename into place (last writer wins for parallel builds) + await safeRename(tmpShared, sharedDir) + await safeRename(tmpCss, cssDir) + + log('info', 'CACHE', `populated cache for ${pluginHash} (theme=${theme}, menu=${menu})`) + } catch (e) { + // Clean up temp dirs on failure + await fs.rm(tmpShared, { recursive: true, force: true }) + await fs.rm(tmpCss, { recursive: true, force: true }) + throw e + } +} + +/** + * Copies cached artifacts to a build directory + * @param {String} cacheRoot Root cache directory + * @param {String} pluginHash Hash of installed plugins + * @param {String} theme Theme name + * @param {String} menu Menu name + * @param {String} destDir Destination build directory + * @return {Promise} + */ +export async function restoreFromCache (cacheRoot, pluginHash, theme, menu, destDir) { + const { sharedDir, cssDir } = getCachePaths(cacheRoot, pluginHash, theme, menu) + await fs.mkdir(destDir, { recursive: true }) + await fs.cp(sharedDir, destDir, { recursive: true }) + await fs.cp(cssDir, destDir, { recursive: true }) + log('info', 'CACHE', `restored from cache for ${pluginHash} (theme=${theme}, menu=${menu})`) +} + +/** + * Removes the entire prebuilt cache directory + * @param {String} cacheRoot Root cache directory + * @return {Promise} + */ +export async function invalidateCache (cacheRoot) { + await fs.rm(cacheRoot, { recursive: true, force: true }) + log('info', 'CACHE', 'invalidated prebuilt cache') +} + +/** + * Renames src to dest atomically. If dest already exists (parallel build), + * removes src instead since dest already has identical content. + */ +async function safeRename (src, dest) { + try { + await fs.rename(src, dest) + } catch (e) { + if (e.code === 'ENOTEMPTY' || e.code === 'EEXIST') { + // Another build already wrote the cache — clean up our temp copy + await fs.rm(src, { recursive: true, force: true }) + } else { + throw e + } + } +} diff --git a/tests/utils-applyBuildReplacements.spec.js b/tests/utils-applyBuildReplacements.spec.js new file mode 100644 index 0000000..1926d3c --- /dev/null +++ b/tests/utils-applyBuildReplacements.spec.js @@ -0,0 +1,57 @@ +import { describe, it, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import os from 'node:os' + +import { applyBuildReplacements } from '../lib/utils/applyBuildReplacements.js' + +describe('applyBuildReplacements()', () => { + let tmpDir + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'aat-replace-test-')) + }) + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }) + }) + + it('should replace all @@placeholders in index.html', async () => { + const template = [ + '', + '', + '' + ].join('\n') + await fs.writeFile(path.join(tmpDir, 'index.html'), template) + + await applyBuildReplacements(tmpDir, { + defaultLanguage: 'fr', + defaultDirection: 'rtl', + buildType: 'development', + timestamp: 1234567890 + }) + + const result = await fs.readFile(path.join(tmpDir, 'index.html'), 'utf8') + assert.ok(result.includes('lang="fr"')) + assert.ok(result.includes('dir="rtl"')) + assert.ok(result.includes('content="development"')) + assert.ok(result.includes('content="1234567890"')) + assert.ok(!result.includes('@@')) + }) + + it('should handle multiple occurrences of the same placeholder', async () => { + const template = '@@config._defaultLanguage @@config._defaultLanguage' + await fs.writeFile(path.join(tmpDir, 'index.html'), template) + + await applyBuildReplacements(tmpDir, { + defaultLanguage: 'de', + defaultDirection: 'ltr', + buildType: 'production', + timestamp: 0 + }) + + const result = await fs.readFile(path.join(tmpDir, 'index.html'), 'utf8') + assert.equal(result, 'de de') + }) +}) diff --git a/tests/utils-computePluginHash.spec.js b/tests/utils-computePluginHash.spec.js new file mode 100644 index 0000000..09df53d --- /dev/null +++ b/tests/utils-computePluginHash.spec.js @@ -0,0 +1,40 @@ +import { before, describe, it, mock } from 'node:test' +import assert from 'node:assert/strict' + +describe('computePluginHash()', () => { + let computePluginHash + + before(async () => { + mock.module('adapt-cli/lib/integration/Project.js', { + defaultExport: class MockProject { + constructor ({ cwd }) { this.cwd = cwd } + async getInstalledDependencies () { + return { + 'adapt-contrib-text': '1.0.0', + 'adapt-contrib-narrative': '2.0.0', + 'adapt-contrib-core': '3.0.0' + } + } + } + }) + ;({ computePluginHash } = await import('../lib/utils/computePluginHash.js')) + }) + + it('should return a 16-character hex string', async () => { + const hash = await computePluginHash('/fake/framework') + assert.match(hash, /^[0-9a-f]{16}$/) + }) + + it('should return the same hash for the same plugin set', async () => { + const hash1 = await computePluginHash('/fake/framework') + const hash2 = await computePluginHash('/fake/framework') + assert.equal(hash1, hash2) + }) + + it('should produce a deterministic hash regardless of insertion order', async () => { + // The mock always returns the same deps — verify stability + const hash1 = await computePluginHash('/path/a') + const hash2 = await computePluginHash('/path/b') + assert.equal(hash1, hash2) + }) +}) diff --git a/tests/utils-generateLanguageManifest.spec.js b/tests/utils-generateLanguageManifest.spec.js new file mode 100644 index 0000000..89586d9 --- /dev/null +++ b/tests/utils-generateLanguageManifest.spec.js @@ -0,0 +1,27 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' + +import { generateLanguageManifest } from '../lib/utils/generateLanguageManifest.js' + +describe('generateLanguageManifest()', () => { + it('should return all filenames except the manifest and assets.json', () => { + const input = ['course.json', 'contentObjects.json', 'articles.json', 'language_data_manifest.js', 'assets.json'] + const result = generateLanguageManifest(input) + assert.deepEqual(result, ['course.json', 'contentObjects.json', 'articles.json']) + }) + + it('should return an empty array when only excluded files are present', () => { + const result = generateLanguageManifest(['language_data_manifest.js', 'assets.json']) + assert.deepEqual(result, []) + }) + + it('should return all filenames when no exclusions apply', () => { + const input = ['course.json', 'blocks.json'] + const result = generateLanguageManifest(input) + assert.deepEqual(result, ['course.json', 'blocks.json']) + }) + + it('should handle an empty input array', () => { + assert.deepEqual(generateLanguageManifest([]), []) + }) +}) diff --git a/tests/utils-prebuiltCache.spec.js b/tests/utils-prebuiltCache.spec.js new file mode 100644 index 0000000..0d0b74e --- /dev/null +++ b/tests/utils-prebuiltCache.spec.js @@ -0,0 +1,113 @@ +import { before, describe, it, beforeEach, afterEach, mock } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import os from 'node:os' + +describe('prebuiltCache', () => { + let getCachePaths, hasCachedBuild, populateCache, restoreFromCache, invalidateCache + let tmpDir, cacheRoot, buildDir + + before(async () => { + // Mock the log utility to suppress output during tests + mock.module('../lib/utils/log.js', { + namedExports: { + log: () => {}, + logDir: () => {}, + logMemory: () => {} + } + }) + ;({ getCachePaths, hasCachedBuild, populateCache, restoreFromCache, invalidateCache } = await import('../lib/utils/prebuiltCache.js')) + }) + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'aat-cache-test-')) + cacheRoot = path.join(tmpDir, 'prebuilt-cache') + buildDir = path.join(tmpDir, 'build') + await fs.mkdir(buildDir, { recursive: true }) + }) + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }) + }) + + describe('getCachePaths()', () => { + it('should return correct shared and CSS directory paths', () => { + const result = getCachePaths('/cache', 'abc123', 'vanilla', 'boxMenu') + assert.equal(result.sharedDir, path.join('/cache', 'abc123')) + assert.equal(result.cssDir, path.join('/cache', 'abc123_vanilla_boxMenu')) + }) + }) + + describe('hasCachedBuild()', () => { + it('should return false when cache does not exist', async () => { + assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), false) + }) + + it('should return true when both shared and CSS dirs exist', async () => { + const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') + await fs.mkdir(sharedDir, { recursive: true }) + await fs.mkdir(cssDir, { recursive: true }) + assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), true) + }) + + it('should return false when only shared dir exists', async () => { + const { sharedDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') + await fs.mkdir(sharedDir, { recursive: true }) + assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), false) + }) + }) + + describe('populateCache()', () => { + it('should copy shared and CSS artifacts to cache', async () => { + // Create mock build output + await fs.mkdir(path.join(buildDir, 'adapt', 'js'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'adapt', 'js', 'adapt.min.js'), 'js-content') + await fs.mkdir(path.join(buildDir, 'adapt', 'css'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'adapt', 'css', 'adapt.css'), 'css-content') + await fs.writeFile(path.join(buildDir, 'index.html'), '@@config._defaultLanguage') + + await populateCache(buildDir, cacheRoot, 'hash1', 'theme', 'menu') + + const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') + const js = await fs.readFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'utf8') + assert.equal(js, 'js-content') + const css = await fs.readFile(path.join(cssDir, 'adapt', 'css', 'adapt.css'), 'utf8') + assert.equal(css, 'css-content') + const html = await fs.readFile(path.join(sharedDir, 'index.html'), 'utf8') + assert.ok(html.includes('@@config._defaultLanguage')) + }) + }) + + describe('restoreFromCache()', () => { + it('should copy cached artifacts to destination', async () => { + // Set up cache + const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') + await fs.mkdir(path.join(sharedDir, 'adapt', 'js'), { recursive: true }) + await fs.writeFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'cached-js') + await fs.mkdir(path.join(cssDir, 'adapt', 'css'), { recursive: true }) + await fs.writeFile(path.join(cssDir, 'adapt', 'css', 'adapt.css'), 'cached-css') + + const destDir = path.join(tmpDir, 'restored') + await restoreFromCache(cacheRoot, 'hash1', 'theme', 'menu', destDir) + + const js = await fs.readFile(path.join(destDir, 'adapt', 'js', 'adapt.min.js'), 'utf8') + assert.equal(js, 'cached-js') + const css = await fs.readFile(path.join(destDir, 'adapt', 'css', 'adapt.css'), 'utf8') + assert.equal(css, 'cached-css') + }) + }) + + describe('invalidateCache()', () => { + it('should remove the cache directory', async () => { + await fs.mkdir(cacheRoot, { recursive: true }) + await fs.writeFile(path.join(cacheRoot, 'test'), 'data') + await invalidateCache(cacheRoot) + await assert.rejects(fs.access(cacheRoot), { code: 'ENOENT' }) + }) + + it('should not throw when cache does not exist', async () => { + await assert.doesNotReject(invalidateCache(path.join(tmpDir, 'nonexistent'))) + }) + }) +}) From c9cd7416cd321fb5e42851139979e2bb5432c712 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 18 Mar 2026 16:43:05 +0000 Subject: [PATCH 04/27] Fix: Defer contentplugin dependency to break circular deadlock --- lib/AdaptFrameworkModule.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index d165da2..20d1543 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -83,10 +83,11 @@ class AdaptFrameworkModule extends AbstractModule { await this.installFramework() - const contentplugin = await this.app.waitForModule('contentplugin') - contentplugin.postInsertHook.tap(() => this.invalidatePrebuiltCache()) - contentplugin.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) - contentplugin.postDeleteHook.tap(() => this.invalidatePrebuiltCache()) + this.app.waitForModule('contentplugin').then(contentplugin => { + contentplugin.postInsertHook.tap(() => this.invalidatePrebuiltCache()) + contentplugin.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) + contentplugin.postDeleteHook.tap(() => this.invalidatePrebuiltCache()) + }) this.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) process.env.BROWSERSLIST_IGNORE_OLD_DATA = '1' From cc0071aaf21befde543171ad5f090f9085cfc969 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 18 Mar 2026 19:59:21 +0000 Subject: [PATCH 05/27] Fix: Cache all build artifacts and apply schema defaults on cache hit - Cache all build root entries except course/ (previously missed required files like connection.txt and SCORM HTML files) - Use CSS_ENTRIES set for theme/menu-specific files (adapt.css, fonts) - Apply schema defaults via jsonschema module on cached builds to replicate grunt's schema-defaults task - Update tests to match actual grunt build output structure --- lib/AdaptFrameworkBuild.js | 17 ++++++++ lib/utils/prebuiltCache.js | 65 ++++++++++++++----------------- tests/utils-prebuiltCache.spec.js | 61 +++++++++++++++++++++++------ 3 files changed, 95 insertions(+), 48 deletions(-) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index 0a9a00f..dc52edc 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -208,6 +208,7 @@ class AdaptFrameworkBuild { if (await hasCachedBuild(prebuiltCacheRoot, pluginHash, theme, menu)) { await restoreFromCache(prebuiltCacheRoot, pluginHash, theme, menu, this.buildDir) + await this.applySchemaDefaults() await this.copyAssets() await this.preBuildHook.invoke(this) await this.writeContentJson() @@ -544,6 +545,22 @@ class AdaptFrameworkBuild { })) } + /** + * Applies schema defaults to the in-memory course and config data using + * the jsonschema module. Replicates what grunt's schema-defaults task does. + * @return {Promise} + */ + async applySchemaDefaults () { + const jsonschema = await App.instance.waitForModule('jsonschema') + const [courseSchema, configSchema] = await Promise.all([ + jsonschema.getSchema('course'), + jsonschema.getSchema('config') + ]) + // Apply defaults without running full validation (which rejects ObjectIds etc.) + courseSchema.compiledWithDefaults(this.courseData.course.data) + configSchema.compiledWithDefaults(this.courseData.config.data) + } + /** * Writes the language_data_manifest.js for each language dir. * Only needed on cache-hit builds where grunt is skipped. diff --git a/lib/utils/prebuiltCache.js b/lib/utils/prebuiltCache.js index 815d7d2..06c721d 100644 --- a/lib/utils/prebuiltCache.js +++ b/lib/utils/prebuiltCache.js @@ -2,17 +2,11 @@ import fs from 'node:fs/promises' import path from 'upath' import { log } from './log.js' -/** Shared artifact paths relative to build dir (JS bundle, templates, plugin assets) */ -const SHARED_GLOBS = [ - 'adapt/js', - 'templates.js', - 'assets', - 'libraries', - 'index.html' -] +/** Entries that are theme/menu-specific and belong in the CSS cache */ +const CSS_ENTRIES = new Set(['adapt.css', 'adapt.css.map', 'fonts']) -/** CSS artifact paths relative to build dir */ -const CSS_PATHS = ['adapt/css'] +/** Entries to skip (rebuilt per-build from course data) */ +const SKIP_ENTRIES = new Set(['course']) /** * Returns the cache directory paths for a given plugin hash, theme, and menu @@ -50,8 +44,25 @@ export async function hasCachedBuild (cacheRoot, pluginHash, theme, menu) { } } +/** + * Copies a file or directory from src to dest, creating parent dirs as needed + * @param {String} src Source path + * @param {String} dest Destination path + * @return {Promise} + */ +async function copyEntry (src, dest) { + const stat = await fs.stat(src) + if (stat.isDirectory()) { + await fs.cp(src, dest, { recursive: true }) + } else { + await fs.mkdir(path.dirname(dest), { recursive: true }) + await fs.copyFile(src, dest) + } +} + /** * Extracts shared artifacts from a completed grunt build into the cache. + * Scans the build root and categorises each entry as shared or CSS-specific. * Uses atomic rename for parallel safety. * @param {String} buildOutputDir The build output directory (contains adapt/, index.html, etc.) * @param {String} cacheRoot Root cache directory @@ -72,32 +83,14 @@ export async function populateCache (buildOutputDir, cacheRoot, pluginHash, them await fs.mkdir(tmpShared, { recursive: true }) await fs.mkdir(tmpCss, { recursive: true }) - // Copy shared artifacts - for (const rel of SHARED_GLOBS) { - const src = path.join(buildOutputDir, rel) - const dest = path.join(tmpShared, rel) - try { - const stat = await fs.stat(src) - if (stat.isDirectory()) { - await fs.cp(src, dest, { recursive: true }) - } else { - await fs.mkdir(path.dirname(dest), { recursive: true }) - await fs.copyFile(src, dest) - } - } catch (e) { - if (e.code !== 'ENOENT') throw e - // Skip missing optional artifacts - } - } - - // Copy CSS artifacts - for (const rel of CSS_PATHS) { - const src = path.join(buildOutputDir, rel) - const dest = path.join(tmpCss, rel) - try { - await fs.cp(src, dest, { recursive: true }) - } catch (e) { - if (e.code !== 'ENOENT') throw e + const entries = await fs.readdir(buildOutputDir) + for (const entry of entries) { + if (SKIP_ENTRIES.has(entry)) continue + const src = path.join(buildOutputDir, entry) + if (CSS_ENTRIES.has(entry)) { + await copyEntry(src, path.join(tmpCss, entry)) + } else { + await copyEntry(src, path.join(tmpShared, entry)) } } diff --git a/tests/utils-prebuiltCache.spec.js b/tests/utils-prebuiltCache.spec.js index 0d0b74e..c244e63 100644 --- a/tests/utils-prebuiltCache.spec.js +++ b/tests/utils-prebuiltCache.spec.js @@ -59,42 +59,79 @@ describe('prebuiltCache', () => { }) describe('populateCache()', () => { - it('should copy shared and CSS artifacts to cache', async () => { - // Create mock build output + it('should cache all build entries except course/', async () => { + // Create mock build output matching actual grunt structure await fs.mkdir(path.join(buildDir, 'adapt', 'js'), { recursive: true }) await fs.writeFile(path.join(buildDir, 'adapt', 'js', 'adapt.min.js'), 'js-content') - await fs.mkdir(path.join(buildDir, 'adapt', 'css'), { recursive: true }) - await fs.writeFile(path.join(buildDir, 'adapt', 'css', 'adapt.css'), 'css-content') - await fs.writeFile(path.join(buildDir, 'index.html'), '@@config._defaultLanguage') + await fs.writeFile(path.join(buildDir, 'adapt.css'), 'css-content') + await fs.writeFile(path.join(buildDir, 'adapt.css.map'), 'map-content') + await fs.mkdir(path.join(buildDir, 'fonts'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'fonts', 'icon.woff2'), 'font-data') + await fs.writeFile(path.join(buildDir, 'index.html'), '') + await fs.writeFile(path.join(buildDir, 'templates.js'), 'templates') + await fs.mkdir(path.join(buildDir, 'assets'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'assets', 'logo.png'), 'img') + await fs.mkdir(path.join(buildDir, 'libraries'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'libraries', 'modernizr.js'), 'lib') + // Required files from plugins (e.g. spoortracking) + await fs.writeFile(path.join(buildDir, 'connection.txt'), '') + await fs.writeFile(path.join(buildDir, 'scorm_test_harness.html'), '') + // course/ should be skipped + await fs.mkdir(path.join(buildDir, 'course', 'en'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'course', 'en', 'course.json'), '{}') await populateCache(buildDir, cacheRoot, 'hash1', 'theme', 'menu') const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') + + // Shared artifacts const js = await fs.readFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'utf8') assert.equal(js, 'js-content') - const css = await fs.readFile(path.join(cssDir, 'adapt', 'css', 'adapt.css'), 'utf8') - assert.equal(css, 'css-content') const html = await fs.readFile(path.join(sharedDir, 'index.html'), 'utf8') - assert.ok(html.includes('@@config._defaultLanguage')) + assert.equal(html, '') + const conn = await fs.readFile(path.join(sharedDir, 'connection.txt'), 'utf8') + assert.equal(conn, '') + const harness = await fs.readFile(path.join(sharedDir, 'scorm_test_harness.html'), 'utf8') + assert.equal(harness, '') + + // CSS artifacts + const css = await fs.readFile(path.join(cssDir, 'adapt.css'), 'utf8') + assert.equal(css, 'css-content') + const cssMap = await fs.readFile(path.join(cssDir, 'adapt.css.map'), 'utf8') + assert.equal(cssMap, 'map-content') + const font = await fs.readFile(path.join(cssDir, 'fonts', 'icon.woff2'), 'utf8') + assert.equal(font, 'font-data') + + // course/ should NOT be cached + await assert.rejects(fs.access(path.join(sharedDir, 'course')), { code: 'ENOENT' }) + await assert.rejects(fs.access(path.join(cssDir, 'course')), { code: 'ENOENT' }) }) }) describe('restoreFromCache()', () => { it('should copy cached artifacts to destination', async () => { - // Set up cache + // Set up cache matching actual structure const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') await fs.mkdir(path.join(sharedDir, 'adapt', 'js'), { recursive: true }) await fs.writeFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'cached-js') - await fs.mkdir(path.join(cssDir, 'adapt', 'css'), { recursive: true }) - await fs.writeFile(path.join(cssDir, 'adapt', 'css', 'adapt.css'), 'cached-css') + await fs.writeFile(path.join(sharedDir, 'index.html'), 'cached-html') + await fs.writeFile(path.join(sharedDir, 'connection.txt'), '') + await fs.mkdir(cssDir, { recursive: true }) + await fs.writeFile(path.join(cssDir, 'adapt.css'), 'cached-css') + await fs.mkdir(path.join(cssDir, 'fonts'), { recursive: true }) + await fs.writeFile(path.join(cssDir, 'fonts', 'icon.woff2'), 'cached-font') const destDir = path.join(tmpDir, 'restored') await restoreFromCache(cacheRoot, 'hash1', 'theme', 'menu', destDir) const js = await fs.readFile(path.join(destDir, 'adapt', 'js', 'adapt.min.js'), 'utf8') assert.equal(js, 'cached-js') - const css = await fs.readFile(path.join(destDir, 'adapt', 'css', 'adapt.css'), 'utf8') + const css = await fs.readFile(path.join(destDir, 'adapt.css'), 'utf8') assert.equal(css, 'cached-css') + const font = await fs.readFile(path.join(destDir, 'fonts', 'icon.woff2'), 'utf8') + assert.equal(font, 'cached-font') + const conn = await fs.readFile(path.join(destDir, 'connection.txt'), 'utf8') + assert.equal(conn, '') }) }) From 30dd01db8ab0f01226666c36c033beacc0ff81e7 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 18 Mar 2026 20:11:44 +0000 Subject: [PATCH 06/27] New: Eager shared cache prebuild after invalidation (refs #176) Adds a `prebuildSharedCache` config option that, when enabled, triggers a background grunt build after cache invalidation to eagerly rebuild the shared prebuilt cache (JS, HTML, templates, libraries). CSS remains lazily built per theme/menu combination on first preview. --- conf/config.schema.json | 5 ++ lib/AdaptFrameworkModule.js | 22 ++++++++- lib/utils.js | 3 +- lib/utils/prebuildSharedCache.js | 81 +++++++++++++++++++++++++++++++ lib/utils/prebuiltCache.js | 43 ++++++++++++++++ tests/utils-prebuiltCache.spec.js | 45 ++++++++++++++++- 6 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 lib/utils/prebuildSharedCache.js diff --git a/conf/config.schema.json b/conf/config.schema.json index bb83336..1bfdf68 100644 --- a/conf/config.schema.json +++ b/conf/config.schema.json @@ -32,6 +32,11 @@ "description": "URL of the Adapt framework git repository to install", "type": "string" }, + "prebuildSharedCache": { + "description": "When enabled, eagerly rebuilds the shared prebuilt cache in the background after invalidation (e.g. plugin install/update/delete)", + "type": "boolean", + "default": false + }, "importMaxFileSize": { "description": "Maximum file upload size for course imports", "type": "string", diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index 20d1543..db6544d 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -4,7 +4,7 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js' import fs from 'node:fs/promises' import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js' import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server' -import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, invalidateCache } from './utils.js' +import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, invalidateCache, prebuildSharedCache } from './utils.js' import path from 'node:path' import semver from 'semver' @@ -243,12 +243,30 @@ class AdaptFrameworkModule extends AbstractModule { } /** - * Invalidates the prebuilt compilation cache + * Invalidates the prebuilt compilation cache and optionally + * triggers an eager rebuild of the shared cache in the background */ invalidatePrebuiltCache () { const cacheRoot = path.join(this.getConfig('buildDir'), 'prebuilt-cache') invalidateCache(cacheRoot) this._pluginHash = null + + if (this.getConfig('prebuildSharedCache')) { + this.prebuildSharedCache() + } + } + + /** + * Eagerly rebuilds the shared prebuilt cache in the background. + * Safe to call multiple times — cancels any in-flight build. + */ + prebuildSharedCache () { + this._eagerBuildPromise = prebuildSharedCache({ + buildDir: this.getConfig('buildDir'), + frameworkDir: this.path + }).catch(e => { + this.log('warn', `eager shared cache build failed: ${e.message}`) + }) } /** diff --git a/lib/utils.js b/lib/utils.js index 6faab23..a07ec2c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -12,6 +12,7 @@ export { collectMigrationScripts } from './utils/collectMigrationScripts.js' export { runContentMigration } from './utils/runContentMigration.js' export { migrateExistingCourses } from './utils/migrateExistingCourses.js' export { computePluginHash } from './utils/computePluginHash.js' -export { hasCachedBuild, populateCache, restoreFromCache, invalidateCache } from './utils/prebuiltCache.js' +export { hasCachedBuild, hasSharedCache, populateCache, populateSharedCacheOnly, restoreFromCache, invalidateCache } from './utils/prebuiltCache.js' +export { prebuildSharedCache } from './utils/prebuildSharedCache.js' export { generateLanguageManifest } from './utils/generateLanguageManifest.js' export { applyBuildReplacements } from './utils/applyBuildReplacements.js' diff --git a/lib/utils/prebuildSharedCache.js b/lib/utils/prebuildSharedCache.js new file mode 100644 index 0000000..21082cf --- /dev/null +++ b/lib/utils/prebuildSharedCache.js @@ -0,0 +1,81 @@ +import { App, ensureDir, writeJson } from 'adapt-authoring-core' +import AdaptCli from 'adapt-cli' +import fs from 'node:fs/promises' +import path from 'upath' +import { copyFrameworkSource } from './copyFrameworkSource.js' +import { hasSharedCache, populateSharedCacheOnly } from './prebuiltCache.js' +import { computePluginHash } from './computePluginHash.js' +import { log } from './log.js' + +/** + * Eagerly rebuilds the shared prebuilt cache in the background. + * Runs a full grunt build with all installed plugins and a minimal + * dummy course, then extracts only the shared artifacts (JS, HTML, + * templates, libraries, required files) into the cache. + * @param {Object} options + * @param {String} options.buildDir Root build directory + * @param {String} options.frameworkDir Path to the adapt_framework source + * @return {Promise} + */ +export async function prebuildSharedCache ({ buildDir, frameworkDir }) { + const app = App.instance + const cacheRoot = path.join(buildDir, 'prebuilt-cache') + const pluginHash = await computePluginHash(frameworkDir) + + if (await hasSharedCache(cacheRoot, pluginHash)) return + + const contentplugin = await app.waitForModule('contentplugin') + const allPlugins = await contentplugin.find({}) + const pluginNames = allPlugins.map(p => p.name) + const theme = allPlugins.find(p => p.type === 'theme')?.name + const menu = allPlugins.find(p => p.type === 'menu')?.name + + if (!theme || !menu) { + throw new Error('Cannot prebuild shared cache: no theme or menu plugin installed') + } + + const tmpDir = path.join(buildDir, `_eager_cache_${Date.now()}`) + try { + log('info', 'CACHE', 'starting eager shared cache build') + + await copyFrameworkSource({ + destDir: tmpDir, + enabledPlugins: pluginNames, + linkNodeModules: true + }) + + // Write minimal course data so grunt can run + const courseDir = path.join(tmpDir, 'src', 'course', 'en') + await ensureDir(courseDir) + await writeJson(path.join(tmpDir, 'src', 'course', 'config.json'), { + _defaultLanguage: 'en', + _theme: theme, + _menu: menu + }) + await writeJson(path.join(courseDir, 'course.json'), { + title: '_eager_cache_build', + _latestTrackingId: 0 + }) + + const outputDir = path.join(tmpDir, 'build') + const cacheDir = path.join(buildDir, 'cache') + await ensureDir(cacheDir) + + await AdaptCli.buildCourse({ + cwd: tmpDir, + sourceMaps: true, + outputDir, + cachePath: path.join(cacheDir, '_eager_cache'), + logger: { log: (...args) => app.logger.log('debug', 'adapt-cli', ...args) } + }) + + // Only extract the shared entries (skip CSS which is theme-specific) + if (!await hasSharedCache(cacheRoot, pluginHash)) { + await populateSharedCacheOnly(outputDir, cacheRoot, pluginHash) + } + + log('info', 'CACHE', 'eager shared cache build complete') + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }) + } +} diff --git a/lib/utils/prebuiltCache.js b/lib/utils/prebuiltCache.js index 06c721d..b1858d1 100644 --- a/lib/utils/prebuiltCache.js +++ b/lib/utils/prebuiltCache.js @@ -124,6 +124,49 @@ export async function restoreFromCache (cacheRoot, pluginHash, theme, menu, dest log('info', 'CACHE', `restored from cache for ${pluginHash} (theme=${theme}, menu=${menu})`) } +/** + * Checks whether the shared cache exists for a given plugin hash + * @param {String} cacheRoot Root cache directory + * @param {String} pluginHash Hash of installed plugins + * @return {Promise} + */ +export async function hasSharedCache (cacheRoot, pluginHash) { + try { + await fs.access(path.join(cacheRoot, pluginHash)) + return true + } catch { + return false + } +} + +/** + * Populates only the shared (plugin-hash-keyed) portion of the cache, + * skipping CSS/theme-specific entries. Used by the eager cache prebuild. + * @param {String} buildOutputDir The build output directory + * @param {String} cacheRoot Root cache directory + * @param {String} pluginHash Hash of installed plugins + * @return {Promise} + */ +export async function populateSharedCacheOnly (buildOutputDir, cacheRoot, pluginHash) { + const sharedDir = path.join(cacheRoot, pluginHash) + await fs.mkdir(cacheRoot, { recursive: true }) + + const tmpShared = `${sharedDir}_tmp_${Date.now()}` + try { + await fs.mkdir(tmpShared, { recursive: true }) + const entries = await fs.readdir(buildOutputDir) + for (const entry of entries) { + if (SKIP_ENTRIES.has(entry) || CSS_ENTRIES.has(entry)) continue + await copyEntry(path.join(buildOutputDir, entry), path.join(tmpShared, entry)) + } + await safeRename(tmpShared, sharedDir) + log('info', 'CACHE', `populated shared cache for ${pluginHash}`) + } catch (e) { + await fs.rm(tmpShared, { recursive: true, force: true }) + throw e + } +} + /** * Removes the entire prebuilt cache directory * @param {String} cacheRoot Root cache directory diff --git a/tests/utils-prebuiltCache.spec.js b/tests/utils-prebuiltCache.spec.js index c244e63..b84cf3a 100644 --- a/tests/utils-prebuiltCache.spec.js +++ b/tests/utils-prebuiltCache.spec.js @@ -5,7 +5,7 @@ import path from 'node:path' import os from 'node:os' describe('prebuiltCache', () => { - let getCachePaths, hasCachedBuild, populateCache, restoreFromCache, invalidateCache + let getCachePaths, hasCachedBuild, hasSharedCache, populateCache, populateSharedCacheOnly, restoreFromCache, invalidateCache let tmpDir, cacheRoot, buildDir before(async () => { @@ -17,7 +17,7 @@ describe('prebuiltCache', () => { logMemory: () => {} } }) - ;({ getCachePaths, hasCachedBuild, populateCache, restoreFromCache, invalidateCache } = await import('../lib/utils/prebuiltCache.js')) + ;({ getCachePaths, hasCachedBuild, hasSharedCache, populateCache, populateSharedCacheOnly, restoreFromCache, invalidateCache } = await import('../lib/utils/prebuiltCache.js')) }) beforeEach(async () => { @@ -135,6 +135,47 @@ describe('prebuiltCache', () => { }) }) + describe('hasSharedCache()', () => { + it('should return false when shared cache does not exist', async () => { + assert.equal(await hasSharedCache(cacheRoot, 'hash1'), false) + }) + + it('should return true when shared cache exists', async () => { + await fs.mkdir(path.join(cacheRoot, 'hash1'), { recursive: true }) + assert.equal(await hasSharedCache(cacheRoot, 'hash1'), true) + }) + }) + + describe('populateSharedCacheOnly()', () => { + it('should cache only shared entries, skipping CSS and course', async () => { + // Create mock build output + await fs.mkdir(path.join(buildDir, 'adapt', 'js'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'adapt', 'js', 'adapt.min.js'), 'js') + await fs.writeFile(path.join(buildDir, 'index.html'), 'html') + await fs.writeFile(path.join(buildDir, 'connection.txt'), '') + // CSS entries — should be excluded + await fs.writeFile(path.join(buildDir, 'adapt.css'), 'css') + await fs.mkdir(path.join(buildDir, 'fonts'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'fonts', 'icon.woff2'), 'font') + // course/ — should be excluded + await fs.mkdir(path.join(buildDir, 'course', 'en'), { recursive: true }) + await fs.writeFile(path.join(buildDir, 'course', 'en', 'course.json'), '{}') + + await populateSharedCacheOnly(buildDir, cacheRoot, 'hash1') + + const sharedDir = path.join(cacheRoot, 'hash1') + // Shared entries present + assert.equal(await fs.readFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'utf8'), 'js') + assert.equal(await fs.readFile(path.join(sharedDir, 'index.html'), 'utf8'), 'html') + assert.equal(await fs.readFile(path.join(sharedDir, 'connection.txt'), 'utf8'), '') + // CSS entries absent + await assert.rejects(fs.access(path.join(sharedDir, 'adapt.css')), { code: 'ENOENT' }) + await assert.rejects(fs.access(path.join(sharedDir, 'fonts')), { code: 'ENOENT' }) + // course/ absent + await assert.rejects(fs.access(path.join(sharedDir, 'course')), { code: 'ENOENT' }) + }) + }) + describe('invalidateCache()', () => { it('should remove the cache directory', async () => { await fs.mkdir(cacheRoot, { recursive: true }) From b2cb53161119d5b2793186ac462f589a6f035410 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 19 Mar 2026 16:43:55 +0000 Subject: [PATCH 07/27] Fix: Apply schema defaults to all content types on cache hit (refs #176) --- lib/AdaptFrameworkBuild.js | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index dc52edc..322d3c7 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -551,14 +551,37 @@ class AdaptFrameworkBuild { * @return {Promise} */ async applySchemaDefaults () { - const jsonschema = await App.instance.waitForModule('jsonschema') + const [jsonschema, contentplugin] = await App.instance.waitForModule('jsonschema', 'contentplugin') + + const enabledPluginSchemas = this.enabledPlugins + .reduce((m, p) => [...m, ...contentplugin.getPluginSchemas(p.name)], []) + const extensionFilter = s => contentplugin.isPluginSchema(s) ? enabledPluginSchemas.includes(s) : true + const getSchema = name => jsonschema.getSchema(name, { useCache: false, extensionFilter }) + + // Apply defaults without running full validation (which rejects ObjectIds etc.) const [courseSchema, configSchema] = await Promise.all([ - jsonschema.getSchema('course'), - jsonschema.getSchema('config') + getSchema('course'), + getSchema('config') ]) - // Apply defaults without running full validation (which rejects ObjectIds etc.) courseSchema.compiledWithDefaults(this.courseData.course.data) configSchema.compiledWithDefaults(this.courseData.config.data) + + for (const type of ['contentObject', 'article', 'block']) { + const schemaName = type === 'contentObject' ? 'contentobject' : type + const schema = await getSchema(schemaName) + for (const item of this.courseData[type].data) { + schema.compiledWithDefaults(item) + } + } + + const componentSchemas = {} + for (const item of this.courseData.component.data) { + const schemaName = `${item._component}-component` + if (!componentSchemas[schemaName]) { + componentSchemas[schemaName] = await getSchema(schemaName) + } + componentSchemas[schemaName].compiledWithDefaults(item) + } } /** From ecd72b98cc5c330f69529d2520d821acba05e577 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Mon, 23 Mar 2026 20:40:19 +0000 Subject: [PATCH 08/27] Update: Replace courseassets dependency with content._assetIds queries (refs adapt-security/adapt-authoring-content#114) --- lib/AdaptFrameworkBuild.js | 6 +++--- lib/AdaptFrameworkImport.js | 12 ++---------- package.json | 1 - 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index 322d3c7..23b92b3 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -314,10 +314,10 @@ class AdaptFrameworkBuild { * @return {Promise} */ async loadAssetData () { - const [assets, courseassets, tags] = await App.instance.waitForModule('assets', 'courseassets', 'tags') + const [assets, content, tags] = await App.instance.waitForModule('assets', 'content', 'tags') - const caRecs = await courseassets.find({ courseId: this.courseId }) - const uniqueAssetIds = new Set(caRecs.map(c => parseObjectId(c.assetId))) + const courseContent = await content.find({ _courseId: this.courseId }, { validate: false }, { projection: { _assetIds: 1 } }) + const uniqueAssetIds = new Set(courseContent.flatMap(c => (c._assetIds ?? []).map(id => parseObjectId(id)))) const usedAssets = await assets.find({ _id: { $in: [...uniqueAssetIds] } }) const usedTagIds = new Set(usedAssets.reduce((m, a) => [...m, ...(a.tags ?? [])], [])) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index 73c5bb8..e1fec05 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -220,10 +220,9 @@ class AdaptFrameworkImport { assets, content, contentplugin, - courseassets, framework, jsonschema - ] = await App.instance.waitForModule('assets', 'content', 'contentplugin', 'courseassets', 'adaptframework', 'jsonschema') + ] = await App.instance.waitForModule('assets', 'content', 'contentplugin', 'adaptframework', 'jsonschema') /** * Cached module instance for easy access * @type {AssetsModule} @@ -239,11 +238,6 @@ class AdaptFrameworkImport { * @type {ContentPluginModule} */ this.contentplugin = contentplugin - /** - * Cached module instance for easy access - * @type {CourseAssetsModule} - */ - this.courseassets = courseassets /** * Cached module instance for easy access * @type {AdaptFrameworkModule} @@ -931,9 +925,7 @@ class AdaptFrameworkImport { const _courseId = parseObjectId(this.idMap[this.contentJson.course._id]) tasks.push( this.content.deleteMany({ _courseId }) - .catch(e => log('warn', 'failed to delete course content', e)), - this.courseassets.deleteMany({ courseId: _courseId }) - .catch(e => log('warn', 'failed to delete course assets', e)) + .catch(e => log('warn', 'failed to delete course content', e)) ) } catch (e) {} // courseId not available, no content to roll back } diff --git a/package.json b/package.json index 52eb2d4..c33317d 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,6 @@ "adapt-authoring-content": "^2.0.0", "adapt-authoring-contentplugin": "^1.0.3", "adapt-authoring-core": "^2.0.0", - "adapt-authoring-courseassets": "^1.0.3", "adapt-authoring-coursetheme": "^1.0.2", "adapt-authoring-mongodb": "^3.0.0", "adapt-authoring-spoortracking": "^1.0.2", From 35317092cb82277ee6627ab4c0af3987c38c44ab Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Mon, 23 Mar 2026 20:59:22 +0000 Subject: [PATCH 09/27] Fix: Update rollback tests to reflect courseassets removal --- lib/AdaptFrameworkBuild.js | 54 ------------------------------ tests/AdaptFrameworkImport.spec.js | 11 +----- 2 files changed, 1 insertion(+), 64 deletions(-) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index 23b92b3..0598588 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -465,60 +465,6 @@ class AdaptFrameworkBuild { })) } - /** - * Applies schema defaults to the in-memory course and config data using - * the jsonschema module. Replicates what grunt's schema-defaults task does. - * - * TODO: replace validateWithDefaults workaround with schema.validate(data, { ignoreErrors: true }) - * once migrated to adapt-schemas v3.x (see #184) - * @return {Promise} - */ - async applySchemaDefaults () { - const [jsonschema, contentplugin] = await App.instance.waitForModule('jsonschema', 'contentplugin') - - const enabledPluginSchemas = this.enabledPlugins - .reduce((m, p) => [...m, ...contentplugin.getPluginSchemas(p.name)], []) - const extensionFilter = s => contentplugin.isPluginSchema(s) ? enabledPluginSchemas.includes(s) : true - const getSchema = name => jsonschema.getSchema(name, { useCache: false, extensionFilter }) - - /** - * Applies defaults via validate(), catching and ignoring validation errors. - * The validated+defaulted data is returned from validate() on success, or - * extracted from the error on failure (validate clones internally). - */ - const validateWithDefaults = (schema, data) => { - try { - return schema.validate(data, { useDefaults: true, ignoreRequired: true }) - } catch (e) { - return e.data.data - } - } - - const [courseSchema, configSchema] = await Promise.all([ - getSchema('course'), - getSchema('config') - ]) - Object.assign(this.courseData.course.data, validateWithDefaults(courseSchema, this.courseData.course.data)) - Object.assign(this.courseData.config.data, validateWithDefaults(configSchema, this.courseData.config.data)) - - for (const type of ['contentObject', 'article', 'block']) { - const schemaName = type === 'contentObject' ? 'contentobject' : type - const schema = await getSchema(schemaName) - for (const item of this.courseData[type].data) { - Object.assign(item, validateWithDefaults(schema, item)) - } - } - - const componentSchemas = {} - for (const item of this.courseData.component.data) { - const schemaName = `${item._component}-component` - if (!componentSchemas[schemaName]) { - componentSchemas[schemaName] = await getSchema(schemaName) - } - Object.assign(item, validateWithDefaults(componentSchemas[schemaName], item)) - } - } - /** * Outputs all course data to the required JSON files * @return {Promise} diff --git a/tests/AdaptFrameworkImport.spec.js b/tests/AdaptFrameworkImport.spec.js index 66d2967..8112c55 100644 --- a/tests/AdaptFrameworkImport.spec.js +++ b/tests/AdaptFrameworkImport.spec.js @@ -407,7 +407,6 @@ describe('AdaptFrameworkImport', () => { contentplugin: null, assets: null, content: null, - courseassets: null, ...overrides } } @@ -442,22 +441,17 @@ describe('AdaptFrameworkImport', () => { assert.deepEqual(deleted.sort(), ['a1', 'a2']) }) - it('should delete course content and course assets', async () => { + it('should delete course content on rollback', async () => { const contentDeleted = [] - const courseAssetsDeleted = [] const ctx = makeRollbackCtx({ content: { deleteMany: async (query) => contentDeleted.push(query) }, - courseassets: { - deleteMany: async (query) => courseAssetsDeleted.push(query) - }, contentJson: { course: { _id: 'oldCourseId' } }, idMap: { oldCourseId: '507f1f77bcf86cd799439011' } }) await rollback.call(ctx) assert.equal(contentDeleted.length, 1) - assert.equal(courseAssetsDeleted.length, 1) }) it('should skip plugin uninstall when contentplugin is not available', async () => { @@ -482,9 +476,6 @@ describe('AdaptFrameworkImport', () => { content: { deleteMany: async (query) => deleted.push(query) }, - courseassets: { - deleteMany: async (query) => deleted.push(query) - }, contentJson: { course: { _id: 'oldCourseId' } }, idMap: {} // no mapping exists }) From 2336a5b603e406fc2d23222b8bff76136d460b83 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Fri, 20 Mar 2026 12:39:42 +0000 Subject: [PATCH 10/27] Update: Adapt to synchronous adapt-schemas v3 API (refs adapt-security/adapt-authoring-jsonschema#58) --- lib/AdaptFrameworkModule.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index db6544d..9749c19 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -289,7 +289,7 @@ class AdaptFrameworkModule extends AbstractModule { async loadSchemas () { const jsonschema = await this.app.waitForModule('jsonschema') const schemas = (await this.runCliCommand('getSchemaPaths')).filter(s => s.includes('/core/')) - await Promise.all(schemas.map(s => jsonschema.registerSchema(s))) + schemas.forEach(s => jsonschema.registerSchema(s)) } /** From 147a2f73546e172d462c77a3b8eba793dadb7157 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 22 Apr 2026 23:00:04 +0100 Subject: [PATCH 11/27] Fix: Make in-memory migration cache prod-writable and concurrency-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, runContentMigration called adapt-migrations load() with no cachePath, letting the library default to /migrations/cache. That broke in two ways: - is read-only on hardened deployments (writes must go through the app's configured tempDir). - Concurrent importers shared the same cache dir, and adapt-migrations wipes *.js on entry — concurrent imports corrupted each other. The default cachePath now resolves to a unique mkdtempSync dir under /migration-cache/run-XXXXXX/, removed in finally. A persistent node_modules symlink is placed one level up at /migration-cache/node_modules → /node_modules, so the bare `import 'adapt-migrations'` inside cached scripts resolves via Node's upward walk without being wiped by the library's own `npm install` step (which runs one level down in the run dir). Unit tests now pass an explicit cachePath so they do not depend on App.instance. --- lib/utils/runContentMigration.js | 68 ++++++++++++++++++++----- tests/utils-runContentMigration.spec.js | 17 +++++-- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/lib/utils/runContentMigration.js b/lib/utils/runContentMigration.js index d9d80ef..7bb4460 100644 --- a/lib/utils/runContentMigration.js +++ b/lib/utils/runContentMigration.js @@ -1,5 +1,11 @@ +import fs from 'node:fs' +import path from 'node:path' +import { createRequire } from 'node:module' +import { App, ensureDir } from 'adapt-authoring-core' import { load, migrate, Journal, Logger } from 'adapt-migrations' +const require = createRequire(import.meta.url) + /** * Runs adapt-migrations on a content array. Shared by framework update, course import, and plugin update. * @param {Object} options @@ -7,26 +13,60 @@ import { load, migrate, Journal, Logger } from 'adapt-migrations' * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before the update * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after the update * @param {String[]} options.scripts Absolute paths to migration scripts - * @param {String} [options.cachePath] Optional cache path for adapt-migrations + * @param {String} [options.cachePath] Optional cache path for adapt-migrations. If omitted, a unique dir under the app's tempDir is created and removed after migration — callers running concurrently MUST either omit this or pass a unique path per call, as adapt-migrations wipes the directory on entry. * @returns {Promise} The migrated content array */ export async function runContentMigration ({ content, fromPlugins, toPlugins, scripts, cachePath }) { const logger = Logger.getInstance() - await load({ scripts, cachePath, logger }) - - const originalFromPlugins = JSON.parse(JSON.stringify(fromPlugins)) - const journal = new Journal({ - logger, - data: { - content, - fromPlugins, - originalFromPlugins, - toPlugins + let resolvedCachePath = cachePath + const usingEphemeralCache = !resolvedCachePath + if (usingEphemeralCache) { + const tempDir = App.instance.getConfig('tempDir') + const baseCacheDir = path.join(tempDir, 'migration-cache') + await ensureDir(baseCacheDir) + // Symlink node_modules at the base so cached migration scripts' bare + // `import 'adapt-migrations'` resolves via Node's upward walk. It must sit + // a level ABOVE the run dir, otherwise adapt-migrations's own `npm install` + // step (which runs in the run dir) wipes the symlink. + const sharedLink = path.join(baseCacheDir, 'node_modules') + if (!fs.existsSync(sharedLink)) { + const sharedNodeModules = path.dirname(path.dirname(require.resolve('adapt-migrations'))) + try { + fs.symlinkSync(sharedNodeModules, sharedLink, 'dir') + } catch (err) { + if (err.code !== 'EEXIST') throw err + } } - }) + resolvedCachePath = fs.mkdtempSync(path.join(baseCacheDir, 'run-')) + } else { + await ensureDir(resolvedCachePath) + } + + try { + await load({ scripts, cachePath: resolvedCachePath, logger }) - await migrate({ journal, logger }) + const originalFromPlugins = JSON.parse(JSON.stringify(fromPlugins)) + const journal = new Journal({ + logger, + data: { + content, + fromPlugins, + originalFromPlugins, + toPlugins + } + }) - return journal.data.content + await migrate({ journal, logger }) + + return journal.data.content + } finally { + if (usingEphemeralCache) { + try { + fs.rmSync(resolvedCachePath, { recursive: true, force: true }) + } catch (err) { + logger.warn(`Failed to clean up migration cache at ${resolvedCachePath}: ${err.message}`) + } + } + } } diff --git a/tests/utils-runContentMigration.spec.js b/tests/utils-runContentMigration.spec.js index 6173754..0206cca 100644 --- a/tests/utils-runContentMigration.spec.js +++ b/tests/utils-runContentMigration.spec.js @@ -1,5 +1,10 @@ import { describe, it, mock } from 'node:test' import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const testCachePath = fs.mkdtempSync(path.join(os.tmpdir(), 'runContentMigration-test-')) const mockLoad = mock.fn(async () => {}) const mockMigrate = mock.fn(async ({ journal }) => { @@ -39,7 +44,8 @@ describe('runContentMigration()', () => { content: [{ _id: 'c1', title: 'old' }], fromPlugins: [{ name: 'core', version: '1.0.0' }], toPlugins: [{ name: 'core', version: '2.0.0' }], - scripts + scripts, + cachePath: testCachePath }) assert.equal(mockLoad.mock.calls.length, 1) assert.deepEqual(mockLoad.mock.calls[0].arguments[0].scripts, scripts) @@ -51,7 +57,8 @@ describe('runContentMigration()', () => { content: [{ _id: 'c1', title: 'old' }], fromPlugins: [{ name: 'core', version: '1.0.0' }], toPlugins: [{ name: 'core', version: '2.0.0' }], - scripts: [] + scripts: [], + cachePath: testCachePath }) assert.equal(mockMigrate.mock.calls.length, 1) const journal = mockMigrate.mock.calls[0].arguments[0].journal @@ -70,7 +77,8 @@ describe('runContentMigration()', () => { content: [{ _id: 'c1', title: 'old' }], fromPlugins: [{ name: 'core', version: '1.0.0' }], toPlugins: [{ name: 'core', version: '2.0.0' }], - scripts: [] + scripts: [], + cachePath: testCachePath }) assert.equal(result[0].title, 'migrated') }) @@ -83,7 +91,8 @@ describe('runContentMigration()', () => { content: [{ _id: 'c1' }], fromPlugins, toPlugins: [{ name: 'core', version: '2.0.0' }], - scripts: [] + scripts: [], + cachePath: testCachePath }) const journal = mockMigrate.mock.calls[0].arguments[0].journal assert.deepEqual(journal.data.originalFromPlugins, fromPlugins) From 778a354a603d624a762d53140bedac465e2187ea Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 23 Apr 2026 00:17:45 +0100 Subject: [PATCH 12/27] Fix: Use JSON-normalized content on write in migrateExistingCourses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deep-equal comparison already used the JSON round-tripped version, but the subsequent content.update() passed the raw migrated item — which still carried MongoDB native types (ObjectId for _id/_courseId/_assetIds, Date for createdAt/updatedAt). The content schema requires those fields to be strings, so every update failed validation with errors like: /_id must be string, /_courseId must be string, /createdAt must be string, /createdBy must be string Using the normalized value for the write (MongoDB accepts string _id in queries too) fixes validation while preserving the existing no-change short-circuit. --- lib/utils/migrateExistingCourses.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils/migrateExistingCourses.js b/lib/utils/migrateExistingCourses.js index a0aa429..9f16b4f 100644 --- a/lib/utils/migrateExistingCourses.js +++ b/lib/utils/migrateExistingCourses.js @@ -57,7 +57,7 @@ export async function migrateExistingCourses ({ fromPlugins, toPlugins, framewor for (let i = 0; i < migratedContent.length; i++) { const normalized = JSON.parse(JSON.stringify(migratedContent[i])) if (!isDeepStrictEqual(originals[i], normalized)) { - await content.update({ _id: migratedContent[i]._id }, migratedContent[i]) + await content.update({ _id: migratedContent[i]._id }, normalized) updatedCount++ } } From 871a11a8d7ca5492ecc5d41f91ca43ad16ea7286 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 23 Apr 2026 00:29:18 +0100 Subject: [PATCH 13/27] Fix: Apply patchCustomStyle/patchThemeName to in-memory content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both patches wrote to the on-disk course.json / config.json but nothing reads those files after the patches run — migration and the DB write both go via this.contentJson. So neither patch actually took effect after the in-memory migration refactor. Mutate this.contentJson directly instead, and drop the now-pointless disk I/O. Also hardened patchThemeName against a missing theme plugin or missing config (previously threw on .name if no theme plugin). --- lib/AdaptFrameworkImport.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index ce7851d..ffafe17 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -339,18 +339,14 @@ class AdaptFrameworkImport { } /** - * Writes the contents of 2-customStyles.less to course.json file. Unfortunately it's necessary to do it this way to ensure it's included in migrations. + * Reads 2-customStyles.less (if present) and injects its contents as customStyle on the in-memory course, so migrations and the DB write see it. Existing customStyle on the course takes precedence. */ async patchCustomStyle () { const [customStylePath] = await glob('**/2-customStyles.less', { cwd: this.path, absolute: true }) - const courseJsonPath = `${this.langPath}/course.json` - if (!customStylePath) { - return - } + if (!customStylePath) return try { const customStyle = await fs.readFile(customStylePath, 'utf8') - const courseJson = await readJson(courseJsonPath) - await writeJson(courseJsonPath, { customStyle, ...courseJson }) + this.contentJson.course = { customStyle, ...this.contentJson.course } log('info', 'patched course customStyle') } catch (e) { log('warn', 'failed to patch course customStyle', e) @@ -358,15 +354,14 @@ class AdaptFrameworkImport { } /** - * Ensures _theme exists on the config + * Ensures _theme exists on the in-memory config */ async patchThemeName () { try { - const configJsonPath = `${this.coursePath}/config.json` - const configJson = await readJson(configJsonPath) - if (configJson._theme) return - configJson._theme = Object.values(this.usedContentPlugins).find(p => p.type === 'theme').name - await writeJson(configJsonPath, configJson) + if (this.contentJson.config?._theme) return + const _theme = Object.values(this.usedContentPlugins).find(p => p.type === 'theme')?.name + if (!_theme || !this.contentJson.config) return + this.contentJson.config._theme = _theme log('info', 'patched config _theme') } catch (e) { log('warn', 'failed to patch config _theme', e) From dc2c1cfcac98699ca23bee449dd00d63ec86a7a3 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 23 Apr 2026 13:30:38 +0100 Subject: [PATCH 14/27] Fix: Guard _shareWithUsers optional chaining in checkContentAccess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous `course?._shareWithUsers.map(...)` had the optional chaining on `course` — redundant, since the surrounding guard already returns when course is falsy — and left `_shareWithUsers.map` to throw when `_shareWithUsers` was absent on the course doc. Move the `?.` onto `_shareWithUsers` and fall back to `[]`. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/AdaptFrameworkModule.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index 63683ac..5a142e5 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -328,7 +328,7 @@ class AdaptFrameworkModule extends AbstractModule { if (!course) { return } - const shareWithUsers = course?._shareWithUsers.map(id => id.toString()) ?? [] + const shareWithUsers = course._shareWithUsers?.map(id => id.toString()) ?? [] const userId = req.auth.user._id.toString() return course.createdBy.toString() === userId || course._isShared || shareWithUsers.includes(userId) } From 0066a5811d495ad184139ebb2dadc8cdea65bd4f Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 29 Apr 2026 15:20:41 +0100 Subject: [PATCH 15/27] Fix: surface real errors in import dry-run and asset resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Initialise statusReport.error so dry-run plugin checks no longer crash with TypeError on the missing array. - In resolveAssets, drop unmappable asset paths instead of leaking them into content data — previously a missed map produced a cryptic INVALID_OBJECTID error downstream. Logs a clear warning and a UNRESOLVED_ASSET_REF status entry. - Log the underlying error (and capture e.message in statusReport) when an asset insert fails, instead of dropping it silently. --- lib/AdaptFrameworkImport.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index ffafe17..e794c13 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -1,5 +1,5 @@ import { App, Hook, readJson, writeJson } from 'adapt-authoring-core' -import { parseObjectId } from 'adapt-authoring-mongodb' +import { isValidObjectId, parseObjectId } from 'adapt-authoring-mongodb' import fs from 'node:fs/promises' import { glob } from 'glob' import path from 'upath' @@ -173,7 +173,8 @@ class AdaptFrameworkImport { */ this.statusReport = { info: [], - warn: [] + warn: [], + error: [] } /** * Summary information for the import run @@ -592,7 +593,8 @@ class AdaptFrameworkImport { const resolved = path.relative(`${this.coursePath}/..`, filepath) this.assetMap[resolved] = e.data.assetId } else { - this.statusReport.warn.push({ code: 'ASSET_IMPORT_FAILED', data: { filepath } }) + log('error', `asset import failed for '${filepath}'`, e) + this.statusReport.warn.push({ code: 'ASSET_IMPORT_FAILED', data: { filepath, reason: e?.message ?? String(e) } }) } } imagesImported++ @@ -835,7 +837,13 @@ class AdaptFrameworkImport { schema.walk(data, field => field?._backboneForms?.type === 'Asset' || field?._backboneForms === 'Asset' ).forEach(({ data: parent, key, value }) => { - value ? parent[key] = this.assetMap[value] ?? value : delete parent[key] + if (!value) return delete parent[key] + const mapped = this.assetMap[value] + if (mapped) return (parent[key] = mapped) + if (isValidObjectId(value)) return (parent[key] = value) + log('warn', `unable to resolve asset reference '${value}' — dropping field`) + this.statusReport.warn.push({ code: 'UNRESOLVED_ASSET_REF', data: { path: value } }) + delete parent[key] }) } From ee49d3c1308f61d36682fcda4873a326ac4013f7 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 29 Apr 2026 15:21:51 +0100 Subject: [PATCH 16/27] New: import migration to drop invalid vanilla _backgroundStyles values Empty-string defaults shipped by adapt-contrib-vanilla's example.json fail enum validation against the same package's schema (the enums for _backgroundRepeat/Size/Position do not include ""), so any course created from the framework example was unimportable. Drop the empty values during import so the schema defaults apply. --- lib/AdaptFrameworkImport.js | 4 +++- lib/migrations/vanilla-background-styles.js | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 lib/migrations/vanilla-background-styles.js diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index e794c13..44e22bb 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -13,6 +13,7 @@ import GraphicSrcTransform from './migrations/graphic-src.js' import NavOrderTransform from './migrations/nav-order.js' import ParentIdTransform from './migrations/parent-id.js' import RemoveUndefTransform from './migrations/remove-undef.js' +import VanillaBackgroundStylesTransform from './migrations/vanilla-background-styles.js' import StartPageTransform from './migrations/start-page.js' import ThemeUndefTransform from './migrations/theme-undef.js' @@ -24,7 +25,8 @@ const ContentMigrations = [ ParentIdTransform, RemoveUndefTransform, StartPageTransform, - ThemeUndefTransform + ThemeUndefTransform, + VanillaBackgroundStylesTransform ] /** diff --git a/lib/migrations/vanilla-background-styles.js b/lib/migrations/vanilla-background-styles.js new file mode 100644 index 0000000..3d55912 --- /dev/null +++ b/lib/migrations/vanilla-background-styles.js @@ -0,0 +1,17 @@ +const FIELDS = ['_backgroundRepeat', '_backgroundSize', '_backgroundPosition'] + +function clean (styles) { + if (!styles || typeof styles !== 'object') return + for (const f of FIELDS) { + if (styles[f] === '' || styles[f] === null) delete styles[f] + } +} + +async function VanillaBackgroundStyles (data) { + const v = data._vanilla + if (!v || typeof v !== 'object') return + clean(v._backgroundStyles) + clean(v._pageHeader?._backgroundStyles) +} + +export default VanillaBackgroundStyles From b4f52b9846b253e0ff2b477d33532162c60ec1d8 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 30 Apr 2026 10:57:38 +0100 Subject: [PATCH 17/27] Fix: include all installed plugins in preview prebuilt cache The prebuilt cache is shared across all courses with the same plugin hash, but the populating build was filtered down to a single course's _enabledPlugins, so the cache contained an incomplete bundle and courses using plugins not enabled in the populating course saw "component not installed" errors at runtime. - copyFrameworkSource: when no enabledPlugins is passed, include all plugin source dirs (not just none). - prebuildSharedCache: drop the enabledPlugins filter so all plugins on disk are compiled, and write the dummy config.json into the build dir (server-build skips the courseJson copy task) including _enabledPlugins so grunt knows which plugins to wire in. - AdaptFrameworkBuild: for previews (which populate the shared cache), include enabledPlugins + disabledPlugins. Publish/export remain filtered to keep shipped bundles small. --- lib/AdaptFrameworkBuild.js | 6 +++++- lib/utils/copyFrameworkSource.js | 4 ++-- lib/utils/prebuildSharedCache.js | 19 ++++++++++--------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index e091c18..a011f43 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -228,9 +228,13 @@ class AdaptFrameworkBuild { const tasks = [this.copyAssets()] if (!contentOnly) { + // preview cache is shared across courses, so include all installed plugins + const pluginsToInclude = this.isPreview + ? [...this.enabledPlugins, ...this.disabledPlugins] + : this.enabledPlugins tasks.push(copyFrameworkSource({ destDir: this.dir, - enabledPlugins: this.enabledPlugins.map(p => p.name), + enabledPlugins: pluginsToInclude.map(p => p.name), linkNodeModules: !this.isExport })) } diff --git a/lib/utils/copyFrameworkSource.js b/lib/utils/copyFrameworkSource.js index 34391fc..c2f90c1 100644 --- a/lib/utils/copyFrameworkSource.js +++ b/lib/utils/copyFrameworkSource.js @@ -17,7 +17,7 @@ export async function copyFrameworkSource (options) { if (options.copyNodeModules !== true) BLACKLIST.push('node_modules') const srcDir = path.join(fwPath, 'src') - const enabledPlugins = options.enabledPlugins ?? [] + const enabledPlugins = options.enabledPlugins await fs.cp(fwPath, options.destDir, { recursive: true, filter: f => { @@ -25,7 +25,7 @@ export async function copyFrameworkSource (options) { const [type, name] = path.relative(srcDir, f).split('/') const isPlugin = f.startsWith(srcDir) && type && type !== 'core' && !!name - if (isPlugin && !enabledPlugins.includes(name)) { + if (isPlugin && enabledPlugins && !enabledPlugins.includes(name)) { return false } return !BLACKLIST.includes(path.basename(f)) diff --git a/lib/utils/prebuildSharedCache.js b/lib/utils/prebuildSharedCache.js index 21082cf..5fe2af5 100644 --- a/lib/utils/prebuildSharedCache.js +++ b/lib/utils/prebuildSharedCache.js @@ -38,26 +38,27 @@ export async function prebuildSharedCache ({ buildDir, frameworkDir }) { try { log('info', 'CACHE', 'starting eager shared cache build') + // pass no enabledPlugins so all installed plugins (incl. dependencies) are included await copyFrameworkSource({ destDir: tmpDir, - enabledPlugins: pluginNames, linkNodeModules: true }) - // Write minimal course data so grunt can run - const courseDir = path.join(tmpDir, 'src', 'course', 'en') - await ensureDir(courseDir) - await writeJson(path.join(tmpDir, 'src', 'course', 'config.json'), { + // server-build skips copy:courseJson — the AT writes course content + // directly into the build dir, so do the same here for the dummy course + const outputDir = path.join(tmpDir, 'build') + const buildCourseDir = path.join(outputDir, 'course', 'en') + await ensureDir(buildCourseDir) + await writeJson(path.join(outputDir, 'course', 'config.json'), { _defaultLanguage: 'en', _theme: theme, - _menu: menu + _menu: menu, + _enabledPlugins: pluginNames }) - await writeJson(path.join(courseDir, 'course.json'), { + await writeJson(path.join(buildCourseDir, 'course.json'), { title: '_eager_cache_build', _latestTrackingId: 0 }) - - const outputDir = path.join(tmpDir, 'build') const cacheDir = path.join(buildDir, 'cache') await ensureDir(cacheDir) From d1f8f13b42829154aae979104d469794dd26fb54 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 30 Apr 2026 11:01:12 +0100 Subject: [PATCH 18/27] New: prebuild shared cache on boot when missing, with build diagnostics - AdaptFrameworkModule: when prebuildSharedCache config is enabled and no cache exists for the current plugin hash, fire prebuildSharedCache in the background during init so the first preview after a cold boot is fast. - AdaptFrameworkModule: when the eager build fails, also log the grunt command, parsed output line, and stderr so the underlying cause is visible instead of just "grunt tasks failed". - utils/log: drop the awaited waitForModule indirection. The existing helper raced with module init and could swallow logs that fired before the framework module was ready; route directly through App.instance.logger so messages are not lost. --- lib/AdaptFrameworkModule.js | 14 +++++++++++++- lib/utils/log.js | 12 +++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index 5a142e5..ca14d91 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -4,7 +4,7 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js' import fs from 'node:fs/promises' import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js' import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server' -import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, invalidateCache, prebuildSharedCache } from './utils.js' +import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, hasSharedCache, invalidateCache, prebuildSharedCache } from './utils.js' import path from 'node:path' import semver from 'semver' @@ -90,6 +90,15 @@ class AdaptFrameworkModule extends AbstractModule { }) this.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) + if (this.getConfig('prebuildSharedCache')) { + const cacheRoot = path.join(this.getConfig('buildDir'), 'prebuilt-cache') + this.getPluginHash() + .then(async pluginHash => { + if (!(await hasSharedCache(cacheRoot, pluginHash))) this.prebuildSharedCache() + }) + .catch(e => this.log('warn', `failed to check prebuilt cache on boot: ${e.message}`)) + } + process.env.BROWSERSLIST_IGNORE_OLD_DATA = '1' if (this.app.args['update-framework'] === true) { @@ -282,6 +291,9 @@ class AdaptFrameworkModule extends AbstractModule { frameworkDir: this.path }).catch(e => { this.log('warn', `eager shared cache build failed: ${e.message}`) + if (e.cmd) this.log('warn', `cmd: ${e.cmd}`) + if (e.raw) this.log('warn', `output: ${e.raw}`) + if (e.stderr) this.log('warn', `stderr: ${e.stderr}`) }).finally(() => { this._eagerBuildPromise = null }) diff --git a/lib/utils/log.js b/lib/utils/log.js index b278867..f6ac80e 100644 --- a/lib/utils/log.js +++ b/lib/utils/log.js @@ -3,15 +3,13 @@ import bytes from 'bytes' import fsSync from 'node:fs' import path from 'upath' -let fw - /** - * Logs a message using the framework module - * @param {...*} args Arguments to be logged + * Logs a message using the framework module's namespace + * @param {String} level Log level + * @param {...*} rest Arguments to be logged */ -export async function log (...args) { - if (!fw) fw = await App.instance.waitForModule('adaptframework') - return fw.log(...args) +export function log (level, ...rest) { + App.instance?.logger?.log(level, 'adaptframework', ...rest) } /** From 18068c7fe50ad51208b6d09c2defeef12ee19bf5 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Tue, 5 May 2026 17:37:49 +0100 Subject: [PATCH 19/27] Fix: Exclude disabled themes/menus from build sources Only one theme/menu can be active per build. The framework's grunt less:dev task globs every theme/menu in src/, which OOMs when more than one is present (see adaptlearning/adapt_framework#3802). Filter disabled themes/menus out of the build sources to keep less:dev working until the framework fix lands. --- lib/AdaptFrameworkBuild.js | 8 +++++++- lib/utils/prebuildSharedCache.js | 11 +++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index a011f43..bab7f56 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -229,8 +229,14 @@ class AdaptFrameworkBuild { const tasks = [this.copyAssets()] if (!contentOnly) { // preview cache is shared across courses, so include all installed plugins + // — except disabled themes/menus, since only one of each can be active per + // build and the framework's less:dev task globs every theme/menu in src/, + // which OOMs when more than one is present (see adapt_framework#3802). const pluginsToInclude = this.isPreview - ? [...this.enabledPlugins, ...this.disabledPlugins] + ? [ + ...this.enabledPlugins, + ...this.disabledPlugins.filter(p => p.type !== 'theme' && p.type !== 'menu') + ] : this.enabledPlugins tasks.push(copyFrameworkSource({ destDir: this.dir, diff --git a/lib/utils/prebuildSharedCache.js b/lib/utils/prebuildSharedCache.js index 5fe2af5..6a6d9cc 100644 --- a/lib/utils/prebuildSharedCache.js +++ b/lib/utils/prebuildSharedCache.js @@ -26,7 +26,6 @@ export async function prebuildSharedCache ({ buildDir, frameworkDir }) { const contentplugin = await app.waitForModule('contentplugin') const allPlugins = await contentplugin.find({}) - const pluginNames = allPlugins.map(p => p.name) const theme = allPlugins.find(p => p.type === 'theme')?.name const menu = allPlugins.find(p => p.type === 'menu')?.name @@ -34,13 +33,21 @@ export async function prebuildSharedCache ({ buildDir, frameworkDir }) { throw new Error('Cannot prebuild shared cache: no theme or menu plugin installed') } + // Only one theme/menu can be active per build — drop the others so + // the framework's less:dev task doesn't glob multiple themes' LESS + // into a single adapt.css (see adapt_framework#3802). + const includedPlugins = allPlugins.filter(p => + (p.type !== 'theme' && p.type !== 'menu') || p.name === theme || p.name === menu + ) + const pluginNames = includedPlugins.map(p => p.name) + const tmpDir = path.join(buildDir, `_eager_cache_${Date.now()}`) try { log('info', 'CACHE', 'starting eager shared cache build') - // pass no enabledPlugins so all installed plugins (incl. dependencies) are included await copyFrameworkSource({ destDir: tmpDir, + enabledPlugins: pluginNames, linkNodeModules: true }) From b485c239724af26f90691c1019dbe5fcfdfdab77 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 6 May 2026 17:10:49 +0100 Subject: [PATCH 20/27] Update: Collapse prebuilt cache to single dir, prebuild every theme/menu combo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prebuilt cache previously split into two dirs per build: a shared half (HTML/JS/templates) keyed by pluginHash and a CSS half keyed by (pluginHash, theme, menu). The slow path of AdaptFrameworkBuild never read the shared half and the eager prebuild only wrote it, so the shared dir provided no measurable benefit — only full (shared+CSS) hits were faster. Collapse to one dir per (pluginHash, theme, menu) combo. The eager prebuild now iterates every (theme x menu) combination of installed plugins serially and runs a full grunt build for each, so every per-course build hits an existing cache once the prebuild has covered its combo. - prebuiltCache.js: getCachePaths -> getCachePath; one-dir variants of hasCachedBuild, populateCache, restoreFromCache; drop hasSharedCache, populateSharedCacheOnly, CSS_ENTRIES. - prebuildSharedCache.js renamed to prebuildCache.js. Outer theme/menu loop, per-combo skip via hasCachedBuild (idempotent), per-iteration errors logged but non-fatal. - AdaptFrameworkModule: prebuildSharedCache method -> prebuildCache; config key prebuildSharedCache -> prebuildCache; boot trigger simplified to call prebuildCache directly (function is itself idempotent). - AdaptFrameworkBuild call sites unchanged — populateCache / hasCachedBuild / restoreFromCache signatures are stable. - Tests rewritten for the single-dir shape. --- conf/config.schema.json | 4 +- lib/AdaptFrameworkModule.js | 28 ++-- lib/utils.js | 4 +- ...rebuildSharedCache.js => prebuildCache.js} | 69 +++++---- lib/utils/prebuiltCache.js | 119 +++------------ tests/utils-prebuiltCache.spec.js | 136 ++++-------------- 6 files changed, 104 insertions(+), 256 deletions(-) rename lib/utils/{prebuildSharedCache.js => prebuildCache.js} (51%) diff --git a/conf/config.schema.json b/conf/config.schema.json index 1bfdf68..74b3361 100644 --- a/conf/config.schema.json +++ b/conf/config.schema.json @@ -32,8 +32,8 @@ "description": "URL of the Adapt framework git repository to install", "type": "string" }, - "prebuildSharedCache": { - "description": "When enabled, eagerly rebuilds the shared prebuilt cache in the background after invalidation (e.g. plugin install/update/delete)", + "prebuildCache": { + "description": "When enabled, eagerly rebuilds the prebuilt cache for every (theme, menu) combination in the background after invalidation (e.g. plugin install/update/delete)", "type": "boolean", "default": false }, diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index ca14d91..61b13cc 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -4,7 +4,7 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js' import fs from 'node:fs/promises' import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js' import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server' -import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, hasSharedCache, invalidateCache, prebuildSharedCache } from './utils.js' +import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, invalidateCache, prebuildCache } from './utils.js' import path from 'node:path' import semver from 'semver' @@ -90,13 +90,8 @@ class AdaptFrameworkModule extends AbstractModule { }) this.postUpdateHook.tap(() => this.invalidatePrebuiltCache()) - if (this.getConfig('prebuildSharedCache')) { - const cacheRoot = path.join(this.getConfig('buildDir'), 'prebuilt-cache') - this.getPluginHash() - .then(async pluginHash => { - if (!(await hasSharedCache(cacheRoot, pluginHash))) this.prebuildSharedCache() - }) - .catch(e => this.log('warn', `failed to check prebuilt cache on boot: ${e.message}`)) + if (this.getConfig('prebuildCache')) { + this.prebuildCache() } process.env.BROWSERSLIST_IGNORE_OLD_DATA = '1' @@ -271,26 +266,27 @@ class AdaptFrameworkModule extends AbstractModule { this.log('warn', `failed to invalidate prebuilt cache: ${e.message}`) } - if (this.getConfig('prebuildSharedCache')) { - this.prebuildSharedCache() + if (this.getConfig('prebuildCache')) { + this.prebuildCache() } } /** - * Eagerly rebuilds the shared prebuilt cache in the background. - * Safe to call multiple times — if a build is already in progress, - * it will be reused instead of starting a new one. + * Eagerly rebuilds the prebuilt cache in the background, iterating every + * (theme, menu) combination of installed plugins. Safe to call multiple + * times — if a build is already in progress, it will be reused. + * Idempotent — already-cached combos are skipped. * @return {Promise} */ - prebuildSharedCache () { + prebuildCache () { if (this._eagerBuildPromise) { return this._eagerBuildPromise } - this._eagerBuildPromise = prebuildSharedCache({ + this._eagerBuildPromise = prebuildCache({ buildDir: this.getConfig('buildDir'), frameworkDir: this.path }).catch(e => { - this.log('warn', `eager shared cache build failed: ${e.message}`) + this.log('warn', `eager prebuild failed: ${e.message}`) if (e.cmd) this.log('warn', `cmd: ${e.cmd}`) if (e.raw) this.log('warn', `output: ${e.raw}`) if (e.stderr) this.log('warn', `stderr: ${e.stderr}`) diff --git a/lib/utils.js b/lib/utils.js index a07ec2c..a6cd855 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -12,7 +12,7 @@ export { collectMigrationScripts } from './utils/collectMigrationScripts.js' export { runContentMigration } from './utils/runContentMigration.js' export { migrateExistingCourses } from './utils/migrateExistingCourses.js' export { computePluginHash } from './utils/computePluginHash.js' -export { hasCachedBuild, hasSharedCache, populateCache, populateSharedCacheOnly, restoreFromCache, invalidateCache } from './utils/prebuiltCache.js' -export { prebuildSharedCache } from './utils/prebuildSharedCache.js' +export { hasCachedBuild, populateCache, restoreFromCache, invalidateCache } from './utils/prebuiltCache.js' +export { prebuildCache } from './utils/prebuildCache.js' export { generateLanguageManifest } from './utils/generateLanguageManifest.js' export { applyBuildReplacements } from './utils/applyBuildReplacements.js' diff --git a/lib/utils/prebuildSharedCache.js b/lib/utils/prebuildCache.js similarity index 51% rename from lib/utils/prebuildSharedCache.js rename to lib/utils/prebuildCache.js index 6a6d9cc..6b15e4c 100644 --- a/lib/utils/prebuildSharedCache.js +++ b/lib/utils/prebuildCache.js @@ -3,47 +3,73 @@ import AdaptCli from 'adapt-cli' import fs from 'node:fs/promises' import path from 'upath' import { copyFrameworkSource } from './copyFrameworkSource.js' -import { hasSharedCache, populateSharedCacheOnly } from './prebuiltCache.js' +import { hasCachedBuild, populateCache } from './prebuiltCache.js' import { computePluginHash } from './computePluginHash.js' import { log } from './log.js' /** - * Eagerly rebuilds the shared prebuilt cache in the background. - * Runs a full grunt build with all installed plugins and a minimal - * dummy course, then extracts only the shared artifacts (JS, HTML, - * templates, libraries, required files) into the cache. + * Eagerly populates the prebuilt cache for every (theme, menu) combination + * of installed plugins. Iterates serially: each iteration runs a full grunt + * build with the chosen theme/menu and caches the output. + * + * Idempotent — combos that already have a cache entry are skipped, so + * re-runs only build what's missing. Per-iteration failures are logged + * but don't abort the whole prebuild. * @param {Object} options * @param {String} options.buildDir Root build directory * @param {String} options.frameworkDir Path to the adapt_framework source * @return {Promise} */ -export async function prebuildSharedCache ({ buildDir, frameworkDir }) { +export async function prebuildCache ({ buildDir, frameworkDir }) { const app = App.instance const cacheRoot = path.join(buildDir, 'prebuilt-cache') const pluginHash = await computePluginHash(frameworkDir) - if (await hasSharedCache(cacheRoot, pluginHash)) return - const contentplugin = await app.waitForModule('contentplugin') const allPlugins = await contentplugin.find({}) - const theme = allPlugins.find(p => p.type === 'theme')?.name - const menu = allPlugins.find(p => p.type === 'menu')?.name + const themes = allPlugins.filter(p => p.type === 'theme') + const menus = allPlugins.filter(p => p.type === 'menu') + + if (!themes.length || !menus.length) { + throw new Error('Cannot prebuild cache: no theme or menu plugin installed') + } + + log('info', 'CACHE', `starting eager prebuild for ${themes.length * menus.length} (theme,menu) combinations`) + + for (const theme of themes) { + for (const menu of menus) { + try { + await prebuildOne({ buildDir, cacheRoot, pluginHash, theme, menu, allPlugins }) + } catch (e) { + log('warn', 'CACHE', `eager prebuild failed for theme=${theme.name} menu=${menu.name}: ${e.message}`) + if (e.cmd) log('warn', 'CACHE', `cmd: ${e.cmd}`) + if (e.stderr) log('warn', 'CACHE', `stderr: ${e.stderr}`) + } + } + } + + log('info', 'CACHE', 'eager prebuild complete') +} - if (!theme || !menu) { - throw new Error('Cannot prebuild shared cache: no theme or menu plugin installed') +async function prebuildOne ({ buildDir, cacheRoot, pluginHash, theme, menu, allPlugins }) { + const app = App.instance + + if (await hasCachedBuild(cacheRoot, pluginHash, theme.name, menu.name)) { + log('info', 'CACHE', `skipping cached combo theme=${theme.name} menu=${menu.name}`) + return } // Only one theme/menu can be active per build — drop the others so // the framework's less:dev task doesn't glob multiple themes' LESS // into a single adapt.css (see adapt_framework#3802). const includedPlugins = allPlugins.filter(p => - (p.type !== 'theme' && p.type !== 'menu') || p.name === theme || p.name === menu + (p.type !== 'theme' && p.type !== 'menu') || p.name === theme.name || p.name === menu.name ) const pluginNames = includedPlugins.map(p => p.name) - const tmpDir = path.join(buildDir, `_eager_cache_${Date.now()}`) + const tmpDir = path.join(buildDir, `_eager_cache_${Date.now()}_${theme.name}_${menu.name}`) try { - log('info', 'CACHE', 'starting eager shared cache build') + log('info', 'CACHE', `building combo theme=${theme.name} menu=${menu.name}`) await copyFrameworkSource({ destDir: tmpDir, @@ -51,15 +77,13 @@ export async function prebuildSharedCache ({ buildDir, frameworkDir }) { linkNodeModules: true }) - // server-build skips copy:courseJson — the AT writes course content - // directly into the build dir, so do the same here for the dummy course const outputDir = path.join(tmpDir, 'build') const buildCourseDir = path.join(outputDir, 'course', 'en') await ensureDir(buildCourseDir) await writeJson(path.join(outputDir, 'course', 'config.json'), { _defaultLanguage: 'en', - _theme: theme, - _menu: menu, + _theme: theme.name, + _menu: menu.name, _enabledPlugins: pluginNames }) await writeJson(path.join(buildCourseDir, 'course.json'), { @@ -77,12 +101,9 @@ export async function prebuildSharedCache ({ buildDir, frameworkDir }) { logger: { log: (...args) => app.logger.log('debug', 'adapt-cli', ...args) } }) - // Only extract the shared entries (skip CSS which is theme-specific) - if (!await hasSharedCache(cacheRoot, pluginHash)) { - await populateSharedCacheOnly(outputDir, cacheRoot, pluginHash) + if (!await hasCachedBuild(cacheRoot, pluginHash, theme.name, menu.name)) { + await populateCache(outputDir, cacheRoot, pluginHash, theme.name, menu.name) } - - log('info', 'CACHE', 'eager shared cache build complete') } finally { await fs.rm(tmpDir, { recursive: true, force: true }) } diff --git a/lib/utils/prebuiltCache.js b/lib/utils/prebuiltCache.js index b1858d1..43ad186 100644 --- a/lib/utils/prebuiltCache.js +++ b/lib/utils/prebuiltCache.js @@ -2,29 +2,23 @@ import fs from 'node:fs/promises' import path from 'upath' import { log } from './log.js' -/** Entries that are theme/menu-specific and belong in the CSS cache */ -const CSS_ENTRIES = new Set(['adapt.css', 'adapt.css.map', 'fonts']) - /** Entries to skip (rebuilt per-build from course data) */ const SKIP_ENTRIES = new Set(['course']) /** - * Returns the cache directory paths for a given plugin hash, theme, and menu + * Returns the cache directory path for a given (pluginHash, theme, menu) combo. * @param {String} cacheRoot Root cache directory * @param {String} pluginHash Hash of installed plugins * @param {String} theme Theme name * @param {String} menu Menu name - * @return {{ sharedDir: String, cssDir: String }} + * @return {String} */ -export function getCachePaths (cacheRoot, pluginHash, theme, menu) { - return { - sharedDir: path.join(cacheRoot, pluginHash), - cssDir: path.join(cacheRoot, `${pluginHash}_${theme}_${menu}`) - } +export function getCachePath (cacheRoot, pluginHash, theme, menu) { + return path.join(cacheRoot, `${pluginHash}_${theme}_${menu}`) } /** - * Checks whether a cached build exists for the given parameters + * Checks whether a cached build exists for the given combo. * @param {String} cacheRoot Root cache directory * @param {String} pluginHash Hash of installed plugins * @param {String} theme Theme name @@ -32,24 +26,14 @@ export function getCachePaths (cacheRoot, pluginHash, theme, menu) { * @return {Promise} */ export async function hasCachedBuild (cacheRoot, pluginHash, theme, menu) { - const { sharedDir, cssDir } = getCachePaths(cacheRoot, pluginHash, theme, menu) try { - await Promise.all([ - fs.access(sharedDir), - fs.access(cssDir) - ]) + await fs.access(getCachePath(cacheRoot, pluginHash, theme, menu)) return true } catch { return false } } -/** - * Copies a file or directory from src to dest, creating parent dirs as needed - * @param {String} src Source path - * @param {String} dest Destination path - * @return {Promise} - */ async function copyEntry (src, dest) { const stat = await fs.stat(src) if (stat.isDirectory()) { @@ -61,10 +45,9 @@ async function copyEntry (src, dest) { } /** - * Extracts shared artifacts from a completed grunt build into the cache. - * Scans the build root and categorises each entry as shared or CSS-specific. - * Uses atomic rename for parallel safety. - * @param {String} buildOutputDir The build output directory (contains adapt/, index.html, etc.) + * Copies the build output (minus per-course content) into the cache for the given combo. + * Uses a temp dir + atomic rename for parallel safety. + * @param {String} buildOutputDir The build output directory * @param {String} cacheRoot Root cache directory * @param {String} pluginHash Hash of installed plugins * @param {String} theme Theme name @@ -72,43 +55,27 @@ async function copyEntry (src, dest) { * @return {Promise} */ export async function populateCache (buildOutputDir, cacheRoot, pluginHash, theme, menu) { - const { sharedDir, cssDir } = getCachePaths(cacheRoot, pluginHash, theme, menu) + const cacheDir = getCachePath(cacheRoot, pluginHash, theme, menu) await fs.mkdir(cacheRoot, { recursive: true }) - // Write to temp dirs first, then rename atomically - const tmpShared = `${sharedDir}_tmp_${Date.now()}` - const tmpCss = `${cssDir}_tmp_${Date.now()}` - + const tmpDir = `${cacheDir}_tmp_${Date.now()}` try { - await fs.mkdir(tmpShared, { recursive: true }) - await fs.mkdir(tmpCss, { recursive: true }) - + await fs.mkdir(tmpDir, { recursive: true }) const entries = await fs.readdir(buildOutputDir) for (const entry of entries) { if (SKIP_ENTRIES.has(entry)) continue - const src = path.join(buildOutputDir, entry) - if (CSS_ENTRIES.has(entry)) { - await copyEntry(src, path.join(tmpCss, entry)) - } else { - await copyEntry(src, path.join(tmpShared, entry)) - } + await copyEntry(path.join(buildOutputDir, entry), path.join(tmpDir, entry)) } - - // Atomic rename into place (last writer wins for parallel builds) - await safeRename(tmpShared, sharedDir) - await safeRename(tmpCss, cssDir) - + await safeRename(tmpDir, cacheDir) log('info', 'CACHE', `populated cache for ${pluginHash} (theme=${theme}, menu=${menu})`) } catch (e) { - // Clean up temp dirs on failure - await fs.rm(tmpShared, { recursive: true, force: true }) - await fs.rm(tmpCss, { recursive: true, force: true }) + await fs.rm(tmpDir, { recursive: true, force: true }) throw e } } /** - * Copies cached artifacts to a build directory + * Copies cached artifacts to a build directory. * @param {String} cacheRoot Root cache directory * @param {String} pluginHash Hash of installed plugins * @param {String} theme Theme name @@ -117,58 +84,13 @@ export async function populateCache (buildOutputDir, cacheRoot, pluginHash, them * @return {Promise} */ export async function restoreFromCache (cacheRoot, pluginHash, theme, menu, destDir) { - const { sharedDir, cssDir } = getCachePaths(cacheRoot, pluginHash, theme, menu) await fs.mkdir(destDir, { recursive: true }) - await fs.cp(sharedDir, destDir, { recursive: true }) - await fs.cp(cssDir, destDir, { recursive: true }) + await fs.cp(getCachePath(cacheRoot, pluginHash, theme, menu), destDir, { recursive: true }) log('info', 'CACHE', `restored from cache for ${pluginHash} (theme=${theme}, menu=${menu})`) } /** - * Checks whether the shared cache exists for a given plugin hash - * @param {String} cacheRoot Root cache directory - * @param {String} pluginHash Hash of installed plugins - * @return {Promise} - */ -export async function hasSharedCache (cacheRoot, pluginHash) { - try { - await fs.access(path.join(cacheRoot, pluginHash)) - return true - } catch { - return false - } -} - -/** - * Populates only the shared (plugin-hash-keyed) portion of the cache, - * skipping CSS/theme-specific entries. Used by the eager cache prebuild. - * @param {String} buildOutputDir The build output directory - * @param {String} cacheRoot Root cache directory - * @param {String} pluginHash Hash of installed plugins - * @return {Promise} - */ -export async function populateSharedCacheOnly (buildOutputDir, cacheRoot, pluginHash) { - const sharedDir = path.join(cacheRoot, pluginHash) - await fs.mkdir(cacheRoot, { recursive: true }) - - const tmpShared = `${sharedDir}_tmp_${Date.now()}` - try { - await fs.mkdir(tmpShared, { recursive: true }) - const entries = await fs.readdir(buildOutputDir) - for (const entry of entries) { - if (SKIP_ENTRIES.has(entry) || CSS_ENTRIES.has(entry)) continue - await copyEntry(path.join(buildOutputDir, entry), path.join(tmpShared, entry)) - } - await safeRename(tmpShared, sharedDir) - log('info', 'CACHE', `populated shared cache for ${pluginHash}`) - } catch (e) { - await fs.rm(tmpShared, { recursive: true, force: true }) - throw e - } -} - -/** - * Removes the entire prebuilt cache directory + * Removes the entire prebuilt cache directory. * @param {String} cacheRoot Root cache directory * @return {Promise} */ @@ -177,16 +99,11 @@ export async function invalidateCache (cacheRoot) { log('info', 'CACHE', 'invalidated prebuilt cache') } -/** - * Renames src to dest atomically. If dest already exists (parallel build), - * removes src instead since dest already has identical content. - */ async function safeRename (src, dest) { try { await fs.rename(src, dest) } catch (e) { if (e.code === 'ENOTEMPTY' || e.code === 'EEXIST') { - // Another build already wrote the cache — clean up our temp copy await fs.rm(src, { recursive: true, force: true }) } else { throw e diff --git a/tests/utils-prebuiltCache.spec.js b/tests/utils-prebuiltCache.spec.js index 6bc0797..6128ee6 100644 --- a/tests/utils-prebuiltCache.spec.js +++ b/tests/utils-prebuiltCache.spec.js @@ -6,11 +6,10 @@ import upath from 'upath' import os from 'node:os' describe('prebuiltCache', () => { - let getCachePaths, hasCachedBuild, hasSharedCache, populateCache, populateSharedCacheOnly, restoreFromCache, invalidateCache + let getCachePath, hasCachedBuild, populateCache, restoreFromCache, invalidateCache let tmpDir, cacheRoot, buildDir before(async () => { - // Mock the log utility to suppress output during tests mock.module('../lib/utils/log.js', { namedExports: { log: () => {}, @@ -18,7 +17,7 @@ describe('prebuiltCache', () => { logMemory: () => {} } }) - ;({ getCachePaths, hasCachedBuild, hasSharedCache, populateCache, populateSharedCacheOnly, restoreFromCache, invalidateCache } = await import('../lib/utils/prebuiltCache.js')) + ;({ getCachePath, hasCachedBuild, populateCache, restoreFromCache, invalidateCache } = await import('../lib/utils/prebuiltCache.js')) }) beforeEach(async () => { @@ -32,36 +31,26 @@ describe('prebuiltCache', () => { await fs.rm(tmpDir, { recursive: true, force: true }) }) - describe('getCachePaths()', () => { - it('should return correct shared and CSS directory paths', () => { - const result = getCachePaths('/cache', 'abc123', 'vanilla', 'boxMenu') - assert.equal(result.sharedDir, upath.join('/cache', 'abc123')) - assert.equal(result.cssDir, upath.join('/cache', 'abc123_vanilla_boxMenu')) + describe('getCachePath()', () => { + it('returns one combo-keyed directory path', () => { + const result = getCachePath('/cache', 'abc123', 'vanilla', 'boxMenu') + assert.equal(result, upath.join('/cache', 'abc123_vanilla_boxMenu')) }) }) describe('hasCachedBuild()', () => { - it('should return false when cache does not exist', async () => { + it('returns false when cache does not exist', async () => { assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), false) }) - it('should return true when both shared and CSS dirs exist', async () => { - const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') - await fs.mkdir(sharedDir, { recursive: true }) - await fs.mkdir(cssDir, { recursive: true }) + it('returns true when the combo dir exists', async () => { + await fs.mkdir(getCachePath(cacheRoot, 'hash1', 'theme', 'menu'), { recursive: true }) assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), true) }) - - it('should return false when only shared dir exists', async () => { - const { sharedDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') - await fs.mkdir(sharedDir, { recursive: true }) - assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), false) - }) }) describe('populateCache()', () => { - it('should cache all build entries except course/', async () => { - // Create mock build output matching actual grunt structure + it('caches all build entries except course/', async () => { await fs.mkdir(path.join(buildDir, 'adapt', 'js'), { recursive: true }) await fs.writeFile(path.join(buildDir, 'adapt', 'js', 'adapt.min.js'), 'js-content') await fs.writeFile(path.join(buildDir, 'adapt.css'), 'css-content') @@ -70,122 +59,47 @@ describe('prebuiltCache', () => { await fs.writeFile(path.join(buildDir, 'fonts', 'icon.woff2'), 'font-data') await fs.writeFile(path.join(buildDir, 'index.html'), '') await fs.writeFile(path.join(buildDir, 'templates.js'), 'templates') - await fs.mkdir(path.join(buildDir, 'assets'), { recursive: true }) - await fs.writeFile(path.join(buildDir, 'assets', 'logo.png'), 'img') await fs.mkdir(path.join(buildDir, 'libraries'), { recursive: true }) await fs.writeFile(path.join(buildDir, 'libraries', 'modernizr.js'), 'lib') - // Required files from plugins (e.g. spoortracking) - await fs.writeFile(path.join(buildDir, 'connection.txt'), '') - await fs.writeFile(path.join(buildDir, 'scorm_test_harness.html'), '') // course/ should be skipped await fs.mkdir(path.join(buildDir, 'course', 'en'), { recursive: true }) await fs.writeFile(path.join(buildDir, 'course', 'en', 'course.json'), '{}') await populateCache(buildDir, cacheRoot, 'hash1', 'theme', 'menu') - const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') - - // Shared artifacts - const js = await fs.readFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'utf8') - assert.equal(js, 'js-content') - const html = await fs.readFile(path.join(sharedDir, 'index.html'), 'utf8') - assert.equal(html, '') - const conn = await fs.readFile(path.join(sharedDir, 'connection.txt'), 'utf8') - assert.equal(conn, '') - const harness = await fs.readFile(path.join(sharedDir, 'scorm_test_harness.html'), 'utf8') - assert.equal(harness, '') - - // CSS artifacts - const css = await fs.readFile(path.join(cssDir, 'adapt.css'), 'utf8') - assert.equal(css, 'css-content') - const cssMap = await fs.readFile(path.join(cssDir, 'adapt.css.map'), 'utf8') - assert.equal(cssMap, 'map-content') - const font = await fs.readFile(path.join(cssDir, 'fonts', 'icon.woff2'), 'utf8') - assert.equal(font, 'font-data') - - // course/ should NOT be cached - await assert.rejects(fs.access(path.join(sharedDir, 'course')), { code: 'ENOENT' }) - await assert.rejects(fs.access(path.join(cssDir, 'course')), { code: 'ENOENT' }) + const cacheDir = getCachePath(cacheRoot, 'hash1', 'theme', 'menu') + assert.equal(await fs.readFile(path.join(cacheDir, 'adapt', 'js', 'adapt.min.js'), 'utf8'), 'js-content') + assert.equal(await fs.readFile(path.join(cacheDir, 'index.html'), 'utf8'), '') + assert.equal(await fs.readFile(path.join(cacheDir, 'adapt.css'), 'utf8'), 'css-content') + assert.equal(await fs.readFile(path.join(cacheDir, 'fonts', 'icon.woff2'), 'utf8'), 'font-data') + await assert.rejects(fs.access(path.join(cacheDir, 'course')), { code: 'ENOENT' }) }) }) describe('restoreFromCache()', () => { - it('should copy cached artifacts to destination', async () => { - // Set up cache matching actual structure - const { sharedDir, cssDir } = getCachePaths(cacheRoot, 'hash1', 'theme', 'menu') - await fs.mkdir(path.join(sharedDir, 'adapt', 'js'), { recursive: true }) - await fs.writeFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'cached-js') - await fs.writeFile(path.join(sharedDir, 'index.html'), 'cached-html') - await fs.writeFile(path.join(sharedDir, 'connection.txt'), '') - await fs.mkdir(cssDir, { recursive: true }) - await fs.writeFile(path.join(cssDir, 'adapt.css'), 'cached-css') - await fs.mkdir(path.join(cssDir, 'fonts'), { recursive: true }) - await fs.writeFile(path.join(cssDir, 'fonts', 'icon.woff2'), 'cached-font') + it('copies cached artifacts to destination', async () => { + const cacheDir = getCachePath(cacheRoot, 'hash1', 'theme', 'menu') + await fs.mkdir(path.join(cacheDir, 'adapt', 'js'), { recursive: true }) + await fs.writeFile(path.join(cacheDir, 'adapt', 'js', 'adapt.min.js'), 'cached-js') + await fs.writeFile(path.join(cacheDir, 'adapt.css'), 'cached-css') const destDir = path.join(tmpDir, 'restored') await restoreFromCache(cacheRoot, 'hash1', 'theme', 'menu', destDir) - const js = await fs.readFile(path.join(destDir, 'adapt', 'js', 'adapt.min.js'), 'utf8') - assert.equal(js, 'cached-js') - const css = await fs.readFile(path.join(destDir, 'adapt.css'), 'utf8') - assert.equal(css, 'cached-css') - const font = await fs.readFile(path.join(destDir, 'fonts', 'icon.woff2'), 'utf8') - assert.equal(font, 'cached-font') - const conn = await fs.readFile(path.join(destDir, 'connection.txt'), 'utf8') - assert.equal(conn, '') - }) - }) - - describe('hasSharedCache()', () => { - it('should return false when shared cache does not exist', async () => { - assert.equal(await hasSharedCache(cacheRoot, 'hash1'), false) - }) - - it('should return true when shared cache exists', async () => { - await fs.mkdir(path.join(cacheRoot, 'hash1'), { recursive: true }) - assert.equal(await hasSharedCache(cacheRoot, 'hash1'), true) - }) - }) - - describe('populateSharedCacheOnly()', () => { - it('should cache only shared entries, skipping CSS and course', async () => { - // Create mock build output - await fs.mkdir(path.join(buildDir, 'adapt', 'js'), { recursive: true }) - await fs.writeFile(path.join(buildDir, 'adapt', 'js', 'adapt.min.js'), 'js') - await fs.writeFile(path.join(buildDir, 'index.html'), 'html') - await fs.writeFile(path.join(buildDir, 'connection.txt'), '') - // CSS entries — should be excluded - await fs.writeFile(path.join(buildDir, 'adapt.css'), 'css') - await fs.mkdir(path.join(buildDir, 'fonts'), { recursive: true }) - await fs.writeFile(path.join(buildDir, 'fonts', 'icon.woff2'), 'font') - // course/ — should be excluded - await fs.mkdir(path.join(buildDir, 'course', 'en'), { recursive: true }) - await fs.writeFile(path.join(buildDir, 'course', 'en', 'course.json'), '{}') - - await populateSharedCacheOnly(buildDir, cacheRoot, 'hash1') - - const sharedDir = path.join(cacheRoot, 'hash1') - // Shared entries present - assert.equal(await fs.readFile(path.join(sharedDir, 'adapt', 'js', 'adapt.min.js'), 'utf8'), 'js') - assert.equal(await fs.readFile(path.join(sharedDir, 'index.html'), 'utf8'), 'html') - assert.equal(await fs.readFile(path.join(sharedDir, 'connection.txt'), 'utf8'), '') - // CSS entries absent - await assert.rejects(fs.access(path.join(sharedDir, 'adapt.css')), { code: 'ENOENT' }) - await assert.rejects(fs.access(path.join(sharedDir, 'fonts')), { code: 'ENOENT' }) - // course/ absent - await assert.rejects(fs.access(path.join(sharedDir, 'course')), { code: 'ENOENT' }) + assert.equal(await fs.readFile(path.join(destDir, 'adapt', 'js', 'adapt.min.js'), 'utf8'), 'cached-js') + assert.equal(await fs.readFile(path.join(destDir, 'adapt.css'), 'utf8'), 'cached-css') }) }) describe('invalidateCache()', () => { - it('should remove the cache directory', async () => { + it('removes the cache directory', async () => { await fs.mkdir(cacheRoot, { recursive: true }) await fs.writeFile(path.join(cacheRoot, 'test'), 'data') await invalidateCache(cacheRoot) await assert.rejects(fs.access(cacheRoot), { code: 'ENOENT' }) }) - it('should not throw when cache does not exist', async () => { + it('does not throw when cache does not exist', async () => { await assert.doesNotReject(invalidateCache(path.join(tmpDir, 'nonexistent'))) }) }) From b04b23906c89ba96928bfdadf4aeceb1c2c29937 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Wed, 6 May 2026 17:28:17 +0100 Subject: [PATCH 21/27] Refactor: Promote prebuiltCache utils to lib/BuildCache.js class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache primitives don't fit the lib/utils/.js "one exported function per file" convention — they're a cohesive set of operations on a single cacheRoot. Promote to a top-level class to match the pattern of other lib/ root files (PascalCase = class) and to drop the repeated cacheRoot argument from every call. - New lib/BuildCache.js exports a default class with constructor(cacheRoot) and methods getPath/has/populate/restore/invalidate. - prebuildCache orchestrator (still in lib/utils/) instantiates a single BuildCache and calls methods rather than passing cacheRoot around. - AdaptFrameworkBuild and AdaptFrameworkModule do the same at their call sites. - lib/utils.js no longer re-exports the cache primitives — consumers import BuildCache directly. - Test file renamed tests/BuildCache.spec.js, exercises the class. No behaviour change. --- lib/AdaptFrameworkBuild.js | 13 +- lib/AdaptFrameworkModule.js | 6 +- lib/BuildCache.js | 105 ++++++++++++++++ lib/utils.js | 1 - lib/utils/prebuildCache.js | 14 +-- lib/utils/prebuiltCache.js | 112 ------------------ ...ebuiltCache.spec.js => BuildCache.spec.js} | 41 +++---- 7 files changed, 143 insertions(+), 149 deletions(-) create mode 100644 lib/BuildCache.js delete mode 100644 lib/utils/prebuiltCache.js rename tests/{utils-prebuiltCache.spec.js => BuildCache.spec.js} (72%) diff --git a/lib/AdaptFrameworkBuild.js b/lib/AdaptFrameworkBuild.js index bab7f56..83888ac 100644 --- a/lib/AdaptFrameworkBuild.js +++ b/lib/AdaptFrameworkBuild.js @@ -3,7 +3,8 @@ import { App, Hook, ensureDir, writeJson } from 'adapt-authoring-core' import { parseObjectId } from 'adapt-authoring-mongodb' import { createWriteStream } from 'node:fs' import AdaptCli from 'adapt-cli' -import { log, logDir, logMemory, copyFrameworkSource, hasCachedBuild, populateCache, restoreFromCache, generateLanguageManifest, applyBuildReplacements } from './utils.js' +import { log, logDir, logMemory, copyFrameworkSource, generateLanguageManifest, applyBuildReplacements } from './utils.js' +import BuildCache from './BuildCache.js' import fs from 'node:fs/promises' import path from 'upath' import semver from 'semver' @@ -201,13 +202,13 @@ class AdaptFrameworkBuild { // Check for cached preview build if (this.isPreview && !contentOnly) { - const prebuiltCacheRoot = path.join(framework.getConfig('buildDir'), 'prebuilt-cache') + const cache = new BuildCache(path.join(framework.getConfig('buildDir'), 'prebuilt-cache')) const pluginHash = await framework.getPluginHash() const theme = this.courseData.config.data._theme const menu = this.courseData.config.data._menu - if (await hasCachedBuild(prebuiltCacheRoot, pluginHash, theme, menu)) { - await restoreFromCache(prebuiltCacheRoot, pluginHash, theme, menu, this.buildDir) + if (await cache.has(pluginHash, theme, menu)) { + await cache.restore(pluginHash, theme, menu, this.buildDir) await this.applySchemaDefaults() await this.copyAssets() await this.preBuildHook.invoke(this) @@ -272,12 +273,12 @@ class AdaptFrameworkBuild { } // Populate prebuilt cache after successful grunt build for preview if (this.isPreview && !contentOnly) { - const prebuiltCacheRoot = path.join(framework.getConfig('buildDir'), 'prebuilt-cache') + const cache = new BuildCache(path.join(framework.getConfig('buildDir'), 'prebuilt-cache')) const pluginHash = await framework.getPluginHash() const theme = this.courseData.config.data._theme const menu = this.courseData.config.data._menu try { - await populateCache(this.buildDir, prebuiltCacheRoot, pluginHash, theme, menu) + await cache.populate(this.buildDir, pluginHash, theme, menu) } catch (e) { log('warn', 'CACHE', `failed to populate prebuilt cache: ${e.message}`) } diff --git a/lib/AdaptFrameworkModule.js b/lib/AdaptFrameworkModule.js index 61b13cc..3e73e89 100644 --- a/lib/AdaptFrameworkModule.js +++ b/lib/AdaptFrameworkModule.js @@ -4,7 +4,8 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js' import fs from 'node:fs/promises' import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js' import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server' -import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, invalidateCache, prebuildCache } from './utils.js' +import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, prebuildCache } from './utils.js' +import BuildCache from './BuildCache.js' import path from 'node:path' import semver from 'semver' @@ -257,11 +258,10 @@ class AdaptFrameworkModule extends AbstractModule { * triggers an eager rebuild of the shared cache in the background */ async invalidatePrebuiltCache () { - const cacheRoot = path.join(this.getConfig('buildDir'), 'prebuilt-cache') this._pluginHash = null try { - await invalidateCache(cacheRoot) + await new BuildCache(path.join(this.getConfig('buildDir'), 'prebuilt-cache')).invalidate() } catch (e) { this.log('warn', `failed to invalidate prebuilt cache: ${e.message}`) } diff --git a/lib/BuildCache.js b/lib/BuildCache.js new file mode 100644 index 0000000..eb44840 --- /dev/null +++ b/lib/BuildCache.js @@ -0,0 +1,105 @@ +import fs from 'node:fs/promises' +import path from 'upath' +import { log } from './utils/log.js' + +/** Build output entries that aren't cached (rebuilt per-build from course data) */ +const SKIP_ENTRIES = new Set(['course']) + +/** + * Filesystem-level cache of grunt build output, keyed by (pluginHash, theme, menu). + * One instance per cache root; methods are stateless beyond the root path. + */ +class BuildCache { + /** + * @param {String} cacheRoot Root cache directory + */ + constructor (cacheRoot) { + this.cacheRoot = cacheRoot + } + + /** + * @returns {String} The cache directory path for the given combo + */ + getPath (pluginHash, theme, menu) { + return path.join(this.cacheRoot, `${pluginHash}_${theme}_${menu}`) + } + + /** + * @returns {Promise} Whether a cached build exists for the given combo + */ + async has (pluginHash, theme, menu) { + try { + await fs.access(this.getPath(pluginHash, theme, menu)) + return true + } catch { + return false + } + } + + /** + * Copies the build output (minus per-course content) into the cache for the given combo. + * Uses a temp dir + atomic rename for parallel safety. + * @param {String} buildOutputDir The build output directory + */ + async populate (buildOutputDir, pluginHash, theme, menu) { + const cacheDir = this.getPath(pluginHash, theme, menu) + await fs.mkdir(this.cacheRoot, { recursive: true }) + + const tmpDir = `${cacheDir}_tmp_${Date.now()}` + try { + await fs.mkdir(tmpDir, { recursive: true }) + const entries = await fs.readdir(buildOutputDir) + for (const entry of entries) { + if (SKIP_ENTRIES.has(entry)) continue + await copyEntry(path.join(buildOutputDir, entry), path.join(tmpDir, entry)) + } + await safeRename(tmpDir, cacheDir) + log('info', 'CACHE', `populated cache for ${pluginHash} (theme=${theme}, menu=${menu})`) + } catch (e) { + await fs.rm(tmpDir, { recursive: true, force: true }) + throw e + } + } + + /** + * Copies cached artifacts to a build directory. + * @param {String} destDir Destination build directory + */ + async restore (pluginHash, theme, menu, destDir) { + await fs.mkdir(destDir, { recursive: true }) + await fs.cp(this.getPath(pluginHash, theme, menu), destDir, { recursive: true }) + log('info', 'CACHE', `restored from cache for ${pluginHash} (theme=${theme}, menu=${menu})`) + } + + /** + * Removes the entire cache root. + */ + async invalidate () { + await fs.rm(this.cacheRoot, { recursive: true, force: true }) + log('info', 'CACHE', 'invalidated prebuilt cache') + } +} + +async function copyEntry (src, dest) { + const stat = await fs.stat(src) + if (stat.isDirectory()) { + await fs.cp(src, dest, { recursive: true }) + } else { + await fs.mkdir(path.dirname(dest), { recursive: true }) + await fs.copyFile(src, dest) + } +} + +async function safeRename (src, dest) { + try { + await fs.rename(src, dest) + } catch (e) { + if (e.code === 'ENOTEMPTY' || e.code === 'EEXIST') { + await fs.rm(src, { recursive: true, force: true }) + } else { + throw e + } + } +} + +export default BuildCache diff --git a/lib/utils.js b/lib/utils.js index a6cd855..0013b47 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -12,7 +12,6 @@ export { collectMigrationScripts } from './utils/collectMigrationScripts.js' export { runContentMigration } from './utils/runContentMigration.js' export { migrateExistingCourses } from './utils/migrateExistingCourses.js' export { computePluginHash } from './utils/computePluginHash.js' -export { hasCachedBuild, populateCache, restoreFromCache, invalidateCache } from './utils/prebuiltCache.js' export { prebuildCache } from './utils/prebuildCache.js' export { generateLanguageManifest } from './utils/generateLanguageManifest.js' export { applyBuildReplacements } from './utils/applyBuildReplacements.js' diff --git a/lib/utils/prebuildCache.js b/lib/utils/prebuildCache.js index 6b15e4c..c9a11a3 100644 --- a/lib/utils/prebuildCache.js +++ b/lib/utils/prebuildCache.js @@ -3,7 +3,7 @@ import AdaptCli from 'adapt-cli' import fs from 'node:fs/promises' import path from 'upath' import { copyFrameworkSource } from './copyFrameworkSource.js' -import { hasCachedBuild, populateCache } from './prebuiltCache.js' +import BuildCache from '../BuildCache.js' import { computePluginHash } from './computePluginHash.js' import { log } from './log.js' @@ -22,7 +22,7 @@ import { log } from './log.js' */ export async function prebuildCache ({ buildDir, frameworkDir }) { const app = App.instance - const cacheRoot = path.join(buildDir, 'prebuilt-cache') + const cache = new BuildCache(path.join(buildDir, 'prebuilt-cache')) const pluginHash = await computePluginHash(frameworkDir) const contentplugin = await app.waitForModule('contentplugin') @@ -39,7 +39,7 @@ export async function prebuildCache ({ buildDir, frameworkDir }) { for (const theme of themes) { for (const menu of menus) { try { - await prebuildOne({ buildDir, cacheRoot, pluginHash, theme, menu, allPlugins }) + await prebuildOne({ buildDir, cache, pluginHash, theme, menu, allPlugins }) } catch (e) { log('warn', 'CACHE', `eager prebuild failed for theme=${theme.name} menu=${menu.name}: ${e.message}`) if (e.cmd) log('warn', 'CACHE', `cmd: ${e.cmd}`) @@ -51,10 +51,10 @@ export async function prebuildCache ({ buildDir, frameworkDir }) { log('info', 'CACHE', 'eager prebuild complete') } -async function prebuildOne ({ buildDir, cacheRoot, pluginHash, theme, menu, allPlugins }) { +async function prebuildOne ({ buildDir, cache, pluginHash, theme, menu, allPlugins }) { const app = App.instance - if (await hasCachedBuild(cacheRoot, pluginHash, theme.name, menu.name)) { + if (await cache.has(pluginHash, theme.name, menu.name)) { log('info', 'CACHE', `skipping cached combo theme=${theme.name} menu=${menu.name}`) return } @@ -101,8 +101,8 @@ async function prebuildOne ({ buildDir, cacheRoot, pluginHash, theme, menu, allP logger: { log: (...args) => app.logger.log('debug', 'adapt-cli', ...args) } }) - if (!await hasCachedBuild(cacheRoot, pluginHash, theme.name, menu.name)) { - await populateCache(outputDir, cacheRoot, pluginHash, theme.name, menu.name) + if (!await cache.has(pluginHash, theme.name, menu.name)) { + await cache.populate(outputDir, pluginHash, theme.name, menu.name) } } finally { await fs.rm(tmpDir, { recursive: true, force: true }) diff --git a/lib/utils/prebuiltCache.js b/lib/utils/prebuiltCache.js deleted file mode 100644 index 43ad186..0000000 --- a/lib/utils/prebuiltCache.js +++ /dev/null @@ -1,112 +0,0 @@ -import fs from 'node:fs/promises' -import path from 'upath' -import { log } from './log.js' - -/** Entries to skip (rebuilt per-build from course data) */ -const SKIP_ENTRIES = new Set(['course']) - -/** - * Returns the cache directory path for a given (pluginHash, theme, menu) combo. - * @param {String} cacheRoot Root cache directory - * @param {String} pluginHash Hash of installed plugins - * @param {String} theme Theme name - * @param {String} menu Menu name - * @return {String} - */ -export function getCachePath (cacheRoot, pluginHash, theme, menu) { - return path.join(cacheRoot, `${pluginHash}_${theme}_${menu}`) -} - -/** - * Checks whether a cached build exists for the given combo. - * @param {String} cacheRoot Root cache directory - * @param {String} pluginHash Hash of installed plugins - * @param {String} theme Theme name - * @param {String} menu Menu name - * @return {Promise} - */ -export async function hasCachedBuild (cacheRoot, pluginHash, theme, menu) { - try { - await fs.access(getCachePath(cacheRoot, pluginHash, theme, menu)) - return true - } catch { - return false - } -} - -async function copyEntry (src, dest) { - const stat = await fs.stat(src) - if (stat.isDirectory()) { - await fs.cp(src, dest, { recursive: true }) - } else { - await fs.mkdir(path.dirname(dest), { recursive: true }) - await fs.copyFile(src, dest) - } -} - -/** - * Copies the build output (minus per-course content) into the cache for the given combo. - * Uses a temp dir + atomic rename for parallel safety. - * @param {String} buildOutputDir The build output directory - * @param {String} cacheRoot Root cache directory - * @param {String} pluginHash Hash of installed plugins - * @param {String} theme Theme name - * @param {String} menu Menu name - * @return {Promise} - */ -export async function populateCache (buildOutputDir, cacheRoot, pluginHash, theme, menu) { - const cacheDir = getCachePath(cacheRoot, pluginHash, theme, menu) - await fs.mkdir(cacheRoot, { recursive: true }) - - const tmpDir = `${cacheDir}_tmp_${Date.now()}` - try { - await fs.mkdir(tmpDir, { recursive: true }) - const entries = await fs.readdir(buildOutputDir) - for (const entry of entries) { - if (SKIP_ENTRIES.has(entry)) continue - await copyEntry(path.join(buildOutputDir, entry), path.join(tmpDir, entry)) - } - await safeRename(tmpDir, cacheDir) - log('info', 'CACHE', `populated cache for ${pluginHash} (theme=${theme}, menu=${menu})`) - } catch (e) { - await fs.rm(tmpDir, { recursive: true, force: true }) - throw e - } -} - -/** - * Copies cached artifacts to a build directory. - * @param {String} cacheRoot Root cache directory - * @param {String} pluginHash Hash of installed plugins - * @param {String} theme Theme name - * @param {String} menu Menu name - * @param {String} destDir Destination build directory - * @return {Promise} - */ -export async function restoreFromCache (cacheRoot, pluginHash, theme, menu, destDir) { - await fs.mkdir(destDir, { recursive: true }) - await fs.cp(getCachePath(cacheRoot, pluginHash, theme, menu), destDir, { recursive: true }) - log('info', 'CACHE', `restored from cache for ${pluginHash} (theme=${theme}, menu=${menu})`) -} - -/** - * Removes the entire prebuilt cache directory. - * @param {String} cacheRoot Root cache directory - * @return {Promise} - */ -export async function invalidateCache (cacheRoot) { - await fs.rm(cacheRoot, { recursive: true, force: true }) - log('info', 'CACHE', 'invalidated prebuilt cache') -} - -async function safeRename (src, dest) { - try { - await fs.rename(src, dest) - } catch (e) { - if (e.code === 'ENOTEMPTY' || e.code === 'EEXIST') { - await fs.rm(src, { recursive: true, force: true }) - } else { - throw e - } - } -} diff --git a/tests/utils-prebuiltCache.spec.js b/tests/BuildCache.spec.js similarity index 72% rename from tests/utils-prebuiltCache.spec.js rename to tests/BuildCache.spec.js index 6128ee6..5bec235 100644 --- a/tests/utils-prebuiltCache.spec.js +++ b/tests/BuildCache.spec.js @@ -5,9 +5,9 @@ import path from 'node:path' import upath from 'upath' import os from 'node:os' -describe('prebuiltCache', () => { - let getCachePath, hasCachedBuild, populateCache, restoreFromCache, invalidateCache - let tmpDir, cacheRoot, buildDir +describe('BuildCache', () => { + let BuildCache + let tmpDir, cacheRoot, buildDir, cache before(async () => { mock.module('../lib/utils/log.js', { @@ -17,7 +17,7 @@ describe('prebuiltCache', () => { logMemory: () => {} } }) - ;({ getCachePath, hasCachedBuild, populateCache, restoreFromCache, invalidateCache } = await import('../lib/utils/prebuiltCache.js')) + ;({ default: BuildCache } = await import('../lib/BuildCache.js')) }) beforeEach(async () => { @@ -25,31 +25,31 @@ describe('prebuiltCache', () => { cacheRoot = path.join(tmpDir, 'prebuilt-cache') buildDir = path.join(tmpDir, 'build') await fs.mkdir(buildDir, { recursive: true }) + cache = new BuildCache(cacheRoot) }) afterEach(async () => { await fs.rm(tmpDir, { recursive: true, force: true }) }) - describe('getCachePath()', () => { + describe('getPath()', () => { it('returns one combo-keyed directory path', () => { - const result = getCachePath('/cache', 'abc123', 'vanilla', 'boxMenu') - assert.equal(result, upath.join('/cache', 'abc123_vanilla_boxMenu')) + assert.equal(cache.getPath('abc123', 'vanilla', 'boxMenu'), upath.join(cacheRoot, 'abc123_vanilla_boxMenu')) }) }) - describe('hasCachedBuild()', () => { + describe('has()', () => { it('returns false when cache does not exist', async () => { - assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), false) + assert.equal(await cache.has('hash1', 'theme', 'menu'), false) }) it('returns true when the combo dir exists', async () => { - await fs.mkdir(getCachePath(cacheRoot, 'hash1', 'theme', 'menu'), { recursive: true }) - assert.equal(await hasCachedBuild(cacheRoot, 'hash1', 'theme', 'menu'), true) + await fs.mkdir(cache.getPath('hash1', 'theme', 'menu'), { recursive: true }) + assert.equal(await cache.has('hash1', 'theme', 'menu'), true) }) }) - describe('populateCache()', () => { + describe('populate()', () => { it('caches all build entries except course/', async () => { await fs.mkdir(path.join(buildDir, 'adapt', 'js'), { recursive: true }) await fs.writeFile(path.join(buildDir, 'adapt', 'js', 'adapt.min.js'), 'js-content') @@ -65,9 +65,9 @@ describe('prebuiltCache', () => { await fs.mkdir(path.join(buildDir, 'course', 'en'), { recursive: true }) await fs.writeFile(path.join(buildDir, 'course', 'en', 'course.json'), '{}') - await populateCache(buildDir, cacheRoot, 'hash1', 'theme', 'menu') + await cache.populate(buildDir, 'hash1', 'theme', 'menu') - const cacheDir = getCachePath(cacheRoot, 'hash1', 'theme', 'menu') + const cacheDir = cache.getPath('hash1', 'theme', 'menu') assert.equal(await fs.readFile(path.join(cacheDir, 'adapt', 'js', 'adapt.min.js'), 'utf8'), 'js-content') assert.equal(await fs.readFile(path.join(cacheDir, 'index.html'), 'utf8'), '') assert.equal(await fs.readFile(path.join(cacheDir, 'adapt.css'), 'utf8'), 'css-content') @@ -76,31 +76,32 @@ describe('prebuiltCache', () => { }) }) - describe('restoreFromCache()', () => { + describe('restore()', () => { it('copies cached artifacts to destination', async () => { - const cacheDir = getCachePath(cacheRoot, 'hash1', 'theme', 'menu') + const cacheDir = cache.getPath('hash1', 'theme', 'menu') await fs.mkdir(path.join(cacheDir, 'adapt', 'js'), { recursive: true }) await fs.writeFile(path.join(cacheDir, 'adapt', 'js', 'adapt.min.js'), 'cached-js') await fs.writeFile(path.join(cacheDir, 'adapt.css'), 'cached-css') const destDir = path.join(tmpDir, 'restored') - await restoreFromCache(cacheRoot, 'hash1', 'theme', 'menu', destDir) + await cache.restore('hash1', 'theme', 'menu', destDir) assert.equal(await fs.readFile(path.join(destDir, 'adapt', 'js', 'adapt.min.js'), 'utf8'), 'cached-js') assert.equal(await fs.readFile(path.join(destDir, 'adapt.css'), 'utf8'), 'cached-css') }) }) - describe('invalidateCache()', () => { + describe('invalidate()', () => { it('removes the cache directory', async () => { await fs.mkdir(cacheRoot, { recursive: true }) await fs.writeFile(path.join(cacheRoot, 'test'), 'data') - await invalidateCache(cacheRoot) + await cache.invalidate() await assert.rejects(fs.access(cacheRoot), { code: 'ENOENT' }) }) it('does not throw when cache does not exist', async () => { - await assert.doesNotReject(invalidateCache(path.join(tmpDir, 'nonexistent'))) + const missing = new BuildCache(path.join(tmpDir, 'nonexistent')) + await assert.doesNotReject(missing.invalidate()) }) }) }) From ebb1c11bebecc171e52e4dcaba401f237c2e81a3 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Thu, 7 May 2026 17:08:13 +0100 Subject: [PATCH 22/27] Fix: Defer updateEnabledPlugins until import completes (refs #109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-insert updateEnabledPlugins fires a full content-tree fetch, config re-validation and re-default sweep over every affected content item. On a 200+ item import this becomes O(n²) and was contributing to the runaway slowdown / OOM during course import. Disable updateEnabledPlugins/updateSortOrder per insert (matching the opt-out lost in 3436f00) and run a single forceUpdate sweep once all content is in place. --- lib/AdaptFrameworkImport.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index 44e22bb..48ec49d 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -757,6 +757,8 @@ class AdaptFrameworkImport { } } if (errors.length) throw App.instance.errors.FW_IMPORT_CONTENT_FAILED.setData({ errors }) + // single-pass sweep now all content is in place; per-insert sweep was disabled to avoid O(n²) work + await this.content.updateEnabledPlugins({ _courseId: this.idMap.course }, { forceUpdate: true }) log('debug', 'imported course data successfully') } @@ -807,7 +809,7 @@ class AdaptFrameworkImport { } insertData = schema.sanitise(insertData) let doc - const opts = { schemaName, validate: true, useCache: false } + const opts = { schemaName, validate: true, useCache: false, updateEnabledPlugins: false, updateSortOrder: false } if (options.isUpdate) { doc = await this.content.update({ _id: data._id }, insertData, opts) } else { From 97987c8163915554a4ffa89d57a459431bf12d35 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Fri, 8 May 2026 12:35:22 +0100 Subject: [PATCH 23/27] Fix: Address several import correctness issues - importCourseAssets now reads the source file size and passes it to assets.insert. Without it, the duplicate-detection size pre-check ran with size=undefined and threw TOO_MANY_RESULTS, causing every asset insert to fail and assetMap to be left empty (so resolveAssets later dropped every asset reference). - importContentObject strips _assetIds from the incoming data so that ContentModule.insert always recomputes it from the resolved asset references. Exports ship _assetIds as path strings, not ObjectIds, and the existing insert path trusts whatever's there. - The second-phase course update (line 739) now passes ignoreRequired so the post-config re-validation doesn't fail on plugin-declared required properties with no default that Ajv can't materialise (e.g. adapt-contrib-glossary's _glossary). --- lib/AdaptFrameworkImport.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index 48ec49d..346d5e9 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -577,13 +577,15 @@ class AdaptFrameworkImport { if (this.settings.isDryRun) { return } + const stats = await fs.stat(filepath) try { const asset = await this.assets.insert({ ...data, createdBy: this.userId, file: { filepath, - originalFilename: filepath + originalFilename: filepath, + size: stats.size }, tags: data.tags }) @@ -735,8 +737,9 @@ class AdaptFrameworkImport { try { const course = await this.importContentObject({ ...this.contentJson.course, tags: this.tags }) /* config */ await this.importContentObject(this.contentJson.config) - // we need to run an update with the same data to make sure all extension schema settings are applied - await this.importContentObject({ ...this.contentJson.course, _id: course._id }, { isUpdate: true }) + // we need to run an update with the same data to make sure all extension schema settings are applied; + // ignoreRequired because some plugins declare top-level required properties with no default that Ajv can't materialise (e.g. adapt-contrib-glossary) + await this.importContentObject({ ...this.contentJson.course, _id: course._id }, { isUpdate: true, ignoreRequired: true }) } catch (e) { throw App.instance.errors.FW_IMPORT_CONTENT_FAILED.setData({ errors: [formatError(e)] }) } @@ -797,6 +800,7 @@ class AdaptFrameworkImport { let insertData = await this.transformData({ ...data, _id: undefined, + _assetIds: undefined, // recompute from resolved asset references; export ships paths, not ObjectIds _courseId: this.idMap.course, createdBy: this.userId }) @@ -809,7 +813,7 @@ class AdaptFrameworkImport { } insertData = schema.sanitise(insertData) let doc - const opts = { schemaName, validate: true, useCache: false, updateEnabledPlugins: false, updateSortOrder: false } + const opts = { schemaName, validate: true, useCache: false, updateEnabledPlugins: false, updateSortOrder: false, ignoreRequired: options.ignoreRequired } if (options.isUpdate) { doc = await this.content.update({ _id: data._id }, insertData, opts) } else { From 5a47f41075147cf1ac221a3f386bb36fa6679f25 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Fri, 8 May 2026 13:33:43 +0100 Subject: [PATCH 24/27] Fix: Use hierarchy index as canonical _sortOrder during import Per-insert updateSortOrder is disabled for performance, so any duplicate, missing, or undefined _sortOrder shipped in the export JSON would persist into the imported course. The hierarchy-deduced value is unique per parent and respects the export's array order, so it's a safer source of truth than the export's own _sortOrder. --- lib/AdaptFrameworkImport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/AdaptFrameworkImport.js b/lib/AdaptFrameworkImport.js index 346d5e9..72ebc62 100644 --- a/lib/AdaptFrameworkImport.js +++ b/lib/AdaptFrameworkImport.js @@ -751,8 +751,8 @@ class AdaptFrameworkImport { try { const itemJson = this.contentJson.contentObjects[_id] await this.importContentObject({ - _sortOrder: hierarchy[itemJson._parentId].indexOf(_id) + 1, - ...itemJson // note that JSON sort order will override the deduced one + ...itemJson, + _sortOrder: hierarchy[itemJson._parentId].indexOf(_id) + 1 // trust the hierarchy: per-insert updateSortOrder is disabled, so we can't rely on bad export values being normalised later }) } catch (e) { errors.push(formatError(e)) From a87b310f75a51cf14c313fe599e335c008e902f2 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Fri, 8 May 2026 23:49:51 +0100 Subject: [PATCH 25/27] Chore: Update resolveAssets test for drop-and-warn behaviour Commit 0066a58 changed resolveAssets to drop unresolvable asset refs and surface them via statusReport.warn (instead of leaving them in place). The "should keep value when not in assetMap" test still asserted the old behaviour and lacked a statusReport mock, so it crashed with TypeError: Cannot read properties of undefined. Renames the test, updates assertions, and adds statusReport to makeCtx. --- tests/AdaptFrameworkImport.spec.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/AdaptFrameworkImport.spec.js b/tests/AdaptFrameworkImport.spec.js index 4a80d3e..aca8ece 100644 --- a/tests/AdaptFrameworkImport.spec.js +++ b/tests/AdaptFrameworkImport.spec.js @@ -150,7 +150,7 @@ describe('AdaptFrameworkImport', () => { describe('#resolveAssets()', () => { function makeCtx (assetMap) { - const ctx = { assetMap } + const ctx = { assetMap, statusReport: { warn: [] } } ctx.resolveAssets = AdaptFrameworkImport.prototype.resolveAssets.bind(ctx) return ctx } @@ -207,14 +207,17 @@ describe('AdaptFrameworkImport', () => { assert.equal('src' in data._graphic, false) }) - it('should keep value when not in assetMap', () => { + it('should drop unresolved asset refs and surface them in statusReport.warn', () => { const ctx = makeCtx({}) const schema = makeSchema({ img: { _backboneForms: { type: 'Asset' } } }) const data = { img: 'unknown/path.png' } ctx.resolveAssets(schema, data) - assert.equal(data.img, 'unknown/path.png') + assert.equal('img' in data, false) + assert.equal(ctx.statusReport.warn.length, 1) + assert.equal(ctx.statusReport.warn[0].code, 'UNRESOLVED_ASSET_REF') + assert.equal(ctx.statusReport.warn[0].data.path, 'unknown/path.png') }) it('should recurse into nested properties', () => { From a60e41e0d306cb2df2ef8b72751501a2194a6291 Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Sat, 9 May 2026 00:01:27 +0100 Subject: [PATCH 26/27] Build: Bump deps to released majors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - adapt-authoring-content: ^2.0.0 → ^3.0.0 (uses content._assetIds queries) - adapt-authoring-core: ^2.0.0 → ^3.0.0 (depends on the bootstrap library architecture) - adapt-migrations: ^1.4.0 → ^2.0.0 (boot-phase runner) --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 822c861..48bff96 100644 --- a/package.json +++ b/package.json @@ -11,14 +11,14 @@ "test": "node --test --test-force-exit --experimental-test-module-mocks 'tests/**/*.spec.js'" }, "dependencies": { - "adapt-authoring-content": "^2.0.0", + "adapt-authoring-content": "^3.0.0", "adapt-authoring-contentplugin": "^1.0.3", - "adapt-authoring-core": "^2.0.0", + "adapt-authoring-core": "^3.0.0", "adapt-authoring-coursetheme": "^1.0.2", "adapt-authoring-mongodb": "^3.0.0", "adapt-authoring-spoortracking": "^1.0.2", "adapt-cli": "^3.3.3", - "adapt-migrations": "^1.4.0", + "adapt-migrations": "^2.0.0", "bytes": "^3.1.2", "fs-extra": "11.3.3", "glob": "^13.0.0", From 345864bc4a9619c2fd371ee06101d611fc09bc3e Mon Sep 17 00:00:00 2001 From: Thomas Taylor Date: Sat, 9 May 2026 00:22:17 +0100 Subject: [PATCH 27/27] Build: Revert errant adapt-migrations bump Confused adapt-migrations (third-party content-migration library, latest 1.4.2 on npm) with adapt-authoring-migrations (the boot-phase runner) in a60e41e. The dep stays at ^1.4.0; only content and core needed bumping to ^3.0.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 48bff96..eee1b03 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "adapt-authoring-mongodb": "^3.0.0", "adapt-authoring-spoortracking": "^1.0.2", "adapt-cli": "^3.3.3", - "adapt-migrations": "^2.0.0", + "adapt-migrations": "^1.4.0", "bytes": "^3.1.2", "fs-extra": "11.3.3", "glob": "^13.0.0",