Breaking: Migrations, prebuilt cache, and import improvements

conf/config.schema.json

@@ -32,6 +32,11 @@
       "description": "URL of the Adapt framework git repository to install",
       "type": "string"
     },
+    "prebuildCache": {
+      "description": "When enabled, eagerly rebuilds the prebuilt cache for every (theme, menu) combination in the background after invalidation (e.g. plugin install/update/delete)",
+      "type": "boolean",
+      "default": false
+    },
     "importMaxFileSize": {
       "description": "Maximum file upload size for course imports",
       "type": "string",

index.js

@@ -3,6 +3,6 @@
  * @namespace adaptframework
  */
 export { default } from './lib/AdaptFrameworkModule.js'
-export { copyFrameworkSource } from './lib/utils.js'
+export { copyFrameworkSource, readFrameworkPluginVersions } from './lib/utils.js'
 export { default as AdaptFrameworkBuild } from './lib/AdaptFrameworkBuild.js'
 export { default as AdaptFrameworkImport } from './lib/AdaptFrameworkImport.js'

lib/AdaptFrameworkBuild.js

@@ -3,7 +3,8 @@ import { App, Hook, ensureDir, writeJson } from 'adapt-authoring-core'
 import { parseObjectId } from 'adapt-authoring-mongodb'
 import { createWriteStream } from 'node:fs'
 import AdaptCli from 'adapt-cli'
-import { log, logDir, logMemory, copyFrameworkSource } from './utils.js'
+import { log, logDir, logMemory, copyFrameworkSource, generateLanguageManifest, applyBuildReplacements } from './utils.js'
+import BuildCache from './BuildCache.js'
 import fs from 'node:fs/promises'
 import path from 'upath'
 import semver from 'semver'
@@ -199,11 +200,48 @@ class AdaptFrameworkBuild {
 
     await this.loadCourseData()
 
+    // Check for cached preview build
+    if (this.isPreview && !contentOnly) {
+      const cache = new BuildCache(path.join(framework.getConfig('buildDir'), 'prebuilt-cache'))
+      const pluginHash = await framework.getPluginHash()
+      const theme = this.courseData.config.data._theme
+      const menu = this.courseData.config.data._menu
+
+      if (await cache.has(pluginHash, theme, menu)) {
+        await cache.restore(pluginHash, theme, menu, this.buildDir)
+        await this.applySchemaDefaults()
+        await this.copyAssets()
+        await this.preBuildHook.invoke(this)
+        await this.writeContentJson()
+        await this.writeLanguageManifest()
+        await applyBuildReplacements(this.buildDir, {
+          defaultLanguage: this.courseData.config.data._defaultLanguage ?? 'en',
+          defaultDirection: this.courseData.config.data._defaultDirection ?? 'ltr',
+          buildType: 'development',
+          timestamp: Date.now()
+        })
+        this.location = path.join(this.dir, 'build')
+        await this.postBuildHook.invoke(this)
+        this.buildData = await this.recordBuildAttempt()
+        return this
+      }
+    }
+
     const tasks = [this.copyAssets()]
     if (!contentOnly) {
+      // preview cache is shared across courses, so include all installed plugins
+      // — except disabled themes/menus, since only one of each can be active per
+      // build and the framework's less:dev task globs every theme/menu in src/,
+      // which OOMs when more than one is present (see adapt_framework#3802).
+      const pluginsToInclude = this.isPreview
+        ? [
+            ...this.enabledPlugins,
+            ...this.disabledPlugins.filter(p => p.type !== 'theme' && p.type !== 'menu')
+          ]
+        : this.enabledPlugins
       tasks.push(copyFrameworkSource({
         destDir: this.dir,
-        enabledPlugins: this.enabledPlugins.map(p => p.name),
+        enabledPlugins: pluginsToInclude.map(p => p.name),
         linkNodeModules: !this.isExport
       }))
     }
@@ -233,6 +271,18 @@ class AdaptFrameworkBuild {
           .setData(e)
       }
     }
+    // Populate prebuilt cache after successful grunt build for preview
+    if (this.isPreview && !contentOnly) {
+      const cache = new BuildCache(path.join(framework.getConfig('buildDir'), 'prebuilt-cache'))
+      const pluginHash = await framework.getPluginHash()
+      const theme = this.courseData.config.data._theme
+      const menu = this.courseData.config.data._menu
+      try {
+        await cache.populate(this.buildDir, pluginHash, theme, menu)
+      } catch (e) {
+        log('warn', 'CACHE', `failed to populate prebuilt cache: ${e.message}`)
+      }
+    }
     if (this.compress) {
       this.location = await this.prepareZip()
     } else {
@@ -275,10 +325,10 @@ class AdaptFrameworkBuild {
    * @return {Promise}
    */
   async loadAssetData () {
-    const [assets, courseassets, tags] = await App.instance.waitForModule('assets', 'courseassets', 'tags')
+    const [assets, content, tags] = await App.instance.waitForModule('assets', 'content', 'tags')
 
-    const caRecs = await courseassets.find({ courseId: this.courseId })
-    const uniqueAssetIds = new Set(caRecs.map(c => parseObjectId(c.assetId)))
+    const courseContent = await content.find({ _courseId: this.courseId }, { validate: false }, { projection: { _assetIds: 1 } })
+    const uniqueAssetIds = new Set(courseContent.flatMap(c => (c._assetIds ?? []).map(id => parseObjectId(id))))
     const usedAssets = await assets.find({ _id: { $in: [...uniqueAssetIds] } })
 
     const usedTagIds = new Set(usedAssets.reduce((m, a) => [...m, ...(a.tags ?? [])], []))
@@ -426,12 +476,35 @@ class AdaptFrameworkBuild {
     }))
   }
 
+  /**
+   * Outputs all course data to the required JSON files
+   * @return {Promise}
+   */
+  async writeContentJson () {
+    const data = Object.values(this.courseData)
+    if (this.isExport && this.assetData.data.length) {
+      this.assetData.data = this.assetData.data.map(d => {
+        return {
+          title: d.title,
+          description: d.description,
+          filename: d.path,
+          tags: d.tags
+        }
+      })
+      data.push(this.assetData)
+    }
+    return Promise.all(data.map(async ({ dir, fileName, data }) => {
+      await ensureDir(dir)
+      const filepath = path.join(dir, fileName)
+      const returnData = await writeJson(filepath, data)
+      log('verbose', 'WRITE', filepath)
+      return returnData
+    }))
+  }
+
   /**
    * Applies schema defaults to the in-memory course and config data using
    * the jsonschema module. Replicates what grunt's schema-defaults task does.
-   *
-   * TODO: replace validateWithDefaults workaround with schema.validate(data, { ignoreErrors: true })
-   * once migrated to adapt-schemas v3.x (see #184)
    * @return {Promise}
    */
   async applySchemaDefaults () {
@@ -442,31 +515,19 @@ class AdaptFrameworkBuild {
     const extensionFilter = s => contentplugin.isPluginSchema(s) ? enabledPluginSchemas.includes(s) : true
     const getSchema = name => jsonschema.getSchema(name, { useCache: false, extensionFilter })
 
-    /**
-     * Applies defaults via validate(), catching and ignoring validation errors.
-     * The validated+defaulted data is returned from validate() on success, or
-     * extracted from the error on failure (validate clones internally).
-     */
-    const validateWithDefaults = (schema, data) => {
-      try {
-        return schema.validate(data, { useDefaults: true, ignoreRequired: true })
-      } catch (e) {
-        return e.data.data
-      }
-    }
-
+    // Apply defaults without running full validation (which rejects ObjectIds etc.)
     const [courseSchema, configSchema] = await Promise.all([
       getSchema('course'),
       getSchema('config')
     ])
-    Object.assign(this.courseData.course.data, validateWithDefaults(courseSchema, this.courseData.course.data))
-    Object.assign(this.courseData.config.data, validateWithDefaults(configSchema, this.courseData.config.data))
+    courseSchema.compiledWithDefaults(this.courseData.course.data)
+    configSchema.compiledWithDefaults(this.courseData.config.data)
 
     for (const type of ['contentObject', 'article', 'block']) {
       const schemaName = type === 'contentObject' ? 'contentobject' : type
       const schema = await getSchema(schemaName)
       for (const item of this.courseData[type].data) {
-        Object.assign(item, validateWithDefaults(schema, item))
+        schema.compiledWithDefaults(item)
       }
     }
 
@@ -476,34 +537,23 @@ class AdaptFrameworkBuild {
       if (!componentSchemas[schemaName]) {
         componentSchemas[schemaName] = await getSchema(schemaName)
       }
-      Object.assign(item, validateWithDefaults(componentSchemas[schemaName], item))
+      componentSchemas[schemaName].compiledWithDefaults(item)
     }
   }
 
   /**
-   * Outputs all course data to the required JSON files
+   * Writes the language_data_manifest.js for each language dir.
+   * Only needed on cache-hit builds where grunt is skipped.
    * @return {Promise}
    */
-  async writeContentJson () {
-    const data = Object.values(this.courseData)
-    if (this.isExport && this.assetData.data.length) {
-      this.assetData.data = this.assetData.data.map(d => {
-        return {
-          title: d.title,
-          description: d.description,
-          filename: d.path,
-          tags: d.tags
-        }
-      })
-      data.push(this.assetData)
-    }
-    return Promise.all(data.map(async ({ dir, fileName, data }) => {
-      await ensureDir(dir)
-      const filepath = path.join(dir, fileName)
-      const returnData = await writeJson(filepath, data)
-      log('verbose', 'WRITE', filepath)
-      return returnData
-    }))
+  async writeLanguageManifest () {
+    const langDir = this.courseData.course.dir
+    const fileNames = Object.values(this.courseData)
+      .filter(d => d.dir === langDir)
+      .map(d => d.fileName)
+    const manifest = generateLanguageManifest(fileNames)
+    await ensureDir(langDir)
+    await writeJson(path.join(langDir, 'language_data_manifest.js'), manifest)
   }
 
   /**