New: Add --capturedir and --rootdir to grunt migration tasks (fixes #3667)

grunt/helpers.js

@@ -194,15 +194,17 @@ module.exports = function(grunt) {
       return [].concat(exports.defaults.includes, buildIncludes, dependencies);
     },
 
-    generateConfigData: function() {
+    generateConfigData: function({
+      rootDir = __dirname.split(path.sep).slice(0, -1).join(path.sep)
+    } = {}) {
 
       const localConfigPath = exports.getLocalConfig();
       const localConfig = localConfigPath
         ? fs.readJSONSync(localConfigPath)
         : {};
       const defaults = Object.assign({}, exports.defaults, localConfig);
 
-      const root = __dirname.split(path.sep).slice(0, -1).join(path.sep);
+      const root = rootDir;
       const adaptJSON = fs.readJSONSync(`${root}/adapt.json`);
       const sourcedir = appendSlash(grunt.option('sourcedir')) || defaults.sourcedir;
       const outputdir = appendSlash(grunt.option('outputdir')) || defaults.outputdir;
@@ -223,14 +225,11 @@ module.exports = function(grunt) {
       // add root path if necessary, and point to course/config.json
 
       const configPath = path.join(path.resolve(root, configDir), coursedir, 'config.' + jsonext);
-      let buildConfig;
+      let buildConfig = {};
 
       try {
         buildConfig = grunt.file.readJSON(configPath).build || {};
-      } catch (error) {
-        grunt.log.error(error);
-        process.exit();
-      }
+      } catch (error) {}
 
       const isDevelopmentBuild = process.argv.some(arg => (arg === 'dev' || arg.includes(':dev') || arg.includes('--dev')));
       const cacheAge = isNaN(grunt.option('cacheage'))
@@ -282,7 +281,10 @@ module.exports = function(grunt) {
       });
       framework.load();
 
-      data.availableLanguageNames = framework.getData().languageNames;
+      data.availableLanguageNames = [];
+      try {
+        data.availableLanguageNames = framework.getData().languageNames;
+      } catch (err) {}
 
       return data;
     },
@@ -369,8 +371,11 @@ module.exports = function(grunt) {
     },
 
     /** @returns {Framework} */
-    getFramework: function({ useOutputData = Boolean(grunt.option('outputdir')) } = {}) {
-      const buildConfig = exports.generateConfigData();
+    getFramework: function({
+      useOutputData = Boolean(grunt.option('outputdir')),
+      rootDir = process.cwd()
+    } = {}) {
+      const buildConfig = exports.generateConfigData({ rootDir });
       const framework = new Framework({
         rootPath: buildConfig.root,
         outputPath: buildConfig.outputdir,

grunt/helpers/Data.js

@@ -54,12 +54,13 @@ class Data {
     this.configFile = null;
     /** @type {[Language]} */
     this.languages = null;
+    /** @type {string} */
+    this.coursePath = path.join(this.sourcePath, this.courseDir).replace(/\\/g, '/');
   }
 
   /** @returns {Data} */
   load() {
-    const coursePath = path.join(this.sourcePath, this.courseDir).replace(/\\/g, '/');
-    this.languages = globs.sync(path.join(coursePath, '*/')).map(languagePath => {
+    this.languages = globs.sync(path.join(this.coursePath, '*/')).map(languagePath => {
       const language = new Language({
         framework: this.framework,
         languagePath,
@@ -73,7 +74,8 @@ class Data {
     }).filter(lang => lang.isValid);
     this.configFile = new JSONFile({
       framework: this.framework,
-      path: path.join(coursePath, `config.${this.jsonext}`)
+      path: path.join(this.coursePath, `config.${this.jsonext}`),
+      jsonext: this.jsonext
     });
     this.configFile.load();
     return this;

grunt/helpers/Framework.js

@@ -29,8 +29,8 @@ class Framework {
    */
   constructor({
     rootPath = process.cwd(),
-    outputPath = process.cwd() + '/build/',
-    sourcePath = process.cwd() + '/src/',
+    outputPath = path.join(rootPath, '/build/'),
+    sourcePath = path.join(rootPath, '/src/'),
     courseDir = 'course',
     includedFilter = function() { return true; },
     jsonext = 'json',
@@ -42,9 +42,9 @@ class Framework {
     /** @type {string} */
     this.rootPath = rootPath.replace(/\\/g, '/');
     /** @type {string} */
-    this.outputPath = outputPath.replace(/\\/g, '/');
+    this.outputPath = path.join(this.rootPath, outputPath).replace(/\\/g, '/');
     /** @type {string} */
-    this.sourcePath = sourcePath.replace(/\\/g, '/');
+    this.sourcePath = path.join(this.rootPath, sourcePath).replace(/\\/g, '/');
     /** @type {string} */
     this.courseDir = courseDir;
     /** @type {function} */
@@ -90,7 +90,8 @@ class Framework {
    * @returns {Data}
    */
   getData({
-    useOutputData = this.useOutputData
+    useOutputData = this.useOutputData,
+    performLoad = true
   } = {}) {
     const data = new Data({
       framework: this,
@@ -100,7 +101,7 @@ class Framework {
       trackingIdType: this.trackingIdType,
       log: this.log
     });
-    data.load();
+    if (performLoad) data.load();
     return data;
   }
 

grunt/helpers/data/Language.js

@@ -80,6 +80,7 @@ class Language {
       const file = new LanguageFile({
         framework: this.framework,
         language: this,
+        jsonext: this.jsonext,
         path: jsonFilePath,
         data: null,
         hasChanged: false

grunt/helpers/data/LanguageFile.js

@@ -15,17 +15,19 @@ class LanguageFile extends JSONFile {
    * @param {Framework} options.framework
    * @param {Language} options.language
    * @param {string} options.path
+   * @param {string} options.jsonext
    * @param {Object} options.data
    * @param {boolean} options.hasChanged
    */
   constructor({
     framework = null,
     language = null,
     path = null,
+    jsonext = null,
     data = null,
     hasChanged = false
   } = {}) {
-    super({ framework, path, data, hasChanged });
+    super({ framework, path, jsonext, data, hasChanged });
     /** @type {Language} */
     this.language = language;
   }

grunt/helpers/lib/JSONFile.js

@@ -22,13 +22,16 @@ class JSONFile {
   constructor({
     framework = null,
     path = null,
+    jsonext = null,
     data = null,
     hasChanged = false
   } = {}) {
     /** @type {Framework} */
     this.framework = framework;
     /** @type {string} */
     this.path = path;
+    /** @type {string} */
+    this.jsonext = jsonext;
     /** @type {Object|Array} */
     this.data = data;
     /** @type {boolean} */

grunt/tasks/migration.js

@@ -15,14 +15,16 @@ module.exports = function(grunt) {
     return {
       ...fileItem.item,
       __index__: fileItem.index,
-      __path__: unix(fileItem.file.path)
+      __path__: unix(fileItem.file.path),
+      __jsonext__: fileItem.file.jsonext
     };
   }
 
   function undressPathIndex(object) {
     const clone = { ...object };
     delete clone.__index__;
     delete clone.__path__;
+    delete clone.__jsonext__;
     return clone;
   }
 
@@ -35,17 +37,48 @@ module.exports = function(grunt) {
       const migrations = await import('adapt-migrations');
       const logger = migrations.Logger.getInstance();
       const cwd = process.cwd();
-      const outputPath = path.join(cwd, './migrations/');
+      const tempPath = path.join(cwd, './migrations/');
       const cache = new migrations.CacheManager();
       const cachePath = await cache.getCachePath({
         outputPath: buildConfig.outputdir,
-        tempPath: outputPath
+        tempPath
       });
 
-      const framework = Helpers.getFramework();
-      logger.debug(`Using ${framework.useOutputData ? framework.outputPath : framework.sourcePath} folder for course data...`);
+      if (mode === 'capture') {
+        const capturePath = grunt.option('capturedir') || tempPath;
+        if (!fs.existsSync(capturePath)) fs.mkdirSync(capturePath);
+        const fromFramework = Helpers.getFramework({
+          rootDir: path.resolve(grunt.option('rootdir') || process.cwd())
+        });
+        const fromPlugins = fromFramework.getPlugins().getAllPackageJSONFileItems().map(fileItem => fileItem.item);
+        const languages = fromFramework.getData().languages.map((language) => language.name);
+        const languageFile = path.join(capturePath, 'captureLanguages.json');
+        fs.writeJSONSync(languageFile, languages);
+        languages.forEach(async (language, index) => {
+          logger.debug(`Migration -- Capture ${language}`);
+          const data = fromFramework.getData();
+          // get all items from config.json file and all language files, append __index__ and __path__ to each item
+          const content = [
+            ...data.configFile.fileItems,
+            ...data.languages[index].getAllFileItems()
+          ].map(dressPathIndex).map(obj => {
+            // reduce __path__ to relative paths about src/course/ or build/course/ so
+            // that they're easier to restore elsewhere later
+            obj.__path__ = obj.__path__.slice(data.coursePath.length).replace(/^\//, '');
+            return obj;
+          });
+          const captured = await migrations.capture({ content, fromPlugins, logger });
+          const outputFile = path.join(capturePath, `capture_${language}.json`);
+          fs.writeJSONSync(outputFile, captured);
+        });
 
-      const plugins = framework.getPlugins().getAllPackageJSONFileItems().map(fileItem => fileItem.item);
+        logger.output(capturePath, 'capture');
+        return next();
+      }
+
+      const toFramework = Helpers.getFramework();
+      logger.debug(`Using ${toFramework.useOutputData ? toFramework.outputPath : toFramework.sourcePath} folder for course data...`);
+      const plugins = toFramework.getPlugins().getAllPackageJSONFileItems().map(fileItem => fileItem.item);
       const migrationScripts = Array.from(await new Promise(resolve => {
         globs([
           '*/*/migrations/**/*.js',
@@ -67,40 +100,20 @@ module.exports = function(grunt) {
         logger
       });
 
-      if (mode === 'capture') {
-
-        if (!fs.existsSync(outputPath)) fs.mkdirSync(outputPath);
-        const languages = framework.getData().languages.map((language) => language.name);
-        const languageFile = path.join(outputPath, 'captureLanguages.json');
-        fs.writeJSONSync(languageFile, languages);
-        languages.forEach(async (language, index) => {
-          logger.debug(`Migration -- Capture ${language}`);
-          const data = framework.getData();
-          // get all items from config.json file and all language files, append __index__ and __path__ to each item
-          const content = [
-            ...data.configFile.fileItems,
-            ...data.languages[index].getAllFileItems()
-          ].map(dressPathIndex);
-          const captured = await migrations.capture({ content, fromPlugins: plugins, logger });
-          const outputFile = path.join(outputPath, `capture_${language}.json`);
-          fs.writeJSONSync(outputFile, captured);
-        });
-
-        logger.output(outputPath, 'capture');
-        return next();
-      }
-
       if (mode === 'migrate') {
+        const capturePath = grunt.option('capturedir') || tempPath;
+        const toFrameworkData = toFramework.getData({ performLoad: false });
+        const coursePath = toFrameworkData.coursePath;
         try {
-          const languagesFile = path.join(outputPath, 'captureLanguages.json');
+          const languagesFile = path.join(capturePath, 'captureLanguages.json');
           const languages = fs.readJSONSync(languagesFile);
 
           for (const language of languages) {
             logger.debug(`Migration -- Migrate ${language}`);
             const Journal = migrations.Journal;
-            if (!fs.existsSync(outputPath)) fs.mkdirSync(outputPath);
-            const outputFile = path.join(outputPath, `capture_${language}.json`);
-            const { content, fromPlugins } = fs.readJSONSync(outputFile);
+            if (!fs.existsSync(capturePath)) fs.mkdirSync(capturePath);
+            const outputFile = path.join(capturePath, `capture_${language}.json`);
+            let { content, fromPlugins } = fs.readJSONSync(outputFile);
             const originalFromPlugins = JSON.parse(JSON.stringify(fromPlugins));
             const journal = new Journal({
               logger,
@@ -113,6 +126,11 @@ module.exports = function(grunt) {
             });
             await migrations.migrate({ journal, logger });
 
+            // change out jsonext
+            content = content.map(item => {
+              item.__path__ = item.__path__.replace('.' + item.__jsonext__, '.' + buildConfig.jsonext);
+              return item;
+            });
             // group all content items by path
             const outputFilePathItems = _.groupBy(content, '__path__');
             // sort items inside each path
@@ -127,13 +145,17 @@ module.exports = function(grunt) {
               const stripped = isSingleObject
                 ? undressPathIndex(outputItems[0]) // config.json, course.json
                 : outputItems.map(undressPathIndex); // contentObjects.json, articles.json, blocks.json, components.json
-              fs.writeJSONSync(outputPath, stripped, { replacer: null, spaces: 2 });
+              // write files to specified --outputdir= location
+              const outputFilePath = path.join(coursePath, outputPath);
+              const outputDir = path.parse(outputFilePath).dir;
+              fs.ensureDirSync(outputDir);
+              fs.writeJSONSync(outputFilePath, stripped, { replacer: null, spaces: 2 });
             });
           }
         } catch (error) {
           logger.error(error.stack);
         }
-        logger.output(outputPath, 'migrate');
+        logger.output(capturePath, 'migrate');
         return next();
       }