Version 0.2.9

Author: SferaDev

package.json

@@ -1,6 +1,6 @@
 {
     "name": "advanced-export",
-    "version": "0.2.8",
+    "version": "0.2.9",
     "private": true,
     "dependencies": {
         "@devexpress/dx-react-core": "1.4.0",
@@ -11,6 +11,7 @@
         "@material-ui/core": "^3.9.2",
         "@material-ui/icons": "^3.0.2",
         "axios": "^0.18.0",
+        "axios-retry": "^3.1.2",
         "babel-preset-es2015": "^6.24.1",
         "babel-preset-stage-2": "^6.24.1",
         "btoa": "^1.2.1",
@@ -48,7 +49,7 @@
     "manifest.webapp": {
         "name": "Advanced Export",
         "description": "Export App with recursive metadata fetching",
-        "version": "0.2.8",
+        "version": "0.2.9",
         "developer": {
             "name": "Alexis Rico",
             "url": "https://github.com/SferaDev"

src/index.js

@@ -68,7 +68,7 @@ function parseMetadataTypes(d2) {
         if (result.pager.hasNextPage()) result.pager.getNextPage().then(result => insertMetadata(model, result));
     };
     metadataTypes.forEach((model) => {
-        d2.models[model].list({paging: false, fields: ['id', 'displayName']}).then(result => {
+        d2.models[model].list({paging: false, fields: ['id', 'displayName', 'code']}).then(result => {
             insertMetadata(model, result);
             if (--parsedElements === 1) store.dispatch({type: actionTypes.LOADING, loading: false});
         }).catch(() => {

src/logic/extractor.js

@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import axios from "axios";
+import axiosRetry from 'axios-retry';
 import * as traverse from "traverse";
 import * as FileSaver from "file-saver";
 import moment from "moment";
@@ -9,7 +10,10 @@ import * as actionTypes from "../actions/actionTypes";
 import * as settingsAction from "../actions/settingsAction";
 import * as configuration from "./configuration";
 
+axiosRetry(axios, { retries: 10 });
+
 const timeout = ms => new Promise(res => setTimeout(res, ms));
+const mergeCustomizer = (obj, src) => _.isArray(obj) ? obj.concat(src) : src;
 
 export let Extractor = (function () {
     let instance;
@@ -50,29 +54,32 @@ ExtractorClass.prototype.initialFetchAndRetrieve = async function (elements) {
 };
 
 ExtractorClass.prototype.fetchAndRetrieve = async function (json) {
-    for (const type in json) {
-        if (Array.isArray(json[type])) {
-            let elements = json[type].filter(e => e.id !== undefined && e.code !== 'default');
-            for (const element of elements) {
-                // Insert on the metadata map
-                this.metadataMap.set(element.id, {...element, type} );
-
-                if (this.debug) console.log('fetchAndRetrieve: Parsing ' + element.id);
-
-                // Traverse references and call recursion
-                let references = await this.recursiveParse(element, this.d2.models[type].name);
-                let newJson = await this.parseElements(references);
-                await this.fetchAndRetrieve(newJson);
-            }
+    const metadataTypes = _.keys(json).filter(type => _.isArray(json[type]));
+    for (const metadataType of metadataTypes) {
+        let references = [];
+        let elements = json[metadataType].filter(e => e.id !== undefined);
+        if (this.debug) console.log('fetchAndRetrieve: Parsing ' + elements.map(e => e.id));
+
+        for (const element of elements) {
+            // Insert on the metadata map
+            this.metadataMap.set(element.id, {metadataType, ...element} );
+
+            // Traverse and store references
+            const innerReferences = await this.recursiveParse(element, this.d2.models[metadataType].name);
+            references.push(...innerReferences);
         }
+
+        // Call recursion
+        const newJson = await this.parseElements(references);
+        await this.fetchAndRetrieve(newJson);
     }
 };
 
 ExtractorClass.prototype.recursiveParse = async function (element, type) {
     let context = this;
     let references = [];
     traverse(element).forEach(function (item) {
-        if (this.isLeaf && this.key === 'id' && item !== '') {
+        if (this.isLeaf && this.key === 'id' && isValidUid(item)) {
             let parent = this.parent;
             while (parent.level > 1 && context.d2.models[parent.key] === undefined) parent = parent.parent;
             if (parent.key !== undefined) {
@@ -92,12 +99,13 @@ ExtractorClass.prototype.parseElements = async function (elementsArray) {
     let promises = [];
     for (let i = 0; i < elements.length; i += 100) {
         let requestUrl = this.d2.Api.getApi().baseUrl +
-            '/metadata.json?fields=:all&filter=id:in:[' + elements.slice(i, i + 100).toString() + ']';
+            '/metadata.json?fields=:all&defaults=EXCLUDE&filter=id:in:[' + elements.slice(i, i + 100).toString() + ']';
         if (this.debug) console.log('parseElements: ' + requestUrl);
         promises.push(axios.get(requestUrl));
     }
     let result = await Promise.all(promises);
-    return _.merge({}, ...result.map(result => result.data));
+    const data = result.map(result => result.data);
+    return _.mergeWith({}, ...data, mergeCustomizer);
 };
 
 ExtractorClass.prototype.handleCreatePackage = async function (elements, dependencies) {
@@ -121,7 +129,7 @@ ExtractorClass.prototype.createPackage = async function (elements, dependencies)
     for (const id of elementSet) {
         if (this.metadataMap.has(id)) {
             let element = this.metadataMap.get(id);
-            let elementType = this.d2.models[element.type].plural;
+            let elementType = this.d2.models[element.metadataType].plural;
             if (resultObject[elementType] === undefined) resultObject[elementType] = [];
             resultObject[elementType].push(cleanJson(element));
         } else if(this.debug) {
@@ -168,6 +176,7 @@ function cleanJson(json) {
     if (store.getState().settings[actionTypes.SETTINGS_USER_CLEAN_UP] === settingsAction.USER_CLEAN_UP_REMOVE_OPTION) {
         traverse(result).forEach(function (item) {
             if (this.key === 'user') this.update({});
+            if (this.key === 'users') this.update([]);
             if (this.key === 'userGroupAccesses') this.update([]);
             if (this.key === 'userAccesses') this.update([]);
             if (this.key === 'lastUpdatedBy') this.update({});
@@ -179,4 +188,9 @@ function cleanJson(json) {
         });
     }
     return result;
+}
+
+function isValidUid(code) {
+    const CODE_PATTERN = /^[a-zA-Z][a-zA-Z0-9]{10}$/;
+    return code !== null && CODE_PATTERN.test(code);
 }

yarn.lock

@@ -617,6 +617,13 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
   integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
 
+axios-retry@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.1.2.tgz#4f4dcbefb0b434e22b72bd5e28a027d77b8a3458"
+  integrity sha512-+X0mtJ3S0mmia1kTVi1eA3DAC+oWnT2A29g3CpkzcBPMT6vJm+hn/WiV9wPt/KXLHVmg5zev9mWqkPx7bHMovg==
+  dependencies:
+    is-retry-allowed "^1.1.0"
+
 axios@^0.18.0:
   version "0.18.0"
   resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102"
@@ -4756,7 +4763,7 @@ is-resolvable@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88"
   integrity sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==
 
-is-retry-allowed@^1.0.0:
+is-retry-allowed@^1.0.0, is-retry-allowed@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz#11a060568b67339444033d0125a61a20d564fb34"
   integrity sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=