Write data too large fix

package.json

@@ -27,7 +27,7 @@
     "clean": "node ./src/bin/clean.js",
     "tests": "mocha -r ts-node/register tests/**/*.spec.ts",
     "snyk-protect": "snyk protect",
-    "prepare": "npm run build && npm run snyk-protect"
+    "prepare": "npm run clean && tsc && npm run snyk-protect"
   },
   "bin": {
     "firestore-clear": "./dist/bin/firestore-clear.js",
@@ -62,5 +62,8 @@
     "ts-node": "^8.0.2",
     "typescript": "^3.1.6"
   },
+  "resolutions":{
+	  "@grpc/grpc-js":"1.6.7"
+  },
   "snyk": true
 }

src/bin/bin-common.ts

@@ -23,6 +23,12 @@ const commandLineParams: { [param: string]: Params } =
       args: '<path>',
       description: 'Filename to store backup. (e.g. backups/full-backup.json).',
     },
+    excludeNodePath: {
+      shortKey: 'e',
+      key: 'excludeNodePath',
+      args: '<path>',
+      description: `Path to database node (has to be a collection) ito exclude`,
+    },
     nodePath: {
       shortKey: 'n',
       key: 'nodePath',
@@ -44,11 +50,22 @@ const commandLineParams: { [param: string]: Params } =
       key: 'noWait',
       description: 'Use with unattended confirmation to remove the 5 second delay.',
     },
+    maxConcurrency: {
+      shortKey: 'm',
+      key: 'maxConcurrency',
+      args: '<maxConcurrency>',
+      description: `Maximum import concurrency to prevent bandwidth exhausted and other load errors. The default (0) is unlimited.`,
+    },
     prettyPrint: {
       shortKey: 'p',
       key: 'prettyPrint',
       description: 'JSON backups done with pretty-printing.',
     },
+    sortKeys: {
+      shortKey: 's',
+      key: 'sortKeys',
+      description: 'Sort keys to get a deterministic key order in the backup.',
+    },
   };
 
 const buildOption = ({shortKey, key, args = '', description}: Params): [string, string] => [`-${shortKey} --${key} ${args}`, description];

src/bin/firestore-clear.ts

@@ -13,6 +13,7 @@ import {
   commandLineParams as params,
   packageInfo,
 } from './bin-common';
+import {measureTimeAsync} from "../lib/helpers";
 
 
 commander.version(packageInfo.version)
@@ -64,7 +65,7 @@ const noWait = commander[params.yesToNoWait.key];
     await sleep(5000);
   }
   console.log(colors.bold(colors.green('Starting clearing of records 🏋️')));
-  await firestoreClear(pathReference, true);
+  await measureTimeAsync("firestore-clear", () => firestoreClear(pathReference, true));
   console.log(colors.bold(colors.green('All done 🎉')));
 })().catch((error) => {
   if (error instanceof ActionAbortedError) {

src/bin/firestore-export.ts

@@ -6,12 +6,15 @@ import fs from 'fs';
 import {firestoreExport} from '../lib';
 import {getCredentialsFromFile, getDBReferenceFromPath, getFirestoreDBReference} from '../lib/firestore-helpers';
 import {accountCredentialsEnvironmentKey, buildOption, commandLineParams as params, packageInfo} from './bin-common';
+import {measureTimeAsync, stableStringify} from "../lib/helpers";
 
 commander.version(packageInfo.version)
+  .option(...buildOption(params.excludeNodePath))
   .option(...buildOption(params.accountCredentialsPath))
   .option(...buildOption(params.backupFileExport))
   .option(...buildOption(params.nodePath))
   .option(...buildOption(params.prettyPrint))
+  .option(...buildOption(params.sortKeys))
   .parse(process.argv);
 
 const accountCredentialsPath = commander[params.accountCredentialsPath.key] || process.env[accountCredentialsEnvironmentKey];
@@ -47,16 +50,20 @@ const writeResults = (results: string, filename: string): Promise<string> => {
 };
 
 const prettyPrint = Boolean(commander[params.prettyPrint.key]);
+const sortKeys = Boolean(commander[params.sortKeys.key]);
 const nodePath = commander[params.nodePath.key];
+const excludeNodePath = commander[params.excludeNodePath.key];
 
 (async () => {
   const credentials = await getCredentialsFromFile(accountCredentialsPath);
   const db = getFirestoreDBReference(credentials);
   const pathReference = getDBReferenceFromPath(db, nodePath);
   console.log(colors.bold(colors.green('Starting Export 🏋️')));
-  const results = await firestoreExport(pathReference, true);
-  const stringResults = JSON.stringify(results, undefined, prettyPrint ? 2 : undefined);
-  await writeResults(stringResults, backupFile);
+  await measureTimeAsync("firestore-export", async () => {
+  const results = await firestoreExport(pathReference, true, excludeNodePath);
+  const stringResults = stringify(results, prettyPrint ? 2 : undefined);
+    await writeResults(stringResults, backupFile);
+  });
   console.log(colors.yellow(`Results were saved to ${backupFile}`));
   console.log(colors.bold(colors.green('All done 🎉')));
 })().catch((error) => {
@@ -68,5 +75,11 @@ const nodePath = commander[params.nodePath.key];
   }
 });
 
+function stringify(results: unknown, space?: number): string {
+  if (sortKeys) {
+    return stableStringify(results, space)
+  }
+  return JSON.stringify(results, undefined, space);
+}
 
 

src/bin/firestore-import.ts

@@ -14,12 +14,14 @@ import {
   commandLineParams as params,
   packageInfo,
 } from './bin-common';
+import {measureTimeAsync} from "../lib/helpers";
 
 commander.version(packageInfo.version)
   .option(...buildOption(params.accountCredentialsPath))
   .option(...buildOption(params.backupFileImport))
   .option(...buildOption(params.nodePath))
   .option(...buildOption(params.yesToImport))
+  .option(...buildOption(params.maxConcurrency))
   .parse(process.argv);
 
 const accountCredentialsPath = commander[params.accountCredentialsPath.key] || process.env[accountCredentialsEnvironmentKey];
@@ -52,6 +54,8 @@ const nodePath = commander[params.nodePath.key];
 
 const unattendedConfirmation = commander[params.yesToImport.key];
 
+const maxConcurrency = parseInt(commander[params.maxConcurrency.key]) || 0;
+
 (async () => {
   const credentials = await getCredentialsFromFile(accountCredentialsPath);
   const db = getFirestoreDBReference(credentials);
@@ -79,7 +83,7 @@ const unattendedConfirmation = commander[params.yesToImport.key];
   }
 
   console.log(colors.bold(colors.green('Starting Import 🏋️')));
-  await firestoreImport(data, pathReference, true, true);
+  await measureTimeAsync("firestore-import", () => firestoreImport(data, pathReference, true, maxConcurrency, true));
   console.log(colors.bold(colors.green('All done 🎉')));
 })().catch((error) => {
   if (error instanceof ActionAbortedError) {

src/lib/export.ts

@@ -10,9 +10,9 @@ import {serializeSpecialTypes} from './helpers';
 
 const exportData = async (startingRef: admin.firestore.Firestore |
   FirebaseFirestore.DocumentReference |
-  FirebaseFirestore.CollectionReference, logs = false) => {
+  FirebaseFirestore.CollectionReference, logs = false, excludeNodePath:string) => {
   if (isLikeDocument(startingRef)) {
-    const collectionsPromise = getCollections(startingRef, logs);
+    const collectionsPromise = getCollections(startingRef, logs, excludeNodePath);
     let dataPromise: Promise<any>;
     if (isRootOfDatabase(startingRef)) {
       dataPromise = Promise.resolve({});
@@ -25,17 +25,17 @@ const exportData = async (startingRef: admin.firestore.Firestore |
       return {'__collections__': res[0], ...res[1]};
     });
   } else {
-    return await getDocuments(<FirebaseFirestore.CollectionReference>startingRef, logs);
+    return await getDocuments(<FirebaseFirestore.CollectionReference>startingRef, logs, excludeNodePath);
   }
 };
 
-const getCollections = async (startingRef: admin.firestore.Firestore | FirebaseFirestore.DocumentReference, logs = false) => {
+const getCollections = async (startingRef: admin.firestore.Firestore | FirebaseFirestore.DocumentReference, logs = false, excludeNodePath:string) => {
   const collectionNames: Array<string> = [];
   const collectionPromises: Array<Promise<any>> = [];
   const collectionsSnapshot = await safelyGetCollectionsSnapshot(startingRef, logs);
   collectionsSnapshot.map((collectionRef: FirebaseFirestore.CollectionReference) => {
     collectionNames.push(collectionRef.id);
-    collectionPromises.push(getDocuments(collectionRef, logs));
+    collectionPromises.push(getDocuments(collectionRef, logs, excludeNodePath));
   });
   const results = await batchExecutor(collectionPromises);
   const zipped: any = {};
@@ -45,9 +45,15 @@ const getCollections = async (startingRef: admin.firestore.Firestore | FirebaseF
   return zipped;
 };
 
-const getDocuments = async (collectionRef: FirebaseFirestore.CollectionReference, logs = false) => {
+const getDocuments = async (collectionRef: FirebaseFirestore.CollectionReference, logs = false,
+  excludeNodePath:string
+  ) => {
   logs && console.log(`Retrieving documents from ${collectionRef.path}`);
   const results: any = {};
+  if (!!excludeNodePath && collectionRef.path.startsWith(excludeNodePath)) {
+    logs && console.log("stopping at:", collectionRef.path)
+    return results;
+  }
   const documentPromises: Array<Promise<object>> = [];
   const allDocuments = await safelyGetDocumentReferences(collectionRef, logs);
   allDocuments.forEach((doc) => {
@@ -60,7 +66,7 @@ const getDocuments = async (collectionRef: FirebaseFirestore.CollectionReference
       } else {
         docDetails[docSnapshot.id] = {};
       }
-      docDetails[docSnapshot.id]['__collections__'] = await getCollections(docSnapshot.ref, logs);
+      docDetails[docSnapshot.id]['__collections__'] = await getCollections(docSnapshot.ref, logs, excludeNodePath);
       resolve(docDetails);
     }));
   });

src/lib/helpers.ts

@@ -4,6 +4,7 @@ import {IGeopoint} from '../interfaces/IGeopoint';
 import {IDocumentReference} from '../interfaces/IDocumentReference';
 import DocumentReference = admin.firestore.DocumentReference;
 import GeoPoint = admin.firestore.GeoPoint;
+import Timeout = NodeJS.Timeout;
 
 // From https://stackoverflow.com/questions/8495687/split-array-into-chunks
 const array_chunks = (array: Array<any>, chunk_size: number): Array<Array<any>> => {
@@ -93,4 +94,109 @@ const isScalar = (val: any) => (typeof val === 'string' || val instanceof String
   || (val === null)
   || (typeof val === 'boolean');
 
-export {array_chunks, serializeSpecialTypes, unserializeSpecialTypes};
+
+// Reduced and typed version of https://github.com/substack/json-stable-stringify
+const stableStringify = (obj: any, space?: number) => {
+  const spaceString = space ? Array(space+1).join(' ') : '';
+  const seen: any[] = [];
+  return (function stringify (parent: any, node: any, level: number): string {
+    const indent = spaceString ? ('\n' + new Array(level + 1).join(spaceString)) : '';
+    const colonSeparator = spaceString ? ': ' : ':';
+
+    if (node && node.toJSON && typeof node.toJSON === 'function') {
+      node = node.toJSON();
+    }
+
+    if (typeof node !== 'object' || node === null) {
+      return JSON.stringify(node);
+    }
+
+    if (Array.isArray(node)) {
+      const out = node.map(item => {
+        const itemString = stringify(node,  item, level+1) || JSON.stringify(null);
+        return indent + spaceString + itemString
+      })
+      return '[' + out.join(',') + indent + ']';
+    }
+    else {
+      if (seen.indexOf(node) !== -1) {
+        throw new TypeError('Converting circular structure to JSON');
+      }
+      else seen.push(node);
+
+      const keys = Object.keys(node).sort();
+
+      const out = keys.map(key => {
+        const value = stringify(node, node[key], level+1);
+
+        if(!value) return;
+
+        const keyValue = JSON.stringify(key)
+          + colonSeparator
+          + value;
+        return indent + spaceString + keyValue;
+      }).filter(value => value !== undefined);
+
+      seen.splice(seen.indexOf(node), 1);
+      return '{' + out.join(',') + indent + '}';
+    }
+  })({ '': obj }, obj, 0);
+};
+
+interface ConcurrencyLimit {
+ wait(): Promise<void>;
+ done(): void
+}
+
+function limitConcurrency(maxConcurrency: number = 0, interval: number = 10): ConcurrencyLimit {
+  if (maxConcurrency === 0) {
+    return {
+      async wait(): Promise<void> { },
+      done() { }
+    }
+  }
+  let unfinishedCount = 0;
+  let resolveQueue: Function[] = [];
+  let intervalId: Timeout;
+  let started = false;
+
+  function start() {
+    started = true;
+    intervalId = setInterval(() => {
+      if (resolveQueue.length === 0) {
+        started = false;
+        clearInterval(intervalId);
+        return;
+      }
+
+      while (unfinishedCount <= maxConcurrency && resolveQueue.length > 0) {
+        const resolveFn = resolveQueue.shift();
+        unfinishedCount++;
+        if (resolveFn) resolveFn();
+      }
+
+    }, interval);
+  }
+
+  return {
+    wait(): Promise<void> {
+      return new Promise(resolve => {
+        if (!started) start();
+        resolveQueue.push(resolve)
+      });
+    },
+    done() {
+      unfinishedCount--;
+    }
+  }
+}
+
+const measureTimeAsync = async <T>( info: string, fn: () => Promise<T>): Promise<T> => {
+  const startTime = Date.now();
+  const result = await fn();
+  const timeDiff = Date.now() - startTime;
+  console.log(`${info} took ${timeDiff}ms`);
+  return result;
+}
+
+export {array_chunks, serializeSpecialTypes, unserializeSpecialTypes, ConcurrencyLimit, limitConcurrency, measureTimeAsync, stableStringify};