diff --git a/lib/sync/dataAdapters/index.js b/lib/sync/dataAdapters/index.js index 3f39e5c..73c69b4 100644 --- a/lib/sync/dataAdapters/index.js +++ b/lib/sync/dataAdapters/index.js @@ -1,66 +1,98 @@ -const mysqlDataAdapter = require('./mysql/mysql') -const solrDataAdapter = require('./solr/solr') -const devnullDataAdapter = require('./devnull') +const mysqlDataAdapter = require("./mysql/mysql"); +const mongodbDataAdapter = require("./mongoDB/mongodb"); +const solrDataAdapter = require("./solr/solr"); +const devnullDataAdapter = require("./devnull"); // I'm not happy about this. Previously, I was using require() in the buildDataAdapter() method below. // As in, I was doing conditional requires. Jest tests would break with the message MODULE NOT FOUND. // I think it was happening because require() was being called after a test had started, so it wasn't // using the real require() but rather Jest's monkey-patched version. I don't want to fight it right now. -const bridgeInteractivePlatformAdapter = require('../platformAdapters/bridgeInteractive') -const trestlePlatformAdapter = require('../platformAdapters/trestle') -const mlsGridPlatformAdapter = require('../platformAdapters/mlsGrid') -const rmlsPlatformAdapter = require('../platformAdapters/rmls') -const bridgeInteractiveMysqlPlatformDataAdapter = require('../platformDataAdapters/bridgeInteractive/mysql') -const trestleMysqlPlatformDataAdapter = require('../platformDataAdapters/trestle/mysql') -const mlsGridMysqlPlatformDataAdapter = require('../platformDataAdapters/mlsGrid/mysql') -const rmlsMysqlPlatformDataAdapter = require('../platformDataAdapters/rmls/mysql') +const bridgeInteractivePlatformAdapter = require("../platformAdapters/bridgeInteractive"); +const trestlePlatformAdapter = require("../platformAdapters/trestle"); +const mlsGridPlatformAdapter = require("../platformAdapters/mlsGrid"); +const rmlsPlatformAdapter = require("../platformAdapters/rmls"); +const bridgeInteractiveMysqlPlatformDataAdapter = require("../platformDataAdapters/bridgeInteractive/mysql"); +const bridgeInteractiveMongodbPlatformDataAdapter = require("../platformDataAdapters/bridgeInteractive/mongodb"); +const trestleMysqlPlatformDataAdapter = require("../platformDataAdapters/trestle/mysql"); +const mlsGridMysqlPlatformDataAdapter = require("../platformDataAdapters/mlsGrid/mysql"); +const rmlsMysqlPlatformDataAdapter = require("../platformDataAdapters/rmls/mysql"); function buildDataAdapter({ destinationConfig, platformAdapterName }) { - const dataAdapterType = destinationConfig.type - let dataAdapter - if (dataAdapterType === 'mysql') { - dataAdapter = mysqlDataAdapter({ destinationConfig: destinationConfig.config }) - } else if (dataAdapterType === 'solr') { - dataAdapter = solrDataAdapter({ destinationConfig: destinationConfig.config }) - } else if (dataAdapterType === 'devnull') { - dataAdapter = devnullDataAdapter() + const dataAdapterType = destinationConfig.type; + let dataAdapter; + if (dataAdapterType === "mysql") { + dataAdapter = mysqlDataAdapter({ + destinationConfig: destinationConfig.config + }); + } + // ------- Updated to support MongoDB + if (dataAdapterType === "mongodb") { + dataAdapter = mongodbDataAdapter({ + destinationConfig: destinationConfig.config + }); + } else if (dataAdapterType === "solr") { + dataAdapter = solrDataAdapter({ + destinationConfig: destinationConfig.config + }); + } else if (dataAdapterType === "devnull") { + dataAdapter = devnullDataAdapter(); } else { - throw new Error('Unknown data adapter: ' + dataAdapterType) + throw new Error("Unknown data adapter: " + dataAdapterType); } - let platformAdapter - if (platformAdapterName === 'bridgeInteractive') { - platformAdapter = bridgeInteractivePlatformAdapter() - } else if (platformAdapterName === 'trestle') { - platformAdapter = trestlePlatformAdapter() - } else if (platformAdapterName === 'mlsGrid') { - platformAdapter = mlsGridPlatformAdapter() - } else if (platformAdapterName === 'rmls') { - platformAdapter = rmlsPlatformAdapter() + let platformAdapter; + if (platformAdapterName === "bridgeInteractive") { + platformAdapter = bridgeInteractivePlatformAdapter(); + } else if (platformAdapterName === "trestle") { + platformAdapter = trestlePlatformAdapter(); + } else if (platformAdapterName === "mlsGrid") { + platformAdapter = mlsGridPlatformAdapter(); + } else if (platformAdapterName === "rmls") { + platformAdapter = rmlsPlatformAdapter(); } else { - throw new Error('Unknown platform adapter: ' + platformAdapterName) + throw new Error("Unknown platform adapter: " + platformAdapterName); } - dataAdapter.setPlatformAdapter(platformAdapter) + dataAdapter.setPlatformAdapter(platformAdapter); - let platformDataAdapter - if (dataAdapterType === 'mysql') { - if (platformAdapterName === 'bridgeInteractive') { - platformDataAdapter = bridgeInteractiveMysqlPlatformDataAdapter() - } else if (platformAdapterName === 'trestle') { - platformDataAdapter = trestleMysqlPlatformDataAdapter() - } else if (platformAdapterName === 'mlsGrid') { - platformDataAdapter = mlsGridMysqlPlatformDataAdapter() - } else if (platformAdapterName === 'rmls') { - platformDataAdapter = rmlsMysqlPlatformDataAdapter() + let platformDataAdapter; + if (dataAdapterType === "mysql") { + if (platformAdapterName === "bridgeInteractive") { + platformDataAdapter = bridgeInteractiveMysqlPlatformDataAdapter(); + } else if (platformAdapterName === "trestle") { + platformDataAdapter = trestleMysqlPlatformDataAdapter(); + } else if (platformAdapterName === "mlsGrid") { + platformDataAdapter = mlsGridMysqlPlatformDataAdapter(); + } else if (platformAdapterName === "rmls") { + platformDataAdapter = rmlsMysqlPlatformDataAdapter(); } else { - throw new Error(`Unknown platform adapter and data adapter combo: ${platformAdapterName}, ${dataAdapterType}`) + throw new Error( + `Unknown platform adapter and data adapter combo: ${platformAdapterName}, ${dataAdapterType}` + ); + } + dataAdapter.setPlatformDataAdapter(platformDataAdapter); + } else if (dataAdapterType === "mongodb") { + if (platformAdapterName === "bridgeInteractive") { + platformDataAdapter = bridgeInteractiveMongodbPlatformDataAdapter(); } - dataAdapter.setPlatformDataAdapter(platformDataAdapter) - } - return dataAdapter + //--------Pending for now----- + // else if (platformAdapterName === "trestle") { + // platformDataAdapter = trestleMysqlPlatformDataAdapter(); + // } else if (platformAdapterName === "mlsGrid") { + // platformDataAdapter = mlsGridMysqlPlatformDataAdapter(); + // } else if (platformAdapterName === "rmls") { + // platformDataAdapter = rmlsMysqlPlatformDataAdapter(); + // } + else { + throw new Error( + `Unknown platform adapter and data adapter combo: ${platformAdapterName}, ${dataAdapterType}` + ); + } + dataAdapter.setPlatformDataAdapter(platformDataAdapter); + } + return dataAdapter; } module.exports = { - buildDataAdapter, -} + buildDataAdapter +}; diff --git a/lib/sync/dataAdapters/mongoDB/mongodb.js b/lib/sync/dataAdapters/mongoDB/mongodb.js new file mode 100644 index 0000000..dbf95f5 --- /dev/null +++ b/lib/sync/dataAdapters/mongoDB/mongodb.js @@ -0,0 +1,490 @@ +const { MongoClient } = require("mongodb"); +const _ = require("lodash"); + +const moment = require("moment"); +const { + getPrimaryKeyField, + getTimestampFields, + shouldIncludeField, + getFieldNamesThatShouldBeIncluded +} = require("../../utils"); +const { Worker } = require("worker_threads"); +const pathLib = require("path"); + +module.exports = ({ destinationConfig }) => { + let platformAdapter; + let platformDataAdapter; + + let client = new MongoClient(destinationConfig.connectionString); + client.connect(); + const userMakeCollectionName = destinationConfig.makeCollectionName; + const userMakeFieldName = destinationConfig.makeFieldName; + const userMakeForeignKeyFieldName = destinationConfig.makeForeignKeyFieldName; + const userTransform = destinationConfig.transform; + const userShouldSyncTableSchema = destinationConfig.shouldSyncTableSchema; + const userMakePrimaryKeyValueForDestination = + destinationConfig.makePrimaryKeyValueForDestination; + const userMakePrimaryKeyValueForMls = + destinationConfig.makePrimaryKeyValueForMls; + + async function setPlatformAdapter(adapter) { + platformAdapter = adapter; + } + + function setPlatformDataAdapter(adapter) { + platformDataAdapter = adapter; + } + + + async function syncStructure(mlsResourceObj, metadata) { + const schemas = metadata["edmx:Edmx"]["edmx:DataServices"][0].Schema; + const entityTypes = platformAdapter.getEntityTypes(schemas); + const collections = await client + .db() + .listCollections() + .toArray(); + const collectionNames = collections.map(x => x.name); + if (shouldSyncTableSchema(mlsResourceObj.name)) { + await effectCollection( + mlsResourceObj, + collectionNames, + entityTypes, + client + ); + } + // if (mlsResourceObj.expand) { + // for (const subMlsResourceObj of mlsResourceObj.expand) { + // if (shouldSyncTableSchema(subMlsResourceObj.name)) { + // await effectCollection(subMlsResourceObj, tableNames, entityTypes) + // } + // } + // } + } + + async function effectCollection( + mlsResourceObj, + collectionNames, + entityTypes, + client + ) { + const collectionName = makeCollectionName(mlsResourceObj.name); + const entityType = entityTypes.find(x => x.$.Name === mlsResourceObj.name); + const indexes = platformAdapter.getIndexes(mlsResourceObj.name); + if (!collectionNames.includes(collectionName)) { + await createCollection(mlsResourceObj, entityType, indexes, client); + await createIndexes(collectionName, indexes, mlsResourceObj.name); + // await syncTableFields(mlsResourceObj, entityType, indexes); + } + } + + async function createCollection(mlsResourceObj, entityType, indexes, client) { + const collectionName = makeCollectionName(mlsResourceObj.name); + return await client.db().createCollection(collectionName); + } + + async function createIndexes(collectionName, indexesToAdd, mlsResourceName) { + for (const [indexName, indexProps] of Object.entries(indexesToAdd)) { + return client + .db() + .collection(collectionName) + .createIndex( + { + [indexName]: 1 + }, + { + unique: indexProps.isPrimary ? true : false + } + ); + } + } + + function makeName(name) { + return name; + } + + function makeCollectionName(name) { + return userMakeCollectionName + ? userMakeCollectionName(name) + : makeName(name); + } + + function makeFieldName(mlsResourceName, name) { + return userMakeFieldName + ? userMakeFieldName(mlsResourceName, name) + : makeName(name); + } + + function makeForeignKeyFieldName( + parentMlsResourceName, + mlsResourceName, + name + ) { + return userMakeForeignKeyFieldName + ? userMakeForeignKeyFieldName( + parentMlsResourceName, + mlsResourceName, + name + ) + : makeName(name); + } + + function transform( + mlsResourceName, + record, + metadata, + parentTransformedMlsData, + cache + ) { + return userTransform + ? userTransform( + mlsResourceName, + record, + metadata, + parentTransformedMlsData, + cache + ) + : record; + } + + function shouldSyncTableSchema(mlsResourceName) { + return userShouldSyncTableSchema + ? userShouldSyncTableSchema(mlsResourceName) + : true; + } + + // The intent is originally for MLS Grid, whose primary keys look like e.g. RTC123: the primary key of 123 prefixed + // with the MLS abbreviation of RTC for Realtracs, but you might want just the 123 part in your destination. + function makePrimaryKeyValueForDestination(mlsResourceName, id) { + return userMakePrimaryKeyValueForDestination + ? userMakePrimaryKeyValueForDestination(mlsResourceName, id) + : id; + } + + // This is basically the opposite of makePrimaryKeyValueForDestination. + function makePrimaryKeyValueForMls(mlsResourceName, id) { + return userMakePrimaryKeyValueForMls + ? userMakePrimaryKeyValueForMls(mlsResourceName, id) + : id; + } + + + async function syncData( + mlsResourceObj, + mlsData, + metadata, + transaction = null, + mapFromSubToTransformedParent = null + ) { + if (!mlsData.length) { + return; + } + // TODO: Now that we have the metadata passed to us, should use it to know field types rather than guessing by the + // field name. + for (const d of mlsData) { + for (const key in d) { + if (key.endsWith("Timestamp") && d[key]) { + d[key] = moment.utc(d[key]).format("YYYY-MM-DD HH:mm:ss.SSS"); + } + } + } + + const collectionName = makeCollectionName(mlsResourceObj.name); + const indexes = platformAdapter.getIndexes(mlsResourceObj.name); + + const fieldNames = getFieldNamesThatShouldBeIncluded( + mlsResourceObj, + metadata, + indexes, + shouldIncludeField, + platformAdapter, + makeFieldName + ); + // The "cache" is an (initially empty) object that we pass each time to the transform function. This allows the + // transform function to e.g. do lookup work when it chooses, storing it on the cache object. For example, it could + // do it all on the first pass and not again, or it could potentially do it only on-demand somehow. But we don't + // have to force it to do it at any particular time. + const cache = {}; + const transformedMlsData = mlsData.map(x => { + const val = _.pick(x, fieldNames); + const transformedParentRecord = mapFromSubToTransformedParent + ? mapFromSubToTransformedParent.get(x) + : null; + const transformedRecord = transform( + mlsResourceObj.name, + val, + metadata, + transformedParentRecord, + cache + ); + // I'm probably doing something wrong, but Knex doesn't seem to handle array values properly. For example, if the + // transformed record has these values: { ListingId: 'abc123', MyColumn: ['hello', 'world'] }, it would make an + // insert statement like this: insert into MyTable (ListingId, MyColumn) values ('abc123', 'hello', 'world'), + // which produces the error: Column count doesn't match value count at row 1. + // To fix, we assume any value that is an object is meant for a JSON field, so we stringify it. + return transformedRecord; + }); + + function upsert() { + return client + .db() + .collection(collectionName) + .insertMany(transformedMlsData); + } + + if (transaction) { + await upsert(); + } else { + await upsert(); + + for (const subMlsResourceObj of mlsResourceObj.expand || []) { + // Delete the records of the expanded resource. We then re-sync them with the recursive call to syncData. + if (subMlsResourceObj.purgeFromParent) { + const officialFieldName = getPrimaryKeyField( + mlsResourceObj.name, + indexes + ); + const idsToPurge = mlsData.map(x => x[officialFieldName]); + const modifiedIdsToPurge = idsToPurge.map(x => + makePrimaryKeyValueForDestination(mlsResourceObj.name, x) + ); + await purgeFromParent( + mlsResourceObj.name, + subMlsResourceObj.name, + modifiedIdsToPurge, + officialFieldName, + t + ); + } + // MLS Grid doesn't have foreign keys on the subresource data, which we need for relational data. So we create + // a map here, from sub to parent, so that it can be used later. The transformed parent record will be passed + // to the user transform function (if supplied). We could also pass the original/non-transformed parent, but + // we don't yet just because I haven't needed it. + const subData = []; + const mapFromSubToTransformedParent = new Map(); + for (let i = 0; i < mlsData.length; i++) { + const parentRecord = mlsData[i]; + const subRecords = parentRecord[subMlsResourceObj.fieldName]; + if (!subRecords) { + continue; + } + subData.push(...subRecords); + const transformedParent = transformedMlsData[i]; + for (const subRecord of subRecords) { + mapFromSubToTransformedParent.set(subRecord, transformedParent); + } + } + await syncData( + subMlsResourceObj, + subData, + metadata, + t, + mapFromSubToTransformedParent + ); + } + } + } + + async function getTimestamps(mlsResourceName, indexes) { + const collectionName = makeCollectionName(mlsResourceName); + const updateTimestampFields = _.pickBy(indexes, v => v.isUpdateTimestamp); + const fieldsString = _.map(updateTimestampFields, (v, k) => + makeFieldName(mlsResourceName, k) + ); + + const rows = await client + .db() + .collection(collectionName) + .find({}) + .sort({ [fieldsString[0]]: -1 }) + .limit(1) + .toArray(); + return _.mapValues( + updateTimestampFields, + (v, k) => (rows.length && rows[0][k]) || new Date(0) + ); + } + + async function getAllMlsIds(mlsResourceName, indexes) { + const collectionName = makeCollectionName(mlsResourceName); + const officialFieldName = getPrimaryKeyField(mlsResourceName, indexes); + const userFieldName = makeFieldName(mlsResourceName, officialFieldName); + const rows = await client + .db() + .collection(collectionName) + .find({}, { [userFieldName]: 1 }) + .sort({ [userFieldName]: -1 }); + const ids = rows.map(x => + makePrimaryKeyValueForMls(mlsResourceName, x[userFieldName]) + ); + return ids; + } + + async function getCount(mlsResourceName) { + const collectionName = makeCollectionName(mlsResourceName); + const count = await client + .db() + .collection(collectionName) + .count({}); + return count; + } + + async function getMostRecentTimestamp(mlsResourceName) { + const collectionName = makeCollectionName(mlsResourceName); + const rows = await client + .db() + .collection(collectionName) + .find({}, { ModificationTimestamp: 1 }) + .sort({ ModificationTimestamp: -1 }) + .limit(1) + .toArray(); + if (!rows.length) { + return null; + } + return rows[0].ModificationTimestamp; + } + + async function purge(mlsResourceObj, mlsIdsToPurge, getIndexes) { + const mlsResourceName = mlsResourceObj.name; + const collectionName = makeCollectionName(mlsResourceName); + const indexes = getIndexes(mlsResourceName); + const officialFieldName = getPrimaryKeyField(mlsResourceName, indexes); + const userFieldName = makeFieldName(mlsResourceName, officialFieldName); + const modifiedIdsToPurge = mlsIdsToPurge.map(x => + makePrimaryKeyValueForDestination(mlsResourceName, x) + ); + + await client + .db() + .collection(collectionName) + .deleteMany({ [userFieldName]: { $in: [modifiedIdsToPurge] } }); + + if (mlsResourceObj.expand) { + for (const expandedMlsResourceObj of mlsResourceObj.expand) { + if (expandedMlsResourceObj.purgeFromParent) { + await purgeFromParent( + mlsResourceObj.name, + expandedMlsResourceObj.name, + modifiedIdsToPurge, + officialFieldName + ); + } + } + } + } + + // I'm not loving this way of doing things. But to explain it: + // My current use case is to purge Media records that were originally synced as part of syncing Property records with + // the expand feature. We need to delete from the Media table using the ResourceRecordKey field (or, what the user + // maps it to in their table), using the parentIds from the parent table (Property). + async function purgeFromParent( + parentMlsResourceName, + mlsResourceName, + parentIds, + officialFieldName + ) { + const collectionName = makeCollectionName(mlsResourceName); + const userFieldName = makeForeignKeyFieldName( + parentMlsResourceName, + mlsResourceName, + officialFieldName + ); + + return client + .db() + .collection(collectionName) + .deleteMany({ [userFieldName]: { $in: parentIds } }); + } + + async function closeConnection() { + return client.close(); + } + + // "Missing IDs data" means that the goal is to understand which records are not up to date in a reconcile process. + // So to do that, we look at fields like ModificationTimestamp, PhotosChangeTimestamp, etc. It's those multiple fields + // that we look at that I'm calling the "data". + async function fetchMissingIdsData(mlsResourceName, indexes) { + const collectionName = makeCollectionName(mlsResourceName); + const officialFieldName = getPrimaryKeyField(mlsResourceName, indexes); + const userFieldName = makeFieldName(mlsResourceName, officialFieldName); + const timestampFieldNames = getTimestampFields(mlsResourceName, indexes); + const mongoTimestampFieldNames = timestampFieldNames.map(x => + makeFieldName(mlsResourceName, x) + ); + + const fieldNamesMongo = [userFieldName, ...mongoTimestampFieldNames].reduce( + (prev, curr) => { + prev[curr] = 1; + return prev; + }, + {} + ); + const rows = await client + .db() + .collection(collectionName) + .find({}, fieldNamesMongo) + .sort({ [userFieldName]: -1 }) + .toArray(); + return rows; + } + + function computeMissingIds( + mlsResourceName, + dataInMls, + dataInAdapter, + indexes + ) { + const officialFieldName = getPrimaryKeyField(mlsResourceName, indexes); + const userFieldName = makeFieldName(mlsResourceName, officialFieldName); + const timestampFieldNames = getTimestampFields(mlsResourceName, indexes); + const mongoDBTimestampFieldNames = timestampFieldNames.map(x => + makeFieldName(mlsResourceName, x) + ); + const workerPath = pathLib.resolve(__dirname, "worker.js"); + // We copy the data, so we can (possibly) transform the IDs (originally for MLS Grid). I don't love this idea due to + // the extra memory use. But we can't pass the makePrimaryKeyValueForDestination function to the worker (node throws a + // DataCloneError). The good news is it's only the index/timestamp fields per record. + const modifiedDataInMls = _.cloneDeep(dataInMls); + for (const record of modifiedDataInMls) { + record[officialFieldName] = makePrimaryKeyValueForDestination( + mlsResourceName, + record[officialFieldName] + ); + } + return new Promise((resolve, reject) => { + const worker = new Worker(workerPath, { + workerData: { + dataInAdapter: dataInAdapter, + userFieldName, + dataInMls: modifiedDataInMls, + officialFieldName, + timestampFieldNames, + mongoDBTimestampFieldNames + } + }); + worker.on("message", missingOrOldIds => { + const missingOrOldIdsForMls = missingOrOldIds.map(x => + makePrimaryKeyValueForMls(mlsResourceName, x) + ); + resolve(missingOrOldIdsForMls); + }); + worker.on("error", error => { + reject(error); + }); + }); + } + + return { + syncStructure, + syncData, + getTimestamps, + closeConnection, + setPlatformAdapter, + setPlatformDataAdapter, + getAllMlsIds, + purge, + getCount, + getMostRecentTimestamp, + fetchMissingIdsData, + computeMissingIds + }; +}; diff --git a/lib/sync/dataAdapters/mongoDB/worker.js b/lib/sync/dataAdapters/mongoDB/worker.js new file mode 100644 index 0000000..f9b6a59 --- /dev/null +++ b/lib/sync/dataAdapters/mongoDB/worker.js @@ -0,0 +1,51 @@ +const { parentPort, workerData } = require('worker_threads') + +// Reminder to self: +// The reason I'm not using parentPort.on('message'), even though it seems like the more flexible / future-proof +// approach is that because if I throw in an error in that, it doesn't make its way back to the parent. There's probably +// a way, but I didn't see it initially looking through the Node 12 docs, so this way seems good enough for now. + +const { + dataInAdapter, + userFieldName, + dataInMls, + officialFieldName, + timestampFieldNames, + mongoDBTimestampFieldNames, +} = workerData; + +const rowsObj = dataInAdapter.reduce((a, v) => { + a[v[userFieldName]] = v + return a +}, {}) + +const missingOrOldObjs = dataInMls.reduce((a, v, index) => { + const id = v[officialFieldName] + const dbObj = rowsObj[id] + if (dbObj) { + for (let i = 0; i < timestampFieldNames.length; i++) { + // If both are null, we consider them equal + if (v[timestampFieldNames[i]] === null && dbObj[mongoDBTimestampFieldNames[i]] === null) { + continue + } + // But at this point, if either are null, then they are out of sync + if (v[timestampFieldNames[i]] === null || dbObj[mongoDBTimestampFieldNames[i]] === null) { + a.push(v) + break + } + const mlsTimestamp = new Date(v[timestampFieldNames[i]]) + const dbTimestamp = dbObj[mongoDBTimestampFieldNames[i]] + const equal = mlsTimestamp.getTime() === new Date(dbTimestamp).getTime() + if (!equal) { + a.push(v) + break + } + } + } else { + a.push(v) + } + return a +}, []) + +const missingOrOldIds = missingOrOldObjs.map(x => x[officialFieldName]) +parentPort.postMessage(missingOrOldIds) diff --git a/lib/sync/destinationManager.js b/lib/sync/destinationManager.js index 7d2dd36..9e65d76 100644 --- a/lib/sync/destinationManager.js +++ b/lib/sync/destinationManager.js @@ -5,8 +5,8 @@ const { getBatch, getMlsSourceUserConfig, getMlsSourceInternalConfig -} = require('../config') -const _ = require('lodash') +} = require("../config"); +const _ = require("lodash"); const { unpackErrorForSerialization, getMlsResourceDirFiles, @@ -18,119 +18,162 @@ const { deleteFilesForMlsResource, getPrimaryKeyField, flattenExpandedMlsResources, - removeFile, -} = require('./utils') -const fs = require('fs') -const fsPromises = fs.promises -const pathLib = require('path') -const { catcher: catcherUtil, fetchWithProgress } = require('./utils') -const moment = require('moment') -const { buildDataAdapter } = require('./dataAdapters/index') + removeFile +} = require("./utils"); +const fs = require("fs"); +const fsPromises = fs.promises; +const pathLib = require("path"); +const { catcher: catcherUtil, fetchWithProgress } = require("./utils"); +const moment = require("moment"); +const { buildDataAdapter } = require("./dataAdapters/index"); -const maxErrorCount = 3 +const maxErrorCount = 3; -module.exports = function(mlsSourceName, configBundle, eventEmitter, loggerArg) { - let platformAdapter +module.exports = function( + mlsSourceName, + configBundle, + eventEmitter, + loggerArg +) { + let platformAdapter; - const logger = loggerArg.child({ source: mlsSourceName }) - const { userConfig, internalConfig, flushInternalConfig } = configBundle - const mlsSourceUserConfig = getMlsSourceUserConfig(userConfig, mlsSourceName) - const mlsSourceInternalConfig = getMlsSourceInternalConfig(internalConfig, mlsSourceName) - const destinations = mlsSourceUserConfig.destinations - const platformAdapterName = mlsSourceUserConfig.platformAdapterName - const dataAdapters = destinations.map(destination => buildDataAdapter({ - destinationConfig: destination, - platformAdapterName, - })) + const logger = loggerArg.child({ source: mlsSourceName }); + const { userConfig, internalConfig, flushInternalConfig } = configBundle; + const mlsSourceUserConfig = getMlsSourceUserConfig(userConfig, mlsSourceName); + const mlsSourceInternalConfig = getMlsSourceInternalConfig( + internalConfig, + mlsSourceName + ); + const destinations = mlsSourceUserConfig.destinations; + const platformAdapterName = mlsSourceUserConfig.platformAdapterName; + const dataAdapters = destinations.map(destination => + buildDataAdapter({ + destinationConfig: destination, + platformAdapterName + }) + ); - const catcher = msg => catcherUtil(msg, { destinationManager: this, logger }) + const catcher = msg => catcherUtil(msg, { destinationManager: this, logger }); async function resumeSync(batchType, metadata) { - const eventType = batchType === 'missing' ? 'reconcile' : 'sync' + const eventType = batchType === "missing" ? "reconcile" : "sync"; try { - logger.info('Starting or resuming sync process') - const mlsResources = mlsSourceUserConfig.mlsResources - const sourceFiles = await getSourceFiles(mlsSourceName, mlsResources) - const configObjPath = batchType === 'missing' ? 'processReconcileBatch' : 'processSyncBatch' - let processBatch = _.get(mlsSourceInternalConfig, [configObjPath]) - let batchId + logger.info("Starting or resuming sync process"); + const mlsResources = mlsSourceUserConfig.mlsResources; + const sourceFiles = await getSourceFiles(mlsSourceName, mlsResources); + const configObjPath = + batchType === "missing" ? "processReconcileBatch" : "processSyncBatch"; + let processBatch = _.get(mlsSourceInternalConfig, [configObjPath]); + let batchId; if (processBatch) { - logger.debug(`Found existing process ${eventType} batch data`) - const batchTimestamp = moment.utc(processBatch.batchTimestamp) - batchId = convertTimestampToBatchId(batchTimestamp) + logger.debug(`Found existing process ${eventType} batch data`); + const batchTimestamp = moment.utc(processBatch.batchTimestamp); + batchId = convertTimestampToBatchId(batchTimestamp); // TODO: ensure processSyncBatch's mlsResourcesStatus matches current config. } else { - batchId = getOldestBatchId(sourceFiles, batchType) + batchId = getOldestBatchId(sourceFiles, batchType); if (!batchId) { - logger.info(`Found no ${eventType} files to process. Done.`) - return + logger.info(`Found no ${eventType} files to process. Done.`); + return; } - const batchTimestamp = convertBatchIdToTimestamp(batchId) + const batchTimestamp = convertBatchIdToTimestamp(batchId); processBatch = { batchTimestamp, mlsResourcesStatus: mlsResources.map(mlsResourceObj => ({ name: mlsResourceObj.name, - done: false, - })), - } + done: false + })) + }; - if (eventType === 'sync') { + if (eventType === "sync") { // Ensure this batch is not currently being downloaded. - const downloadSyncBatch = getBatch(mlsSourceInternalConfig, 'downloadSyncBatch') + const downloadSyncBatch = getBatch( + mlsSourceInternalConfig, + "downloadSyncBatch" + ); if (downloadSyncBatch) { - const downloadBatchId = convertTimestampToBatchId(downloadSyncBatch.batchTimestamp) + const downloadBatchId = convertTimestampToBatchId( + downloadSyncBatch.batchTimestamp + ); if (batchId === downloadBatchId) { - const e = new Error(`Refusing to process sync batch ${batchId} because it has not finished downloading`) - logger.error({ err: e }, 'Error syncing') - throw e + const e = new Error( + `Refusing to process sync batch ${batchId} because it has not finished downloading` + ); + logger.error({ err: e }, "Error syncing"); + throw e; } } } - _.set(mlsSourceInternalConfig, [configObjPath], processBatch) - await flushInternalConfig() + _.set(mlsSourceInternalConfig, [configObjPath], processBatch); + await flushInternalConfig(); } - logger.debug({ batchTimestamp: processBatch.batchTimestamp, batchId }, `Timestamp for process ${eventType} batch`) - const sourceFilesForBatch = getSourceFilesForBatch(sourceFiles, batchId, batchType) + logger.debug( + { batchTimestamp: processBatch.batchTimestamp, batchId }, + `Timestamp for process ${eventType} batch` + ); + const sourceFilesForBatch = getSourceFilesForBatch( + sourceFiles, + batchId, + batchType + ); eventEmitter.emit(`ors:${eventType}.start`, { mlsSourceName, - batchId, - }) - const startIndex = processBatch.mlsResourcesStatus.findIndex(x => !x.done) - logger.info(`Start processing ${eventType} data for resources`) + batchId + }); + const startIndex = processBatch.mlsResourcesStatus.findIndex( + x => !x.done + ); + logger.info(`Start processing ${eventType} data for resources`); for (let i = startIndex; i < mlsResources.length; i++) { - const mlsResourceObj = mlsResources[i] - const mlsResourceName = mlsResourceObj.name - logger.info({resource: mlsResourceName}, `Start processing ${eventType} data for resource`) - eventEmitter.emit(`ors:${eventType}.resource.start`, {mlsResourceObj}) - const filesForMlsResource = sourceFilesForBatch[i] - const processBatchMlsResource = processBatch.mlsResourcesStatus[i] + const mlsResourceObj = mlsResources[i]; + const mlsResourceName = mlsResourceObj.name; + logger.info( + { resource: mlsResourceName }, + `Start processing ${eventType} data for resource` + ); + eventEmitter.emit(`ors:${eventType}.resource.start`, { + mlsResourceObj + }); + const filesForMlsResource = sourceFilesForBatch[i]; + const processBatchMlsResource = processBatch.mlsResourcesStatus[i]; if (processBatchMlsResource.currentFilePath) { - if (filesForMlsResource.length && processBatchMlsResource.currentFilePath !== filesForMlsResource[0]) { - const e = new Error(`Existing process batch's currentFilePath (${processBatchMlsResource.currentFilePath}) doesn't match next file to process (${filesForMlsResource[0]}`) - logger.error({ err: e }, 'Error syncing') - throw e + if ( + filesForMlsResource.length && + processBatchMlsResource.currentFilePath !== filesForMlsResource[0] + ) { + const e = new Error( + `Existing process batch's currentFilePath (${processBatchMlsResource.currentFilePath}) doesn't match next file to process (${filesForMlsResource[0]}` + ); + logger.error({ err: e }, "Error syncing"); + throw e; } } - logger.info({resource: mlsResourceName, filesCount: filesForMlsResource.length}, `Processing ${eventType} files`) + logger.info( + { resource: mlsResourceName, filesCount: filesForMlsResource.length }, + `Processing ${eventType} files` + ); for (let i = 0; i < filesForMlsResource.length; i++) { - const filePath = filesForMlsResource[i] - const fileNum = i + 1 - logger.info({resource: mlsResourceName, fileNum, dataFilePath: filePath}, `Processing ${eventType} file`) - let destinationsStatus + const filePath = filesForMlsResource[i]; + const fileNum = i + 1; + logger.info( + { resource: mlsResourceName, fileNum, dataFilePath: filePath }, + `Processing ${eventType} file` + ); + let destinationsStatus; if (processBatchMlsResource.currentFilePath) { - destinationsStatus = processBatchMlsResource.destinationsStatus + destinationsStatus = processBatchMlsResource.destinationsStatus; // TODO: ensure destinationsStatus matches current config. } else { - processBatchMlsResource.currentFilePath = filePath + processBatchMlsResource.currentFilePath = filePath; destinationsStatus = destinations.map(destination => ({ name: destination.name, done: false, error: null, - errorCount: 0, - })) - processBatchMlsResource.destinationsStatus = destinationsStatus - await flushInternalConfig() + errorCount: 0 + })); + processBatchMlsResource.destinationsStatus = destinationsStatus; + await flushInternalConfig(); } const success = await processSync({ mlsResourceObj, @@ -140,27 +183,30 @@ module.exports = function(mlsSourceName, configBundle, eventEmitter, loggerArg) fileCount: filesForMlsResource.length, eventEmitter, metadata, - eventType, - }) + eventType + }); if (!success) { - return + return; } delete processBatchMlsResource.currentFilePath; delete processBatchMlsResource.destinationsStatus; - await flushInternalConfig() + await flushInternalConfig(); } - processBatchMlsResource.done = true - await flushInternalConfig() - eventEmitter.emit(`ors:${eventType}.resource.done`, {mlsResourceObj}) - logger.info({resource: mlsResourceName}, `Done processing ${eventType} data for resource`) + processBatchMlsResource.done = true; + await flushInternalConfig(); + eventEmitter.emit(`ors:${eventType}.resource.done`, { mlsResourceObj }); + logger.info( + { resource: mlsResourceName }, + `Done processing ${eventType} data for resource` + ); } - _.unset(mlsSourceInternalConfig, [configObjPath]) - await flushInternalConfig() - eventEmitter.emit(`ors:${eventType}.done`, { mlsSourceName, batchId }) - logger.info(`Done processing ${eventType} data for resources`) + _.unset(mlsSourceInternalConfig, [configObjPath]); + await flushInternalConfig(); + eventEmitter.emit(`ors:${eventType}.done`, { mlsSourceName, batchId }); + logger.info(`Done processing ${eventType} data for resources`); } catch (error) { - eventEmitter.emit(`ors:${eventType}.error`, error) - throw error + eventEmitter.emit(`ors:${eventType}.error`, error); + throw error; } } @@ -172,294 +218,442 @@ module.exports = function(mlsSourceName, configBundle, eventEmitter, loggerArg) fileCount, eventEmitter, metadata, - eventType, + eventType }) { - const mlsResourceName = mlsResourceObj.name - const fileContents = await fsPromises.readFile(dataFilePath, 'utf8') - const mlsData = JSON.parse(fileContents) - const startIndex = destinationsStatus.findIndex(x => !x.done) + const mlsResourceName = mlsResourceObj.name; + const fileContents = await fsPromises.readFile(dataFilePath, "utf8"); + const mlsData = JSON.parse(fileContents); + const startIndex = destinationsStatus.findIndex(x => !x.done); for (let i = startIndex; i >= 0 && i < destinations.length; i++) { - const destination = destinations[i] - const dataAdapter = dataAdapters[i] - const destinationStatus = destinationsStatus[i] + const destination = destinations[i]; + const dataAdapter = dataAdapters[i]; + const destinationStatus = destinationsStatus[i]; if (destinationStatus.errorCount >= maxErrorCount) { - logger.warn({ resource: mlsResourceName, dataFilePath, destination: destination.name }, `Destination has reached max error count of ${maxErrorCount}. Refusing to process ${eventType}.`) - return false + logger.warn( + { + resource: mlsResourceName, + dataFilePath, + destination: destination.name + }, + `Destination has reached max error count of ${maxErrorCount}. Refusing to process ${eventType}.` + ); + return false; } try { - const recordsToSyncCount = (mlsData.value && mlsData.value.length) || 0 - let existingRecordsInDestinationCount + const recordsToSyncCount = (mlsData.value && mlsData.value.length) || 0; + let existingRecordsInDestinationCount; if (fileNum === 1) { - existingRecordsInDestinationCount = await dataAdapter.getCount(mlsResourceName) - logger.debug({ resource: mlsResourceName, destination: destination.name, recordsCount: existingRecordsInDestinationCount }, 'Starting with X records in destination') + existingRecordsInDestinationCount = await dataAdapter.getCount( + mlsResourceName + ); + logger.debug( + { + resource: mlsResourceName, + destination: destination.name, + recordsCount: existingRecordsInDestinationCount + }, + "Starting with X records in destination" + ); } if (recordsToSyncCount > 0) { - logger.info({ - resource: mlsResourceName, - dataFilePath, - destination: destination.name, - recordsToSyncCount - }, 'Syncing X records to destination') - if (!'debug') { - throw new Error('test sync error') + logger.info( + { + resource: mlsResourceName, + dataFilePath, + destination: destination.name, + recordsToSyncCount + }, + "Syncing X records to destination" + ); + if (!"debug") { + throw new Error("test sync error"); } - await dataAdapter.syncData(mlsResourceObj, mlsData.value, metadata) + await dataAdapter.syncData(mlsResourceObj, mlsData.value, metadata); } else { - logger.info({ resource: mlsResourceName, destination: destination.name }, 'File had 0 records. Skipping sync.') + logger.info( + { resource: mlsResourceName, destination: destination.name }, + "File had 0 records. Skipping sync." + ); } - destinationStatus.done = true - await flushInternalConfig() - eventEmitter.emit(`ors:${eventType}.destination.page`, { destination, recordsSyncedCount: recordsToSyncCount }) + destinationStatus.done = true; + await flushInternalConfig(); + eventEmitter.emit(`ors:${eventType}.destination.page`, { + destination, + recordsSyncedCount: recordsToSyncCount + }); if (fileNum === fileCount) { - const recordsInDestinationCount = await dataAdapter.getCount(mlsResourceName) - logger.debug({ resource: mlsResourceName, destination: destination.name, recordsCount: recordsInDestinationCount }, 'Ended with X records in destination') + const recordsInDestinationCount = await dataAdapter.getCount( + mlsResourceName + ); + logger.debug( + { + resource: mlsResourceName, + destination: destination.name, + recordsCount: recordsInDestinationCount + }, + "Ended with X records in destination" + ); } } catch (e) { - const unpackedError = unpackErrorForSerialization(e) - destinationStatus.error = unpackedError - destinationStatus.errorCount++ - await flushInternalConfig() - logger.error({ err: e }, 'Error syncing') - throw e + const unpackedError = unpackErrorForSerialization(e); + destinationStatus.error = unpackedError; + destinationStatus.errorCount++; + await flushInternalConfig(); + logger.error({ err: e }, "Error syncing"); + throw e; } } - logger.info({ resource: mlsResourceName, dataFilePath }, 'Done syncing file data') - logger.info({ resource: mlsResourceName, dataFilePath }, 'Deleting sync file') - await removeFile(dataFilePath, mlsResourceName) - return true + logger.info( + { resource: mlsResourceName, dataFilePath }, + "Done syncing file data" + ); + logger.info( + { resource: mlsResourceName, dataFilePath }, + "Deleting sync file" + ); + await removeFile(dataFilePath, mlsResourceName); + return true; } async function resumePurge() { try { - logger.info('Starting or resuming purge') - const topLevelMlsResources = mlsSourceUserConfig.mlsResources + logger.info("Starting or resuming purge"); + const topLevelMlsResources = mlsSourceUserConfig.mlsResources; // See more explanation in downloader, but we will only purge top level resources and special subresources. // const mlsResources = flattenExpandedMlsResources(topLevelMlsResources) - const mlsResources = topLevelMlsResources - const sourceFiles = await getSourceFiles(mlsSourceName, mlsResources) - let processPurgeBatch = _.get(mlsSourceInternalConfig, ['processPurgeBatch']) - let batchId + const mlsResources = topLevelMlsResources; + const sourceFiles = await getSourceFiles(mlsSourceName, mlsResources); + let processPurgeBatch = _.get(mlsSourceInternalConfig, [ + "processPurgeBatch" + ]); + let batchId; if (processPurgeBatch) { - logger.debug('Found existing process purge batch data') - const batchTimestamp = moment.utc(processPurgeBatch.batchTimestamp) - batchId = convertTimestampToBatchId(batchTimestamp) + logger.debug("Found existing process purge batch data"); + const batchTimestamp = moment.utc(processPurgeBatch.batchTimestamp); + batchId = convertTimestampToBatchId(batchTimestamp); // TODO: ensure processPurgeBatch's mlsResourcesStatus matches current config. } else { - batchId = getOldestBatchId(sourceFiles, 'purge') + batchId = getOldestBatchId(sourceFiles, "purge"); if (!batchId) { - logger.info('Found no purge files to process. Done.') - return + logger.info("Found no purge files to process. Done."); + return; } - const batchTimestamp = convertBatchIdToTimestamp(batchId) + const batchTimestamp = convertBatchIdToTimestamp(batchId); const destinationsStatus = destinations.map(destination => ({ name: destination.name, done: false, error: null, - errorCount: 0, - })) + errorCount: 0 + })); processPurgeBatch = { batchTimestamp, mlsResourcesStatus: mlsResources.map(mlsResourceObj => ({ name: mlsResourceObj.name, done: false, // Don't use a single reference. Copy the destinationsStatus for each MLS resource. - destinationsStatus: _.cloneDeep(destinationsStatus), - })), - } + destinationsStatus: _.cloneDeep(destinationsStatus) + })) + }; // Ensure this batch is not currently being downloaded. - const downloadPurgeBatch = getBatch(mlsSourceInternalConfig, 'downloadPurgeBatch') + const downloadPurgeBatch = getBatch( + mlsSourceInternalConfig, + "downloadPurgeBatch" + ); if (downloadPurgeBatch) { - const downloadBatchId = convertTimestampToBatchId(downloadPurgeBatch.batchTimestamp) + const downloadBatchId = convertTimestampToBatchId( + downloadPurgeBatch.batchTimestamp + ); if (batchId === downloadBatchId) { - throw new Error(`Refusing to process purge batch ${batchId} because it has not finished downloading`) + throw new Error( + `Refusing to process purge batch ${batchId} because it has not finished downloading` + ); } } - _.set(mlsSourceInternalConfig, ['processPurgeBatch'], processPurgeBatch) - await flushInternalConfig() + _.set( + mlsSourceInternalConfig, + ["processPurgeBatch"], + processPurgeBatch + ); + await flushInternalConfig(); } - - logger.debug({ batchTimestamp: processPurgeBatch.batchTimestamp, batchId }, 'Timestamp for process purge batch') - const sourceFilesForBatch = getSourceFilesForBatch(sourceFiles, batchId, 'purge') - eventEmitter.emit('ors:purge.start', { - mlsSourceName, + logger.debug( + { batchTimestamp: processPurgeBatch.batchTimestamp, batchId }, + "Timestamp for process purge batch" + ); + const sourceFilesForBatch = getSourceFilesForBatch( + sourceFiles, batchId, - }) - const startIndex = processPurgeBatch.mlsResourcesStatus.findIndex(x => !x.done) - logger.info('Start processing purge data for resources') + "purge" + ); + eventEmitter.emit("ors:purge.start", { + mlsSourceName, + batchId + }); + const startIndex = processPurgeBatch.mlsResourcesStatus.findIndex( + x => !x.done + ); + logger.info("Start processing purge data for resources"); for (let i = startIndex; i < mlsResources.length; i++) { - const mlsResourceObj = mlsResources[i] - const mlsResourceName = mlsResources[i].name - logger.info({ resource: mlsResourceName }, 'Start processing purge data for resource') - eventEmitter.emit('ors:purge.resource.start', {mlsResourceObj}) - const filesForMlsResource = sourceFilesForBatch[i] - const processPurgeBatchMlsResource = processPurgeBatch.mlsResourcesStatus[i] - const destinationsStatus = processPurgeBatchMlsResource.destinationsStatus - const mlsIndexes = platformAdapter.getIndexes(mlsResourceName) - const primaryKey = getPrimaryKeyField(mlsResourceName, mlsIndexes) - + const mlsResourceObj = mlsResources[i]; + const mlsResourceName = mlsResources[i].name; + logger.info( + { resource: mlsResourceName }, + "Start processing purge data for resource" + ); + eventEmitter.emit("ors:purge.resource.start", { mlsResourceObj }); + const filesForMlsResource = sourceFilesForBatch[i]; + const processPurgeBatchMlsResource = + processPurgeBatch.mlsResourcesStatus[i]; + const destinationsStatus = + processPurgeBatchMlsResource.destinationsStatus; + const mlsIndexes = platformAdapter.getIndexes(mlsResourceName); + const primaryKey = getPrimaryKeyField(mlsResourceName, mlsIndexes); // We can't process the files one by one, like with sync. Instead, we concat all the data // and compare it to what's in the destinations. - logger.info({ resource: mlsResourceName }, 'Reading IDs from MLS') - let idsInMls = [] + logger.info({ resource: mlsResourceName }, "Reading IDs from MLS"); + let idsInMls = []; for (const filePath of filesForMlsResource) { - logger.trace({ resource: mlsResourceName, dataFilePath: filePath }, 'Reading IDs from data file path') - const fileContents = await fsPromises.readFile(filePath, 'utf8') - const mlsData = JSON.parse(fileContents) - const ids = mlsData.value.map(x => x[primaryKey]) - idsInMls = idsInMls.concat(ids) + logger.trace( + { resource: mlsResourceName, dataFilePath: filePath }, + "Reading IDs from data file path" + ); + const fileContents = await fsPromises.readFile(filePath, "utf8"); + const mlsData = JSON.parse(fileContents); + const ids = mlsData.value.map(x => x[primaryKey]); + idsInMls = idsInMls.concat(ids); } - logger.trace({ resource: mlsResourceName }, 'Sorting IDs') + logger.trace({ resource: mlsResourceName }, "Sorting IDs"); if (!mlsSourceUserConfig.useOrderBy) { - idsInMls.sort() + idsInMls.sort(); } - const success = await processPurge({ mlsResourceObj, idsInMls, destinationsStatus, getIndexes: platformAdapter.getIndexes }) + const success = await processPurge({ + mlsResourceObj, + idsInMls, + destinationsStatus, + getIndexes: platformAdapter.getIndexes + }); if (!success) { - return + return; } - delete processPurgeBatchMlsResource.destinationsStatus - processPurgeBatchMlsResource.done = true - await flushInternalConfig() + delete processPurgeBatchMlsResource.destinationsStatus; + processPurgeBatchMlsResource.done = true; + await flushInternalConfig(); - logger.info({ resource: mlsResourceName }, 'Done processing purge for resource') - logger.info({ resource: mlsResourceName }, 'Deleting purge files') - eventEmitter.emit('ors:purge.resource.done', {mlsResourceObj}) - await deleteFilesForMlsResource(mlsResourceName, filesForMlsResource, logger.child({ resource: mlsResourceName })) + logger.info( + { resource: mlsResourceName }, + "Done processing purge for resource" + ); + logger.info({ resource: mlsResourceName }, "Deleting purge files"); + eventEmitter.emit("ors:purge.resource.done", { mlsResourceObj }); + await deleteFilesForMlsResource( + mlsResourceName, + filesForMlsResource, + logger.child({ resource: mlsResourceName }) + ); } - _.unset(mlsSourceInternalConfig, ['processPurgeBatch']) - await flushInternalConfig() - eventEmitter.emit('ors:purge.done', { mlsSourceName, batchId }) - logger.info('Done processing purge data for resources') + _.unset(mlsSourceInternalConfig, ["processPurgeBatch"]); + await flushInternalConfig(); + eventEmitter.emit("ors:purge.done", { mlsSourceName, batchId }); + logger.info("Done processing purge data for resources"); } catch (error) { - eventEmitter.emit('ors:purge.error', error) - throw error + eventEmitter.emit("ors:purge.error", error); + throw error; } } - async function processPurge({ mlsResourceObj, idsInMls, destinationsStatus, getIndexes }) { - const mlsResourceName = mlsResourceObj.name - logger.info({ resource: mlsResourceName }, 'Starting to purge resource') - const startIndex = destinationsStatus.findIndex(x => !x.done) + async function processPurge({ + mlsResourceObj, + idsInMls, + destinationsStatus, + getIndexes + }) { + const mlsResourceName = mlsResourceObj.name; + logger.info({ resource: mlsResourceName }, "Starting to purge resource"); + const startIndex = destinationsStatus.findIndex(x => !x.done); for (let i = startIndex; i < destinations.length; i++) { - const destination = destinations[i] - const dataAdapter = dataAdapters[i] - const destinationStatus = destinationsStatus[i] + const destination = destinations[i]; + const dataAdapter = dataAdapters[i]; + const destinationStatus = destinationsStatus[i]; if (destinationStatus.errorCount >= maxErrorCount) { - logger.warn({ resource: mlsResourceName, destination: destination.name }, `Destination has reached max error count of ${maxErrorCount}. Refusing to process purge.`) - return false + logger.warn( + { resource: mlsResourceName, destination: destination.name }, + `Destination has reached max error count of ${maxErrorCount}. Refusing to process purge.` + ); + return false; } try { - logger.info({ resource: mlsResourceName, destination: destination.name }, 'Getting existing IDs from destination') - const mlsIndexes = getIndexes(mlsResourceName) - const mlsIdsInDestination = await dataAdapter.getAllMlsIds(mlsResourceName, mlsIndexes) - const mlsIdsToPurge = _.difference(mlsIdsInDestination, idsInMls) + logger.info( + { resource: mlsResourceName, destination: destination.name }, + "Getting existing IDs from destination" + ); + const mlsIndexes = getIndexes(mlsResourceName); + const mlsIdsInDestination = await dataAdapter.getAllMlsIds( + mlsResourceName, + mlsIndexes + ); + const mlsIdsToPurge = _.difference(mlsIdsInDestination, idsInMls); if (mlsIdsToPurge.length) { - logger.info({ resource: mlsResourceName, destination: destination.name, recordsCount: mlsIdsToPurge.length }, 'Purging X records') + logger.info( + { + resource: mlsResourceName, + destination: destination.name, + recordsCount: mlsIdsToPurge.length + }, + "Purging X records" + ); // Let's have a safety net to make sure we don't delete too many due to some bug. // TODO: Make this configurable - const maxAllowedPurgeCount = 5000 + const maxAllowedPurgeCount = 5000; if (mlsIdsToPurge.length > maxAllowedPurgeCount) { - throw new Error(`Refusing to purge ${mlsIdsToPurge.length} records because count is greater than max allowed purge count of ${maxAllowedPurgeCount}`) + throw new Error( + `Refusing to purge ${mlsIdsToPurge.length} records because count is greater than max allowed purge count of ${maxAllowedPurgeCount}` + ); } - if (!'debug') { - throw new Error('test purge error') + if (!"debug") { + throw new Error("test purge error"); } - await dataAdapter.purge(mlsResourceObj, mlsIdsToPurge, getIndexes) - logger.debug({ resource: mlsResourceName, destination: destination.name, recordsCount: mlsIdsInDestination.length - mlsIdsToPurge.length }, 'Successfully purged records. Net is X records.') + await dataAdapter.purge(mlsResourceObj, mlsIdsToPurge, getIndexes); + logger.debug( + { + resource: mlsResourceName, + destination: destination.name, + recordsCount: mlsIdsInDestination.length - mlsIdsToPurge.length + }, + "Successfully purged records. Net is X records." + ); } else { - if (!'debug') { - throw new Error('error to test stopping mid purge') + if (!"debug") { + throw new Error("error to test stopping mid purge"); } - logger.info({ resource: mlsResourceName, destination: destination.name }, 'No IDs to purge') + logger.info( + { resource: mlsResourceName, destination: destination.name }, + "No IDs to purge" + ); } - destinationStatus.done = true - await flushInternalConfig() - eventEmitter.emit('ors:purge.destination.page', { destination, idsPurged: mlsIdsToPurge }) + destinationStatus.done = true; + await flushInternalConfig(); + eventEmitter.emit("ors:purge.destination.page", { + destination, + idsPurged: mlsIdsToPurge + }); } catch (e) { - const unpackedError = unpackErrorForSerialization(e) - destinationStatus.error = unpackedError - destinationStatus.errorCount++ - await flushInternalConfig() - logger.error({ err: e }, 'Error purging') - throw e + const unpackedError = unpackErrorForSerialization(e); + destinationStatus.error = unpackedError; + destinationStatus.errorCount++; + await flushInternalConfig(); + logger.error({ err: e }, "Error purging"); + throw e; } } - logger.info({ resource: mlsResourceName }, 'Done purging') - return true + logger.info({ resource: mlsResourceName }, "Done purging"); + return true; } // Initially, the primary data adapter is used to get the timestamps. Theoretically, all data adapters should agree // on the value(s), so really the term 'primary' here just means to consistently refer to one of the data adapters // used. Might as well use the first one. function getPrimaryDataAdapter() { - return dataAdapters[0] + return dataAdapters[0]; } function closeConnections() { - return Promise.all(dataAdapters.map(x => x.closeConnection())) + return Promise.all(dataAdapters.map(x => x.closeConnection())); } async function syncMetadata(metadata) { - const mlsResources = mlsSourceUserConfig.mlsResources - await Promise.all(_.flatMap(mlsResources, mlsResourceObj => { - return dataAdapters.map(dataAdapter => { - return dataAdapter.syncStructure(mlsResourceObj, metadata) + const mlsResources = mlsSourceUserConfig.mlsResources; + await Promise.all( + _.flatMap(mlsResources, mlsResourceObj => { + return dataAdapters.map(dataAdapter => { + return dataAdapter.syncStructure(mlsResourceObj, metadata); + }); }) - })) + ); } async function getStatsDetails() { - const mlsResources = mlsSourceUserConfig.mlsResources - const d = await Promise.all(mlsResources.map(async resource => { - const destinationData = await Promise.all(destinations.map(async (destination, i) => { - const dataAdapter = dataAdapters[i] - const [existingRecordsInDestinationCount, mostRecentAt] = await Promise.all([ - dataAdapter.getCount(resource.name), - dataAdapter.getMostRecentTimestamp(resource.name), - ]) + const mlsResources = mlsSourceUserConfig.mlsResources; + const d = await Promise.all( + mlsResources.map(async resource => { + const destinationData = await Promise.all( + destinations.map(async (destination, i) => { + const dataAdapter = dataAdapters[i]; + const [ + existingRecordsInDestinationCount, + mostRecentAt + ] = await Promise.all([ + dataAdapter.getCount(resource.name), + dataAdapter.getMostRecentTimestamp(resource.name) + ]); + return { + name: destination.name, + num_records: existingRecordsInDestinationCount, + most_recent_at: mostRecentAt + }; + }) + ); return { - name: destination.name, - num_records: existingRecordsInDestinationCount, - most_recent_at: mostRecentAt, - } - })) - return { - name: resource.name, - destinations: destinationData, - } - })) - return d + name: resource.name, + destinations: destinationData + }; + }) + ); + return d; } async function getMissingIds(mlsResourceName, dataInMls, indexes) { - const allMissingIdsDataArray = await Promise.all(destinations.map(async (destination, i) => { - logger.trace({ resource: mlsResourceName, destination: destination.name }, 'Getting missing IDs from destination for resource') - const dataAdapter = dataAdapters[i] - const missingIdsDataForDestination = await dataAdapter.fetchMissingIdsData(mlsResourceName, indexes) - logger.trace({ resource: mlsResourceName, destination: destination.name }, 'Done fetching missing IDs data from destination for resource') - return missingIdsDataForDestination - })) - const allMissingIdsSet = new Set() + const allMissingIdsDataArray = await Promise.all( + destinations.map(async (destination, i) => { + logger.trace( + { resource: mlsResourceName, destination: destination.name }, + "Getting missing IDs from destination for resource" + ); + const dataAdapter = dataAdapters[i]; + const missingIdsDataForDestination = await dataAdapter.fetchMissingIdsData( + mlsResourceName, + indexes + ); + logger.trace( + { resource: mlsResourceName, destination: destination.name }, + "Done fetching missing IDs data from destination for resource" + ); + return missingIdsDataForDestination; + }) + ); + const allMissingIdsSet = new Set(); for (let i = 0; i < destinations.length; i++) { - const dataAdapter = dataAdapters[i] - const destination = destinations[i] - const dataInAdapter = allMissingIdsDataArray[i] - logger.trace({ resource: mlsResourceName, destination: destination.name }, 'Computing missing IDs for destination for resource') - const missingIds = await dataAdapter.computeMissingIds(mlsResourceName, dataInMls, dataInAdapter, indexes) - logger.trace({ resource: mlsResourceName, destination: destination.name }, 'Done computing missing IDs for destination for resource') + const dataAdapter = dataAdapters[i]; + const destination = destinations[i]; + const dataInAdapter = allMissingIdsDataArray[i]; + logger.trace( + { resource: mlsResourceName, destination: destination.name }, + "Computing missing IDs for destination for resource" + ); + + const missingIds = await dataAdapter.computeMissingIds( + mlsResourceName, + dataInMls, + dataInAdapter, + indexes + ); + logger.trace( + { resource: mlsResourceName, destination: destination.name }, + "Done computing missing IDs for destination for resource" + ); for (const id of missingIds) { - allMissingIdsSet.add(id) + allMissingIdsSet.add(id); } } - const array = Array.from(allMissingIdsSet) - return array + const array = Array.from(allMissingIdsSet); + return array; } function setPlatformAdapter(adapter) { - platformAdapter = adapter + platformAdapter = adapter; } return { @@ -470,6 +664,6 @@ module.exports = function(mlsSourceName, configBundle, eventEmitter, loggerArg) resumePurge, getStatsDetails, getMissingIds, - setPlatformAdapter, - } -} + setPlatformAdapter + }; +}; diff --git a/lib/sync/downloader.js b/lib/sync/downloader.js index 5b903c7..1469265 100644 --- a/lib/sync/downloader.js +++ b/lib/sync/downloader.js @@ -118,7 +118,7 @@ module.exports = function(mlsSourceName, configBundle, eventEmitter, loggerArg) } async function downloadMlsMetadata() { - logger.info('Start downloading MLS metadata') + logger.info('Start downloading MLS metadata') let auth = _.get(mlsSourceInternalConfig, ['auth'], {}) const metadataPath = mlsSourceUserConfig.metadataPath @@ -361,7 +361,7 @@ module.exports = function(mlsSourceName, configBundle, eventEmitter, loggerArg) timestamps = getTimestampsFromMlsData(mostRecentMlsData, indexes) logger.info({ resource: mlsResourceName, ...timestamps }, 'Using timestamps from recently downloaded files') } - if (!timestamps) { + if (!timestamps) { timestamps = await destinationManager.getPrimaryDataAdapter().getTimestamps(mlsResourceName, indexes) logger.info({ resource: mlsResourceName, ...timestamps }, 'Using timestamps from primary data adapter') } diff --git a/lib/sync/platformDataAdapters/bridgeInteractive/mongodb.js b/lib/sync/platformDataAdapters/bridgeInteractive/mongodb.js new file mode 100644 index 0000000..1c7fdd2 --- /dev/null +++ b/lib/sync/platformDataAdapters/bridgeInteractive/mongodb.js @@ -0,0 +1,22 @@ +module.exports = function() { + function getDatabaseType(property) { + const type = property.$.Type; + if (type.match(/Collection\(.*Enums\)/) || type.match(/.*Enums\./)) { + // These are potentially long strings + return "TEXT"; + } else if (type.match(/Collection.*ComplexTypes/)) { + // These are JSON arrays + return "JSON"; + } + return null; + } + + function overridesDatabaseType(property) { + return getDatabaseType(property) !== null; + } + + return { + overridesDatabaseType, + getDatabaseType + }; +}; diff --git a/lib/sync/platformDataAdapters/bridgeInteractive/mysql.js b/lib/sync/platformDataAdapters/bridgeInteractive/mysql.js index cf402e0..b6bd481 100644 --- a/lib/sync/platformDataAdapters/bridgeInteractive/mysql.js +++ b/lib/sync/platformDataAdapters/bridgeInteractive/mysql.js @@ -8,7 +8,6 @@ module.exports = function() { // These are JSON arrays return 'JSON' } - return null } diff --git a/package-lock.json b/package-lock.json index c5abcb6..59a7f10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,12 @@ { "name": "openresync", - "version": "0.2.0", + "version": "0.4.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "version": "0.2.0", + "name": "openresync", + "version": "0.4.0", "dependencies": { "@graphql-tools/schema": "^7.1.3", "@tailwindcss/postcss7-compat": "^2.0.4", @@ -31,6 +32,7 @@ "moment-locales-webpack-plugin": "^1.2.0", "moment-timezone": "^0.5.33", "moment-timezone-data-webpack-plugin": "^1.4.0", + "mongodb": "^4.10.0", "mysql2": "^2.2.5", "objection": "^2.2.14", "pino": "^6.11.0", @@ -5452,6 +5454,11 @@ "integrity": "sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==", "dev": true }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==" + }, "node_modules/@types/webpack": { "version": "4.41.25", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.25.tgz", @@ -5514,6 +5521,15 @@ "node": ">=0.10.0" } }, + "node_modules/@types/whatwg-url": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "dependencies": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "node_modules/@types/ws": { "version": "7.4.7", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", @@ -8747,6 +8763,40 @@ "node-int64": "^0.4.0" } }, + "node_modules/bson": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", + "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/bson/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/buffer": { "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", @@ -11023,14 +11073,18 @@ "dev": true }, "node_modules/define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", "dependencies": { - "object-keys": "^1.0.12" + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/define-property": { @@ -11798,26 +11852,34 @@ } }, "node_modules/es-abstract": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0.tgz", - "integrity": "sha512-LJzK7MrQa8TS0ja2w3YNLzUgJCGPdPOV1yVvezjNnS89D+VR08+Szt2mz3YB2Dck/+w5tfIq/RoUAFqJJGM2yw==", + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.4.tgz", + "integrity": "sha512-0UtvRN79eMe2L+UNEF1BwRe364sj/DXhQ/k5FmivgoSdpM90b8Jc0mDzKMGo7QS0BVbOP/bTwBKNnDc9rNzaPA==", "dependencies": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", - "get-intrinsic": "^1.1.1", + "function.prototype.name": "^1.1.5", + "get-intrinsic": "^1.1.3", + "get-symbol-description": "^1.0.0", "has": "^1.0.3", - "has-symbols": "^1.0.2", - "is-callable": "^1.2.3", - "is-negative-zero": "^2.0.1", - "is-regex": "^1.1.2", - "is-string": "^1.0.5", - "object-inspect": "^1.9.0", + "has-property-descriptors": "^1.0.0", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.7", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.2", "object-keys": "^1.1.1", - "object.assign": "^4.1.2", - "string.prototype.trimend": "^1.0.4", - "string.prototype.trimstart": "^1.0.4", - "unbox-primitive": "^1.0.0" + "object.assign": "^4.1.4", + "regexp.prototype.flags": "^1.4.3", + "safe-regex-test": "^1.0.0", + "string.prototype.trimend": "^1.0.5", + "string.prototype.trimstart": "^1.0.5", + "unbox-primitive": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -13187,14 +13249,13 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/function.prototype.name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.4.tgz", - "integrity": "sha512-iqy1pIotY/RmhdFZygSSlW0wko2yxkSCKqsuv4pr8QESohpYyG/Z7B/XXvPRKTJS//960rgguE5mSRUsDdaJrQ==", - "dev": true, + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", + "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2", + "es-abstract": "^1.19.0", "functions-have-names": "^1.2.2" }, "engines": { @@ -13214,7 +13275,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.2.tgz", "integrity": "sha512-bLgc3asbWdwPbx2mNk2S49kmJCuQeu0nfmaOgbs8WIyzzkw3r4htszdIi9Q9EMezDPTYuJx2wvjZ/EwgAthpnA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -13326,13 +13386,13 @@ } }, "node_modules/get-intrinsic": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", - "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", - "has-symbols": "^1.0.1" + "has-symbols": "^1.0.3" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -13359,6 +13419,21 @@ "node": ">=6" } }, + "node_modules/get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", @@ -13775,9 +13850,9 @@ } }, "node_modules/has-bigints": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", - "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -13814,10 +13889,35 @@ "node": ">=0.10.0" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dependencies": { + "get-intrinsic": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dependencies": { + "has-symbols": "^1.0.2" + }, "engines": { "node": ">= 0.4" }, @@ -14806,7 +14906,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", - "dev": true, "dependencies": { "get-intrinsic": "^1.1.0", "has": "^1.0.3", @@ -14945,9 +15044,12 @@ "dev": true }, "node_modules/is-bigint": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.1.tgz", - "integrity": "sha512-J0ELF4yHFxHy0cmSxZuheDOz2luOdVvqjwmEcj8H/L1JHeuEDSDbeRP+Dk9kFVk5RTFzbucJ2Kb9F7ixY2QaCg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dependencies": { + "has-bigints": "^1.0.1" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -14964,11 +15066,12 @@ } }, "node_modules/is-boolean-object": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.0.tgz", - "integrity": "sha512-a7Uprx8UtD+HWdyYwnD1+ExtTgqQtD2k/1yJgtXP6wnMm8byhkoTZRl+95LLThpzNZJ5aEvi46cdH+ayMFRwmA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", "dependencies": { - "call-bind": "^1.0.0" + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" }, "engines": { "node": ">= 0.4" @@ -14983,9 +15086,9 @@ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "node_modules/is-callable": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", - "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", "engines": { "node": ">= 0.4" }, @@ -15236,9 +15339,9 @@ } }, "node_modules/is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", "engines": { "node": ">= 0.4" }, @@ -15267,9 +15370,12 @@ } }, "node_modules/is-number-object": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.4.tgz", - "integrity": "sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, "engines": { "node": ">= 0.4" }, @@ -15377,12 +15483,12 @@ "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" }, "node_modules/is-regex": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.2.tgz", - "integrity": "sha512-axvdhb5pdhEVThqJzYXwMlVuZwC+FF2DpcOhTS+y/8jVq4trxyPgfcwIxIKiyeuLlSQYKkmUaPQJ8ZE4yNKXDg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", "dependencies": { "call-bind": "^1.0.2", - "has-symbols": "^1.0.1" + "has-tostringtag": "^1.0.0" }, "engines": { "node": ">= 0.4" @@ -15426,6 +15532,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -15436,9 +15553,12 @@ } }, "node_modules/is-string": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", - "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, "engines": { "node": ">= 0.4" }, @@ -15485,6 +15605,17 @@ "node": ">=0.10.0" } }, + "node_modules/is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-whitespace-character": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-whitespace-character/-/is-whitespace-character-1.0.4.tgz", @@ -18906,6 +19037,12 @@ "readable-stream": "^2.0.1" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -19419,6 +19556,71 @@ "semver": "bin/semver.js" } }, + "node_modules/mongodb": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.10.0.tgz", + "integrity": "sha512-My2QxLTw0Cc1O9gih0mz4mqo145Jq4rLAQx0Glk/Ha9iYBzYpt4I2QFNRIh35uNFNfe8KFQcdwY1/HKxXBkinw==", + "dependencies": { + "bson": "^4.7.0", + "denque": "^2.1.0", + "mongodb-connection-string-url": "^2.5.3", + "socks": "^2.7.0" + }, + "engines": { + "node": ">=12.9.0" + }, + "optionalDependencies": { + "saslprep": "^1.0.3" + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.4.tgz", + "integrity": "sha512-SeAxuWs0ez3iI3vvmLk/j2y+zHwigTDKQhtdxTgt5ZCOQQS5+HW4g45/Xw5vzzbn7oQXCNQ24Z40AkJsizEy7w==", + "dependencies": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -19996,9 +20198,9 @@ } }, "node_modules/object-inspect": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.9.0.tgz", - "integrity": "sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -20044,17 +20246,20 @@ } }, "node_modules/object.assign": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", - "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", + "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", "dependencies": { - "call-bind": "^1.0.0", - "define-properties": "^1.1.3", - "has-symbols": "^1.0.1", + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "has-symbols": "^1.0.3", "object-keys": "^1.1.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/object.defaults": { @@ -23486,13 +23691,13 @@ } }, "node_modules/regexp.prototype.flags": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", - "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", - "dev": true, + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" }, "engines": { "node": ">= 0.4" @@ -24117,6 +24322,19 @@ "ret": "~0.1.10" } }, + "node_modules/safe-regex-test": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", + "integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==", + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.3", + "is-regex": "^1.1.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -24167,6 +24385,18 @@ "node": ">=0.10.0" } }, + "node_modules/saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "dependencies": { + "sparse-bitfield": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/sass": { "version": "1.32.8", "resolved": "https://registry.npmjs.org/sass/-/sass-1.32.8.tgz", @@ -24666,7 +24896,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -24733,6 +24962,15 @@ "node": ">=4" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -24913,6 +25151,24 @@ "node": ">=0.8.0" } }, + "node_modules/socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "dependencies": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks/node_modules/ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + }, "node_modules/solr-client": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/solr-client/-/solr-client-0.9.0.tgz", @@ -25013,6 +25269,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "optional": true, + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -25402,13 +25667,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/string.prototype.trimend": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz", + "integrity": "sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/string.prototype.trimstart": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", - "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz", + "integrity": "sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg==", "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -26596,6 +26875,20 @@ "is-typedarray": "^1.0.0" } }, + "node_modules/typescript": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "dev": true, + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, "node_modules/uglify-js": { "version": "3.4.10", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.10.tgz", @@ -26628,14 +26921,17 @@ } }, "node_modules/unbox-primitive": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.0.tgz", - "integrity": "sha512-P/51NX+JXyxK/aigg1/ZgyccdAxm5K1+n8+tvqSntjOivPt19gvm1VC49RWYetsiub8WViUchdxl/KWHHB0kzA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", "dependencies": { - "function-bind": "^1.1.1", - "has-bigints": "^1.0.0", - "has-symbols": "^1.0.0", - "which-boxed-primitive": "^1.0.1" + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/unc-path-regex": { @@ -33638,6 +33934,11 @@ "integrity": "sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==", "dev": true }, + "@types/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==" + }, "@types/webpack": { "version": "4.41.25", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.25.tgz", @@ -33698,6 +33999,15 @@ } } }, + "@types/whatwg-url": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "requires": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "@types/ws": { "version": "7.4.7", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", @@ -36468,6 +36778,25 @@ "node-int64": "^0.4.0" } }, + "bson": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", + "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", + "requires": { + "buffer": "^5.6.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + } + } + }, "buffer": { "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", @@ -38323,11 +38652,12 @@ "dev": true }, "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", "requires": { - "object-keys": "^1.0.12" + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" } }, "define-property": { @@ -38991,26 +39321,34 @@ } }, "es-abstract": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0.tgz", - "integrity": "sha512-LJzK7MrQa8TS0ja2w3YNLzUgJCGPdPOV1yVvezjNnS89D+VR08+Szt2mz3YB2Dck/+w5tfIq/RoUAFqJJGM2yw==", + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.4.tgz", + "integrity": "sha512-0UtvRN79eMe2L+UNEF1BwRe364sj/DXhQ/k5FmivgoSdpM90b8Jc0mDzKMGo7QS0BVbOP/bTwBKNnDc9rNzaPA==", "requires": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", - "get-intrinsic": "^1.1.1", + "function.prototype.name": "^1.1.5", + "get-intrinsic": "^1.1.3", + "get-symbol-description": "^1.0.0", "has": "^1.0.3", - "has-symbols": "^1.0.2", - "is-callable": "^1.2.3", - "is-negative-zero": "^2.0.1", - "is-regex": "^1.1.2", - "is-string": "^1.0.5", - "object-inspect": "^1.9.0", + "has-property-descriptors": "^1.0.0", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.7", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.2", "object-keys": "^1.1.1", - "object.assign": "^4.1.2", - "string.prototype.trimend": "^1.0.4", - "string.prototype.trimstart": "^1.0.4", - "unbox-primitive": "^1.0.0" + "object.assign": "^4.1.4", + "regexp.prototype.flags": "^1.4.3", + "safe-regex-test": "^1.0.0", + "string.prototype.trimend": "^1.0.5", + "string.prototype.trimstart": "^1.0.5", + "unbox-primitive": "^1.0.2" } }, "es-array-method-boxes-properly": { @@ -40120,14 +40458,13 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "function.prototype.name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.4.tgz", - "integrity": "sha512-iqy1pIotY/RmhdFZygSSlW0wko2yxkSCKqsuv4pr8QESohpYyG/Z7B/XXvPRKTJS//960rgguE5mSRUsDdaJrQ==", - "dev": true, + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", + "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2", + "es-abstract": "^1.19.0", "functions-have-names": "^1.2.2" } }, @@ -40140,8 +40477,7 @@ "functions-have-names": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.2.tgz", - "integrity": "sha512-bLgc3asbWdwPbx2mNk2S49kmJCuQeu0nfmaOgbs8WIyzzkw3r4htszdIi9Q9EMezDPTYuJx2wvjZ/EwgAthpnA==", - "dev": true + "integrity": "sha512-bLgc3asbWdwPbx2mNk2S49kmJCuQeu0nfmaOgbs8WIyzzkw3r4htszdIi9Q9EMezDPTYuJx2wvjZ/EwgAthpnA==" }, "fuse.js": { "version": "3.6.1", @@ -40231,13 +40567,13 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-intrinsic": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", - "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", - "has-symbols": "^1.0.1" + "has-symbols": "^1.0.3" } }, "get-package-type": { @@ -40255,6 +40591,15 @@ "pump": "^3.0.0" } }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, "get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", @@ -40593,9 +40938,9 @@ } }, "has-bigints": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", - "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==" }, "has-flag": { "version": "3.0.0", @@ -40622,10 +40967,26 @@ } } }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "requires": { + "get-intrinsic": "^1.1.1" + } + }, "has-symbols": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "requires": { + "has-symbols": "^1.0.2" + } }, "has-unicode": { "version": "2.0.1", @@ -41447,7 +41808,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", - "dev": true, "requires": { "get-intrinsic": "^1.1.0", "has": "^1.0.3", @@ -41550,9 +41910,12 @@ "dev": true }, "is-bigint": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.1.tgz", - "integrity": "sha512-J0ELF4yHFxHy0cmSxZuheDOz2luOdVvqjwmEcj8H/L1JHeuEDSDbeRP+Dk9kFVk5RTFzbucJ2Kb9F7ixY2QaCg==" + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "requires": { + "has-bigints": "^1.0.1" + } }, "is-binary-path": { "version": "2.1.0", @@ -41563,11 +41926,12 @@ } }, "is-boolean-object": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.0.tgz", - "integrity": "sha512-a7Uprx8UtD+HWdyYwnD1+ExtTgqQtD2k/1yJgtXP6wnMm8byhkoTZRl+95LLThpzNZJ5aEvi46cdH+ayMFRwmA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", "requires": { - "call-bind": "^1.0.0" + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" } }, "is-buffer": { @@ -41576,9 +41940,9 @@ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-callable": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", - "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==" + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" }, "is-ci": { "version": "1.2.1", @@ -41763,9 +42127,9 @@ "dev": true }, "is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==" }, "is-npm": { "version": "4.0.0", @@ -41792,9 +42156,12 @@ } }, "is-number-object": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.4.tgz", - "integrity": "sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "requires": { + "has-tostringtag": "^1.0.0" + } }, "is-obj": { "version": "2.0.0", @@ -41864,12 +42231,12 @@ "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" }, "is-regex": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.2.tgz", - "integrity": "sha512-axvdhb5pdhEVThqJzYXwMlVuZwC+FF2DpcOhTS+y/8jVq4trxyPgfcwIxIKiyeuLlSQYKkmUaPQJ8ZE4yNKXDg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", "requires": { "call-bind": "^1.0.2", - "has-symbols": "^1.0.1" + "has-tostringtag": "^1.0.0" } }, "is-relative": { @@ -41898,6 +42265,14 @@ "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", "dev": true }, + "is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "requires": { + "call-bind": "^1.0.2" + } + }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -41905,9 +42280,12 @@ "dev": true }, "is-string": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", - "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "requires": { + "has-tostringtag": "^1.0.0" + } }, "is-svg": { "version": "3.0.0", @@ -41939,6 +42317,14 @@ "unc-path-regex": "^0.1.2" } }, + "is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "requires": { + "call-bind": "^1.0.2" + } + }, "is-whitespace-character": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-whitespace-character/-/is-whitespace-character-1.0.4.tgz", @@ -44667,6 +45053,12 @@ "readable-stream": "^2.0.1" } }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -45071,6 +45463,58 @@ } } }, + "mongodb": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.10.0.tgz", + "integrity": "sha512-My2QxLTw0Cc1O9gih0mz4mqo145Jq4rLAQx0Glk/Ha9iYBzYpt4I2QFNRIh35uNFNfe8KFQcdwY1/HKxXBkinw==", + "requires": { + "bson": "^4.7.0", + "denque": "^2.1.0", + "mongodb-connection-string-url": "^2.5.3", + "saslprep": "^1.0.3", + "socks": "^2.7.0" + }, + "dependencies": { + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + } + } + }, + "mongodb-connection-string-url": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.4.tgz", + "integrity": "sha512-SeAxuWs0ez3iI3vvmLk/j2y+zHwigTDKQhtdxTgt5ZCOQQS5+HW4g45/Xw5vzzbn7oQXCNQ24Z40AkJsizEy7w==", + "requires": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + }, + "dependencies": { + "tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "requires": { + "punycode": "^2.1.1" + } + }, + "webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" + }, + "whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "requires": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + } + } + } + }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -45552,9 +45996,9 @@ "dev": true }, "object-inspect": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.9.0.tgz", - "integrity": "sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw==" + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==" }, "object-is": { "version": "1.1.3", @@ -45585,13 +46029,13 @@ } }, "object.assign": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", - "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", + "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", "requires": { - "call-bind": "^1.0.0", - "define-properties": "^1.1.3", - "has-symbols": "^1.0.1", + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "has-symbols": "^1.0.3", "object-keys": "^1.1.1" } }, @@ -48384,13 +48828,13 @@ } }, "regexp.prototype.flags": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", - "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", - "dev": true, + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" } }, "regexpp": { @@ -48880,6 +49324,16 @@ "ret": "~0.1.10" } }, + "safe-regex-test": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", + "integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==", + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.3", + "is-regex": "^1.1.4" + } + }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -48923,6 +49377,15 @@ } } }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "sass": { "version": "1.32.8", "resolved": "https://registry.npmjs.org/sass/-/sass-1.32.8.tgz", @@ -49329,7 +49792,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, "requires": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -49388,6 +49850,11 @@ } } }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" + }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -49543,6 +50010,22 @@ } } }, + "socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "requires": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "dependencies": { + "ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + } + } + }, "solr-client": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/solr-client/-/solr-client-0.9.0.tgz", @@ -49629,6 +50112,15 @@ "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==", "dev": true }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -49954,13 +50446,24 @@ "es-abstract": "^1.18.0-next.2" } }, + "string.prototype.trimend": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz", + "integrity": "sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" + } + }, "string.prototype.trimstart": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", - "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz", + "integrity": "sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg==", "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" } }, "strip-ansi": { @@ -50893,6 +51396,13 @@ "is-typedarray": "^1.0.0" } }, + "typescript": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "dev": true, + "peer": true + }, "uglify-js": { "version": "3.4.10", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.10.tgz", @@ -50918,14 +51428,14 @@ } }, "unbox-primitive": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.0.tgz", - "integrity": "sha512-P/51NX+JXyxK/aigg1/ZgyccdAxm5K1+n8+tvqSntjOivPt19gvm1VC49RWYetsiub8WViUchdxl/KWHHB0kzA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", "requires": { - "function-bind": "^1.1.1", - "has-bigints": "^1.0.0", - "has-symbols": "^1.0.0", - "which-boxed-primitive": "^1.0.1" + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" } }, "unc-path-regex": { diff --git a/package.json b/package.json index 5d63916..c7d3951 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "moment-locales-webpack-plugin": "^1.2.0", "moment-timezone": "^0.5.33", "moment-timezone-data-webpack-plugin": "^1.4.0", + "mongodb": "^4.10.0", "mysql2": "^2.2.5", "objection": "^2.2.14", "pino": "^6.11.0", diff --git a/server/index.js b/server/index.js index d842d54..7fbb745 100644 --- a/server/index.js +++ b/server/index.js @@ -453,7 +453,7 @@ const jobCountWrapper = (sourceName, type, fn) => { runningJobs, }) const dts = m.format(displayStringFormat) - console.log('Running jobs', runningJobs.length, `Starting job ${type} ${sourceName} at ${dts}`) + console.log('Running jobs', runningJobs.length, `Starting job ${type} ${sourceName} at ${dts} --`) await fn() } finally { const runningJobIndex = runningJobs.findIndex(x => x.sourceName === sourceName && x.type === type) @@ -462,7 +462,7 @@ const jobCountWrapper = (sourceName, type, fn) => { runningJobs, }) const m = moment().format(displayStringFormat) - console.log('Running jobs', runningJobs.length, `Ended job ${type} ${sourceName} at ${m}`) + console.log('Running jobs', runningJobs.length, `Ended job ${type} ${sourceName} at ${m} <<`) } } } diff --git a/vue.config.js b/vue.config.js index cd8d975..2fee20a 100644 --- a/vue.config.js +++ b/vue.config.js @@ -7,10 +7,10 @@ const currentYear = new Date().getFullYear() module.exports = { devServer: { port: 3461, - host: 'openresync.test', + host: 'localhost', proxy: { '.*': { - target: 'http://openresync.test:' + config.server.port, + target: 'http://localhost:' + config.server.port, ws: true, }, },