diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 6ec85f6..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Run mData Web", - "skipFiles": [ - "/**" - ], - "program": "${workspaceFolder}/app/web.js" - }, - { - "type": "node", - "request": "launch", - "name": "Run mData Worker", - "skipFiles": [ - "/**" - ], - "program": "${workspaceFolder}/app/mdata-worker.js" - } - ] -} \ No newline at end of file diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 0000000..34de966 --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,7 @@ +FROM node:12-alpine +WORKDIR /usr/src/app +COPY package*.json ./ +RUN npm install +COPY . . +EXPOSE 3000 +CMD [ "node", "web.js" ] \ No newline at end of file diff --git a/app/app.js b/app/app.js deleted file mode 100644 index 04e504e..0000000 --- a/app/app.js +++ /dev/null @@ -1,43 +0,0 @@ -console.log(`NEO4J_URL: ${process.env.NEO4J_URL}`); -console.log(`REDIS_URL: ${process.env.REDIS_URL}`); -const sleep = require('sleepjs'); -var neo4j = require('neo4j'); -var db = new neo4j.GraphDatabase(process.env.NEO4J_URL); - -var redis = require("redis"), - client = redis.createClient({ - url: process.env.REDIS_URL - }); -client.on("error", function (err) { - console.log("Redis client Error " + err); -}); - -async function runMain() { - console.info('Sleeping for 20 secs'); - await sleep(20000); - db.cypher({ - query: 'MERGE (alice:Person {name : {nameParam} }) RETURN alice.name AS name', - params: { - nameParam: 'Alice', - }, - }, function (err, results) { - if (err) throw err; - console.log(`neo4j results: ${JSON.stringify(results, null, 4)}`); - }); - - client.set('some-key', '42', function (err) { - if (err) { - throw err; - } else { - client.get('some-key', function (err, value) { - if (err) { - throw err; - } else { - console.log(`redis results ${value}`); - } - }); - } - }); -} - -runMain(); \ No newline at end of file diff --git a/app/index.js b/app/index.js deleted file mode 100755 index 14c3b58..0000000 --- a/app/index.js +++ /dev/null @@ -1,11 +0,0 @@ -const SFConnectionUtils = require('./utils/SFConnectionUtils.js'); -const MdataIndexer = require('./MdataIndexer.js'); -async function main() { - var sfConn = new SFConnectionUtils(); - await sfConn.login(); - var indexer = new MdataIndexer(sfConn); - await sfConn.setOrgNamespacePrefix(); - await sfConn.setIdentityInfo(); - await indexer.startIndexing(); -} -main(); \ No newline at end of file diff --git a/app/package.json b/app/package.json index d64495d..93ab48d 100755 --- a/app/package.json +++ b/app/package.json @@ -4,8 +4,7 @@ "description": "Docker version of Mdata", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start-worker":"node mdata-worker.js", - "start-web":"node web.js" + "start":"node web.js" }, "author": "", "license": "ISC", diff --git a/app/web.js b/app/web.js index 77789ad..53470af 100755 --- a/app/web.js +++ b/app/web.js @@ -12,9 +12,9 @@ client = redis.createClient({ }); -const SFConnectionUtils = require('./utils/SFConnectionUtils.js'); +const SFConnectionUtils = require('./worker/utils/SFConnectionUtils.js'); -const PluginFactory = require('./plugins/PluginFactory.js'); +const PluginFactory = require('./worker/plugins/PluginFactory.js'); var pluginFactory = new PluginFactory(); @@ -45,8 +45,6 @@ app.set('port', process.env.PORT || 3000); //Middleware app.use(session({secret: "w585hJqIfL0GWMUbD1WboOuvsjG9Urv1h8cEv8XyFZBPYV582WnLKapj1TboI5gp8sy3hDC53mbDXYDjLrIEvBbsz3MDKmzdLZCw"})); app.use('/plugins/:id/execute',async function (req, res, next) { - - console.log('OAuth Header:', req.get('Authorization')); let oauthHdrValid = {}; if(!process.env.LOCAL){ oauthHdrValid = await validOauthHeader(req); @@ -66,7 +64,7 @@ app.use('/plugins/:id/execute',async function (req, res, next) { } }); -app.use(express.static('./app/public')); +app.use(express.static('./public')); validOauthHeader = async function(req){ let authHdr = req.get('Authorization'); @@ -144,7 +142,6 @@ oauthCallback = function (request, response) { // You can change loginUrl to connect to sandbox or prerelease env. base_url: loginUrl }, function (error, payload) { - console.log('Payload:' + JSON.stringify(payload)); if (payload) { client.rpush('mdata_org_index', JSON.stringify(payload)); response.send('Authorization successful !!!'); @@ -167,6 +164,8 @@ mdataInit = function (request, response) { }); return response.redirect(uri); }; + + app.get('/plugins', listAllPlugins); app.get('/plugins/:id', pluginDetail); app.get('/plugins/:id/execute', runPlugin); @@ -175,9 +174,6 @@ app.get('/oauth/callback', oauthCallback); app.get("/mdata",mdataInit ); - - - // Start listening for HTTP requests var server = app.listen(app.get('port'), function() { var host = server.address().address; diff --git a/app/worker/Dockerfile b/app/worker/Dockerfile new file mode 100644 index 0000000..d324831 --- /dev/null +++ b/app/worker/Dockerfile @@ -0,0 +1,7 @@ +FROM node:12-alpine +WORKDIR /usr/src/app/worker +COPY package*.json ./ +RUN npm install +COPY . . +EXPOSE 3000 +CMD [ "node", "mdata-worker.js" ] \ No newline at end of file diff --git a/app/MdataIndexer.js b/app/worker/MdataIndexer.js similarity index 100% rename from app/MdataIndexer.js rename to app/worker/MdataIndexer.js diff --git a/app/apex-parser/ApexCodeBlock.js b/app/worker/apex-parser/ApexCodeBlock.js similarity index 100% rename from app/apex-parser/ApexCodeBlock.js rename to app/worker/apex-parser/ApexCodeBlock.js diff --git a/app/apex-parser/ApexParserUtils.js b/app/worker/apex-parser/ApexParserUtils.js similarity index 100% rename from app/apex-parser/ApexParserUtils.js rename to app/worker/apex-parser/ApexParserUtils.js diff --git a/app/apex-parser/Apexcode.g4 b/app/worker/apex-parser/Apexcode.g4 similarity index 100% rename from app/apex-parser/Apexcode.g4 rename to app/worker/apex-parser/Apexcode.g4 diff --git a/app/apex-parser/Apexcode.tokens b/app/worker/apex-parser/Apexcode.tokens similarity index 100% rename from app/apex-parser/Apexcode.tokens rename to app/worker/apex-parser/Apexcode.tokens diff --git a/app/apex-parser/ApexcodeLexer.js b/app/worker/apex-parser/ApexcodeLexer.js similarity index 100% rename from app/apex-parser/ApexcodeLexer.js rename to app/worker/apex-parser/ApexcodeLexer.js diff --git a/app/apex-parser/ApexcodeLexer.tokens b/app/worker/apex-parser/ApexcodeLexer.tokens similarity index 100% rename from app/apex-parser/ApexcodeLexer.tokens rename to app/worker/apex-parser/ApexcodeLexer.tokens diff --git a/app/apex-parser/ApexcodeListener.js b/app/worker/apex-parser/ApexcodeListener.js similarity index 100% rename from app/apex-parser/ApexcodeListener.js rename to app/worker/apex-parser/ApexcodeListener.js diff --git a/app/apex-parser/ApexcodeParser.js b/app/worker/apex-parser/ApexcodeParser.js similarity index 100% rename from app/apex-parser/ApexcodeParser.js rename to app/worker/apex-parser/ApexcodeParser.js diff --git a/app/apex-parser/ApexcodeVisitor.js b/app/worker/apex-parser/ApexcodeVisitor.js similarity index 100% rename from app/apex-parser/ApexcodeVisitor.js rename to app/worker/apex-parser/ApexcodeVisitor.js diff --git a/app/config.js b/app/worker/config.js similarity index 100% rename from app/config.js rename to app/worker/config.js diff --git a/app/mdata-worker.js b/app/worker/mdata-worker.js similarity index 82% rename from app/mdata-worker.js rename to app/worker/mdata-worker.js index 7efd30c..7a1de24 100755 --- a/app/mdata-worker.js +++ b/app/worker/mdata-worker.js @@ -23,9 +23,14 @@ async function main(){ async function waitForRequest() { client.brpop(['mdata_org_index', 1], async function (list, items) { if (items && items.length === 2) { - logger.info('Received request to index org ... '); - await handleMdataIndexRequest(JSON.parse(items[1])); - setTimeout(waitForRequest, 3000); + try { + logger.info('Received request to index org ... '); + await handleMdataIndexRequest(JSON.parse(items[1])); + } catch(error) { + console.error(error); + } finally { + setTimeout(waitForRequest, 3000); + } } else { logger.info('No active request...'); setTimeout(waitForRequest, 3000); diff --git a/app/metadata/AbstractMetadataType.js b/app/worker/metadata/AbstractMetadataType.js similarity index 100% rename from app/metadata/AbstractMetadataType.js rename to app/worker/metadata/AbstractMetadataType.js diff --git a/app/metadata/ApexClasses.js b/app/worker/metadata/ApexClasses.js similarity index 100% rename from app/metadata/ApexClasses.js rename to app/worker/metadata/ApexClasses.js diff --git a/app/metadata/ApexTriggers.js b/app/worker/metadata/ApexTriggers.js similarity index 100% rename from app/metadata/ApexTriggers.js rename to app/worker/metadata/ApexTriggers.js diff --git a/app/metadata/CustomFields.js b/app/worker/metadata/CustomFields.js similarity index 100% rename from app/metadata/CustomFields.js rename to app/worker/metadata/CustomFields.js diff --git a/app/metadata/CustomObjects.js b/app/worker/metadata/CustomObjects.js similarity index 100% rename from app/metadata/CustomObjects.js rename to app/worker/metadata/CustomObjects.js diff --git a/app/metadata/EmailTemplates.js b/app/worker/metadata/EmailTemplates.js similarity index 100% rename from app/metadata/EmailTemplates.js rename to app/worker/metadata/EmailTemplates.js diff --git a/app/metadata/FormulaFields.js b/app/worker/metadata/FormulaFields.js similarity index 100% rename from app/metadata/FormulaFields.js rename to app/worker/metadata/FormulaFields.js diff --git a/app/metadata/Layouts.js b/app/worker/metadata/Layouts.js similarity index 100% rename from app/metadata/Layouts.js rename to app/worker/metadata/Layouts.js diff --git a/app/metadata/ListViews.js b/app/worker/metadata/ListViews.js similarity index 100% rename from app/metadata/ListViews.js rename to app/worker/metadata/ListViews.js diff --git a/app/metadata/MetadataFactory.js b/app/worker/metadata/MetadataFactory.js similarity index 100% rename from app/metadata/MetadataFactory.js rename to app/worker/metadata/MetadataFactory.js diff --git a/app/metadata/PicklistValues.js b/app/worker/metadata/PicklistValues.js similarity index 100% rename from app/metadata/PicklistValues.js rename to app/worker/metadata/PicklistValues.js diff --git a/app/metadata/Profiles.js b/app/worker/metadata/Profiles.js similarity index 100% rename from app/metadata/Profiles.js rename to app/worker/metadata/Profiles.js diff --git a/app/metadata/PublicGroups.js b/app/worker/metadata/PublicGroups.js similarity index 100% rename from app/metadata/PublicGroups.js rename to app/worker/metadata/PublicGroups.js diff --git a/app/metadata/RecordTypes.js b/app/worker/metadata/RecordTypes.js similarity index 100% rename from app/metadata/RecordTypes.js rename to app/worker/metadata/RecordTypes.js diff --git a/app/metadata/ReportTypes.js b/app/worker/metadata/ReportTypes.js similarity index 100% rename from app/metadata/ReportTypes.js rename to app/worker/metadata/ReportTypes.js diff --git a/app/metadata/Reports.js b/app/worker/metadata/Reports.js similarity index 100% rename from app/metadata/Reports.js rename to app/worker/metadata/Reports.js diff --git a/app/metadata/Roles.js b/app/worker/metadata/Roles.js similarity index 100% rename from app/metadata/Roles.js rename to app/worker/metadata/Roles.js diff --git a/app/metadata/ValidationRules.js b/app/worker/metadata/ValidationRules.js similarity index 100% rename from app/metadata/ValidationRules.js rename to app/worker/metadata/ValidationRules.js diff --git a/app/metadata/WorkflowAlerts.js b/app/worker/metadata/WorkflowAlerts.js similarity index 100% rename from app/metadata/WorkflowAlerts.js rename to app/worker/metadata/WorkflowAlerts.js diff --git a/app/metadata/WorkflowFieldUpdates.js b/app/worker/metadata/WorkflowFieldUpdates.js similarity index 100% rename from app/metadata/WorkflowFieldUpdates.js rename to app/worker/metadata/WorkflowFieldUpdates.js diff --git a/app/metadata/WorkflowRules.js b/app/worker/metadata/WorkflowRules.js similarity index 100% rename from app/metadata/WorkflowRules.js rename to app/worker/metadata/WorkflowRules.js diff --git a/app/worker/package.json b/app/worker/package.json new file mode 100755 index 0000000..ca8327a --- /dev/null +++ b/app/worker/package.json @@ -0,0 +1,33 @@ +{ + "name": "mdata-worker", + "version": "1.0.0", + "description": "Docker version of Mdata", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start":"node mdata-worker.js" + }, + "author": "", + "license": "ISC", + "dependencies": { + "antlr4": "^4.7.1", + "await-async": "0.0.1", + "body-parser": "^1.18.3", + "dotenv": "^6.0.0", + "express": "^4.16.3", + "express-session": "^1.15.6", + "jsforce": "^1.8.5", + "lodash": "^4.17.10", + "neo4j-driver": "^1.6.2", + "node-rest-client": "^3.1.0", + "redis": "^2.8.0", + "salesforce-oauth2": "^0.2.0", + "sf-formula-ast": "0.0.1", + "sleepjs": "^3.0.1", + "unzip": "^0.1.11", + "unzip2": "^0.2.5", + "unzipper": "^0.10.5", + "winston": "^3.0.0", + "xml2js": "^0.4.19", + "natives": "^1.1.6" + } +} diff --git a/app/plugins/AbstractPlugin.js b/app/worker/plugins/AbstractPlugin.js similarity index 100% rename from app/plugins/AbstractPlugin.js rename to app/worker/plugins/AbstractPlugin.js diff --git a/app/plugins/AllFieldReferencesPlugin.js b/app/worker/plugins/AllFieldReferencesPlugin.js similarity index 100% rename from app/plugins/AllFieldReferencesPlugin.js rename to app/worker/plugins/AllFieldReferencesPlugin.js diff --git a/app/plugins/AllObjectsPlugin.js b/app/worker/plugins/AllObjectsPlugin.js similarity index 100% rename from app/plugins/AllObjectsPlugin.js rename to app/worker/plugins/AllObjectsPlugin.js diff --git a/app/plugins/FieldsNotInLayoutsPlugin.js b/app/worker/plugins/FieldsNotInLayoutsPlugin.js similarity index 100% rename from app/plugins/FieldsNotInLayoutsPlugin.js rename to app/worker/plugins/FieldsNotInLayoutsPlugin.js diff --git a/app/plugins/GlobalPicklistPlugin.js b/app/worker/plugins/GlobalPicklistPlugin.js similarity index 100% rename from app/plugins/GlobalPicklistPlugin.js rename to app/worker/plugins/GlobalPicklistPlugin.js diff --git a/app/plugins/IdenticalProfilesPlugin.js b/app/worker/plugins/IdenticalProfilesPlugin.js similarity index 100% rename from app/plugins/IdenticalProfilesPlugin.js rename to app/worker/plugins/IdenticalProfilesPlugin.js diff --git a/app/plugins/ListViewWithNoFiltersPlugin.js b/app/worker/plugins/ListViewWithNoFiltersPlugin.js similarity index 100% rename from app/plugins/ListViewWithNoFiltersPlugin.js rename to app/worker/plugins/ListViewWithNoFiltersPlugin.js diff --git a/app/plugins/PluginFactory.js b/app/worker/plugins/PluginFactory.js similarity index 100% rename from app/plugins/PluginFactory.js rename to app/worker/plugins/PluginFactory.js diff --git a/app/plugins/ProfileDiffPlugin.js b/app/worker/plugins/ProfileDiffPlugin.js similarity index 100% rename from app/plugins/ProfileDiffPlugin.js rename to app/worker/plugins/ProfileDiffPlugin.js diff --git a/app/plugins/SimilarLayoutsPlugin.js b/app/worker/plugins/SimilarLayoutsPlugin.js similarity index 100% rename from app/plugins/SimilarLayoutsPlugin.js rename to app/worker/plugins/SimilarLayoutsPlugin.js diff --git a/app/plugins/SimilarProfilesPlugin.js b/app/worker/plugins/SimilarProfilesPlugin.js similarity index 100% rename from app/plugins/SimilarProfilesPlugin.js rename to app/worker/plugins/SimilarProfilesPlugin.js diff --git a/app/soql-parser/SOQL.g4 b/app/worker/soql-parser/SOQL.g4 similarity index 100% rename from app/soql-parser/SOQL.g4 rename to app/worker/soql-parser/SOQL.g4 diff --git a/app/soql-parser/SOQL.tokens b/app/worker/soql-parser/SOQL.tokens similarity index 100% rename from app/soql-parser/SOQL.tokens rename to app/worker/soql-parser/SOQL.tokens diff --git a/app/soql-parser/SOQLLexer.js b/app/worker/soql-parser/SOQLLexer.js similarity index 100% rename from app/soql-parser/SOQLLexer.js rename to app/worker/soql-parser/SOQLLexer.js diff --git a/app/soql-parser/SOQLLexer.tokens b/app/worker/soql-parser/SOQLLexer.tokens similarity index 100% rename from app/soql-parser/SOQLLexer.tokens rename to app/worker/soql-parser/SOQLLexer.tokens diff --git a/app/soql-parser/SOQLListener.js b/app/worker/soql-parser/SOQLListener.js similarity index 100% rename from app/soql-parser/SOQLListener.js rename to app/worker/soql-parser/SOQLListener.js diff --git a/app/soql-parser/SOQLParser.js b/app/worker/soql-parser/SOQLParser.js similarity index 100% rename from app/soql-parser/SOQLParser.js rename to app/worker/soql-parser/SOQLParser.js diff --git a/app/soql-parser/SOQLParserUtils.js b/app/worker/soql-parser/SOQLParserUtils.js similarity index 100% rename from app/soql-parser/SOQLParserUtils.js rename to app/worker/soql-parser/SOQLParserUtils.js diff --git a/app/soql-parser/SOQLStatement.js b/app/worker/soql-parser/SOQLStatement.js similarity index 100% rename from app/soql-parser/SOQLStatement.js rename to app/worker/soql-parser/SOQLStatement.js diff --git a/app/utils/JSONUtils.js b/app/worker/utils/JSONUtils.js similarity index 100% rename from app/utils/JSONUtils.js rename to app/worker/utils/JSONUtils.js diff --git a/app/utils/LoggerUtils.js b/app/worker/utils/LoggerUtils.js similarity index 100% rename from app/utils/LoggerUtils.js rename to app/worker/utils/LoggerUtils.js diff --git a/app/utils/MetadataUtils.js b/app/worker/utils/MetadataUtils.js similarity index 100% rename from app/utils/MetadataUtils.js rename to app/worker/utils/MetadataUtils.js diff --git a/app/utils/Neo4JUtils.js b/app/worker/utils/Neo4JUtils.js similarity index 100% rename from app/utils/Neo4JUtils.js rename to app/worker/utils/Neo4JUtils.js diff --git a/app/utils/ParseUtils.js b/app/worker/utils/ParseUtils.js similarity index 100% rename from app/utils/ParseUtils.js rename to app/worker/utils/ParseUtils.js diff --git a/app/utils/SFConnectionUtils.js b/app/worker/utils/SFConnectionUtils.js similarity index 100% rename from app/utils/SFConnectionUtils.js rename to app/worker/utils/SFConnectionUtils.js diff --git a/conf/neo4j.conf b/conf/neo4j.conf old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml index cef7d4f..1489a16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,42 +1,28 @@ -mdata-worker: - image: node:latest - volumes: - - ./app:/app - links: - - neo4j - - redis - working_dir: /app - command: node mdata-worker.js --inspect-brk=0.0.0.0 . - env_file: "mdata.env" - ports: - - "9229:9229" - -mdata-web: - image: node:latest - volumes: - - ./app:/app - links: - - neo4j - - redis - - mdata-worker - working_dir: /app - command: node web.js - env_file: "mdata.env" - ports: - - "3000:3000" -redis: - image: redis - container_name: cache - expose: - - 6379 -neo4j: - image: neo4j:3.5.12 - environment: - - NEO4J_AUTH=neo4j/test - ports: - - "7474:7474" - - "7687:7687" - volumes: - - ./data:/data - - ./conf:/conf - - ./plugins:/plugins +version: '3' +services: + mdata-worker: + build: ./app/worker + env_file: "mdata.env" + ports: + - "9229:9229" + mdata-web: + build: ./app + env_file: "mdata.env" + ports: + - "3000:3000" + redis: + image: redis + container_name: cache + expose: + - 6379 + neo4j: + image: neo4j:3.5.12 + environment: + - NEO4J_AUTH=neo4j/test + ports: + - "7474:7474" + - "7687:7687" + volumes: + - ./data:/data + - ./conf:/conf + - ./plugins:/plugins \ No newline at end of file diff --git a/heroku.yml b/heroku.yml new file mode 100644 index 0000000..eaa531c --- /dev/null +++ b/heroku.yml @@ -0,0 +1,18 @@ +setup: + addons: + - plan: heroku-redis:hobby-dev + - plan: graphenedb:dev-free + config: + LOCAL: true + SKIP_JOB_CREATE: true + BOOTSTRAP_SLEEP_SECS: 15 + SF_CLIENT_ID: 3MVG9WtWSKUDG.x6eKgRnQaY0devdNN4eIspkzn5bM7B4KpnqDMadEWWqFzlZBOzXYhCRnRfONx1n0ekc4wWe + SF_CLIENT_SECRET: BFDE194265A7FC3DBCFD0585FCA3EDA355D8895C913B42739F5FA8B304575874 + SF_CALLBACK_URL: https://mdata-poc.herokuapp.com/oauth/callback +build: + docker: + web: app/Dockerfile + worker: app/worker/Dockerfile +run: + web: npm start + worker: npm start \ No newline at end of file diff --git a/plugins/README.txt b/plugins/README.txt old mode 100644 new mode 100755 diff --git a/plugins/apoc-3.5.0.1.jar b/plugins/apoc-3.5.0.1.jar old mode 100644 new mode 100755 diff --git a/plugins/graphAlgorithms-3.5.2.0.jar b/plugins/graphAlgorithms-3.5.2.0.jar old mode 100644 new mode 100755 diff --git a/run_mdata.sh b/run_mdata.sh deleted file mode 100755 index 7bfce87..0000000 --- a/run_mdata.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -if [ $1 == "up" ]; -then - cd app - npm clean-install - cd .. -fi -docker-compose $@ \ No newline at end of file diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index fb57ccd..0000000 --- a/yarn.lock +++ /dev/null @@ -1,4 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - -