From 6416f2731e016f1b8bcf5a664b34c43fdde69a18 Mon Sep 17 00:00:00 2001 From: Smartek <70715469+smartekIT@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:27:25 +1000 Subject: [PATCH 1/5] use mongodb TTL for 7 days age - reducing history in the database for 30 days instead of 90 days. - added new index in the mongodb to activate TTL feature for faster process of cleaning finished jobs after 7 days. --- db.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/db.js b/db.js index 5985fbc..55002a4 100644 --- a/db.js +++ b/db.js @@ -196,7 +196,8 @@ module.exports.CreateDB = function(meshserver) { }; obj.deleteOldHistory = function() { var nowTime = Math.floor(new Date() / 1000); - var oldTime = nowTime - (86400 * 90); // 90 days + //var oldTime = nowTime - (86400 * 90); // 90 days + var oldTime = nowTime - (86400 * 30); // 30 days return obj.scriptFile.deleteMany( { type: 'job', completeTime: { $lte: oldTime } } ); }; obj.addVariable = function(name, scope, scopeTarget, value) { @@ -263,7 +264,7 @@ module.exports.CreateDB = function(meshserver) { // Check if we need to reset indexes var indexesByName = {}, indexCount = 0; for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; } - if ((indexCount != 6) || (indexesByName['ScriptName1'] == null) || (indexesByName['ScriptPath1'] == null) || (indexesByName['JobTime1'] == null) || (indexesByName['JobNode1'] == null) || (indexesByName['JobScriptID1'] == null)) { + if ((indexCount != 7) || (indexesByName['ScriptName1'] == null) || (indexesByName['ScriptPath1'] == null) || (indexesByName['JobTime1'] == null) || (indexesByName['JobNode1'] == null) || (indexesByName['JobScriptID1'] == null)) || (indexesByName['ClearScriptHist'] == null)) { // Reset all indexes console.log('Resetting plugin (ScriptTask) indexes...'); obj.scriptFile.dropIndexes(function (err) { @@ -272,6 +273,7 @@ module.exports.CreateDB = function(meshserver) { obj.scriptFile.createIndex({ queueTime: 1 }, { name: 'JobTime1' }); obj.scriptFile.createIndex({ node: 1 }, { name: 'JobNode1' }); obj.scriptFile.createIndex({ scriptId: 1 }, { name: 'JobScriptID1' }); + obj.scriptFile.createIndex({ completeTime: 1 }, {name: 'ClearScriptHist', partialFilterExpression: { returnVal: { '$exists': true }, type: { '$eq': 'job' } }, expireAfterSeconds: 604800}); }); } }); @@ -294,6 +296,7 @@ module.exports.CreateDB = function(meshserver) { obj.scriptFilex.ensureIndex({ fieldName: 'queueTime' }); obj.scriptFilex.ensureIndex({ fieldName: 'node' }); obj.scriptFilex.ensureIndex({ fieldName: 'scriptId' }); + obj.scriptFilex.ensureIndex({ fieldName: 'completeTime' }); } obj.scriptFile = new NEMongo(obj.scriptFilex); formatId = function(id) { return id; }; @@ -301,4 +304,4 @@ module.exports.CreateDB = function(meshserver) { } return obj; -} \ No newline at end of file +} From d9b2dbb8e290a4a20acaf77de26cfc0be8cdb1ff Mon Sep 17 00:00:00 2001 From: Smartek <70715469+smartekIT@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:36:08 +1000 Subject: [PATCH 2/5] change completeTime filed format to date in db --- scripttask.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripttask.js b/scripttask.js index 609e720..d152402 100644 --- a/scripttask.js +++ b/scripttask.js @@ -656,7 +656,8 @@ module.exports.scripttask = function (parent) { //obj.debug('ScriptTask', 'jobComplete Triggered', JSON.stringify(command)); var jobNodeHistory = null, scriptHistory = null; var jobId = command.jobId, retVal = command.retVal, errVal = command.errVal, dispatchTime = command.dispatchTime; - var completeTime = Math.floor(new Date() / 1000); + //var completeTime = Math.floor(new Date() / 1000); + var completeTime = (new Date()); obj.db.update(jobId, { completeTime: completeTime, returnVal: retVal, @@ -670,7 +671,8 @@ module.exports.scripttask = function (parent) { }) .then(sId => { if (sId == null) return Promise.resolve(); - return obj.db.update(sId, { lastRun: completeTime } ) + //return obj.db.update(sId, { lastRun: completeTime } ) + return obj.db.update(sId, { lastRun: dispatchTime } ) .then(() => { obj.makeJobsFromSchedules(sId); }); From 189e59402e5f8a78f6082dd3a72aeede846046d9 Mon Sep 17 00:00:00 2001 From: Smartek <70715469+smartekIT@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:44:31 +1000 Subject: [PATCH 3/5] fix time reading after mongodb TTL addition completeTime is no longer part of latestTime to avoid invalid date records after changed to mongodb TTL and completeTime field to datetime type. --- views/user.handlebars | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/views/user.handlebars b/views/user.handlebars index f69a731..bdf0ca1 100644 --- a/views/user.handlebars +++ b/views/user.handlebars @@ -511,7 +511,8 @@ function redrawScriptTree() { } if (message.event.nodeHistory.length) { message.event.nodeHistory.forEach(function(nh) { - nh.latestTime = Math.max(nh.completeTime, nh.queueTime, nh.dispatchTime, nh.dontQueueUntil); + //nh.latestTime = Math.max(nh.completeTime, nh.queueTime, nh.dispatchTime, nh.dontQueueUntil); + nh.latestTime = Math.max(nh.queueTime, nh.dispatchTime, nh.dontQueueUntil); }); message.event.nodeHistory.sort((a, b) => (a.latestTime < b.latestTime) ? 1 : -1); message.event.nodeHistory.forEach(function(nh) { @@ -539,7 +540,8 @@ function redrawScriptTree() { } if (message.event.scriptHistory.length) { message.event.scriptHistory.forEach(function(nh) { - nh.latestTime = Math.max(nh.completeTime, nh.queueTime, nh.dispatchTime, nh.dontQueueUntil); + //nh.latestTime = Math.max(nh.completeTime, nh.queueTime, nh.dispatchTime, nh.dontQueueUntil); + nh.latestTime = Math.max(nh.queueTime, nh.dispatchTime, nh.dontQueueUntil); }); message.event.scriptHistory.sort((a, b) => (a.latestTime < b.latestTime) ? 1 : -1); message.event.scriptHistory.forEach(function(nh) { From 3669851fc47939db04a94b8792008cdefe9c9ac3 Mon Sep 17 00:00:00 2001 From: Smartek <70715469+smartekIT@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:59:31 +1000 Subject: [PATCH 4/5] added info about mongodb TTL --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index a289b34..d11894e 100644 --- a/readme.md +++ b/readme.md @@ -32,7 +32,8 @@ Restart your MeshCentral server after making this change. - Queues are checked / run every minute. - Scheduled jobs only show the *next* scheduled job at any given time. - History is limited to 200 events per node/script in the viewport. -- Historical events that have completed will delete after 90 days. +- Historical events that have completed will delete after 30 days. +- For mongodb it's even faster cleaning by using the TTL feature to delete completed jobs after 30 days. - Jobs only run when the endpoint agent is online. They do *not* queue to the agent when offline, then run at the specified time. - Scripts are cached on the clients and verified via hash at runtime for the latest version. From 8357fa4002a178c6dacd35a0184bd2ba0e9b26ab Mon Sep 17 00:00:00 2001 From: Smartek <70715469+smartekIT@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:04:57 +1000 Subject: [PATCH 5/5] changed mongodb TTL to 30 days making it more satisfying by increase the mongodb TTL to 30 days before deleting the old finished jobs. --- db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db.js b/db.js index 55002a4..20cda9c 100644 --- a/db.js +++ b/db.js @@ -273,7 +273,7 @@ module.exports.CreateDB = function(meshserver) { obj.scriptFile.createIndex({ queueTime: 1 }, { name: 'JobTime1' }); obj.scriptFile.createIndex({ node: 1 }, { name: 'JobNode1' }); obj.scriptFile.createIndex({ scriptId: 1 }, { name: 'JobScriptID1' }); - obj.scriptFile.createIndex({ completeTime: 1 }, {name: 'ClearScriptHist', partialFilterExpression: { returnVal: { '$exists': true }, type: { '$eq': 'job' } }, expireAfterSeconds: 604800}); + obj.scriptFile.createIndex({ completeTime: 1 }, {name: 'ClearScriptHist', partialFilterExpression: { returnVal: { '$exists': true }, type: { '$eq': 'job' } }, expireAfterSeconds: 2592000}); }); } });