diff --git a/src/butler.js b/src/butler.js index 4c03ad1e..910cc6a2 100644 --- a/src/butler.js +++ b/src/butler.js @@ -11,7 +11,32 @@ const build = require('./app'); const udp = require('./udp'); const { mqttInitHandlers } = require('./lib/mqtt_handlers'); +const { + configFileStructureAssert, + configFileYamlAssert, + configFileNewRelicAssert, + configFileInfluxDbAssert, +} = require('./lib/assert/assert_config_file'); + const start = async () => { + // Verify correct structure of config file + configFileStructureAssert(globals.config, globals.logger); + + // Verify that config file is valid YAML + configFileStructureAssert(globals.config, globals.logger); + + // Verify that config file is valid YAML + configFileYamlAssert(globals.configFileExpanded); + + // Verify select parts/values in config file + if (globals.options.qsConnection) { + // Verify that the config file contains the required data related to New Relic + configFileNewRelicAssert(globals.config, globals.configQRS, globals.logger); + + // Verify that the config file contains the required data related to InfluxDb + configFileInfluxDbAssert(globals.config, globals.configQRS, globals.logger); + } + const apps = await build({}); const { restServer } = apps; @@ -132,7 +157,7 @@ const start = async () => { } // Prepare to listen on port Y for incoming UDP connections regarding failed tasks - globals.udpServerTaskFailureSocket = dgram.createSocket({ + globals.udpServerReloadTaskSocket = dgram.createSocket({ type: 'udp4', reuseAddr: true, }); @@ -143,7 +168,7 @@ const start = async () => { udp.udp.udpInitTaskErrorServer(); // Start UDP server for failed task events - globals.udpServerTaskFailureSocket.bind(globals.udpPortTaskFailure, globals.udpHost); + globals.udpServerReloadTaskSocket.bind(globals.udpPortTaskFailure, globals.udpHost); globals.logger.debug(`Server for UDP server: ${globals.udpHost}`); } }; diff --git a/src/config/config-gen-api-docs.yaml b/src/config/config-gen-api-docs.yaml index 23b97bc1..bca40858 100644 --- a/src/config/config-gen-api-docs.yaml +++ b/src/config/config-gen-api-docs.yaml @@ -99,6 +99,21 @@ Butler: dynamic: useAppTags: true # Should app tags be stored in InfluxDB as tags? useTaskTags: true # Should task tags be stored in InfluxDB as tags? + reloadTaskSuccess: + enable: true + allReloadTasks: + enable: false + byCustomProperty: + enable: true + customPropertyName: 'Butler_SuccessReloadTask_InfluxDB' + enabledValue: 'Yes' + tag: + static: # Static attributes/dimensions to attach to events sent to InfluxDb + # - name: event-specific-tag 1 + # value: abc 123 + dynamic: + useAppTags: true # Should app tags be sent to InfluxDb as tags? + useTaskTags: true # Should task tags be sent to InfluxDb as tags? # Store script logs of failed reloads on disk. # The script logs will be stored in daily directories under the specified main directory below diff --git a/src/config/log_appender_xml/scheduler/LocalLogConfig.xml b/src/config/log_appender_xml/scheduler/LocalLogConfig.xml index 26027af2..191a3348 100644 --- a/src/config/log_appender_xml/scheduler/LocalLogConfig.xml +++ b/src/config/log_appender_xml/scheduler/LocalLogConfig.xml @@ -37,6 +37,25 @@ </layout> </appender> + <!-- Appender for detecting successful reload tasks --> + <appender name="ReloadTaskSuccessLogger" type="log4net.Appender.UdpAppender"> + <filter type="log4net.Filter.StringMatchFilter"> + <param name="stringToMatch" value="Reload complete" /> + </filter> + <filter type="log4net.Filter.DenyAllFilter" /> + <param name="remoteAddress" value="<IP of server where Butler is running>" /> + <param name="remotePort" value="9998" /> + <param name="encoding" value="utf-8" /> + <layout type="log4net.Layout.PatternLayout"> + <converter> + <param name="name" value="hostname" /> + <param name="type" value="Qlik.Sense.Logging.log4net.Layout.Pattern.HostNamePatternConverter" /> + </converter> + <param name="conversionpattern" value="/scheduler-reloadtask-success/;%hostname;%property{TaskName};%property{AppName};%property{User};%property{TaskId};%property{AppId};%date;%level;%property{ExecutionId};%message" /> + </layout> + </appender> + + <!-- Mail appender. Not dependent on Butler. Works as a basic solution, but does not support templating, script logs etc that Butler offers --> <!-- <appender name="MailAppender" type="log4net.Appender.SmtpAppender"> --> <!-- <filter type="log4net.Filter.StringMatchFilter"> --> @@ -72,6 +91,11 @@ <appender-ref ref="AbortedReloadTaskLogger" /> </logger> + <!-- Send message to Butler on reload task success --> + <logger name="System.Scheduler.Scheduler.Slave.Tasks.ReloadTask"> + <appender-ref ref="ReloadTaskSuccessLogger" /> + </logger> + <!--Send email on task failure--> <!-- <logger name="System.Scheduler.Scheduler.Slave.Tasks.ReloadTask"> --> diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 5a6f5b2e..193abc13 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -106,6 +106,21 @@ Butler: dynamic: useAppTags: true # Should app tags be stored in InfluxDB as tags? useTaskTags: true # Should task tags be stored in InfluxDB as tags? + reloadTaskSuccess: + enable: true + allReloadTasks: + enable: false + byCustomProperty: + enable: true + customPropertyName: 'Butler_SuccessReloadTask_InfluxDB' + enabledValue: 'Yes' + tag: + static: # Static attributes/dimensions to attach to events sent to InfluxDb + # - name: event-specific-tag 1 + # value: abc 123 + dynamic: + useAppTags: true # Should app tags be sent to InfluxDb as tags? + useTaskTags: true # Should task tags be sent to InfluxDb as tags? # Store script logs of failed reloads on disk. # The script logs will be stored in daily directories under the specified main directory below diff --git a/src/globals.js b/src/globals.js index eb38fa3e..9ecfb1cb 100644 --- a/src/globals.js +++ b/src/globals.js @@ -10,7 +10,6 @@ const winston = require('winston'); // Add dependencies const { Command, Option } = require('commander'); -const { configFileNewRelicAssert, configFileStructureAssert, configFileYamlAssert } = require('./lib/assert/assert_config_file'); require('winston-daily-rotate-file'); @@ -96,9 +95,6 @@ if (options.configfile && options.configfile.length > 0) { configFileExpanded = upath.resolve(__dirname, `./config/${env}.yaml`); } -// Verify that config file is valid YAML -configFileYamlAssert(configFileExpanded); - // Are we running as standalone app or not? const isPkg = typeof process.pkg !== 'undefined'; if (isPkg && configFileOption === undefined) { @@ -217,9 +213,6 @@ logger.verbose( )}` ); -// Verify correct structure of config file -configFileStructureAssert(config, logger); - // Helper function to read the contents of the certificate files: const readCert = (filename) => fs.readFileSync(filename); @@ -265,11 +258,6 @@ if (config.has('Butler.restServerApiDocGenerate') === false || config.get('Butle logger.debug('CONFIG: API doc mode=on'); } -// Verify select parts/values in config file -if (options.qsConnection) { - configFileNewRelicAssert(config, configQRS, logger); -} - // MS Teams notification objects let teamsTaskFailureObj; let teamsTaskAbortedObj; @@ -325,9 +313,9 @@ if ( // UDP server connection parameters const udpHost = config.get('Butler.udpServerConfig.serverHost'); -let udpServerTaskFailureSocket = null; +let udpServerReloadTaskSocket = null; // Prepare to listen on port Y for incoming UDP connections regarding failed tasks -// const udpServerTaskFailureSocket = dgram.createSocket({ +// const udpServerReloadTaskSocket = dgram.createSocket({ // type: 'udp4', // reuseAddr: true, // }); @@ -634,6 +622,11 @@ async function initHostInfo() { } } +function sleep(ms) { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => setTimeout(resolve, ms)); +} + module.exports = { config, configEngine, @@ -644,7 +637,7 @@ module.exports = { teamsUserSessionObj, teamsServiceStoppedMonitorObj, teamsServiceStartedMonitorObj, - udpServerTaskFailureSocket, + udpServerReloadTaskSocket, udpHost, udpPortTaskFailure, // mqttClient, @@ -667,4 +660,5 @@ module.exports = { checkFileExistsSync, options, execPath, + sleep, }; diff --git a/src/lib/assert/assert_config_file.js b/src/lib/assert/assert_config_file.js index dbcb680e..b724111a 100644 --- a/src/lib/assert/assert_config_file.js +++ b/src/lib/assert/assert_config_file.js @@ -1,8 +1,87 @@ const QrsInteract = require('qrs-interact'); const yaml = require('js-yaml'); +const { getReloadTasksCustomProperties } = require('../../qrs_util/task_cp_util'); + +// Veriify InfluxDb related settings in the config file +const configFileInfluxDbAssert = async (config, configQRS, logger) => { + // Set up shared Sense repository service configuration + const cfg = { + hostname: config.get('Butler.configQRS.host'), + portNumber: 4242, + certificates: { + certFile: configQRS.certPaths.certPath, + keyFile: configQRS.certPaths.keyPath, + }, + }; + + cfg.headers = { + 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', + }; + + const qrsInstance = new QrsInteract(cfg); + + // ------------------------------------------ + // The custom property specified by + // Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName + // should be present on reload tasks in the Qlik Sense server + + // Only test if the feature in question is enabled in the config file + if ( + config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable') === true && + config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName') && + config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue') + ) { + // Get custom property values + try { + const res1 = await getReloadTasksCustomProperties(config, configQRS, logger); + logger.debug(`ASSERT CONFIG INFLUXDB: The following custom properties are available for reload tasks: ${res1}`); + + // CEnsure that the CP name specified in the config file is found in the list of available CPs + // CP name is case sensitive and found in the "name" property of the CP object + if ( + res1.findIndex((cp) => cp.name === config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName')) === + -1 + ) { + logger.error( + `ASSERT CONFIG INFLUXDB: Custom property '${config.get( + 'Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName' + )}' not found in Qlik Sense. Aborting.` + ); + process.exit(1); + } + + // Ensure that the CP value specified in the config file is found in the list of available CP values + // CP value is case sensitive and found in the "choiceValues" array of the CP objects in res1 + const res2 = res1.filter( + (cp) => cp.name === config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName') + )[0].choiceValues; + logger.debug( + `ASSERT CONFIG INFLUXDB: The following values are available for custom property '${config.get( + 'Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName' + )}': ${res2}` + ); + + if ( + res2.findIndex((cpValue) => cpValue === config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue')) === + -1 + ) { + logger.error( + `ASSERT CONFIG INFLUXDB: Custom property value '${config.get( + 'Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue' + )}' not found for custom property '${config.get( + 'Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName' + )}'. Aborting.` + ); + process.exit(1); + } + } catch (err) { + logger.error(`ASSERT CONFIG INFLUXDB: ${err}`); + } + } +}; /** - * Verify settings in the config file + * Verify New Relic settings in the config file */ const configFileNewRelicAssert = async (config, configQRS, logger) => { // Set up shared Sense repository service configuration @@ -609,6 +688,46 @@ const configFileStructureAssert = async (config, logger) => { configFileCorrect = false; } + if (!config.has('Butler.influxDb.reloadTaskSuccess.enable')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.enable"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.allReloadTasks.enable')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.allReloadTasks.enable"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.tag.static')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.tag.static"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.tag.dynamic.useAppTags')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.tag.dynamic.useAppTags"'); + configFileCorrect = false; + } + + if (!config.has('Butler.influxDb.reloadTaskSuccess.tag.dynamic.useTaskTags')) { + logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.tag.dynamic.useTaskTags"'); + configFileCorrect = false; + } + if (!config.has('Butler.scriptLog.storeOnDisk.reloadTaskFailure.enable')) { logger.error('ASSERT CONFIG: Missing config file entry "Butler.scriptLog.storeOnDisk.reloadTaskFailure.enable"'); configFileCorrect = false; @@ -2273,4 +2392,5 @@ module.exports = { configFileNewRelicAssert, configFileStructureAssert, configFileYamlAssert, + configFileInfluxDbAssert, }; diff --git a/src/lib/post_to_influxdb.js b/src/lib/post_to_influxdb.js index c2d3f360..cef4a8de 100755 --- a/src/lib/post_to_influxdb.js +++ b/src/lib/post_to_influxdb.js @@ -94,15 +94,116 @@ function postWindowsServiceStatusToInfluxDB(serviceStatus) { }); } -// Store information about failed reload tasks to InfluxDB -function postReloadTaskNotificationInfluxDb(reloadParams) { +// Store information about successful reload tasks to InfluxDB +function postReloadTaskSuccessNotificationInfluxDb(reloadParams) { try { - globals.logger.verbose('TASK FAILED INFLUXDB: Sending reload task notification to InfluxDB'); + globals.logger.verbose('RELOAD TASK SUCCESS INFLUXDB: Sending reload task notification to InfluxDB'); + + // Build InfluxDB datapoint + let datapoint = [ + { + measurement: 'reload_task_success', + tags: { + host: reloadParams.host, + user: reloadParams.user, + task_id: reloadParams.taskId, + task_name: reloadParams.taskName, + app_id: reloadParams.appId, + app_name: reloadParams.appName, + log_level: reloadParams.logLevel, + }, + fields: { + log_timestamp: reloadParams.logTimeStamp, + execution_id: reloadParams.executionId, + log_message: reloadParams.logMessage, + }, + }, + ]; + + // Get task info + const { taskInfo } = reloadParams; + + globals.logger.debug(`RELOAD TASK SUCCESS INFLUXDB: Task info:\n${JSON.stringify(taskInfo, null, 2)}`); + + // Use task info to enrich log entry sent to InfluxDB + datapoint[0].tags.task_executingNodeName = taskInfo.executingNodeName; + datapoint[0].tags.task_executionStatusNum = taskInfo.executionStatusNum; + datapoint[0].tags.task_exeuctionStatusText = taskInfo.executionStatusText; + + datapoint[0].fields.task_executionStartTime_json = JSON.stringify(taskInfo.executionStartTime); + datapoint[0].fields.task_executionStopTime_json = JSON.stringify(taskInfo.executionStopTime); + + datapoint[0].fields.task_executionDuration_json = JSON.stringify(taskInfo.executionDuration); + + // Add execution duration in seconds + datapoint[0].fields.task_executionDuration_sec = + taskInfo.executionDuration.hours * 3600 + taskInfo.executionDuration.minutes * 60 + taskInfo.executionDuration.seconds; + + // Add execution duration in minutes + datapoint[0].fields.task_executionDuration_min = + taskInfo.executionDuration.hours * 60 + taskInfo.executionDuration.minutes + taskInfo.executionDuration.seconds / 60; + + // Add execution duration in hours + datapoint[0].fields.task_executionDuration_h = + taskInfo.executionDuration.hours + taskInfo.executionDuration.minutes / 60 + taskInfo.executionDuration.seconds / 3600; + // Should app tags be included? + if (globals.config.get('Butler.influxDb.reloadTaskSuccess.tag.dynamic.useAppTags') === true) { + // Add app tags to InfluxDB datapoint + // eslint-disable-next-line no-restricted-syntax + for (const item of reloadParams.appTags) { + datapoint[0].tags[`appTag_${item}`] = 'true'; + } + } + + // Should task tags be included? + if (globals.config.get('Butler.influxDb.reloadTaskSuccess.tag.dynamic.useTaskTags') === true) { + // Add task tags to InfluxDB datapoint + // eslint-disable-next-line no-restricted-syntax + for (const item of reloadParams.taskTags) { + datapoint[0].tags[`taskTag_${item}`] = 'true'; + } + } + + // Add any static tags (defined in the config file) + const staticTags = globals.config.get('Butler.influxDb.reloadTaskSuccess.tag.static'); + if (staticTags) { + // eslint-disable-next-line no-restricted-syntax + for (const item of staticTags) { + datapoint[0].tags[item.name] = item.value; + } + } + + const deepClonedDatapoint = _.cloneDeep(datapoint); + + // Send to InfluxDB + globals.influx + .writePoints(deepClonedDatapoint) + + .then(() => { + globals.logger.silly( + `RELOAD TASK SUCCESS INFLUXDB: Influxdb datapoint for reload task notification: ${JSON.stringify(datapoint, null, 2)}` + ); + + datapoint = null; + globals.logger.verbose('RELOAD TASK SUCCESS INFLUXDB: Sent reload task notification to InfluxDB'); + }) + .catch((err) => { + globals.logger.error(`RELOAD TASK SUCCESS INFLUXDB: Error saving reload task notification to InfluxDB! ${err.stack}`); + }); + } catch (err) { + globals.logger.error(`RELOAD TASK SUCCESS INFLUXDB: ${err}`); + } +} + +// Store information about failed reload tasks to InfluxDB +function postReloadTaskFailureNotificationInfluxDb(reloadParams) { + try { + globals.logger.verbose('RELOAD TASK FAILED INFLUXDB: Sending reload task notification to InfluxDB'); // Build InfluxDB datapoint let datapoint = [ { - measurement: 'task_failed', + measurement: 'reload_task_failed', tags: { host: reloadParams.host, user: reloadParams.user, @@ -136,7 +237,7 @@ function postReloadTaskNotificationInfluxDb(reloadParams) { scriptLogData.scriptLogTail = ''; } - globals.logger.debug(`TASK FAILED INFLUXDB: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); + globals.logger.debug(`RELOAD TASK FAILED INFLUXDB: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); // Use script log data to enrich log entry sent to InfluxDB datapoint[0].tags.task_executingNodeName = scriptLogData.executingNodeName; @@ -206,21 +307,22 @@ function postReloadTaskNotificationInfluxDb(reloadParams) { .then(() => { globals.logger.silly( - `TASK FAILED INFLUXDB: Influxdb datapoint for reload task notification: ${JSON.stringify(datapoint, null, 2)}` + `RELOAD TASK FAILED INFLUXDB: Influxdb datapoint for reload task notification: ${JSON.stringify(datapoint, null, 2)}` ); datapoint = null; - globals.logger.verbose('TASK FAILED INFLUXDB: Sent reload task notification to InfluxDB'); + globals.logger.verbose('RELOAD TASK FAILED INFLUXDB: Sent reload task notification to InfluxDB'); }) .catch((err) => { - globals.logger.error(`TASK FAILED INFLUXDB: Error saving reload task notification to InfluxDB! ${err.stack}`); + globals.logger.error(`RELOAD TASK FAILED INFLUXDB: Error saving reload task notification to InfluxDB! ${err.stack}`); }); } catch (err) { - globals.logger.error(`TASK FAILED INFLUXDB: ${err}`); + globals.logger.error(`RELOAD TASK FAILED INFLUXDB: ${err}`); } } module.exports = { postButlerMemoryUsageToInfluxdb, postWindowsServiceStatusToInfluxDB, - postReloadTaskNotificationInfluxDb, + postReloadTaskFailureNotificationInfluxDb, + postReloadTaskSuccessNotificationInfluxDb, }; diff --git a/src/lib/scriptlog.js b/src/lib/scriptlog.js index c97dfe75..7221177a 100644 --- a/src/lib/scriptlog.js +++ b/src/lib/scriptlog.js @@ -40,226 +40,247 @@ function delay(milliseconds) { }); } -function getScriptLog(reloadTaskId, headLineCount, tailLineCount) { - return new Promise((resolve, reject) => { - try { - // Set up Sense repository service configuration - const configQRS = { - hostname: globals.config.get('Butler.configQRS.host'), - portNumber: 4242, - certificates: { - certFile: globals.configQRS.certPaths.certPath, - keyFile: globals.configQRS.certPaths.keyPath, - }, +// Function to get reload task execution results +async function getReloadTaskExecutionResults(reloadTaskId) { + try { + // Set up Sense repository service configuration + const configQRS = { + hostname: globals.config.get('Butler.configQRS.host'), + portNumber: 4242, + certificates: { + certFile: globals.configQRS.certPaths.certPath, + keyFile: globals.configQRS.certPaths.keyPath, + }, + }; + + configQRS.headers = { + 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', + }; + + const qrsInstance = new QrsInteract(configQRS); + + // Step 1 + globals.logger.debug(`GETSCRIPTLOG 1: reloadTaskId: ${reloadTaskId}`); + + const result1 = await qrsInstance.Get(`reloadtask/${reloadTaskId}`); + + globals.logger.debug(`GETSCRIPTLOG 1: body: ${JSON.stringify(result1.body)}`); + + const taskInfo = { + fileReferenceId: result1.body.operational.lastExecutionResult.fileReferenceID, + executingNodeName: result1.body.operational.lastExecutionResult.executingNodeName, + executionDetailsSorted: result1.body.operational.lastExecutionResult.details.sort(compareTaskDetails), + executionDetailsConcatenated: '', + executionStatusNum: result1.body.operational.lastExecutionResult.status, + executionStatusText: taskStatusLookup[result1.body.operational.lastExecutionResult.status], + // scriptLogAvailable = result1.body.operational.lastExecutionResult.scriptLogAvailable, + scriptLogSize: result1.body.operational.lastExecutionResult.scriptLogSize, + }; + + // Get execution details as a single string ny concatenating the individual execution step details + // eslint-disable-next-line no-restricted-syntax + for (const execDetail of taskInfo.executionDetailsSorted) { + taskInfo.executionDetailsConcatenated = `${taskInfo.executionDetailsConcatenated + execDetail.detailCreatedDate}\t${ + execDetail.message + }\n`; + } + + // Add duration as JSON + const taskDuration = luxon.Duration.fromMillis(result1.body.operational.lastExecutionResult.duration); + taskInfo.executionDuration = taskDuration.shiftTo('hours', 'minutes', 'seconds').toObject(); + taskInfo.executionDuration.seconds = Math.floor(taskInfo.executionDuration.seconds); + + // Add start datetime in various formats + if (result1.body.operational.lastExecutionResult.startTime.substring(0, 4) === '1753') { + taskInfo.executionStartTime = { + startTimeUTC: '-', + startTimeLocal1: '-', + startTimeLocal2: '-', + startTimeLocal3: '-', + startTimeLocal4: '-', + startTimeLocal5: '-', }; + } else { + const luxonDT = luxon.DateTime.fromISO(result1.body.operational.lastExecutionResult.startTime); + taskInfo.executionStartTime = { + startTimeUTC: result1.body.operational.lastExecutionResult.startTime, + startTimeLocal1: luxonDT.toFormat('yyyy-LL-dd HH:mm:ss'), + startTimeLocal2: luxonDT.toLocaleString(luxon.DateTime.DATETIME_SHORT_WITH_SECONDS), + startTimeLocal3: luxonDT.toLocaleString(luxon.DateTime.DATETIME_MED_WITH_SECONDS), + startTimeLocal4: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), + startTimeLocal5: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), + }; + } - configQRS.headers = { - 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', + // Add stop datetime in various formats + if (result1.body.operational.lastExecutionResult.stopTime.substring(0, 4) === '1753') { + taskInfo.executionStopTime = { + stopTimeUTC: '-', + stopTimeLocal1: '-', + stopTimeLocal2: '-', + stopTimeLocal3: '-', + stopTimeLocal4: '-', + stopTimeLocal5: '-', + }; + } else { + const luxonDT = luxon.DateTime.fromISO(result1.body.operational.lastExecutionResult.stopTime); + taskInfo.executionStopTime = { + stopTimeUTC: result1.body.operational.lastExecutionResult.stopTime, + stopTimeLocal1: luxonDT.toFormat('yyyy-LL-dd HH:mm:ss'), + stopTimeLocal2: luxonDT.toLocaleString(luxon.DateTime.DATETIME_SHORT_WITH_SECONDS), + stopTimeLocal3: luxonDT.toLocaleString(luxon.DateTime.DATETIME_MED_WITH_SECONDS), + stopTimeLocal4: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), + stopTimeLocal5: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), }; + } - const qrsInstance = new QrsInteract(configQRS); - - // Step 1 - globals.logger.debug(`GETSCRIPTLOG 1: reloadTaskId: ${reloadTaskId}`); - qrsInstance.Get(`reloadtask/${reloadTaskId}`).then((result1) => { - const taskInfo = { - fileReferenceId: result1.body.operational.lastExecutionResult.fileReferenceID, - executingNodeName: result1.body.operational.lastExecutionResult.executingNodeName, - executionDetailsSorted: result1.body.operational.lastExecutionResult.details.sort(compareTaskDetails), - executionDetailsConcatenated: '', - executionStatusNum: result1.body.operational.lastExecutionResult.status, - executionStatusText: taskStatusLookup[result1.body.operational.lastExecutionResult.status], - // scriptLogAvailable = result1.body.operational.lastExecutionResult.scriptLogAvailable, - scriptLogSize: result1.body.operational.lastExecutionResult.scriptLogSize, + // Add various datetime formats to task history entries + taskInfo.executionDetailsSorted = taskInfo.executionDetailsSorted.map((item) => { + if (item.detailCreatedDate.substring(0, 4) === '1753') { + return { + timestampUTC: '-', + timestampLocal1: '-', + timestampLocal2: '-', + timestampLocal3: '-', + timestampLocal4: '-', + timestampLocal5: '-', + message: item.message, + detailsType: item.detailsType, }; + } + + const luxonDT = luxon.DateTime.fromISO(item.detailCreatedDate); + return { + timestampUTC: item.detailCreatedDate, + timestampLocal1: luxonDT.toFormat('yyyy-LL-dd HH:mm:ss'), + timestampLocal2: luxonDT.toLocaleString(luxon.DateTime.DATETIME_SHORT_WITH_SECONDS), + timestampLocal3: luxonDT.toLocaleString(luxon.DateTime.DATETIME_MED_WITH_SECONDS), + timestampLocal4: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), + timestampLocal5: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), + message: item.message, + detailsType: item.detailsType, + }; + }); + + return taskInfo; + } catch (err) { + globals.logger.error(`SCRIPTLOG: ${err}`); + return false; + } +} - // Get execution details as a single string ny concatenating the individual execution step details - // eslint-disable-next-line no-restricted-syntax - for (const execDetail of taskInfo.executionDetailsSorted) { - taskInfo.executionDetailsConcatenated = `${taskInfo.executionDetailsConcatenated + execDetail.detailCreatedDate}\t${ - execDetail.message - }\n`; - } - - // Add duration as JSON - const taskDuration = luxon.Duration.fromMillis(result1.body.operational.lastExecutionResult.duration); - taskInfo.executionDuration = taskDuration.shiftTo('hours', 'minutes', 'seconds').toObject(); - taskInfo.executionDuration.seconds = Math.floor(taskInfo.executionDuration.seconds); - - // Add start datetime in various formats - if (result1.body.operational.lastExecutionResult.startTime.substring(0, 4) === '1753') { - taskInfo.executionStartTime = { - startTimeUTC: '-', - startTimeLocal1: '-', - startTimeLocal2: '-', - startTimeLocal3: '-', - startTimeLocal4: '-', - startTimeLocal5: '-', - }; - } else { - const luxonDT = luxon.DateTime.fromISO(result1.body.operational.lastExecutionResult.startTime); - taskInfo.executionStartTime = { - startTimeUTC: result1.body.operational.lastExecutionResult.startTime, - startTimeLocal1: luxonDT.toFormat('yyyy-LL-dd HH:mm:ss'), - startTimeLocal2: luxonDT.toLocaleString(luxon.DateTime.DATETIME_SHORT_WITH_SECONDS), - startTimeLocal3: luxonDT.toLocaleString(luxon.DateTime.DATETIME_MED_WITH_SECONDS), - startTimeLocal4: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), - startTimeLocal5: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), - }; - } - - // Add stop datetime in various formats - if (result1.body.operational.lastExecutionResult.stopTime.substring(0, 4) === '1753') { - taskInfo.executionStopTime = { - stopTimeUTC: '-', - stopTimeLocal1: '-', - stopTimeLocal2: '-', - stopTimeLocal3: '-', - stopTimeLocal4: '-', - stopTimeLocal5: '-', - }; - } else { - const luxonDT = luxon.DateTime.fromISO(result1.body.operational.lastExecutionResult.stopTime); - taskInfo.executionStopTime = { - stopTimeUTC: result1.body.operational.lastExecutionResult.stopTime, - stopTimeLocal1: luxonDT.toFormat('yyyy-LL-dd HH:mm:ss'), - stopTimeLocal2: luxonDT.toLocaleString(luxon.DateTime.DATETIME_SHORT_WITH_SECONDS), - stopTimeLocal3: luxonDT.toLocaleString(luxon.DateTime.DATETIME_MED_WITH_SECONDS), - stopTimeLocal4: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), - stopTimeLocal5: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), - }; - } - - // Add various datetime formats to task history entries - taskInfo.executionDetailsSorted = taskInfo.executionDetailsSorted.map((item) => { - if (item.detailCreatedDate.substring(0, 4) === '1753') { - return { - timestampUTC: '-', - timestampLocal1: '-', - timestampLocal2: '-', - timestampLocal3: '-', - timestampLocal4: '-', - timestampLocal5: '-', - message: item.message, - detailsType: item.detailsType, - }; - } - - const luxonDT = luxon.DateTime.fromISO(item.detailCreatedDate); - return { - timestampUTC: item.detailCreatedDate, - timestampLocal1: luxonDT.toFormat('yyyy-LL-dd HH:mm:ss'), - timestampLocal2: luxonDT.toLocaleString(luxon.DateTime.DATETIME_SHORT_WITH_SECONDS), - timestampLocal3: luxonDT.toLocaleString(luxon.DateTime.DATETIME_MED_WITH_SECONDS), - timestampLocal4: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), - timestampLocal5: luxonDT.toLocaleString(luxon.DateTime.DATETIME_FULL_WITH_SECONDS), - message: item.message, - detailsType: item.detailsType, - }; - }); - - // Step 2 - // Only get script log if there is a valid fileReferenceId - globals.logger.debug(`GETSCRIPTLOG 2: taskInfo.fileReferenceId: ${taskInfo.fileReferenceId}`); - if (taskInfo.fileReferenceId !== '00000000-0000-0000-0000-000000000000') { - globals.logger.debug( - `GETSCRIPTLOG 3: reloadtask/${reloadTaskId}/scriptlog?fileReferenceId=${taskInfo.fileReferenceId}` - ); - qrsInstance - .Get(`reloadtask/${reloadTaskId}/scriptlog?fileReferenceId=${taskInfo.fileReferenceId}`) - .then(async (result2) => { - // Step 3 - // Use Axios for final call to QRS, as QRS-Interact has a bug that prevents downloading of script logs - const httpsAgent = new https.Agent({ - rejectUnauthorized: globals.config.get('Butler.configQRS.rejectUnauthorized'), - cert: globals.configQRS.cert, - key: globals.configQRS.key, - }); - - const axiosConfig = { - url: `/qrs/download/reloadtask/${result2.body.value}/scriptlog.txt?xrfkey=abcdefghijklmnop`, - method: 'get', - baseURL: `https://${globals.configQRS.host}:${globals.configQRS.port}`, - headers: { - 'x-qlik-xrfkey': 'abcdefghijklmnop', - 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', - }, - timeout: 10000, - responseType: 'text', - httpsAgent, - // passphrase: "YYY" - }; - - axios - .request(axiosConfig) - .then((result3) => { - const scriptLogFull = result3.data.split('\r\n'); - - let scriptLogHead = ''; - let scriptLogTail = ''; - - if (headLineCount > 0) { - scriptLogHead = scriptLogFull.slice(0, headLineCount).join('\r\n'); - } - - if (tailLineCount > 0) { - scriptLogTail = scriptLogFull.slice(Math.max(scriptLogFull.length - tailLineCount, 0)).join('\r\n'); - } - - globals.logger.debug(`SCRIPTLOG: Script log head:\n${scriptLogHead}`); - globals.logger.debug(`SCRIPTLOG: Script log tails:\n${scriptLogTail}`); - - globals.logger.verbose('SCRIPTLOG: Done getting script log'); - - resolve({ - executingNodeName: taskInfo.executingNodeName, - executionDetails: taskInfo.executionDetailsSorted, - executionDetailsConcatenated: taskInfo.executionDetailsConcatenated, - executionDuration: taskInfo.executionDuration, - executionStartTime: taskInfo.executionStartTime, - executionStopTime: taskInfo.executionStopTime, - executionStatusNum: taskInfo.executionStatusNum, - executionStatusText: taskInfo.executionStatusText, - scriptLogFull, - scriptLogSize: taskInfo.scriptLogSize, - scriptLogHead, - scriptLogHeadCount: headLineCount, - scriptLogTail, - scriptLogTailCount: tailLineCount, - }); - }) - .catch((err) => { - globals.logger.error(`SCRIPTLOG ERROR: ${err}`); - if (err.response.data) { - globals.logger.error(`SCRIPTLOG ERROR: ${err.response.data}`); - } - }); - }); - } else { - // No script log is available - resolve({ - executingNodeName: taskInfo.executingNodeName, - executionDetails: taskInfo.executionDetailsSorted, - executionDetailsConcatenated: taskInfo.executionDetailsConcatenated, - executionDuration: taskInfo.executionDuration, - executionStartTime: taskInfo.executionStartTime, - executionStopTime: taskInfo.executionStopTime, - executionStatusNum: taskInfo.executionStatusNum, - executionStatusText: taskInfo.executionStatusText, - scriptLogFull: '', - scriptLogSize: 0, - scriptLogHead: '', - scriptLogHeadCount: 0, - scriptLogTail: '', - scriptLogTailCount: 0, - }); - } +// Function to get: +// - reload task execution results +// - reload task script log +async function getScriptLog(reloadTaskId, headLineCount, tailLineCount) { + try { + // Step 1 + const taskInfo = await getReloadTaskExecutionResults(reloadTaskId); + + // Step 2 + // Set up Sense repository service configuration + const configQRS = { + hostname: globals.config.get('Butler.configQRS.host'), + portNumber: 4242, + certificates: { + certFile: globals.configQRS.certPaths.certPath, + keyFile: globals.configQRS.certPaths.keyPath, + }, + }; + + configQRS.headers = { + 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', + }; + + const qrsInstance = new QrsInteract(configQRS); + + // Only get script log if there is a valid fileReferenceId + globals.logger.debug(`GETSCRIPTLOG 2: taskInfo.fileReferenceId: ${taskInfo.fileReferenceId}`); + if (taskInfo.fileReferenceId !== '00000000-0000-0000-0000-000000000000') { + globals.logger.debug(`GETSCRIPTLOG 3: reloadtask/${reloadTaskId}/scriptlog?fileReferenceId=${taskInfo.fileReferenceId}`); + + const result2 = await qrsInstance.Get(`reloadtask/${reloadTaskId}/scriptlog?fileReferenceId=${taskInfo.fileReferenceId}`); + + // Step 3 + // Use Axios for final call to QRS, as QRS-Interact has a bug that prevents downloading of script logs + const httpsAgent = new https.Agent({ + rejectUnauthorized: globals.config.get('Butler.configQRS.rejectUnauthorized'), + cert: globals.configQRS.cert, + key: globals.configQRS.key, }); - } catch (err) { - globals.logger.error(`SCRIPTLOG: ${err}`); - reject(); + + const axiosConfig = { + url: `/qrs/download/reloadtask/${result2.body.value}/scriptlog.txt?xrfkey=abcdefghijklmnop`, + method: 'get', + baseURL: `https://${globals.configQRS.host}:${globals.configQRS.port}`, + headers: { + 'x-qlik-xrfkey': 'abcdefghijklmnop', + 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', + }, + timeout: 10000, + responseType: 'text', + httpsAgent, + // passphrase: "YYY" + }; + + const result3 = await axios.request(axiosConfig); + + const scriptLogFull = result3.data.split('\r\n'); + + let scriptLogHead = ''; + let scriptLogTail = ''; + + if (headLineCount > 0) { + scriptLogHead = scriptLogFull.slice(0, headLineCount).join('\r\n'); + } + + if (tailLineCount > 0) { + scriptLogTail = scriptLogFull.slice(Math.max(scriptLogFull.length - tailLineCount, 0)).join('\r\n'); + } + + globals.logger.debug(`SCRIPTLOG: Script log head:\n${scriptLogHead}`); + globals.logger.debug(`SCRIPTLOG: Script log tails:\n${scriptLogTail}`); + + globals.logger.verbose('SCRIPTLOG: Done getting script log'); + + return { + executingNodeName: taskInfo.executingNodeName, + executionDetails: taskInfo.executionDetailsSorted, + executionDetailsConcatenated: taskInfo.executionDetailsConcatenated, + executionDuration: taskInfo.executionDuration, + executionStartTime: taskInfo.executionStartTime, + executionStopTime: taskInfo.executionStopTime, + executionStatusNum: taskInfo.executionStatusNum, + executionStatusText: taskInfo.executionStatusText, + scriptLogFull, + scriptLogSize: taskInfo.scriptLogSize, + scriptLogHead, + scriptLogHeadCount: headLineCount, + scriptLogTail, + scriptLogTailCount: tailLineCount, + }; } - }); + // No script log is available + return { + executingNodeName: taskInfo.executingNodeName, + executionDetails: taskInfo.executionDetailsSorted, + executionDetailsConcatenated: taskInfo.executionDetailsConcatenated, + executionDuration: taskInfo.executionDuration, + executionStartTime: taskInfo.executionStartTime, + executionStopTime: taskInfo.executionStopTime, + executionStatusNum: taskInfo.executionStatusNum, + executionStatusText: taskInfo.executionStatusText, + scriptLogFull: '', + scriptLogSize: 0, + scriptLogHead: '', + scriptLogHeadCount: 0, + scriptLogTail: '', + scriptLogTailCount: 0, + }; + } catch (err) { + globals.logger.error(`SCRIPTLOG: ${err}`); + return false; + } } async function failedTaskStoreLogOnDisk(reloadParams) { @@ -296,4 +317,5 @@ async function failedTaskStoreLogOnDisk(reloadParams) { module.exports = { getScriptLog, failedTaskStoreLogOnDisk, + getReloadTaskExecutionResults, }; diff --git a/src/qrs_util/task_cp_util.js b/src/qrs_util/task_cp_util.js index 9fc4e080..10a78a0f 100644 --- a/src/qrs_util/task_cp_util.js +++ b/src/qrs_util/task_cp_util.js @@ -10,8 +10,10 @@ const globals = require('../globals'); * @param {*} cpValue * @returns */ -async function isCustomPropertyValueSet(taskId, cpName, cpValue) { - globals.logger.debug(`Checking if value "${cpValue}" is set for custom property "${cpName}"`); +async function isCustomPropertyValueSet(taskId, cpName, cpValue, logger) { + const localLogger = logger !== undefined ? logger : globals.logger; + + localLogger.debug(`Checking if value "${cpValue}" is set for custom property "${cpName}"`); try { const qrsInstance = new QrsInteract({ @@ -28,14 +30,14 @@ async function isCustomPropertyValueSet(taskId, cpName, cpValue) { // Get info about the task try { - globals.logger.debug( + localLogger.debug( `ISCPVALUESET: task/full?filter=id eq ${taskId} and customProperties.definition.name eq '${cpName}' and customProperties.value eq '${cpValue}'` ); const result = await qrsInstance.Get( `task/full?filter=id eq ${taskId} and customProperties.definition.name eq '${cpName}' and customProperties.value eq '${cpValue}'` ); - globals.logger.debug(`ISCPVALUESET: Got response: ${result.statusCode} for CP ${cpName}`); + localLogger.debug(`ISCPVALUESET: Got response: ${result.statusCode} for CP ${cpName}`); if (result.body.length === 1) { // Yes, the CP/value exists for this task @@ -45,11 +47,11 @@ async function isCustomPropertyValueSet(taskId, cpName, cpValue) { // Value not set for the CP return false; } catch (err) { - globals.logger.error(`ISCPVALUESET: Error while getting CP: ${err.message}`); + localLogger.error(`ISCPVALUESET: Error while getting CP: ${err.message}`); return false; } } catch (err) { - globals.logger.error(`ISCPVALUESET: Error while getting CP: ${err}`); + localLogger.error(`ISCPVALUESET: Error while getting CP: ${err}`); return false; } } @@ -107,7 +109,52 @@ async function getTaskCustomPropertyValues(taskId, cpName) { } } +// Function to get all custom properties that are available for reload tasks +async function getReloadTasksCustomProperties(config, configQRS, logger) { + logger.debug('GETRELOADTASKSCP: Retrieving all custom properties that are available for reload tasks'); + + try { + const cfg = { + hostname: config.get('Butler.configQRS.host'), + portNumber: 4242, + certificates: { + certFile: configQRS.certPaths.certPath, + keyFile: configQRS.certPaths.keyPath, + }, + }; + + cfg.headers = { + 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', + }; + + const qrsInstance = new QrsInteract(cfg); + + // Get info about the task + try { + logger.debug('GETRELOADTASKSCP: custompropertydefinition/full?filter=objectType eq ReloadTask'); + + const result = await qrsInstance.Get(`custompropertydefinition/full?filter=objectTypes eq 'ReloadTask'`); + logger.debug(`GETRELOADTASKSCP: Got response: ${result.statusCode} for CP`); + + if (result.body.length > 0) { + // At least one CP exists for reload tasks. + return result.body; + } + + // The task and/or the CP does not exist + return []; + } catch (err) { + logger.error(`GETRELOADTASKSCP: Error while getting CP: ${err.message}`); + return []; + } + } catch (err) { + logger.error(`GETRELOADTASKSCP: Error while getting CP: ${err}`); + return false; + } +} + module.exports = { isCustomPropertyValueSet, getTaskCustomPropertyValues, + getReloadTasksCustomProperties, }; diff --git a/src/udp/udp_handlers.js b/src/udp/udp_handlers.js index d6a41263..2ba1c794 100644 --- a/src/udp/udp_handlers.js +++ b/src/udp/udp_handlers.js @@ -6,10 +6,12 @@ const webhookOut = require('../lib/webhook_notification'); const msteams = require('../lib/msteams_notification'); const signl4 = require('../lib/incident_mgmt/signl4'); const newRelic = require('../lib/incident_mgmt/new_relic'); -const { failedTaskStoreLogOnDisk, getScriptLog } = require('../lib/scriptlog'); +const { failedTaskStoreLogOnDisk, getScriptLog, getReloadTaskExecutionResults } = require('../lib/scriptlog'); const { getTaskTags } = require('../qrs_util/task_tag_util'); const { getAppTags } = require('../qrs_util/app_tag_util'); -const { postReloadTaskNotificationInfluxDb } = require('../lib/post_to_influxdb'); +const { doesTaskExist } = require('../qrs_util/does_task_exist'); +const qrsUtil = require('../qrs_util'); +const { postReloadTaskFailureNotificationInfluxDb, postReloadTaskSuccessNotificationInfluxDb } = require('../lib/post_to_influxdb'); // Handler for failed scheduler initiated reloads const schedulerAborted = async (msg) => { @@ -262,7 +264,8 @@ const schedulerFailed = async (msg, legacyFlag) => { (globals.config.has('Butler.incidentTool.newRelic.enable') && globals.config.get('Butler.incidentTool.newRelic.enable') === true) || (globals.config.has('Butler.slackNotification.enable') && globals.config.get('Butler.slackNotification.enable') === true) || (globals.config.has('Butler.teamsNotification.enable') && globals.config.get('Butler.teamsNotification.enable') === true) || - (globals.config.has('Butler.influxDb.reloadTaskFailure.enable') && globals.config.get('Butler.influxDb.reloadTaskFailure.enable') === true) || + (globals.config.has('Butler.influxDb.reloadTaskFailure.enable') && + globals.config.get('Butler.influxDb.reloadTaskFailure.enable') === true) || (globals.config.has('Butler.emailNotification.enable') && globals.config.get('Butler.emailNotification.enable') === true) ) { if (legacyFlag) { @@ -607,7 +610,7 @@ const schedulerFailed = async (msg, legacyFlag) => { globals.config.get('Butler.influxDb.enable') === true && globals.config.get('Butler.influxDb.reloadTaskFailure.enable') === true ) { - postReloadTaskNotificationInfluxDb({ + postReloadTaskFailureNotificationInfluxDb({ host: msg[1], user: msg[4].replace(/\\/g, '/'), taskName: msg[2], @@ -762,14 +765,163 @@ const schedulerFailed = async (msg, legacyFlag) => { } }; +// -------------------------------------------------------- +// Handler for successful scheduler initiated reloads +// -------------------------------------------------------- +const schedulerReloadTaskSuccess = async (msg) => { + globals.logger.verbose( + `RELOAD TASK SUCCESS: Received reload task success UDP message from scheduler: UDP msg=${msg[0]}, Host=${msg[1]}, App name=${msg[3]}, Task name=${msg[2]}, Log level=${msg[8]}, Log msg=${msg[10]}` + ); + + const reloadTaskId = msg[5]; + + // Does task ID exist in Sense? + const taskExists = await doesTaskExist(reloadTaskId); + if (taskExists.exists !== true) { + globals.logger.warn(`RELOAD TASK SUCCESS: Task ID ${reloadTaskId} does not exist in Sense`); + return false; + } + + // Determine if this task should be stored in InflixDB + let storeInInfluxDb = false; + if ( + globals.config.get('Butler.influxDb.enable') === true && + globals.config.get('Butler.influxDb.reloadTaskSuccess.enable') === true && + globals.config.get('Butler.influxDb.reloadTaskSuccess.allReloadTasks.enable') === true + ) { + storeInInfluxDb = true; + } else if ( + // Is storing of data in InfluxDB enabled for this specific task, via custom property? + globals.config.get('Butler.influxDb.enable') === true && + globals.config.get('Butler.influxDb.reloadTaskSuccess.enable') === true && + globals.config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable') === true + ) { + // Is the custom property set for this specific task? + // Get custom property name and value from config + const customPropertyName = globals.config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName'); + const customPropertyValue = globals.config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue'); + + // Get custom property value for this task + const customPropertyValueForTask = await qrsUtil.customPropertyUtil.isCustomPropertyValueSet( + reloadTaskId, + customPropertyName, + customPropertyValue, + globals.logger + ); + + if (customPropertyValueForTask) { + storeInInfluxDb = true; + } + } + + if (storeInInfluxDb) { + // Get results from last reload task execution + // It may take a seconds or two from the finished-successfully log message is written until the execution results are available via the QRS. + // Specifically, it may take a while for the last "FinishedSuccess" meesage to appear in the executionDetails array. + // + // Try at most five times, with a 1 second delay between each attempt. + // Check if the message property of the last entry in the taskInfo.executionDetailsSorted array is "Changing task state from Started to FinishedSuccess" + // Then give up and don't store anything in InfluxDB, but show a log warning. + let taskInfo; + let retryCount = 0; + while (retryCount < 5) { + // eslint-disable-next-line no-await-in-loop + taskInfo = await getReloadTaskExecutionResults(reloadTaskId); + + if ( + taskInfo?.executionDetailsSorted[taskInfo.executionDetailsSorted.length - 1]?.message === + 'Changing task state from Started to FinishedSuccess' + ) { + // Is duration longer than 0 seconds? + // I.e. is executionDuration.hours, executionDuration.minutes or executionDuration.seconds > 0? + // Warn if not, as this is likely caused by the QRS not having updated the execution details yet + if ( + taskInfo.executionDuration.hours === 0 && + taskInfo.executionDuration.minutes === 0 && + taskInfo.executionDuration.seconds === 0 + ) { + globals.logger.warn( + `RELOAD TASK SUCCESS: Task info for reload task ${reloadTaskId} retrieved successfully after ${retryCount} attempts, but duration is 0 seconds. This is likely caused by the QRS not having updated the execution details yet.` + ); + } + + globals.logger.debug( + `RELOAD TASK SUCCESS: Task info for reload task ${reloadTaskId} retrieved successfully after ${retryCount} attempts` + ); + break; + } + + retryCount += 1; + + globals.logger.verbose( + `RELOAD TASK SUCCESS: Unable to get task info for reload task ${reloadTaskId}. Attempt ${retryCount} of 5. Waiting 1 second before trying again` + ); + + // eslint-disable-next-line no-await-in-loop + await globals.sleep(1000); + } + + if (!taskInfo) { + globals.logger.warn( + `RELOAD TASK SUCCESS: Unable to get task info for reload task ${reloadTaskId}. Not storing task info in InfluxDB` + ); + return false; + } + globals.logger.verbose(`RELOAD TASK SUCCESS: Task info for reload task ${reloadTaskId}: ${JSON.stringify(taskInfo, null, 2)}`); + + // Get app/task tags so they can be included in data sent to alert destinations + let appTags = []; + let taskTags = []; + + // Get tags for the app that was reloaded + appTags = await getAppTags(msg[6]); + globals.logger.verbose(`Tags for app ${msg[6]}: ${JSON.stringify(appTags, null, 2)}`); + + // Get tags for the task that finished reloading successfully + taskTags = await getTaskTags(msg[5]); + globals.logger.verbose(`Tags for task ${msg[5]}: ${JSON.stringify(taskTags, null, 2)}`); + + // Post to InfluxDB when a reload task has finished successfully + if ( + globals.config.has('Butler.influxDb.enable') && + globals.config.has('Butler.influxDb.reloadTaskSuccess.enable') && + globals.config.get('Butler.influxDb.enable') === true && + globals.config.get('Butler.influxDb.reloadTaskSuccess.enable') === true + ) { + postReloadTaskSuccessNotificationInfluxDb({ + host: msg[1], + user: msg[4].replace(/\\/g, '/'), + taskName: msg[2], + taskId: msg[5], + appName: msg[3], + appId: msg[6], + logTimeStamp: msg[7], + logLevel: msg[8], + executionId: msg[9], + logMessage: msg[10], + appTags, + taskTags, + taskInfo, + }); + + globals.logger.info(`RELOAD TASK SUCCESS: Reload info for reload task ${reloadTaskId} stored in InfluxDB`); + } + + return true; + } + + globals.logger.verbose(`RELOAD TASK SUCCESS: Not storing task info in InfluxDB`); + return false; +}; + // -------------------------------------------------------- // Set up UDP server handlers for acting on Sense failed task events // -------------------------------------------------------- module.exports.udpInitTaskErrorServer = () => { // Handler for UDP server startup event // eslint-disable-next-line no-unused-vars - globals.udpServerTaskFailureSocket.on('listening', (message, remote) => { - const address = globals.udpServerTaskFailureSocket.address(); + globals.udpServerReloadTaskSocket.on('listening', (message, remote) => { + const address = globals.udpServerReloadTaskSocket.address(); globals.logger.info(`TASKFAILURE: UDP server listening on ${address.address}:${address.port}`); @@ -789,8 +941,8 @@ module.exports.udpInitTaskErrorServer = () => { // Handler for UDP error event // eslint-disable-next-line no-unused-vars - globals.udpServerTaskFailureSocket.on('error', (message, remote) => { - const address = globals.udpServerTaskFailureSocket.address(); + globals.udpServerReloadTaskSocket.on('error', (message, remote) => { + const address = globals.udpServerReloadTaskSocket.address(); globals.logger.error(`TASKFAILURE: UDP server error on ${address.address}:${address.port}`); // Publish MQTT message that UDP server has reported an error @@ -809,7 +961,7 @@ module.exports.udpInitTaskErrorServer = () => { // Main handler for UDP messages relating to failed tasks // eslint-disable-next-line no-unused-vars - globals.udpServerTaskFailureSocket.on('message', async (message, remote) => { + globals.udpServerReloadTaskSocket.on('message', async (message, remote) => { // --------------------------------------------------------- // === Message from Scheduler reload failed log appender === // @@ -891,6 +1043,9 @@ module.exports.udpInitTaskErrorServer = () => { } else if (msg[0].toLowerCase() === '/scheduler-reload-aborted/') { // Scheduler log appender detecting aborted scheduler-started reload schedulerAborted(msg); + } else if (msg[0].toLowerCase() === '/scheduler-reloadtask-success/') { + // Scheduler log appender detecting successful scheduler-started reload task + schedulerReloadTaskSuccess(msg); } else { // Scheduler log appender detecting failed scheduler-started reload. // This is default to better support legacy Butler installations. See above.