feat: Store reload task success info in InfluxBD

Implements #870

src/butler.js

@@ -11,7 +11,32 @@ const build = require('./app');
 const udp = require('./udp');
 const { mqttInitHandlers } = require('./lib/mqtt_handlers');
 
+const {
+    configFileStructureAssert,
+    configFileYamlAssert,
+    configFileNewRelicAssert,
+    configFileInfluxDbAssert,
+} = require('./lib/assert/assert_config_file');
+
 const start = async () => {
+    // Verify correct structure of config file
+    configFileStructureAssert(globals.config, globals.logger);
+
+    // Verify that config file is valid YAML
+    configFileStructureAssert(globals.config, globals.logger);
+
+    // Verify that config file is valid YAML
+    configFileYamlAssert(globals.configFileExpanded);
+
+    // Verify select parts/values in config file
+    if (globals.options.qsConnection) {
+        // Verify that the config file contains the required data related to New Relic
+        configFileNewRelicAssert(globals.config, globals.configQRS, globals.logger);
+
+        // Verify that the config file contains the required data related to InfluxDb
+        configFileInfluxDbAssert(globals.config, globals.configQRS, globals.logger);
+    }
+
     const apps = await build({});
 
     const { restServer } = apps;
@@ -132,7 +157,7 @@ const start = async () => {
     }
 
     // Prepare to listen on port Y for incoming UDP connections regarding failed tasks
-    globals.udpServerTaskFailureSocket = dgram.createSocket({
+    globals.udpServerReloadTaskSocket = dgram.createSocket({
         type: 'udp4',
         reuseAddr: true,
     });
@@ -143,7 +168,7 @@ const start = async () => {
         udp.udp.udpInitTaskErrorServer();
 
         // Start UDP server for failed task events
-        globals.udpServerTaskFailureSocket.bind(globals.udpPortTaskFailure, globals.udpHost);
+        globals.udpServerReloadTaskSocket.bind(globals.udpPortTaskFailure, globals.udpHost);
         globals.logger.debug(`Server for UDP server: ${globals.udpHost}`);
     }
 };

src/config/config-gen-api-docs.yaml

@@ -99,6 +99,21 @@ Butler:
         dynamic:
           useAppTags: true      # Should app tags be stored in InfluxDB as tags?
           useTaskTags: true     # Should task tags be stored in InfluxDB as tags?      
+    reloadTaskSuccess:
+      enable: true
+      allReloadTasks:
+        enable: false
+      byCustomProperty:
+        enable: true
+        customPropertyName: 'Butler_SuccessReloadTask_InfluxDB'
+        enabledValue: 'Yes'
+      tag: 
+        static:                 # Static attributes/dimensions to attach to events sent to InfluxDb
+          # - name: event-specific-tag 1
+          #   value: abc 123
+        dynamic:
+          useAppTags: true      # Should app tags be sent to InfluxDb as tags?
+          useTaskTags: true     # Should task tags be sent to InfluxDb as tags?
 
   # Store script logs of failed reloads on disk.
   # The script logs will be stored in daily directories under the specified main directory below

src/config/log_appender_xml/scheduler/LocalLogConfig.xml

@@ -37,6 +37,25 @@
         </layout>
     </appender>
 
+    <!-- Appender for detecting successful reload tasks -->
+    <appender name="ReloadTaskSuccessLogger" type="log4net.Appender.UdpAppender">
+        <filter type="log4net.Filter.StringMatchFilter">
+            <param name="stringToMatch" value="Reload complete" />
+        </filter>
+        <filter type="log4net.Filter.DenyAllFilter" />
+        <param name="remoteAddress" value="<IP of server where Butler is running>" />
+        <param name="remotePort" value="9998" />
+        <param name="encoding" value="utf-8" />
+        <layout type="log4net.Layout.PatternLayout">
+            <converter>
+                <param name="name" value="hostname" />
+                <param name="type" value="Qlik.Sense.Logging.log4net.Layout.Pattern.HostNamePatternConverter" />
+            </converter>
+            <param name="conversionpattern" value="/scheduler-reloadtask-success/;%hostname;%property{TaskName};%property{AppName};%property{User};%property{TaskId};%property{AppId};%date;%level;%property{ExecutionId};%message" />
+        </layout>
+    </appender>
+
+
     <!-- Mail appender. Not dependent on Butler. Works as a basic solution, but does not support templating, script logs etc that Butler offers -->
     <!-- <appender name="MailAppender" type="log4net.Appender.SmtpAppender"> -->
         <!-- <filter type="log4net.Filter.StringMatchFilter"> -->
@@ -72,6 +91,11 @@
         <appender-ref ref="AbortedReloadTaskLogger" />
     </logger>
 
+    <!-- Send message to Butler on reload task success -->
+    <logger name="System.Scheduler.Scheduler.Slave.Tasks.ReloadTask">
+        <appender-ref ref="ReloadTaskSuccessLogger" />
+    </logger>
+
 
     <!--Send email on task failure-->
     <!-- <logger name="System.Scheduler.Scheduler.Slave.Tasks.ReloadTask"> -->

src/config/production_template.yaml

@@ -106,6 +106,21 @@ Butler:
         dynamic:
           useAppTags: true      # Should app tags be stored in InfluxDB as tags?
           useTaskTags: true     # Should task tags be stored in InfluxDB as tags?      
+    reloadTaskSuccess:
+      enable: true
+      allReloadTasks:
+        enable: false
+      byCustomProperty:
+        enable: true
+        customPropertyName: 'Butler_SuccessReloadTask_InfluxDB'
+        enabledValue: 'Yes'
+      tag: 
+        static:                 # Static attributes/dimensions to attach to events sent to InfluxDb
+          # - name: event-specific-tag 1
+          #   value: abc 123
+        dynamic:
+          useAppTags: true      # Should app tags be sent to InfluxDb as tags?
+          useTaskTags: true     # Should task tags be sent to InfluxDb as tags?
 
   # Store script logs of failed reloads on disk.
   # The script logs will be stored in daily directories under the specified main directory below

src/globals.js

@@ -10,7 +10,6 @@ const winston = require('winston');
 
 // Add dependencies
 const { Command, Option } = require('commander');
-const { configFileNewRelicAssert, configFileStructureAssert, configFileYamlAssert } = require('./lib/assert/assert_config_file');
 
 require('winston-daily-rotate-file');
 
@@ -96,9 +95,6 @@ if (options.configfile && options.configfile.length > 0) {
     configFileExpanded = upath.resolve(__dirname, `./config/${env}.yaml`);
 }
 
-// Verify that config file is valid YAML
-configFileYamlAssert(configFileExpanded);
-
 // Are we running as standalone app or not?
 const isPkg = typeof process.pkg !== 'undefined';
 if (isPkg && configFileOption === undefined) {
@@ -217,9 +213,6 @@ logger.verbose(
     )}`
 );
 
-// Verify correct structure of config file
-configFileStructureAssert(config, logger);
-
 // Helper function to read the contents of the certificate files:
 const readCert = (filename) => fs.readFileSync(filename);
 
@@ -265,11 +258,6 @@ if (config.has('Butler.restServerApiDocGenerate') === false || config.get('Butle
     logger.debug('CONFIG: API doc mode=on');
 }
 
-// Verify select parts/values in config file
-if (options.qsConnection) {
-    configFileNewRelicAssert(config, configQRS, logger);
-}
-
 // MS Teams notification objects
 let teamsTaskFailureObj;
 let teamsTaskAbortedObj;
@@ -325,9 +313,9 @@ if (
 // UDP server connection parameters
 const udpHost = config.get('Butler.udpServerConfig.serverHost');
 
-let udpServerTaskFailureSocket = null;
+let udpServerReloadTaskSocket = null;
 // Prepare to listen on port Y for incoming UDP connections regarding failed tasks
-// const udpServerTaskFailureSocket = dgram.createSocket({
+// const udpServerReloadTaskSocket = dgram.createSocket({
 //     type: 'udp4',
 //     reuseAddr: true,
 // });
@@ -634,6 +622,11 @@ async function initHostInfo() {
     }
 }
 
+function sleep(ms) {
+    // eslint-disable-next-line no-promise-executor-return
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 module.exports = {
     config,
     configEngine,
@@ -644,7 +637,7 @@ module.exports = {
     teamsUserSessionObj,
     teamsServiceStoppedMonitorObj,
     teamsServiceStartedMonitorObj,
-    udpServerTaskFailureSocket,
+    udpServerReloadTaskSocket,
     udpHost,
     udpPortTaskFailure,
     // mqttClient,
@@ -667,4 +660,5 @@ module.exports = {
     checkFileExistsSync,
     options,
     execPath,
+    sleep,
 };

src/lib/assert/assert_config_file.js

@@ -1,8 +1,87 @@
 const QrsInteract = require('qrs-interact');
 const yaml = require('js-yaml');
+const { getReloadTasksCustomProperties } = require('../../qrs_util/task_cp_util');
+
+// Veriify InfluxDb related settings in the config file
+const configFileInfluxDbAssert = async (config, configQRS, logger) => {
+    // Set up shared Sense repository service configuration
+    const cfg = {
+        hostname: config.get('Butler.configQRS.host'),
+        portNumber: 4242,
+        certificates: {
+            certFile: configQRS.certPaths.certPath,
+            keyFile: configQRS.certPaths.keyPath,
+        },
+    };
+
+    cfg.headers = {
+        'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository',
+    };
+
+    const qrsInstance = new QrsInteract(cfg);
+
+    // ------------------------------------------
+    // The custom property specified by
+    // Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName
+    // should be present on reload tasks in the Qlik Sense server
+
+    // Only test if the feature in question is enabled in the config file
+    if (
+        config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable') === true &&
+        config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName') &&
+        config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue')
+    ) {
+        // Get custom property values
+        try {
+            const res1 = await getReloadTasksCustomProperties(config, configQRS, logger);
+            logger.debug(`ASSERT CONFIG INFLUXDB: The following custom properties are available for reload tasks: ${res1}`);
+
+            // CEnsure that the CP name specified in the config file is found in the list of available CPs
+            // CP name is case sensitive and found in the "name" property of the CP object
+            if (
+                res1.findIndex((cp) => cp.name === config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName')) ===
+                -1
+            ) {
+                logger.error(
+                    `ASSERT CONFIG INFLUXDB: Custom property '${config.get(
+                        'Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName'
+                    )}' not found in Qlik Sense. Aborting.`
+                );
+                process.exit(1);
+            }
+
+            // Ensure that the CP value specified in the config file is found in the list of available CP values
+            // CP value is case sensitive and found in the "choiceValues" array of the CP objects in res1
+            const res2 = res1.filter(
+                (cp) => cp.name === config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName')
+            )[0].choiceValues;
+            logger.debug(
+                `ASSERT CONFIG INFLUXDB: The following values are available for custom property '${config.get(
+                    'Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName'
+                )}': ${res2}`
+            );
+
+            if (
+                res2.findIndex((cpValue) => cpValue === config.get('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue')) ===
+                -1
+            ) {
+                logger.error(
+                    `ASSERT CONFIG INFLUXDB: Custom property value '${config.get(
+                        'Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue'
+                    )}' not found for custom property '${config.get(
+                        'Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName'
+                    )}'. Aborting.`
+                );
+                process.exit(1);
+            }
+        } catch (err) {
+            logger.error(`ASSERT CONFIG INFLUXDB: ${err}`);
+        }
+    }
+};
 
 /**
- * Verify settings in the config file
+ * Verify New Relic settings in the config file
  */
 const configFileNewRelicAssert = async (config, configQRS, logger) => {
     // Set up shared Sense repository service configuration
@@ -609,6 +688,46 @@ const configFileStructureAssert = async (config, logger) => {
         configFileCorrect = false;
     }
 
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.enable')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.enable"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.allReloadTasks.enable')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.allReloadTasks.enable"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.byCustomProperty.enable"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.byCustomProperty.customPropertyName"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.byCustomProperty.enabledValue"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.tag.static')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.tag.static"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.tag.dynamic.useAppTags')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.tag.dynamic.useAppTags"');
+        configFileCorrect = false;
+    }
+
+    if (!config.has('Butler.influxDb.reloadTaskSuccess.tag.dynamic.useTaskTags')) {
+        logger.error('ASSERT CONFIG: Missing config file entry "Butler.influxDb.reloadTaskSuccess.tag.dynamic.useTaskTags"');
+        configFileCorrect = false;
+    }
+
     if (!config.has('Butler.scriptLog.storeOnDisk.reloadTaskFailure.enable')) {
         logger.error('ASSERT CONFIG: Missing config file entry "Butler.scriptLog.storeOnDisk.reloadTaskFailure.enable"');
         configFileCorrect = false;
@@ -2273,4 +2392,5 @@ module.exports = {
     configFileNewRelicAssert,
     configFileStructureAssert,
     configFileYamlAssert,
+    configFileInfluxDbAssert,
 };