fix!: New config file structure. Not backwards compatible!

ptarmiganlabs · Mar 10, 2023 · 9885d68 · 9885d68
1 parent 22fe139
commit 9885d68
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 87 deletions.
diff --git a/README.md b/README.md
@@ -11,7 +11,6 @@ The tool will
 - Extract complete info for all data connections.
 - Run once or recurring at a configurable interval.
 
-
 ## Table of contents
 
 - [Butler Spyglass](#butler-spyglass)
@@ -69,16 +68,16 @@ In addition to the above, complete definitions (except credentials, passwords et
 
 ### Extracting load scripts
 
-Whether or not to extract app load scripts is controlled by the configuration parameter `ButlerSpyglass.script.enableScriptExtract`. Set to true/false as needed.
+Whether or not to extract app load scripts is controlled by the configuration parameter `ButlerSpyglass.scriptExtract.enableScriptExtract`. Set to true/false as needed.
 
-Each app's load script is extracted and stored in a file in a folder as defined by the `ButlerSpyglass.script.exportDir` configuration parameter.
+Each app's load script is extracted and stored in a file in a folder as defined by the `ButlerSpyglass.scriptExtract.exportDir` configuration parameter.
 Each file will be use the app ID as file name.
 
 ### Extracting data lineage
 
-Whether or not to extract data lineage info for apps is controlled by the configuration parameter `ButlerSpyglass.lineage.enableLineageExtract`. Set to true/false as needed.
+Whether or not to extract data lineage info for apps is controlled by the configuration parameter `ButlerSpyglass.lineageExtract.enableLineageExtract`. Set to true/false as needed.
 
-Data lineage information is stored in a single CSV (`lineage.csv`) file in a folder defined by the `ButlerSpyglass.lineage.exportDir` configuration parameter.
+Data lineage information is stored in a single CSV (`lineage.csv`) file in a folder defined by the `ButlerSpyglass.lineageExtract.exportDir` configuration parameter.
 
 More info about discriminators and statements is found [here](https://help.qlik.com/en-US/sense-developer/February2023/Subsystems/EngineJSONAPI/Content/models-lineageinfo.htm).
 
@@ -92,36 +91,36 @@ All parameters must be defined in the config file - run time errors will occur o
 
 | Parameter | Description |
 | --------- | ----------- |
-| **General** |  |
-| logLevel | The level of details in the logs. Possible values are silly, debug, verbose, info, warn, error (in order of decreasing level of detail). Milliseconds |
+| logLevel | The level of details in the logs. Possible values are silly, debug, verbose, info, warn, error (in order of decreasing level of detail). |
 | fileLogging | true/false to enable/disable logging to disk file |
 | logDirectory | Subdirectory where log files are stored |
-| extractFrequency | Time between extraction runs. 60000 means that the next extraction run will start 60 seconds after the previous one ends. Milliseconds |
-| extractItemInterval | Time between two sets of apps are extracted. The number of apps in a set is defined by concurrentTasks (below). For example, if set to 500 there will be a 0.5 sec delay between sets of apps are sent to the Qlik Sense engine API. Milliseconds |
-| extractItemTimeout | Timeout for the call to the engine API. For example, if set to 5000 and no response has been received from the engine API within 5 seconds, an error will be thrown. Milliseconds.   |
-| concurrentTasks | Number of apps that will be sent in parallel to the engine API. Use with caution! You can easily affect performance of a Sense environment by setting this parameter too high. Start setting it low, then increase it while at the same time monitoring the realtime performance (mainly CPU) of the target server, to ensure it is not too heavily loaded by the data extraction tasks. |
-| enableScheduledExecution | true=start an extraction run extractFrequency milliseconds after the previous one finished. false=only run once, then exit |
-|  |  |
-| **Lineage specific** |  |
-| enableLineageExtract | Control whether to extract lineage info or not. true/false |
-| exportDir | Folder where lineage files should be stored. Files are stored in a subfolder `lineage` |
-| maxLengthDiscriminator | Max characters of discriminator field (=source or destination of data) to store in per-app lineage disk file |
-| maxLengthStatement | Max characters of statement field (e.g. SQL statement) to store in per-app lineage disk file |
-|  |  |
-| **Script specific** | Control whether to extract lineage info or not. true/false |
-| enableScriptExtract: true |  |
-| exportDir | Folder where script files should be stored. Files are stored in a subfolder `script`  |
-|  |  |
-| **Parameters for connecting to Qlik Sense engine API** |  |
-| engineVersion | Version of the Qlik Sense engine running on the target server. Sense February 2019 has version number 12.170.2 |
-| server | Fully qualified domain name (=FQDN) of Qlik Sense Enterprise server from which data should be retrieved. |
-| serverPort | Should be 4747, unless configured otherwise in the QMC. |
-| useSSL | Set to true if https is used to communicate with the engine API. |
-| X-Qlik-User | Sense user directory and user to be used when connecting to the engine API. `UserDirectory=Internal;UserId=sa_repository` is a system account that will give access to all apps |
-| ca | Root certificate, as exported from the QMC |
-| cert | Client certificate, as exported from the QMC |
-| key | Client certificate key, as exported from the QMC |
-| rejectUnauthorized | If set to true, strict checking will be done with respect to ssl certificates etc when connecting to the engine API. |
+| extract.frequency | Time between extraction runs. 60000 means that the next extraction run will start 60 seconds after the previous one ends. Milliseconds |
+| extract.itemInterval | Time between two sets of apps are extracted. The number of apps in a set is defined by `extract.concurrentTasks` (below). For example, if set to 500 there will be a 0.5 sec delay between sets of apps are sent to the Qlik Sense engine API. Milliseconds |
+| extract.itemTimeout | Timeout for the call to the engine API. For example, if set to 5000 and no response has been received from the engine API within 5 seconds, an error will be thrown. Milliseconds   |
+| extract.concurrentTasks | Number of apps that will be sent in parallel to the engine API. Use with caution! You can easily affect performance of a Sense environment by setting this parameter too high. Start setting it low, then increase it while at the same time monitoring the realtime performance (mainly CPU) of the target server, to ensure it is not too heavily loaded by the data extraction tasks. |
+| extract.enableScheduledExecution | true=start an extraction run extractFrequency milliseconds after the previous one finished. false=only run once, then exit |
+| lineageExtract.enable | Controls whether to extract lineage info or not. true/false |
+| lineageExtract.exportDir | Directory where lineage files should be stored. |
+| lineageExtract.maxLengthDiscriminator | Max characters of discriminator field (=source or destination of data) to store in per-app lineage disk file |
+| lineageExtract.maxLengthStatement | Max characters of statement field (e.g. SQL statement) to store in per-app lineage disk file |
+| scriptExtract.enable | Controls whether load scripts are extracted to text files or not. true/false |
+| scriptExtract.exportDir | Directory where script files will be stored. |
+| dataConnectionExtract.enable | Controls whether data connections are extracted to JSON file not. true/false |
+| dataConnectionExtract.exportDir | Directory where data connections JSON file will be stored. |
+| configEngine.engineVersion | Version of the Qlik Sense engine running on the target server. Version 12.612.0 should work with any Qlik Sense server from 2020 February and later. |
+| configEngine.host | Host name, fully qualified domain name (=FQDN) or IP address of Qlik Sense Enterprise server where Qlik Engine Service (QES) is running. |
+| configEngine.port | Should be 4747, unless configured otherwise in the QMC. |
+| configEngine.useSSL | Set to true if https is used to communicate with the engine API. |
+| configEngine.headers.X-Qlik-User | Sense user directory and user to be used when connecting to the engine API. `UserDirectory=Internal;UserId=sa_repository` is a system account. |
+| configEngine.rejectUnauthorized | If set to true, strict checking will be done with respect to ssl certificates etc when connecting to the engine API. |
+| configQRS.authentication | Method to authenticate with Qlik Repository Service. Valid options are: `certificates`. |
+| configQRS.host | Host name, fully qualified domain name (=FQDN) or IP address of Qlik Sense Enterprise server where Qlik Repository Service (QRS) is running. |
+| configQRS.port | Should be 4242, unless configured otherwise in the QMC. |
+| configQRS.useSSL | Set to true if https is used to communicate with the repository API. | |
+| configQRS.headers.X-Qlik-User | Sense user directory and user to be used when connecting to the engine API. `UserDirectory=Internal;UserId=sa_repository` is a system account. |
+| cert.clientCerCA | Root certificate, as exported from the QMC |
+| cert.clientCert | Client certificate, as exported from the QMC |
+| cert.clientCertKey | Client certificate key, as exported from the QMC |
 
 ## Logging
 

diff --git a/butler-spyglass.js b/butler-spyglass.js
@@ -24,8 +24,8 @@ const readCert = (filename) => fs.readFileSync(filename);
 //  Engine config
 const configEngine = {
     engineVersion: config.get('ButlerSpyglass.configEngine.engineVersion'),
-    host: config.get('ButlerSpyglass.configEngine.server'),
-    port: config.get('ButlerSpyglass.configEngine.serverPort'),
+    host: config.get('ButlerSpyglass.configEngine.host'),
+    port: config.get('ButlerSpyglass.configEngine.port'),
     isSecure: config.get('ButlerSpyglass.configEngine.useSSL'),
     headers: config.get('ButlerSpyglass.configEngine.headers'),
     ca: readCert(config.get('ButlerSpyglass.cert.clientCertCA')),
@@ -46,15 +46,15 @@ logger.info(`| Log level  : ${getLoggingLevel()}`);
 logger.info(`|  `);
 logger.info(`--------------------------------------`);
 logger.info(``);
-logger.info(`Extracting metadata from server: ${config.get('ButlerSpyglass.configEngine.server')}`);
-if (config.get('ButlerSpyglass.lineage.enableLineageExtract') === true) {
-    logger.info(`Data linage files will be stored in                : ${config.get('ButlerSpyglass.lineage.exportDir')}`);
+logger.info(`Extracting metadata from server: ${config.get('ButlerSpyglass.configEngine.host')}`);
+if (config.get('ButlerSpyglass.lineageExtract.enable') === true) {
+    logger.info(`Data linage files will be stored in                : ${config.get('ButlerSpyglass.lineageExtract.exportDir')}`);
 }
-if (config.get('ButlerSpyglass.script.enableScriptExtract')) {
-    logger.info(`Load script files will be stored in                : ${config.get('ButlerSpyglass.script.exportDir')}`);
+if (config.get('ButlerSpyglass.scriptExtract.enable')) {
+    logger.info(`Load script files will be stored in                : ${config.get('ButlerSpyglass.scriptExtract.exportDir')}`);
 }
-if (config.get('ButlerSpyglass.dataconnection.enableDataConnectionExtract')) {
-    logger.info(`Data connection definitions files will be stored in: ${config.get('ButlerSpyglass.dataconnection.exportDir')}`);
+if (config.get('ButlerSpyglass.dataConnectionExtract.enable')) {
+    logger.info(`Data connection definitions files will be stored in: ${config.get('ButlerSpyglass.dataConnectionExtract.exportDir')}`);
 }
 logger.verbose(``);
 logger.verbose(`Butler Spyglass was started from ${execPath}`);
@@ -80,9 +80,9 @@ const q = new Queue(
         // cb();
     },
     {
-        concurrent: config.get('ButlerSpyglass.concurrentTasks'), // Number of tasks to process in parallel
-        maxTimeout: config.get('ButlerSpyglass.extractItemTimeout'), // Max time allowed for each app extract, before timeout error is thrown
-        afterProcessDelay: config.get('ButlerSpyglass.extractItemInterval'), // Delay between each task
+        concurrent: config.get('ButlerSpyglass.extract.concurrentTasks'), // Number of tasks to process in parallel
+        maxTimeout: config.get('ButlerSpyglass.extract.itemTimeout'), // Max time allowed for each app extract, before timeout error is thrown
+        afterProcessDelay: config.get('ButlerSpyglass.extract.itemInterval'), // Delay between each task
         filo: true,
     }
 );
@@ -101,14 +101,14 @@ const scheduledExtract = function scheduledExtract() {
     logger.info(`Extraction run started`);
 
     // Get data connections
-    if (config.get('ButlerSpyglass.dataconnection.enableDataConnectionExtract') === true) {
+    if (config.get('ButlerSpyglass.dataConnectionExtract.enable') === true) {
         getDataConnections();
     }
 
     // Empty output folders
-    fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.lineage.exportDir')}/`)));
-    fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.script.exportDir')}/`)));
-    fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.dataconnection.exportDir')}/`)));
+    fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.lineageExtract.exportDir')}/`)));
+    fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.scriptExtract.exportDir')}/`)));
+    fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.dataConnectionExtract.exportDir')}/`)));
 
     // create a new session
     const configEnigma = {
@@ -196,9 +196,9 @@ q.on('drain', () => {
 
     // Schedule next extraction run after configured time period
     // Only do this if enable in the config file though!
-    if (config.get('ButlerSpyglass.enableScheduledExecution')) {
-        logger.info(`Waiting ${config.get('ButlerSpyglass.extractFrequency') / 1000} seconds until next extraction run`);
-        setTimeout(scheduledExtract, config.get('ButlerSpyglass.extractFrequency'));
+    if (config.get('ButlerSpyglass.extract.enableScheduledExecution')) {
+        logger.info(`Waiting ${config.get('ButlerSpyglass.extract.frequency') / 1000} seconds until next extraction run`);
+        setTimeout(scheduledExtract, config.get('ButlerSpyglass.extract.frequency'));
     } else {
         logger.info(`All done - exiting.`);
         process.exit(0);

diff --git a/config/production-template.yaml b/config/production-template.yaml
@@ -1,36 +1,51 @@
 ---
 ButlerSpyglass:
   # Logging configuration
-  logLevel: info          # Log level. Possible log levels are silly, debug, verbose, info, warn, error
-  fileLogging: true       # true/false to enable/disable logging to disk file
-  logDirectory: logs      # Subdirectory where log files are stored
+  logLevel: info                    # Log level. Possible log levels are silly, debug, verbose, info, warn, error
+  fileLogging: true                 # true/false to enable/disable logging to disk file
+  logDirectory: ./log               # Subdirectory where log files are stored. Either absolute path or relative to where Butler Spyglass was started
 
   # Extract configuration
-  extractFrequency: 60000     # Time between extraction runs. Milliseconds
-  extractItemInterval: 500    # Time between requests to the engine API. Milliseconds
-  extractItemTimeout: 5000    # Timeout for calls to the engine API. Milliseconds
-  concurrentTasks: 1          # Simultaneous calls to the engine API. Example: If set to 3, this means 3 calls will be done at the same time, every extractItemInterval milliseconds.
-  enableScheduledExecution: true   # true=start an extraction run extractFrequency milliseconds after the previous one finished. false=only run once, then exit
+  extract:
+    frequency: 60000000             # Time between extraction runs. Milliseconds
+    itemInterval: 250               # Time between requests to the engine API. Milliseconds
+    itemTimeout: 15000              # Timeout for calls to the engine API. Milliseconds
+    concurrentTasks: 3              # Simultaneous calls to the engine API. Example: If set to 3, this means 3 calls will be done at the same time, every extractItemInterval milliseconds.
+    enableScheduledExecution: true  # true=start an extraction run extractFrequency milliseconds after the previous one finished. false=only run once, then exit
 
-  lineage:
-    enableLineageExtract: true
-    exportDir: ./out/lineage
-    maxLengthDiscriminator: 1000       # Max characters of discriminator field (=source or destination of data) to store in per-app lineage disk file
-    maxLengthStatement: 1000           # Max characters of statemenf field (e.g. SQL statement) to store in per-app lineage disk file
+  lineageExtract:
+    enable: true                    # Should data lineage files be created?
+    exportDir: ./out/lineage        # Directory where data lineage files will be stored.
+    maxLengthDiscriminator: 1000    # Max characters of discriminator field (=source or destination of data) to store in per-app lineage disk file
+    maxLengthStatement: 1000        # Max characters of statemenf field (e.g. SQL statement) to store in per-app lineage disk file
 
-  script:
-    enableScriptExtract: true
-    exportDir: ./out/script
+  scriptExtract:
+    enable: true                    # Should app load scripts be saved to files?
+    exportDir: ./out/script         # Directory where load script files will be stored.
+
+  dataConnectionExtract:
+    enable: true                    # Should data connections definitions be saved to files? One JSON file with all data connections will be created.
+    exportDir: ./out/dataconnection # Directory where data connection JSON definitions file will be stored.
 
   configEngine:
-    engineVersion: 12.170.2        # Qlik Associative Engine version to use with Enigma.js. ver 12.170.2 works with Feb 2019 
-    server: <Fully qualified domain name of Qlik Sense Enterprise server from which data should be retrieved>
-    serverPort: 4747
-    isSecure: true
+    engineVersion: 12.612.0         # Qlik Associative Engine version to use with Enigma.js. ver 12.612.0 works with Feb 2020 and later 
+    host: 192.168.100.109
+    port: 4747
+    useSSL: true
     headers:
       X-Qlik-User: UserDirectory=Internal;UserId=sa_repository
-    ca: /path/to/certificate/experted/from/sense/root.pem
-    cert: /path/to/certificate/experted/from/sense/client.pem
-    key: /path/to/certificate/experted/from/sense/client_key.pem
     rejectUnauthorized: false
 
+  configQRS: 
+    authentication: certificates
+    host: 192.168.100.109
+    port: 4242
+    useSSL: true
+    headers:
+      X-Qlik-User: UserDirectory=Internal;UserId=sa_repository
+
+  # Certificates to use when connecting to Sense. Get these from the Certificate Export in QMC.
+  cert:
+    clientCert: /Users/goran/code/secret/pro2win1-nopwd/client.pem
+    clientCertKey: /Users/goran/code/secret/pro2win1-nopwd/client_key.pem
+    clientCertCA: /Users/goran/code/secret/pro2win1-nopwd/root.pem