Skip to content

Commit

Permalink
feat: Create pre-built binaries for Windows, macOS and Linux
Browse files Browse the repository at this point in the history 
Implements #69
  • Loading branch information
@mountaindude
mountaindude committed Mar 10, 2023
1 parent 1d01139 commit 2be9842
Show file tree
Hide file tree
Showing 8 changed files with 685 additions and 321 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@ config/development.json
npm-debug.log
.DS_Store
out/*
release/*


# ---> Node
# Logs
logs
log
*.log
npm-debug.log*

Expand Down
392 changes: 265 additions & 127 deletions README.md
View file

Large diffs are not rendered by default.

207 changes: 116 additions & 91 deletions butler-spyglass.js
View file
Original file line number Diff line number Diff line change
@@ -1,184 +1,209 @@
var config = require('config');
var fs = require('fs-extra');
var Queue = require('better-queue');
const config = require('config');
const fs = require('fs-extra');
const Queue = require('better-queue');
const enigma = require('enigma.js');
const WebSocket = require('ws');
const path = require('path');

// const path = require('path');
const upath = require('upath');

// Load our own code
const extractApp = require('./src/extract_app.js');
const logger = require('./src/logger.js');
const extractApp = require('./src/extract_app');
const { getDataConnections } = require('./src/get_dataconnection');
const { logger, getLoggingLevel } = require('./src/logger');

// Get app version from package.json file
var appVersion = require('./package.json').version;
var appName = require('./package.json').name;
const appVersion = require('./package.json').version;
const appName = require('./package.json').name;

// Are we running as standalone app or not?
const isPkg = typeof process.pkg !== 'undefined';
const execPath = isPkg ? upath.dirname(process.execPath) : __dirname;

// Helper function to read the contents of the certificate files:
// const readCert = filename => fs.readFileSync(path.resolve(__dirname, certificatesPath, filename));
const readCert = filename => fs.readFileSync(filename);

const readCert = (filename) => fs.readFileSync(filename);

// Engine config
var configEngine = {
const configEngine = {
engineVersion: config.get('ButlerSpyglass.configEngine.engineVersion'),
host: config.get('ButlerSpyglass.configEngine.server'),
port: config.get('ButlerSpyglass.configEngine.serverPort'),
isSecure: config.get('ButlerSpyglass.configEngine.isSecure'),
isSecure: config.get('ButlerSpyglass.configEngine.useSSL'),
headers: config.get('ButlerSpyglass.configEngine.headers'),
ca: readCert(config.get('ButlerSpyglass.configEngine.ca')),
cert: readCert(config.get('ButlerSpyglass.configEngine.cert')),
key: readCert(config.get('ButlerSpyglass.configEngine.key')),
rejectUnauthorized: config.get('ButlerSpyglass.configEngine.rejectUnauthorized')
ca: readCert(config.get('ButlerSpyglass.cert.clientCertCA')),
cert: readCert(config.get('ButlerSpyglass.cert.clientCert')),
key: readCert(config.get('ButlerSpyglass.cert.clientCertKey')),
rejectUnauthorized: config.get('ButlerSpyglass.configEngine.rejectUnauthorized'),
};

// Set up enigma.js configuration
const qixSchema = require('enigma.js/schemas/' + configEngine.engineVersion);

logger.logger.info(`--------------------------------------`);
logger.logger.info(`Starting ${appName}`);
logger.logger.info(`App version is: ${appVersion}`);
logger.logger.info(`Log level is: ${logger.getLoggingLevel()}`);
logger.logger.info(`Extracting metadata from server: ${config.get('ButlerSpyglass.configEngine.server')}`);
logger.logger.info(`--------------------------------------`);
const qixSchema = require(`enigma.js/schemas/${configEngine.engineVersion}`);

logger.info(`--------------------------------------`);
logger.info(`| ${appName}`);
logger.info(`| `);
logger.info(`| Version : ${appVersion}`);
logger.info(`| Log level : ${getLoggingLevel()}`);
logger.info(`| `);
logger.info(`--------------------------------------`);
logger.info(``);
logger.info(`Extracting metadata from server: ${config.get('ButlerSpyglass.configEngine.server')}`);
if (config.get('ButlerSpyglass.lineage.enableLineageExtract') === true) {
logger.info(`Data linage files will be stored in : ${config.get('ButlerSpyglass.lineage.exportDir')}`);
}
if (config.get('ButlerSpyglass.script.enableScriptExtract')) {
logger.info(`Load script files will be stored in : ${config.get('ButlerSpyglass.script.exportDir')}`);
}
if (config.get('ButlerSpyglass.dataconnection.enableDataConnectionExtract')) {
logger.info(`Data connection definitions files will be stored in: ${config.get('ButlerSpyglass.dataconnection.exportDir')}`);
}
logger.verbose(``);
logger.verbose(`Butler Spyglass was started from ${execPath}`);
logger.verbose(`Butler Spyglass was started as a stand-alone binary: ${isPkg}`);
logger.verbose(``);
logger.debug(`Options: ${JSON.stringify(config, null, 2)}`);
logger.debug(``);

// Log info about what Qlik Sense certificates are being used
logger.logger.debug(`Engine client cert: ${config.get('ButlerSpyglass.configEngine.cert')}`);
logger.logger.debug(`Engine client cert key: ${config.get('ButlerSpyglass.configEngine.key')}`);
logger.logger.debug(`Engine CA cert: ${config.get('ButlerSpyglass.configEngine.ca')}`);


logger.debug(`Engine client cert: ${config.get('ButlerSpyglass.cert.clientCert')}`);
logger.debug(`Engine client cert key: ${config.get('ButlerSpyglass.cert.clientCertKey')}`);
logger.debug(`Engine CA cert: ${config.get('ButlerSpyglass.cert.clientCertCA')}`);

// Set up task queue
var q = new Queue(async function (taskItem, cb) {
logger.logger.debug(`Dumping app: ${taskItem.qDocId} <<>> ${taskItem.qTitle}`);

let _self = this;
const newLocal = extractApp.appExtractMetadata(_self, q, taskItem, cb);

// cb();
}, {
concurrent: config.get('ButlerSpyglass.concurrentTasks'), // Number of tasks to process in parallel
maxTimeout: config.get('ButlerSpyglass.extractItemTimeout'), // Max time allowed for each app extract, before timeout error is thrown
afterProcessDelay: config.get('ButlerSpyglass.extractItemInterval'), // Delay between each task
filo: true
});
const q = new Queue(
async function dumpApp(taskItem, cb) {
logger.debug(`Dumping app: ${taskItem.qDocId} <<>> ${taskItem.qTitle}`);

const _self = this;
const newLocal = extractApp.appExtractMetadata(_self, q, taskItem, cb);

// cb();
},
{
concurrent: config.get('ButlerSpyglass.concurrentTasks'), // Number of tasks to process in parallel
maxTimeout: config.get('ButlerSpyglass.extractItemTimeout'), // Max time allowed for each app extract, before timeout error is thrown
afterProcessDelay: config.get('ButlerSpyglass.extractItemInterval'), // Delay between each task
filo: true,
}
);

// var qRetry = new Queue(async function (taskItemRetry, cbRetry) {
// logger.logger.debug(`Retrying app ${taskItemRetry.qDocId} <<>> ${taskItemRetry.qTitle}`);
// logger.debug(`Retrying app ${taskItemRetry.qDocId} <<>> ${taskItemRetry.qTitle}`);

// let _selfRetry = this;
// const newLocalRetry =
// const newLocalRetry =
// })

q.on('task_finish', function (taskId, result) {
q.on('task_finish', (taskId, result) => {
// Handle finished result
logger.logger.debug(`Task finished: ${taskId} with result ${result}`);
logger.debug(`Task finished: ${taskId} with result ${result}`);
});

q.on('task_failed', function (taskId, errorMessage, stats) {
q.on('task_failed', (taskId, errorMessage, stats) => {
// Handle error
logger.logger.error(`Task failed: ${taskId} with error ${errorMessage}, stats=${JSON.stringify(stats, null, 2)}`);
logger.error(`Task failed: ${taskId} with error ${errorMessage}, stats=${JSON.stringify(stats, null, 2)}`);
});

// q.on('task_progress', function (taskId, completed, total) {
// // Handle task progress
// logger.logger.debug(`========== Task progress: ${taskId}, ${completed}/${total} done`);
// logger.debug(`========== Task progress: ${taskId}, ${completed}/${total} done`);
// });

// Fires when queue is empty and all tasks have finished.
// Fires when queue is empty and all tasks have finished.
// I.e. when all data in an extraction run is available and can be written to disk.
q.on('drain', () => {
logger.logger.info(`Done writing lineage data and script files to disk`);
logger.info(`Done writing lineage data, script files and data connections to disk`);

// Schedule next extraction run after configured time period
// Only do this if enable in the config file though!
if (config.get('ButlerSpyglass.enableScheduledExecution')) {
logger.logger.info(`Waiting ${config.get('ButlerSpyglass.extractFrequency')/1000} seconds until next extraction run`);
logger.info(`Waiting ${config.get('ButlerSpyglass.extractFrequency') / 1000} seconds until next extraction run`);
setTimeout(scheduledExtract, config.get('ButlerSpyglass.extractFrequency'));
} else {
logger.logger.info(`All done - exiting.`);
logger.info(`All done - exiting.`);
process.exit(0);
}
});


// Define function to be scheduled
var scheduledExtract = function () {
const scheduledExtract = function scheduledExtract() {
// Write separator to separate this run from the previous one
logger.logger.info(`--------------------------------------`);
logger.logger.info(`Script extraction run started`);
logger.info(`--------------------------------------`);
logger.info(`Extraction run started`);

// Empty output folders
fs.emptyDirSync(path.resolve(path.normalize(config.get('ButlerSpyglass.lineage.lineageFolder') + '/')));
fs.emptyDirSync(path.resolve(path.normalize(config.get('ButlerSpyglass.script.scriptFolder') + '/')));
// Get data connections
if (config.get('ButlerSpyglass.dataconnection.enableDataConnectionExtract') === true) {
getDataConnections();
}

// Empty output folders
fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.lineage.exportDir')}/`)));
fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.script.exportDir')}/`)));
fs.emptyDirSync(upath.resolve(upath.normalize(`${config.get('ButlerSpyglass.dataconnection.exportDir')}/`)));

// create a new session
const configEnigma = {
schema: qixSchema,
url: `wss://${configEngine.host}:${configEngine.port}`,
createSocket: url => new WebSocket(url, {
ca: [configEngine.ca],
key: configEngine.key,
cert: configEngine.cert,
headers: {
'X-Qlik-User': 'UserDirectory=Internal;UserId=sa_repository'
},
rejectUnauthorized: false
}),
createSocket: (url) =>
new WebSocket(url, {
ca: [configEngine.ca],
key: configEngine.key,
cert: configEngine.cert,
headers: {
'X-Qlik-User': 'UserDirectory=Internal;UserId=sa_repository',
},
rejectUnauthorized: false,
}),
};

const sessionAppList = enigma.create(configEnigma);

sessionAppList
.open()
.then(global => {

.then((global) => {
// We can now interact with the global object, for example get the document list.
global.getDocList()
.then(list => {
logger.logger.silly(`Apps on this Engine that the configured user can open: ${JSON.stringify(list, null, 2)}`);
logger.logger.info(`Number of apps on server: ${list.length}`);
global
.getDocList()
.then((list) => {
logger.silly(`Apps on this Engine that the configured user can open: ${JSON.stringify(list, null, 2)}`);
logger.info(`Number of apps on server: ${list.length}`);

// Reset # processed apps to zero
extractApp.resetExtractedAppCount();

// Send tasks to queue
list.forEach(element => {
q.push(element)
.on('failed', function (err) {
logger.logger.error('Task FAILED =====' + err);
// Task failed!
});
list.forEach((element) => {
q.push(element).on('failed', (err) => {
// Task failed!
logger.error(`Task FAILED: ${err}`);
});
});

q.on('progress', function (progress) {
// logger.logger.verbose(`========== Task progress: ${taskId}, ${completed}/${total} done`);
logger.logger.verbose(`========== Task progress: ${progress}`);
q.on('progress', (progress) => {
// logger.verbose(`========== Task progress: ${taskId}, ${completed}/${total} done`);
logger.verbose(`========== Task progress: ${progress}`);
// progress.eta - human readable string estimating time remaining
// progress.pct - % complete (out of 100)
// progress.complete - # completed so far
// progress.total - # for completion
// progress.message - status message
});

})

.then(() => {
try {
sessionAppList.close();
} catch (ex) {
logger.logger.error(`Error when closing sessionAppList: ${ex}`);
logger.error(`Error when closing sessionAppList: ${ex}`);
}
});
}).catch((error) => {
logger.logger.error('Failed to open session and/or retrieve the app list:', error);
})
.catch((error) => {
logger.error('Failed to open session and/or retrieve the app list:', error);
process.exit(1);
});

};

// Kick off first extract. Following extracts will be triggered from within the scheduledExtract() function
scheduledExtract();
scheduledExtract();
15 changes: 15 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 7 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"js-yaml": "^4.1.0",
"jshint": "^2.13.6",
"qrs-interact": "^6.3.1",
"upath": "^2.0.1",
"winston": "^3.8.2",
"winston-daily-rotate-file": "^4.7.1",
"ws": "^8.12.1"
Expand Down Expand Up @@ -47,5 +48,10 @@
"devops",
"qliksense",
"data lineage"
]
],
"pkg": {
"scripts": [
"node_modules/enigma.js/**/*.json"
]
}
}
Loading

0 comments on commit 2be9842

Please sign in to comment.