Skip to content

Commit

Permalink
670 wip
Browse files Browse the repository at this point in the history
  • Loading branch information
Göran Sander committed May 28, 2023
1 parent 2c12743 commit 0badfb4
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 38 deletions.
32 changes: 19 additions & 13 deletions src/config/production_template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,6 @@ Butler:
logLevel: verbose # Starting at what log level should uptime messages be shown?
storeInInfluxdb:
enable: false # Should Butler memory usage be logged to InfluxDB?
hostIP: <IP or host name> # Where is InfluxDB server located?
hostPort: 8086 # InfluxDB port
auth:
enable: false # Does InfluxDB require login?
username: user_joe
password: joesecret
dbName: butler # Name of database in InfluxDB to which Butler's data is written
instanceTag: DEV # Tag that can be used to differentiate data from multiple Butler instances
# Default retention policy that should be created in InfluxDB when Butler creates a new database there.
# Any data older than retention policy threshold will be purged from InfluxDB.
retentionPolicy:
name: 10d
duration: 10d
storeNewRelic:
enable: false
destinationAccount:
Expand Down Expand Up @@ -90,6 +77,23 @@ Butler:
insertApiKey: <API key 2 (with insert permissions) from New Relic>
accountId: <New Relic account ID 2>

# InfluxDB settings
influxDb:
enable: true # Master switch for InfluxDB integration. If false, no data will be sent to InfluxDB.
hostIP: <IP or host name> # Where is InfluxDB server located?
hostPort: 8086 # InfluxDB port
auth:
enable: false # Does InfluxDB require login?
username: user_joe
password: joesecret
dbName: butler # Name of database in InfluxDB to which Butler's data is written
instanceTag: DEV # Tag that can be used to differentiate data from multiple Butler instances
# Default retention policy that should be created in InfluxDB when Butler creates a new database there.
# Any data older than retention policy threshold will be purged from InfluxDB.
retentionPolicy:
name: 10d
duration: 10d

# Store script logs of failed reloads on disk.
# The script logs will be stored in daily directories under the specified main directory below
# NOTE: Use an absolute path when running Butler as a standalone executable!
Expand Down Expand Up @@ -663,6 +667,8 @@ Butler:
- QlikSenseSchedulerService
- QlikSenseServiceDispatcher
alertDestination: # Control to thich destinations service related alerts are sent
influxDb: # Send service alerts to InfluxDB
enable: true
newRelic: # Send service alerts to New Relic
enable: true
email: # Send service alerts as emails
Expand Down
42 changes: 20 additions & 22 deletions src/globals.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,28 +363,18 @@ if (config.has('Butler.restServerEndpointsEnable')) {
logger.info(`Enabled API endpoints: ${JSON.stringify(endpointsEnabled, null, 2)}`);

// Set up InfluxDB
logger.info(`CONFIG: Influxdb enabled: ${config.get('Butler.uptimeMonitor.storeInInfluxdb.enable')}`);
logger.info(`CONFIG: Influxdb host IP: ${config.get('Butler.uptimeMonitor.storeInInfluxdb.hostIP')}`);
logger.info(`CONFIG: Influxdb host port: ${config.get('Butler.uptimeMonitor.storeInInfluxdb.hostPort')}`);
logger.info(`CONFIG: Influxdb db name: ${config.get('Butler.uptimeMonitor.storeInInfluxdb.dbName')}`);
logger.info(`CONFIG: Influxdb enabled: ${config.get('Butler.influxDb.enable')}`);
logger.info(`CONFIG: Influxdb host IP: ${config.get('Butler.influxDb.hostIP')}`);
logger.info(`CONFIG: Influxdb host port: ${config.get('Butler.influxDb.hostPort')}`);
logger.info(`CONFIG: Influxdb db name: ${config.get('Butler.influxDb.dbName')}`);

// Set up Influxdb client
const influx = new Influx.InfluxDB({
host: config.get('Butler.uptimeMonitor.storeInInfluxdb.hostIP'),
port: `${
config.has('Butler.uptimeMonitor.storeInInfluxdb.hostPort') ? config.get('Butler.uptimeMonitor.storeInInfluxdb.hostPort') : '8086'
}`,
database: config.get('Butler.uptimeMonitor.storeInInfluxdb.dbName'),
username: `${
config.get('Butler.uptimeMonitor.storeInInfluxdb.auth.enable')
? config.get('Butler.uptimeMonitor.storeInInfluxdb.auth.username')
: ''
}`,
password: `${
config.get('Butler.uptimeMonitor.storeInInfluxdb.auth.enable')
? config.get('Butler.uptimeMonitor.storeInInfluxdb.auth.password')
: ''
}`,
host: config.get('Butler.influxDb.hostIP'),
port: `${config.has('Butler.influxDb.hostPort') ? config.get('Butler.influxDb.hostPort') : '8086'}`,
database: config.get('Butler.influxDb.dbName'),
username: `${config.get('Butler.influxDb.auth.enable') ? config.get('Butler.influxDb.auth.username') : ''}`,
password: `${config.get('Butler.influxDb.auth.enable') ? config.get('Butler.influxDb.auth.password') : ''}`,
schema: [
{
measurement: 'butler_memory_usage',
Expand All @@ -396,12 +386,20 @@ const influx = new Influx.InfluxDB({
},
tags: ['butler_instance'],
},
{
measurement: 'win_service_state',
fields: {
state: Influx.FieldType.INTEGER,
startup_mode: Influx.FieldType.INTEGER,
},
tags: ['butler_instance', 'host', 'service_name', 'service_display_name'],
},
],
});

function initInfluxDB() {
const dbName = config.get('Butler.uptimeMonitor.storeInInfluxdb.dbName');
const enableInfluxdb = config.get('Butler.uptimeMonitor.storeInInfluxdb.enable');
const dbName = config.get('Butler.influxDb.dbName');
const enableInfluxdb = config.get('Butler.influxDb.enable');

if (enableInfluxdb) {
influx
Expand All @@ -413,7 +411,7 @@ function initInfluxDB() {
.then(() => {
logger.info(`CONFIG: Created new InfluxDB database: ${dbName}`);

const newPolicy = config.get('Butler.uptimeMonitor.storeInInfluxdb.retentionPolicy');
const newPolicy = config.get('Butler.influxDb.retentionPolicy');

// Create new default retention policy
influx
Expand Down
54 changes: 54 additions & 0 deletions src/lib/post_to_influxdb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,60 @@ function postButlerMemoryUsageToInfluxdb(memory) {
});
}

// Add function to store windows service status to InfluxDB
function postWindowsServiceStatusToInfluxDB(serviceStatus) {
// Create lookup table for Windows service state to numeric value, starting with 1 for stopped
const serviceStateLookup = {
stopped: 1,
start_pending: 2,
stop_pending: 3,
running: 4,
continue_pending: 5,
pause_pending: 6,
paused: 7,
};

// Create lookup table for Windows service startup mode to numeric value, starting with 0
const serviceStartupModeLookup = {
automatic: 0,
manual: 1,
disabled: 2,
};

let datapoint = [
{
measurement: 'win_service_state',
tags: {
butler_instance: serviceStatus.instanceTag,
host: serviceStatus.host,
service_name: serviceStatus.serviceName,
display_name: serviceStatus.serviceDetails.displayName,
},
fields: {
state: serviceStateLookup[serviceStatus.serviceStatus],
startup_mode: serviceStartupModeLookup[serviceStatus.serviceDetails.startupMode],
},
},
];
const deepClonedDatapoint = _.cloneDeep(datapoint);

globals.influx
.writePoints(deepClonedDatapoint)

.then(() => {
globals.logger.silly(
`WINDOWS SERVICE STATUS: Influxdb datapoint for Windows service status: ${JSON.stringify(datapoint, null, 2)}`
);

datapoint = null;
globals.logger.verbose('WINDOWS SERVICE STATUS: Sent Windows service status data to InfluxDB');
})
.catch((err) => {
globals.logger.error(`WINDOWS SERVICE STATUS: Error saving Windows service status to InfluxDB! ${err.stack}`);
});
}

module.exports = {
postButlerMemoryUsageToInfluxdb,
postWindowsServiceStatusToInfluxDB,
};
24 changes: 24 additions & 0 deletions src/lib/service_monitor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const webhookOut = require('./webhook_notification');
const slack = require('./slack_notification');
const teams = require('./msteams_notification');
const smtp = require('./smtp');
const influxDb = require('./post_to_influxdb');

// One state machines for each service
const serviceStateMachine = [];
Expand Down Expand Up @@ -418,6 +419,26 @@ const checkServiceStatus = async (config, logger) => {
host: host.host,
});
}

// InfluDB
if (
globals.config.has('Butler.emailNotification.enable') &&
globals.config.has('Butler.serviceMonitor.alertDestination.influxDb.enable') &&
globals.config.get('Butler.emailNotification.enable') === true &&
globals.config.get('Butler.serviceMonitor.alertDestination.InfluxDb.enable') === true
) {
const instanceTag = globals.config.has('Butler.influxDbConfig.instanceTag')
? globals.config.get('Butler.influxDbConfig.instanceTag')
: '';

influxDb.postWindowsServiceStatusToInfluxDB({
instanceTag,
serviceName,
serviceStatus,
serviceDetails,
host: host.host,
});
}
});
});
};
Expand Down Expand Up @@ -508,6 +529,9 @@ async function setupServiceMonitorTimer(config, logger) {
}
} catch (err) {
logger.error(`SERVICE MONITOR: ${err}`);
if (err.stack) {
logger.error(`SERVICE MONITOR: ${err.stack}`);
}
}
}

Expand Down
8 changes: 5 additions & 3 deletions src/lib/service_uptime.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,13 @@ function serviceUptimeStart() {
);

// Store to Influxdb if enabled
const butlerMemoryInfluxTag = globals.config.has('Butler.uptimeMonitor.storeInInfluxdb.instanceTag')
? globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.instanceTag')
const butlerMemoryInfluxTag = globals.config.has('Butler.influxDb.instanceTag')
? globals.config.get('Butler.influxDb.instanceTag')
: '';

if (
globals.config.has('Butler.influxDb.enable') &&
globals.config.get('Butler.influxDb.enable') === true &&
globals.config.has('Butler.uptimeMonitor.storeInInfluxdb.enable') &&
globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.enable') === true
) {
Expand All @@ -83,7 +85,7 @@ function serviceUptimeStart() {
});
}

// Do generic http POST if enabled
// Post uptime to New Relic if enabled
if (
globals.config.has('Butler.uptimeMonitor.storeNewRelic.enable') &&
globals.config.get('Butler.uptimeMonitor.storeNewRelic.enable') === true
Expand Down

0 comments on commit 0badfb4

Please sign in to comment.