Skip to content

Commit

Permalink
feat(qseow): Add task filtering to qseow task-vis command, showing …
Browse files Browse the repository at this point in the history 
…only parts of a task network

Implements #581
  • Loading branch information
@mountaindude
mountaindude committed Jan 1, 2025
1 parent 03d7f57 commit 042891a
Show file tree
Hide file tree
Showing 17 changed files with 849 additions and 216 deletions.
18 changes: 17 additions & 1 deletion src/lib/cli/qseow-visualise-task.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Option } from 'commander';

import { catchLog } from '../util/log.js';
import { qseowSharedParamAssertOptions } from '../util/qseow/assert-options.js';
import { qseowSharedParamAssertOptions, visTaskAssertOptions } from '../util/qseow/assert-options.js';
import { visTask } from '../cmd/qseow/vistask.js';

export function setupQseowVisualiseTaskCommand(qseow) {
Expand All @@ -10,6 +10,7 @@ export function setupQseowVisualiseTaskCommand(qseow) {
.description('visualise task network')
.action(async (options) => {
await qseowSharedParamAssertOptions(options);
await visTaskAssertOptions(options);

await visTask(options);
})
Expand Down Expand Up @@ -65,6 +66,21 @@ export function setupQseowVisualiseTaskCommand(qseow) {
.addOption(
new Option('--auth-jwt <jwt>', 'JSON Web Token (JWT) to use for authentication with Qlik Sense server').env('CTRLQ_AUTH_JWT')
)
.addOption(
new Option('--task-type <type...>', 'type of tasks to include').choices(['reload', 'ext-program']).env('CTRLQ_TASK_TYPE')
)
.addOption(new Option('--task-id <ids...>', 'task(s) in task chains to include in the network graph.').env('CTRLQ_TASK_ID'))
.addOption(new Option('--task-tag <tags...>', 'task tag(s) to include in the network graph.').env('CTRLQ_TASK_TAG'))
.addOption(
new Option('--app-id <ids...>', 'app(s) associated with the tasks that should be included in the network graph.').env(
'CTRLQ_APP_ID'
)
)
.addOption(
new Option('--app-tag <tags...>', 'app tag(s) associated with the tasks that should be included in the network graph.').env(
'CTRLQ_APP_TAG'
)
)
.addOption(new Option('--vis-host <host>', 'host for visualisation server').default('localhost').env('CTRLQ_VIS_HOST'))
.addOption(new Option('--vis-port <port>', 'port for visualisation server').default('3000').env('CTRLQ_VIS_PORT'));
}
149 changes: 12 additions & 137 deletions src/lib/cmd/qseow/gettask.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,150 +683,25 @@ async function parseTree(options, qlikSenseTasks) {
const taskModel = qlikSenseTasks.taskNetwork;
let taskTree = [];

// Array to keep track of root nodes of task chains
let rootNodes = [];

// Are any task id filters specified?
// If so get all task chains the tasks are part of,
// then get the root nodes of each chain. They will be the starting points for the task tree.

// Start by checking if any task id filters are specified
if (options.taskId) {
// options.taskId is an array of task ids
// Get all matching tasks in task model
logger.verbose(`Task id filters specified: ${options.taskId}`);

const nodesFiltered = taskModel.nodes.filter((node) => {
if (options.taskId.includes(node.id)) {
return true;
} else {
return false;
}
});

// Method:
// 1. For each node in nodesFiltered, find its root node.
try {
// Did task filters result in any actual tasks/nodes?
if (nodesFiltered.length > 0) {
for (const node of nodesFiltered) {
// node can be isolated, i.e. not part of a chain, or part of a chain
// If isolated, it is by definition a root node
// If part of a chain, it may or may not be a root node

// Method to find root node:
// 1. Check if node is a top level/root node. isTopLevelNode property is true for root nodes.
// 2. Check if node has any upstream nodes.
// 1. Recursively investigate upstream nodes until a root node is found.
// 3. Save all found root nodes.

// Is the node a root node?
if (node.isTopLevelNode) {
// Add the node to rootNodes
rootNodes.push(node);
} else {
const tmpRootNodes = qlikSenseTasks.findRootNodes(node);
rootNodes.push(...tmpRootNodes);
}
}

// Set nodesToVisualize to root nodes
nodesToVisualize.push(...rootNodes);

logger.verbose(`Found ${rootNodes.length} root nodes in task model`);
// Log root node type, id and if available name
rootNodes.forEach((node) => {
if (node.taskName) {
logger.debug(`Root task: [${node.id}] - "${node.taskName}"`);
} else if (node.metaNodeType) {
logger.debug(`Root meta task: [${node.id}] - "${node.metaNodeType}"`);
}
});
} else {
logger.warn('No tasks found matching the specified task id(s)/tag(s). Exiting.');
return false;
}
} catch (error) {
console.error(error);
console.error('Error in parseTree()');
}
}

// Any task tag filters specified?
if (options.taskTag) {
// Get all matching tasks in task model
logger.verbose(`Task tag filters specified: ${options.taskTag}`);

rootNodes = []; // Reset rootNodes array

const nodesFiltered = taskModel.nodes.filter((node) => {
// Are there any tags in this node?
if (!node.taskTags) {
return false;
}

if (node.taskTags.some((tag) => options.taskTag.includes(tag))) {
return true;
} else {
return false;
}
});

// Method:
// 1. For each node in nodesFiltered, find its root node.
try {
// Did task filters result in any actual tasks/nodes?
if (nodesFiltered.length > 0) {
for (const node of nodesFiltered) {
// node can be isolated, i.e. not part of a chain, or part of a chain
// If isolated, it is by definition a root node
// If part of a chain, it may or may not be a root node

// Method to find root node:
// 1. Check if node is a top level/root node. isTopLevelNode property is true for root nodes.
// 2. Check if node has any upstream nodes.
// 1. Recursively investigate upstream nodes until a root node is found.
// 3. Save all found root nodes.

// Is the node a root node?
if (node.isTopLevelNode) {
// Add the node to rootNodes
rootNodes.push(node);
} else {
const tmpRootNodes = qlikSenseTasks.findRootNodes(node);
rootNodes.push(...tmpRootNodes);
}
}

// Set nodesToVisualize to root nodes
nodesToVisualize.push(...rootNodes);

logger.verbose(`Found ${rootNodes.length} root nodes in task model`);
// Log root node type, id and if available name
rootNodes.forEach((node) => {
if (node.taskName) {
logger.debug(`Root task: [${node.id}] - "${node.taskName}"`);
} else if (node.metaNodeType) {
logger.debug(`Root meta task: [${node.id}] - "${node.metaNodeType}"`);
}
});
} else {
logger.warn('No tasks found matching the specified task id(s)/tag(s). Exiting.');
return false;
}
} catch (error) {
console.error(error);
console.error('Error in parseTree()');
}
}

// If no task id or tag filters specified, visualize all nodes in task model
if (!options.taskId && !options.taskTag) {
// No task id filters specified
// Visualize all nodes in task model
logger.verbose('No task id or tag filters specified. Visualizing all nodes in task model.');

nodesToVisualize.push(...taskModel.nodes);
} else {
// Task id filters specified.
// Get all task chains the tasks are part of,
// then get the root nodes of each chain. They will be the starting points for the task tree.

// Array to keep track of root nodes of task chains
const rootNodes = await qlikSenseTasks.getRootNodesFromFilter();

// Set nodesToVisualize to root nodes, if there are any
if (rootNodes) {
nodesToVisualize.push(...rootNodes);
}
}

// De-duplicate nodesToVisualize array, using id as the key
Expand Down
120 changes: 113 additions & 7 deletions src/lib/cmd/qseow/vistask.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,12 +240,41 @@ const prepareFile = async (url) => {

const template = handlebars.compile(file, { noEscape: true });

// Debug logging of task network, which consists of three properties:
// 1. nodes: Array of nodes
// 2. edges: Array of edges
// 3. tasks: Array of tasks
logger.debug(`Tasks found: ${taskNetwork?.tasks?.length}`);
for (const task of taskNetwork.tasks) {
// Log task type
if (task.metaNode === false) {
logger.debug(`Task: [${task.id}] - "${task.taskName}"`);
} else {
logger.debug(`Meta node: [${task.id}], Meta node type: ${task.metaNodeType}`);
}
}

// Log nodes
logger.debug(`Nodes found: ${taskNetwork?.nodes?.length}`);
for (const node of taskNetwork.nodes) {
if (node.metaNode === true) {
logger.debug(`Meta node: [${node.id}] - "${node.label}"`);
} else {
logger.debug(`Task node: [${node.id}] - "${node.label}"`);
}
}

// Log edges
logger.debug(`Edges found: ${taskNetwork?.edges?.length}`);
for (const edge of taskNetwork.edges) {
logger.debug(`Edge: ${JSON.stringify(edge)}`);
}

// Get task network model
const taskModel = taskNetwork;

// Add schema nodes
const nodes = taskModel.nodes.filter((node) => node.metaNode === true);
// let nodes = taskModel.nodes.filter((node) => node.metaNodeType === 'schedule');
let nodesNetwork = nodes.map((node) => {
const newNode = {};
if (node.metaNodeType === 'schedule') {
Expand Down Expand Up @@ -407,7 +436,7 @@ const requestHandler = async (req, res) => {
}
};

// Set up http server for serviing html pages with the task visualization
// Set up http server for serving html pages with the task visualization
const startHttpServer = async (options) => {
const server = http.createServer(requestHandler);

Expand All @@ -419,6 +448,12 @@ const startHttpServer = async (options) => {
});
};

/**
* Start an HTTP server for visualizing QSEoW tasks as a network diagram.
*
* @param {Object} options - Options for the visTask function.
* @returns {Promise<boolean>} - A promise that resolves to true if the server was started successfully, false otherwise.
*/
export async function visTask(options) {
// Set log level
setLoggingLevel(options.logLevel);
Expand Down Expand Up @@ -492,23 +527,94 @@ export async function visTask(options) {
logger.error('Failed to get task model from QSEoW');
return false;
}
taskNetwork = qlikSenseTasks.taskNetwork;

// Filter tasks based on CLI options. Two possible
// 1. If no filters specified, show all tasks.
// 2. At least one filter specified.
// - If --task-id <id...> specified
// - Get root task(s) for each specified task id
// - Include in network diagram all tasks that are children of the root tasks
// - If --task-tag <tag...> specified
// - Get all tasks that have the specified tag(s)
// - Get root task(s) for each task that has the specified tag(s)§
// - If --app-id <id...> specified
// - Get all tasks that are associated with the specified app id(s)
// - Get root task(s) for each task that is associated with the specified app id(s)
// - Include in network diagram all tasks that are children of the root tasks
// - If --app-tag <tag...> specified
// - Get all apps that are associated with the specified app tag(s)
// - Get all tasks that are associated with the apps that have the specified app tag(s)
// - Get root task(s) for each task that is associated with the apps that have the specified app tag(s)
// - Include in network diagram all tasks that are children of the root tasks
//
// Filters above are additive, i.e. all tasks that match any of the filters are included in the network diagram.
// Make sure to de-duplicate root tasks.

// Arrays to keep track of which nodes in task model to visualize
// const nodesToVisualize = [];
// const edgesToVisualize = [];

// If no task id or tag filters specified, visualize all nodes in task model
if (!options.taskId && !options.taskTag) {
// No task id filters specified
// Visualize all nodes in task model
logger.verbose('No task id or tag filters specified. Visualizing all nodes in task model.');

taskNetwork = qlikSenseTasks.taskNetwork;
} else {
// Task id filters specified.
// Get all task chains the tasks are part of,
// then get the rMeta nodeoot nodes of each chain. They will be the starting points for the task tree.

// Array to keep track of root nodes of task chains
const rootNodes = await qlikSenseTasks.getRootNodesFromFilter();

// List root nodes to console
logger.verbose(`${rootNodes.length} root nodes sent to visualizer:`);
rootNodes.forEach((node) => {
// Meta node?
if (node.metaNode === true) {
// Reload task?
if (node.taskType === 'reloadTask') {
logger.verbose(
`Meta node: metanode type=${node.metaNodeType} id=[${node.id}] task type=${node.taskType} task name="${node.completeSchemaEvent.reloadTask.name}"`
);
}
} else {
logger.verbose(`Root node: [${node.id}] "${node.taskName}"`);
}
});

// Get all nodes that are children of the root nodes
const { nodes, edges, tasks } = await qlikSenseTasks.getNodesAndEdgesFromRootNodes(rootNodes);
// nodesToVisualize.push(...nodes);
// edgesToVisualize.push(...edges);
// tasksToVisualize.push(...tasks);

// taskNetwork = { nodes: nodesToVisualize, edges: edgesToVisualize, tasks)

taskNetwork = { nodes, edges, tasks };
}

// Add additional values to Handlebars template
templateData.visTaskHost = options.visHost;
templateData.visTaskPort = options.visPort;

// Get reload task count, i.e. tasks where taskType === 0
templateData.reloadTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 0).length;
// templateData.reloadTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 0).length;
templateData.reloadTaskCount = taskNetwork.tasks.filter((task) => task.taskType === 0).length;

// Get external program task count, i.e. tasks where taskType === 1
templateData.externalProgramTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 1).length;
// templateData.externalProgramTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 1).length;
templateData.externalProgramTaskCount = taskNetwork.tasks.filter((task) => task.taskType === 1).length;

// Get schema trigger count
templateData.schemaTriggerCount = qlikSenseTasks.qlikSenseSchemaEvents.schemaEventList.length;
// Count taskNetwork.nodes events where metaNodeType === 'schedule'
templateData.schemaTriggerCount = taskNetwork.nodes.filter((node) => node.metaNodeType === 'schedule').length;

// Get composite trigger count
templateData.compositeTaskCount = qlikSenseTasks.qlikSenseCompositeEvents.compositeEventList.length;
// Count taskNetwork.nodes events where metaNodeType === 'composite'
templateData.compositeTaskCount = taskNetwork.nodes.filter((node) => node.metaNodeType === 'composite').length;

startHttpServer(optionsNew);
return true;
Expand Down
Loading

0 comments on commit 042891a

Please sign in to comment.