diff --git a/.vscode/launch.json b/.vscode/launch.json index 2c98751..7489d52 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -58,44 +58,44 @@ // ------------------------------------ // Import tasks from Excel file // ------------------------------------ - "args": [ - "task-import", - "--auth-type", - "cert", - "--host", - "192.168.100.109", - "--auth-cert-file", - "./cert/client.pem", - "--auth-cert-key-file", - "./cert/client_key.pem", - "--auth-user-dir", - "LAB", - "--auth-user-id", - "goran", + // "args": [ + // "task-import", + // "--auth-type", + // "cert", + // "--host", + // "192.168.100.109", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", - "--file-type", - "excel", + // "--file-type", + // "excel", - "--file-name", - "testdata/tasks.xlsx", - "--sheet-name", - "5" + // "--file-name", + // "testdata/tasks.xlsx", + // "--sheet-name", + // "7" - // "--import-app", - // "--import-app-sheet-name", - // "App import", + // // "--import-app", + // // "--import-app-sheet-name", + // // "App import", - // "--qvf-overwrite", - // "no", + // // "--qvf-overwrite", + // // "no", - // "--limit-import-count", - // "2", + // // "--limit-import-count", + // // "2", - // "--sleep-app-upload", - // "500", + // // "--sleep-app-upload", + // // "500", - // "--dry-run" - ] + // // "--dry-run" + // ] // ------------------------------------ // Import tasks from CSV file @@ -121,7 +121,8 @@ // "--file-name", // // "tasks2source.csv", // // "task-chain.csv", - // "testdata/reload-tasks.csv", + // // "testdata/reload-tasks.csv", + // "./tasks5.csv", // // "--qvf-overwrite", // // "no", @@ -318,45 +319,46 @@ // "comprule" // ] - // ------------------------------------ - // Get reload tasks as CSV/Excel/JSON file - // ------------------------------------ - // "args": [ - // "task-get", - // "--auth-type", - // "cert", - // "--host", - // "192.168.100.109", - // // "--auth-cert-file", - // // "./cert/client.pem", - // // "--auth-cert-key-file", - // // "./cert/client_key.pem", - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", - // // "--task-id", - // // "82bc3e66-c899-4e44-b52f-552145da5ee0", - // // "82bc3e66-c899-4e44-b52f-552145da5ee1", - // // "5748afa9-3abe-43ab-bb1f-127c48ced075", - // // "5520e710-91ad-41d2-aeb6-434cafbf366b", - // // "--task-tag", - // // "Ctrl-Q demo", - // // "Butler 5.0 demo", - - // "--output-format", - // "table", - // "--output-dest", - // "file", - // "--output-file-name", - // // "tasks.xlsx", - // "tasks.csv", - // // "tasks2.json", - // // "tasks2.xlsx", - // "--output-file-format", - // // "excel", - // // "json", - // "csv", + ------------------------------------ + Get reload tasks as CSV/Excel/JSON file + ------------------------------------ + "args": [ + "task-get", + "--auth-type", + "cert", + "--host", + "192.168.100.109", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + "--auth-user-dir", + "LAB", + "--auth-user-id", + "goran", + // "--task-id", + // "82bc3e66-c899-4e44-b52f-552145da5ee0", + // "82bc3e66-c899-4e44-b52f-552145da5ee1", + // "5748afa9-3abe-43ab-bb1f-127c48ced075", + // "5520e710-91ad-41d2-aeb6-434cafbf366b", + // "--task-tag", + // "Ctrl-Q demo", + // "Butler 5.0 demo", + + "--output-format", + "table", + "--output-dest", + "file", + "--output-file-name", + // "tasks.xlsx", + // "tasks.csv", + "tasks_all.csv", + // "tasks2.json", + // "tasks2.xlsx", + "--output-file-format", + // "excel", + // "json", + "csv", // "--output-file-name", // // "reload-tasks.xlsx", diff --git a/src/lib/cmd/gettask.js b/src/lib/cmd/gettask.js index 1ef8380..04f6c22 100644 --- a/src/lib/cmd/gettask.js +++ b/src/lib/cmd/gettask.js @@ -437,10 +437,25 @@ const getTask = async (options) => { taskTable = taskTable.concat([row]); // Find all triggers for this task - const schemaEventsForThisTask = schemaEventList.filter((item) => item.schemaEvent?.reloadTask?.id === task.taskId); - const compositeEventsForThisTask = compositeEventList.filter( - (item) => item.compositeEvent?.reloadTask?.id === task.taskId - ); + const schemaEventsForThisTask = schemaEventList.filter((item) => { + if (item.schemaEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.schemaEvent?.externalProgramTask?.id === task.taskId) { + return true; + } + return false; + }); + + const compositeEventsForThisTask = compositeEventList.filter((item) => { + if (item.compositeEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.compositeEvent?.externalProgramTask?.id === task.taskId) { + return true; + } + return false; + }); // Write schema events to table if (columnBlockShow.schematrigger) { @@ -690,7 +705,7 @@ const getTask = async (options) => { } if (columnBlockShow.extprogram) { - headerRow = headerRow.concat(['Path', 'Parameters']); + headerRow = headerRow.concat(['Ext program path', 'Ext program parameters']); } if (columnBlockShow.lastexecution) { diff --git a/src/lib/task/class_alltasks.js b/src/lib/task/class_alltasks.js index c10ed1a..eddbbd8 100644 --- a/src/lib/task/class_alltasks.js +++ b/src/lib/task/class_alltasks.js @@ -272,6 +272,7 @@ class QlikSenseTasks { // Get schema events for this task, storing the info using the same structure as returned from QRS API currentTask.schemaEvents = this.parseSchemaEvents({ + taskType: 'reload', taskRows: param.taskRows, taskFileColumnHeaders: param.taskFileColumnHeaders, taskCounter: param.taskCounter, @@ -411,6 +412,7 @@ class QlikSenseTasks { // Get schema events for this task, storing the info using the same structure as returned from QRS API currentTask.schemaEvents = this.parseSchemaEvents({ + taskType: 'external program', taskRows: param.taskRows, taskFileColumnHeaders: param.taskFileColumnHeaders, taskCounter: param.taskCounter, @@ -435,6 +437,7 @@ class QlikSenseTasks { // Function to get schema events for a specific task // Parameters: + // - taskType: Type of task. Possible values: "reload", "external program" // - taskRows: Array of rows associated with the task. All rows associated with the task are passed to this function // - taskFileColumnHeaders: Object containing info about which column contains what data // - taskCounter: Counter for the current task @@ -443,7 +446,7 @@ class QlikSenseTasks { // - nodesWithEvents: Set of nodes that have associated events parseSchemaEvents(param) { // Get schema events for this task, storing the info using the same structure as returned from QRS API - const prelAchemaEvents = []; + const prelSchemaEvents = []; const schemaEventRows = param.taskRows.filter( (item) => @@ -472,12 +475,22 @@ class QlikSenseTasks { schemaFilterDescription: [schemaEventRow[param.taskFileColumnHeaders.schemaFilterDescription.pos]], incrementDescription: schemaEventRow[param.taskFileColumnHeaders.schemaIncrementDescription.pos], incrementOption: mapIncrementOption.get(schemaEventRow[param.taskFileColumnHeaders.schemaIncrementOption.pos]), - reloadTask: { - id: param.fakeTaskId, - }, schemaPath: 'SchemaEvent', }; + if (param.taskType === 'reload') { + schemaEvent.reloadTask = { + id: param.fakeTaskId, + }; + } else if (param.taskType === 'external program') { + schemaEvent.externalProgramTask = { + id: param.fakeTaskId, + }; + } else { + logger.error(`(${param.taskCounter}) PARSE TASKS FROM FILE: Incorrect task type "${param.taskType}". Exiting.`); + process.exit(1); + } + this.qlikSenseSchemaEvents.addSchemaEvent(schemaEvent); // Add schema event to network representation of tasks @@ -496,24 +509,39 @@ class QlikSenseTasks { completeSchemaEvent: schemaEvent, }); - this.taskNetwork.edges.push({ - from: nodeId, - to: schemaEvent.reloadTask.id, - }); + // Add edge from schema trigger node to current task, taking into account task type + if (param.taskType === 'reload') { + this.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.reloadTask.id, + }); - // Keep a note that this node has associated events - param.nodesWithEvents.add(schemaEvent.reloadTask.id); + // Keep a note that this node has associated events + param.nodesWithEvents.add(schemaEvent.reloadTask.id); - // Add this schema event to the current task - // Remove reference to task ID first though - delete schemaEvent.reloadTask.id; - delete schemaEvent.reloadTask; + // Remove reference to task ID + delete schemaEvent.reloadTask.id; + delete schemaEvent.reloadTask; + } else if (param.taskType === 'external program') { + this.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.externalProgramTask.id, + }); - prelAchemaEvents.push(schemaEvent); + // Keep a note that this node has associated events + param.nodesWithEvents.add(schemaEvent.externalProgramTask.id); + + // Remove reference to task ID + delete schemaEvent.externalProgramTask.id; + delete schemaEvent.externalProgramTask; + } + + // Add this schema event to the current task + prelSchemaEvents.push(schemaEvent); } } - return prelAchemaEvents; + return prelSchemaEvents; } // Function to get composite events for a specific task @@ -613,7 +641,7 @@ class QlikSenseTasks { } else { // The task pointed to by the composite event rule does not exist logger.error( - `(${param.taskCounter}) PARSE TASKS FROM FILE: Task "${ + `(${param.taskCounter}) PARSE COMPOSITE EVENT RULE FROM FILE: Task "${ rule[param.taskFileColumnHeaders.ruleTaskId.pos] }" does not exist. Exiting.` ); @@ -828,6 +856,12 @@ class QlikSenseTasks { return -1; } + // Is first column empty? + if (item[taskFileColumnHeaders.taskCounter.pos] === undefined) { + // Empty task counter column + return -1; + } + if (item[taskFileColumnHeaders.taskCounter.pos] === taskFileColumnHeaders.taskCounter.name) { // This is the header row return -1; @@ -1112,10 +1146,10 @@ class QlikSenseTasks { // Set task ID for the composite event itself, i.e. which task is the event associated with (i.e. the downstream task) // Handle different task types differently - if (item.compositeEvent.reloadTask.id) { + if (item.compositeEvent?.reloadTask?.id) { // Reload task a.compositeEvent.reloadTask.id = this.taskIdMap.get(item.compositeEvent.reloadTask.id); - } else if (item.compositeEvent.externalProgramTask.id) { + } else if (item.compositeEvent?.externalProgramTask?.id) { // External program task a.compositeEvent.externalProgramTask.id = this.taskIdMap.get(item.compositeEvent.externalProgramTask.id); } @@ -1128,6 +1162,14 @@ class QlikSenseTasks { // Get triggering/upstream task id const id = this.taskIdMap.get(b.task.id); + + // If id is not found in the mapping table, it means that the task + // referenced by the rule (i.e. the upstream teask) is neither a task + // that existed before this execution of Ctrl-Q, nor a task that was + // created during this execution of Ctrl-Q. + // This is an error - the task ID should exist. + // Most likely the error is caused by an invalid value in the "Rule task id" + // column in the source file. if (id !== undefined && validate(id) === true) { // Determine what kind of task this is. Options are: // - reload @@ -1157,23 +1199,31 @@ class QlikSenseTasks { // Use mapTaskType to get the string variant of the task type. Convert to lower case. const taskTypeString = mapTaskType.get(taskType).trim().toLowerCase(); + // Ensure we got a valid task type + if (!['reload', 'externalprogram'].includes(taskTypeString)) { + logger.error( + `PREPARING COMPOSITE EVENT: Invalid task type "${taskTypeString}" for upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` + ); + process.exit(1); + } + if (taskTypeString === 'reload') { b.reloadTask = { id }; } else if (taskTypeString === 'externalprogram') { b.externalProgramTask = { id }; } - } else if (this.options.dryRun === false || this.options.dryRun === undefined) { + } else if (id === undefined) { + // (this.options.dryRun === false || this.options.dryRun === undefined) { logger.error( - `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.reloadTask.id}" in rule for composite event "${a.compositeEvent.name}" ` + `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` ); - b.reloadTask.id = null; + process.exit(1); } return b; }); return a; }); - // Loop over all composite events in the source file, create missing ones where needed logger.info('-------------------------------------------------------------------'); logger.info('Creating composite events for the just created tasks...'); @@ -1233,9 +1283,17 @@ class QlikSenseTasks { .then((result) => { if (result.status === 201) { const response = JSON.parse(result.data); - logger.info( - `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.reloadTask.id}. Result: ${result.status}/${result.statusText}.` - ); + + if (response?.reloadTask) { + logger.info( + `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.reloadTask.id}. Result: ${result.status}/${result.statusText}.` + ); + } else if (response?.externalProgramTask) { + logger.info( + `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.externalProgramTask.id}. Result: ${result.status}/${result.statusText}.` + ); + } + resolve(response.id); } else { reject(); @@ -1325,26 +1383,27 @@ class QlikSenseTasks { // Build a body for the API call const body = { - name: newTask.name, - taskType: 1, - enabled: newTask.enabled, - taskSessionTimeout: newTask.taskSessionTimeout, - maxRetries: newTask.maxRetries, - path: newTask.path, - parameters: newTask.parameters, - tags: newTask.tags, - customProperties: newTask.customProperties, - schemaPath: 'ExternalProgramTask', - // schemaPath: 'ExternalProgramTask', + task: { + name: newTask.name, + taskType: 1, + enabled: newTask.enabled, + taskSessionTimeout: newTask.taskSessionTimeout, + maxRetries: newTask.maxRetries, + path: newTask.path, + parameters: newTask.parameters, + tags: newTask.tags, + customProperties: newTask.customProperties, + schemaPath: 'ExternalProgramTask', + }, + schemaEvents: newTask.schemaEvents, }; - // schemaEvents: newTask.schemaEvents, // Save task to QSEoW const axiosConfig = setupQRSConnection(this.options, { method: 'post', fileCert: this.fileCert, fileCertKey: this.fileCertKey, - path: '/qrs/externalprogramtask', + path: '/qrs/externalprogramtask/create', body, }); @@ -2041,7 +2100,7 @@ class QlikSenseTasks { // Add edges from upstream tasks to the new meta node // eslint-disable-next-line no-restricted-syntax for (const rule of compositeEvent.compositeEvent.compositeRules) { - if (validate(rule.reloadTask.id)) { + if (validate(rule?.reloadTask?.id)) { // Upstream task is a reload task logger.debug( `Composite event "${compositeEvent.compositeEvent.name}" is triggered by reload task with ID=${rule.reloadTask.id}.` @@ -2057,14 +2116,14 @@ class QlikSenseTasks { completeCompositeEvent: compositeEvent.compositeEvent, rule, }); - } else if (validate(rule.externalProgramTask.id)) { + } else if (validate(rule?.externalProgramTask?.id)) { // Upstream task is an external program task logger.debug( `Composite event "${compositeEvent.compositeEvent.name}" is triggered by external program task with ID=${rule.externalProgramTask.id}.` ); this.taskNetwork.edges.push({ - from: rule.reloadTask.id, + from: rule.externalProgramTask.id, fromTaskType: 'ExternalProgram', to: nodeId, toTaskType: 'Composite', @@ -2201,7 +2260,7 @@ class QlikSenseTasks { ); this.taskNetwork.edges.push({ - from: rule.reloadTask.id, + from: rule.externalProgramTask.id, fromTaskType: 'ExternalProgram', to: nodeId, toTaskType: 'Composite', @@ -2222,10 +2281,17 @@ class QlikSenseTasks { `Added edge from new meta composite event node "${nodeId}" to reload task ID=${compositeEvent.compositeEvent?.reloadTask?.id}.` ); - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.compositeEvent.reloadTask.id, - }); + if (compositeEvent.compositeEvent?.reloadTask) { + this.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.reloadTask.id, + }); + } else if (compositeEvent.compositeEvent?.externalProgramTask) { + this.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.externalProgramTask.id, + }); + } } } } diff --git a/src/lib/task/class_task.js b/src/lib/task/class_task.js index fdb57ca..f3aed7d 100644 --- a/src/lib/task/class_task.js +++ b/src/lib/task/class_task.js @@ -20,15 +20,23 @@ class QlikSenseTask { // Data in the "task" parameter was loaded from a Qlik Sense (QSEoW) server if (task.schemaPath === 'ReloadTask') { this.sourceType = 'from_qseow'; - this.taskId = task.id; + + if (task.id) { + this.taskId = task.id; + } if (anonymizeTaskNames === true) { this.taskName = randomWords({ min: 2, max: 5, join: ' ' }); - this.appName = randomWords({ min: 2, max: 5, join: ' ' }); + if (task.app.name) { + this.appName = randomWords({ min: 2, max: 5, join: ' ' }); + } } else { this.taskName = task.name; - this.appName = task.app.name; + if (task.app.name) { + this.appName = task.app.name; + } } + this.taskEnabled = task.enabled; this.appId = task.app.id; this.appPublished = task.app.published; @@ -37,6 +45,7 @@ class QlikSenseTask { this.taskSessionTimeout = task.taskSessionTimeout; this.isPartialReload = task.isPartialReload; this.isManuallyTriggered = task.isManuallyTriggered; + this.taskLastExecutionStartTimestamp = task.operational.lastExecutionResult.startTime === '1753-01-01T00:00:00.000Z' ? '' @@ -49,6 +58,7 @@ class QlikSenseTask { this.taskLastExecutionExecutingNodeName = task.operational.lastExecutionResult.executingNodeName; this.taskNextExecutionTimestamp = task.operational.nextExecution === '1753-01-01T00:00:00.000Z' ? '' : task.operational.nextExecution; + this.taskTags = task.tags; this.taskTagsFriendly = task.tags.map((tag) => tag.name); this.taskCustomProperties = task.customProperties; @@ -61,6 +71,7 @@ class QlikSenseTask { } this.completeTaskObject = task; + this.taskType = 0; logger.silly(`Initialised reload task object from QSEoW: ${JSON.stringify(task)}`); } else if (task.schemaPath === 'ExternalProgramTask') { this.sourceType = 'from_qseow'; @@ -77,6 +88,7 @@ class QlikSenseTask { this.taskEnabled = task.enabled; this.taskMaxRetries = task.maxRetries; this.taskSessionTimeout = task.taskSessionTimeout; + this.taskLastExecutionStartTimestamp = task?.operational?.lastExecutionResult?.startTime === '1753-01-01T00:00:00.000Z' ? '' @@ -89,6 +101,7 @@ class QlikSenseTask { this.taskLastExecutionExecutingNodeName = task?.operational?.lastExecutionResult?.executingNodeName; this.taskNextExecutionTimestamp = task?.operational?.nextExecution === '1753-01-01T00:00:00.000Z' ? '' : task?.operational?.nextExecution; + this.taskTags = task.tags; this.taskTagsFriendly = task.tags.map((tag) => tag.name); this.taskCustomProperties = task.customProperties; @@ -101,26 +114,18 @@ class QlikSenseTask { } this.completeTaskObject = task; + this.taskType = 1; logger.silly(`Initialised external program task object from QSEoW: ${JSON.stringify(task)}`); } } else if (source.toLowerCase() === 'from_file') { // Data in the "task" parameter was loaded from a task definition file on disk - this.sourceType = 'from_file'; - if (task.taskType === 0) { - this.taskType = task.taskType; + if (task.schemaPath === 'ReloadTask') { + this.sourceType = 'from_file'; if (task.id) { this.taskId = task.id; } - this.taskName = task.name; - this.taskEnabled = task.enabled; - this.taskSessionTimeout = task.taskSessionTimeout; - this.taskMaxRetries = task.maxRetries; - this.isPartialReload = task.isPartialReload; - this.isManuallyTriggered = task.isManuallyTriggered; - - this.appId = task.app.id; if (anonymizeTaskNames === true) { this.taskName = randomWords({ min: 2, max: 5, join: ' ' }); if (task.app.name) { @@ -133,6 +138,15 @@ class QlikSenseTask { } } + this.taskEnabled = task.enabled; + this.appId = task.app.id; + this.appPublished = task.app.published; + this.appStream = task.app.published ? task.app.stream.name : ''; + this.taskMaxRetries = task.maxRetries; + this.taskSessionTimeout = task.taskSessionTimeout; + this.isPartialReload = task.isPartialReload; + this.isManuallyTriggered = task.isManuallyTriggered; + this.taskTags = task.tags; this.taskTagsFriendly = task.tags.map((tag) => tag.name); this.taskCustomProperties = task.customProperties; @@ -142,8 +156,33 @@ class QlikSenseTask { this.compositeEvents = task.compositeEvents; this.completeTaskObject = task; + this.taskType = 0; + logger.silly(`Initialised task object from file: ${JSON.stringify(task)}`); + } else if (task.schemaPath === 'ExternalProgramTask') { + this.sourceType = 'from_file'; + this.taskId = task.id; + + this.path = task.path; + this.parameters = task.parameters; + + if (anonymizeTaskNames === true) { + this.taskName = randomWords({ min: 2, max: 5, join: ' ' }); + } else { + this.taskName = task.name; + } + this.taskEnabled = task.enabled; + this.taskMaxRetries = task.maxRetries; + this.taskSessionTimeout = task.taskSessionTimeout; + + this.taskTags = task.tags; + this.taskTagsFriendly = task.tags.map((tag) => tag.name); + this.taskCustomProperties = task.customProperties; + this.taskCustomPropertiesFriendly = task.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`); + + this.completeTaskObject = task; + this.taskType = 1; + logger.silly(`Initialised external program task object from file: ${JSON.stringify(task)}`); } - logger.silly(`Initialised task object from file: ${JSON.stringify(task)}`); } } } diff --git a/src/lib/util/lookups.js b/src/lib/util/lookups.js index 878bb90..35cb0cd 100644 --- a/src/lib/util/lookups.js +++ b/src/lib/util/lookups.js @@ -116,6 +116,7 @@ const taskFileColumnHeaders = { eventCreatedDate: { name: 'Event created date', pos: -1 }, eventModifiedDate: { name: 'Event modified date', pos: -1 }, eventModifiedBy: { name: 'Event modified by', pos: -1 }, + schemaIncrementOption: { name: 'Schema increment option', pos: -1 }, schemaIncrementDescription: { name: 'Schema increment description', pos: -1 }, daylightSavingsTime: { name: 'Daylight savings time', pos: -1 }, @@ -123,6 +124,7 @@ const taskFileColumnHeaders = { scheamExpiration: { name: 'Schema expiration', pos: -1 }, schemaFilterDescription: { name: 'Schema filter description', pos: -1 }, schemaTimeZone: { name: 'Schema time zone', pos: -1 }, + timeConstraintSeconds: { name: 'Time contstraint seconds', pos: -1 }, timeConstraintMinutes: { name: 'Time contstraint minutes', pos: -1 }, timeConstraintHours: { name: 'Time contstraint hours', pos: -1 },