From 62d296905368480494ca7361417a3f87dd412860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Wed, 8 Nov 2023 07:21:00 +0000 Subject: [PATCH 1/4] Fix typos --- src/lib/app/class_allapps.js | 2 +- src/lib/util/customproperties.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lib/app/class_allapps.js b/src/lib/app/class_allapps.js index 4194378..50ce93b 100644 --- a/src/lib/app/class_allapps.js +++ b/src/lib/app/class_allapps.js @@ -300,7 +300,7 @@ class QlikSenseApps { logger.error( `IMPORT APP TO QSEOW: Custom property "${tmpCustomProperties[0]}" for app "${ appRow[0][appFileColumnHeaders.appName.pos] - }" not found or not valid for object type "Reload task". Exiting.` + }" not found or not valid for task type "Reload task". Exiting.` ); process.exit(1); } diff --git a/src/lib/util/customproperties.js b/src/lib/util/customproperties.js index 59ac11e..5b13954 100644 --- a/src/lib/util/customproperties.js +++ b/src/lib/util/customproperties.js @@ -47,7 +47,7 @@ function getCustomPropertyIdByName(objectType, customPropertyName, cpExisting) { // The custom property exists, but is it enabled for this object type (task, app etc)? const correctObjectType = cp[0].objectTypes.find((item) => objectType.toLowerCase() === item.toLowerCase()); if (!correctObjectType) { - logger.warn(`Custom property "${customPropertyName}" is not valid for object type "${objectType}".`); + logger.warn(`Custom property "${customPropertyName}" is not valid for task type "${objectType}".`); resolve(false); } @@ -82,7 +82,7 @@ function getCustomPropertyIdByName(objectType, customPropertyName, cpExisting) { // if (result.status === 200 && result.data.length === 1) { // const correctObjectType = result.data[0].objectTypes.find((item) => objectType.toLowerCase() === item.toLowerCase()); // if (!correctObjectType) { -// logger.warn(`Custom property "${customPropertyName}" is not valid for object type "${objectType}".`); +// logger.warn(`Custom property "${customPropertyName}" is not valid for task type "${objectType}".`); // resolve(false); // } @@ -108,7 +108,7 @@ function getCustomPropertyDefinitionByName(objectType, customPropertyName, cpExi // The custom property exists, but is it enabled for this object type (task, app etc)? const correctObjectType = cp[0].objectTypes.find((item) => objectType.toLowerCase() === item.toLowerCase()); if (!correctObjectType) { - logger.warn(`Custom property "${customPropertyName}" is not valid for object type "${objectType}".`); + logger.warn(`Custom property "${customPropertyName}" is not valid for task type "${objectType}".`); resolve(false); } @@ -143,7 +143,7 @@ function getCustomPropertyDefinitionByName(objectType, customPropertyName, cpExi // if (result.status === 200 && result.data.length === 1) { // const correctObjectType = result.data[0].objectTypes.find((item) => objectType.toLowerCase() === item.toLowerCase()); // if (!correctObjectType) { -// logger.warn(`Custom property "${customPropertyName}" is not valid for object type "${objectType}".`); +// logger.warn(`Custom property "${customPropertyName}" is not valid for task type "${objectType}".`); // resolve(false); // } @@ -171,7 +171,7 @@ function doesCustomPropertyValueExist(objectType, customPropertyName, customProp // The custom property exists, but is it enabled for this object type (task, app etc)? const correctObjectType = cp[0].objectTypes.find((item) => objectType.toLowerCase() === item.toLowerCase()); if (!correctObjectType) { - logger.warn(`Custom property "${customPropertyName}" is not valid for object type "${objectType}".`); + logger.warn(`Custom property "${customPropertyName}" is not valid for task type "${objectType}".`); resolve(false); } @@ -217,7 +217,7 @@ function doesCustomPropertyValueExist(objectType, customPropertyName, customProp // if (result.status === 200 && result.data.length === 1) { // const correctObjectType = result.data[0].objectTypes.find((item) => objectType.toLowerCase() === item.toLowerCase()); // if (!correctObjectType) { -// logger.warn(`Custom property "${customPropertyName}" is not valid for object type "${objectType}".`); +// logger.warn(`Custom property "${customPropertyName}" is not valid for task type "${objectType}".`); // resolve(false); // } From ac8f8b9d5b9f83259edeefb4361216e215fd85e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Wed, 8 Nov 2023 07:27:33 +0000 Subject: [PATCH 2/4] ext pgm tasks wip --- .vscode/launch.json | 84 ++++----- src/lib/task/class_alltasks.js | 326 ++++++++++++++++++++++++++++++--- src/lib/util/lookups.js | 11 ++ 3 files changed, 358 insertions(+), 63 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 9ae4cf1..2c98751 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -58,44 +58,44 @@ // ------------------------------------ // Import tasks from Excel file // ------------------------------------ - // "args": [ - // "task-import", - // "--auth-type", - // "cert", - // "--host", - // "192.168.100.109", - // "--auth-cert-file", - // "./cert/client.pem", - // "--auth-cert-key-file", - // "./cert/client_key.pem", - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", + "args": [ + "task-import", + "--auth-type", + "cert", + "--host", + "192.168.100.109", + "--auth-cert-file", + "./cert/client.pem", + "--auth-cert-key-file", + "./cert/client_key.pem", + "--auth-user-dir", + "LAB", + "--auth-user-id", + "goran", - // "--file-type", - // "excel", + "--file-type", + "excel", - // "--file-name", - // "testdata/tasks.xlsx", - // "--sheet-name", - // "4" + "--file-name", + "testdata/tasks.xlsx", + "--sheet-name", + "5" - // // "--import-app", - // // "--import-app-sheet-name", - // // "App import", + // "--import-app", + // "--import-app-sheet-name", + // "App import", - // // "--qvf-overwrite", - // // "no", + // "--qvf-overwrite", + // "no", - // // "--limit-import-count", - // // "2", + // "--limit-import-count", + // "2", - // // "--sleep-app-upload", - // // "500", + // "--sleep-app-upload", + // "500", - // // "--dry-run" - // ] + // "--dry-run" + ] // ------------------------------------ // Import tasks from CSV file @@ -771,17 +771,17 @@ // ------------------------------------ // Connection test // ------------------------------------ - "args": [ - "connection-test", - "--host", - "192.168.100.109", - "--auth-user-dir", - "LAB", - "--auth-user-id", - "goran", - "--log-level", - "info" - ] + // "args": [ + // "connection-test", + // "--host", + // "192.168.100.109", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", + // "--log-level", + // "info" + // ] } ] diff --git a/src/lib/task/class_alltasks.js b/src/lib/task/class_alltasks.js index e075b47..c10ed1a 100644 --- a/src/lib/task/class_alltasks.js +++ b/src/lib/task/class_alltasks.js @@ -191,7 +191,9 @@ class QlikSenseTasks { taskCreationOption.trim() !== '' && !['if-exists-add-another', 'if-exists-update-existing'].includes(taskCreationOption) ) { - logger.error(`(${param.taskCounter}) PARSE TASKS FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.`); + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.` + ); process.exit(1); } @@ -248,17 +250,22 @@ class QlikSenseTasks { .filter((item2) => item2.trim().length !== 0) .map((cp) => cp.trim()); + // Do we have two items in the array? First is the custom property name, second is the value if (tmpCustomProperty?.length === 2) { // eslint-disable-next-line no-await-in-loop const customPropertyId = await getCustomPropertyIdByName('ReloadTask', tmpCustomProperty[0], param.cpExisting); - currentTask.customProperties.push({ - definition: { - id: customPropertyId, - name: tmpCustomProperty[0].trim(), - }, - value: tmpCustomProperty[1].trim(), - }); + // If previous call returned false, it means the custom property does not exist in Sense + // or cannot be used with this task type. In that case, skip it. + if (customPropertyId) { + currentTask.customProperties.push({ + definition: { + id: customPropertyId, + name: tmpCustomProperty[0].trim(), + }, + value: tmpCustomProperty[1].trim(), + }); + } } } } @@ -287,6 +294,145 @@ class QlikSenseTasks { return { currentTask, taskCreationOption }; } + // Function to parse the rows associated with a specific external program task in the source file + // Properties in the param object: + // - taskRows: Array of rows associated with the task. All rows associated with the task are passed to this function + // - taskFileColumnHeaders: Object containing info about which column contains what data + // - taskCounter: Counter for the current task + // - tagsExisting: Array of existing tags in QSEoW + // - cpExisting: Array of existing custom properties in QSEoW + // - fakeTaskId: Fake task ID used to associate task with schema/composite events + // - nodesWithEvents: Set of nodes that have associated events + // + // Returns: + // Object with two properties: + // - currentTask: Object containing task data + // - taskCreationOption: Task creation option. Possible values: "if-exists-add-another", "if-exists-update-existing" + async parseExternalProgramTask(param) { + let currentTask = null; + let taskCreationOption; + + // Create task object using same structure as results from QRS API + + // Get task import options + if (param.taskFileColumnHeaders.importOptions.pos === 999) { + // No task creation options column in the file + // Use the default task creation option + taskCreationOption = 'if-exists-update-existing'; + } else { + // Task creation options column exists in the file + // Use the value from the file + taskCreationOption = param.taskRows[0][param.taskFileColumnHeaders.importOptions.pos]; + } + + // Ensure task creation option is valid. Allow empty option + if ( + taskCreationOption && + taskCreationOption.trim() !== '' && + !['if-exists-add-another', 'if-exists-update-existing'].includes(taskCreationOption) + ) { + logger.error( + `(${param.taskCounter}) PARSE EXTERNAL PROGRAM TASK FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.` + ); + process.exit(1); + } + + currentTask = { + id: param.taskRows[0][param.taskFileColumnHeaders.taskId.pos], + name: param.taskRows[0][param.taskFileColumnHeaders.taskName.pos], + taskType: mapTaskType.get(param.taskRows[0][param.taskFileColumnHeaders.taskType.pos]), + enabled: param.taskRows[0][param.taskFileColumnHeaders.taskEnabled.pos], + taskSessionTimeout: param.taskRows[0][param.taskFileColumnHeaders.taskSessionTimeout.pos], + maxRetries: param.taskRows[0][param.taskFileColumnHeaders.taskMaxRetries.pos], + + path: param.taskRows[0][param.taskFileColumnHeaders.extPgmPath.pos], + parameters: param.taskRows[0][param.taskFileColumnHeaders.extPgmParam.pos], + + tags: [], + customProperties: [], + + schemaPath: 'ExternalProgramTask', + schemaEvents: [], + compositeEvents: [], + prelCompositeEvents: [], + }; + + // Add tags to task object + if (param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos]) { + const tmpTags = param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos] + .split('/') + .filter((item) => item.trim().length !== 0) + .map((item) => item.trim()); + + // eslint-disable-next-line no-restricted-syntax + for (const item of tmpTags) { + // eslint-disable-next-line no-await-in-loop + const tagId = await getTagIdByName(item, param.tagsExisting); + currentTask.tags.push({ + id: tagId, + name: item, + }); + } + } + + // Add custom properties to task object + if (param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos]) { + const tmpCustomProperties = param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos] + .split('/') + .filter((item) => item.trim().length !== 0) + .map((cp) => cp.trim()); + + // eslint-disable-next-line no-restricted-syntax + for (const item of tmpCustomProperties) { + const tmpCustomProperty = item + .split('=') + .filter((item2) => item2.trim().length !== 0) + .map((cp) => cp.trim()); + + // Do we have two items in the array? First is the custom property name, second is the value + if (tmpCustomProperty?.length === 2) { + // eslint-disable-next-line no-await-in-loop + const customPropertyId = await getCustomPropertyIdByName('ExternalProgramTask', tmpCustomProperty[0], param.cpExisting); + + // If previous call returned false, it means the custom property does not exist in Sense + // or cannot be used with this task type. In that case, skip it. + if (customPropertyId) { + currentTask.customProperties.push({ + definition: { + id: customPropertyId, + name: tmpCustomProperty[0].trim(), + }, + value: tmpCustomProperty[1].trim(), + }); + } + } + } + } + + // Get schema events for this task, storing the info using the same structure as returned from QRS API + currentTask.schemaEvents = this.parseSchemaEvents({ + taskRows: param.taskRows, + taskFileColumnHeaders: param.taskFileColumnHeaders, + taskCounter: param.taskCounter, + currentTask, + fakeTaskId: param.fakeTaskId, + nodesWithEvents: param.nodesWithEvents, + }); + + // Get composite events for this task + currentTask.prelCompositeEvents = await this.parseCompositeEvents({ + taskType: 'external program', + taskRows: param.taskRows, + taskFileColumnHeaders: param.taskFileColumnHeaders, + taskCounter: param.taskCounter, + currentTask, + fakeTaskId: param.fakeTaskId, + nodesWithEvents: param.nodesWithEvents, + }); + + return { currentTask, taskCreationOption }; + } + // Function to get schema events for a specific task // Parameters: // - taskRows: Array of rows associated with the task. All rows associated with the task are passed to this function @@ -759,7 +905,7 @@ class QlikSenseTasks { this.taskNetwork.nodes.push({ id: res.currentTask.id, metaNode: false, - isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === res.currentTask.taskId), + isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === res.currentTask.id), label: res.currentTask.name, enabled: res.currentTask.enabled, @@ -787,7 +933,7 @@ class QlikSenseTasks { // We now have a basic task object including tags and custom properties. // Schema events are included but composite events are only partially there, as they may need // IDs of tasks that have not yet been created. - // Still, store all info for composite evets and then do another loop where those events are created for + // Still, store all info for composite events and then do another loop where those events are created for // tasks for which they are defined. // // The strategey is to first create all tasks, then add composite events. @@ -801,7 +947,9 @@ class QlikSenseTasks { if (this.options.dryRun === false || this.options.dryRun === undefined) { // eslint-disable-next-line no-await-in-loop const newTaskId = await this.createReloadTaskInQseow(res.currentTask, taskCounter); - logger.info(`(${taskCounter}) Created new task "${res.currentTask.name}", new task id: ${newTaskId}.`); + logger.info( + `(${taskCounter}) Created new reload task "${res.currentTask.name}", new task id: ${newTaskId}.` + ); // Add mapping between fake task ID used when creating task network and actual, newly created task ID this.taskIdMap.set(fakeTaskId, newTaskId); @@ -863,17 +1011,91 @@ class QlikSenseTasks { ); } } else if (taskType === 'external program') { - logger.info(`(${taskCounter}) PARSE TASKS FROM FILE: External program task. Not yet implemented.`); // External program task + + // Create a fake ID for this task. Used to associate task with schema/composite events + const fakeTaskId = `ext-pgm-task-${uuidv4()}`; + // eslint-disable-next-line no-await-in-loop - // await this.parseExternalProgramTask( - // taskRows, - // taskFileColumnHeaders, - // taskCounter, - // tagsExisting, - // cpExisting, - // nodesWithEvents - // ); + const res = await this.parseExternalProgramTask({ + taskRows, + taskFileColumnHeaders, + taskCounter, + tagsExisting, + cpExisting, + fakeTaskId, + nodesWithEvents, + }); + + // Add external program task as node in task network + // NB: A top level node is defined as: + // 1. A task whose taskID does not show up in the "to" field of any edge. + + // eslint-disable-next-line no-restricted-syntax + this.taskNetwork.nodes.push({ + id: res.currentTask.id, + metaNode: false, + isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === res.currentTask.id), + label: res.currentTask.name, + enabled: res.currentTask.enabled, + + completeTaskObject: res.currentTask, + + taskTags: res.currentTask.tags.map((tag) => tag.name), + taskCustomProperties: res.currentTask.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), + }); + + // We now have a basic task object including tags and custom properties. + // Schema events are included but composite events are only partially there, as they may need + // IDs of tasks that have not yet been created. + // Still, store all info for composite events and then do another loop where those events are created for + // tasks for which they are defined. + // + // The strategey is to first create all tasks, then add composite events. + // Only then can we be sure all composite events refer to existing tasks. + + // Create new or update existing external program task in QSEoW + // Should we create a new task? + // Controlled by option --update-mode + if (this.options.updateMode === 'create') { + // Create new task + if (this.options.dryRun === false || this.options.dryRun === undefined) { + // eslint-disable-next-line no-await-in-loop + const newTaskId = await this.createExternalProgramTaskInQseow(res.currentTask, taskCounter); + logger.info( + `(${taskCounter}) Created new external program task "${res.currentTask.name}", new task id: ${newTaskId}.` + ); + + // Add mapping between fake task ID used when creating task network and actual, newly created task ID + this.taskIdMap.set(fakeTaskId, newTaskId); + + // Add mapping between fake task ID specified in source file and actual, newly created task ID + if (res.currentTask.id) { + this.taskIdMap.set(res.currentTask.id, newTaskId); + } + + res.currentTask.idRef = res.currentTask.id; + res.currentTask.id = newTaskId; + + // eslint-disable-next-line no-await-in-loop + await this.addTask('from_file', res.currentTask, false); + } else { + logger.info(`(${taskCounter}) DRY RUN: Creating external program task in QSEoW "${res.currentTask.name}"`); + } + } else if (this.options.updateMode === 'update-if-exists') { + // Update existing task + // TODO + // // Verify task ID is a valid UUID + // // If it's not a valid UUID, the ID specified in the source file will be treated as a task name + // if (!validate(res.currentTask.id)) { + // // eslint-disable-next-line no-await-in-loop + // const task = await + } else { + // Invalid combination of import options + throw new Error( + `Invalid task update mode. Valid values are "create" and "update-if-exists". You specified "${this.options.updateMode}".` + ); + } } } @@ -1029,6 +1251,7 @@ class QlikSenseTasks { }); } + // Function to create new reload task in QSEoW createReloadTaskInQseow(newTask, taskCounter) { // eslint-disable-next-line no-async-promise-executor return new Promise(async (resolve, reject) => { @@ -1044,7 +1267,7 @@ class QlikSenseTasks { name: newTask.name, isManuallyTriggered: newTask.isManuallyTriggered, isPartialReload: newTask.isPartialReload, - taskType: newTask.taskType, + taskType: 0, enabled: newTask.enabled, taskSessionTimeout: newTask.taskSessionTimeout, maxRetries: newTask.maxRetries, @@ -1090,6 +1313,67 @@ class QlikSenseTasks { }); } + // Function to create new external program task in QSEoW + // Parameters: + // - newTask: Object containing task data + // - taskCounter: Task counter, unique for each task in the source file + createExternalProgramTaskInQseow(newTask, taskCounter) { + // eslint-disable-next-line no-async-promise-executor + return new Promise(async (resolve, reject) => { + try { + logger.debug(`(${taskCounter}) CREATE EXTERNAL PROGRAM TASK IN QSEOW: Starting`); + + // Build a body for the API call + const body = { + name: newTask.name, + taskType: 1, + enabled: newTask.enabled, + taskSessionTimeout: newTask.taskSessionTimeout, + maxRetries: newTask.maxRetries, + path: newTask.path, + parameters: newTask.parameters, + tags: newTask.tags, + customProperties: newTask.customProperties, + schemaPath: 'ExternalProgramTask', + // schemaPath: 'ExternalProgramTask', + }; + // schemaEvents: newTask.schemaEvents, + + // Save task to QSEoW + const axiosConfig = setupQRSConnection(this.options, { + method: 'post', + fileCert: this.fileCert, + fileCertKey: this.fileCertKey, + path: '/qrs/externalprogramtask', + body, + }); + + axios + .request(axiosConfig) + .then((result) => { + const response = JSON.parse(result.data); + + logger.debug( + `(${taskCounter}) CREATE EXTERNAL PROGRAM TASK IN QSEOW: "${newTask.name}", new task id: ${response.id}. Result: ${result.status}/${result.statusText}.` + ); + + if (result.status === 201) { + resolve(response.id); + } else { + reject(); + } + }) + .catch((err) => { + logger.error(`CREATE EXTERNAL PROGRAM TASK IN QSEOW 1: ${err}`); + reject(err); + }); + } catch (err) { + logger.error(`CREATE EXTERNAL PROGRAM TASK IN QSEOW 2: ${err}`); + reject(err); + } + }); + } + saveTaskModelToQseow() { // eslint-disable-next-line no-async-promise-executor return new Promise(async (resolve, reject) => { diff --git a/src/lib/util/lookups.js b/src/lib/util/lookups.js index 7505441..878bb90 100644 --- a/src/lib/util/lookups.js +++ b/src/lib/util/lookups.js @@ -91,17 +91,24 @@ const taskFileColumnHeaders = { taskEnabled: { name: 'Task enabled', pos: -1 }, taskSessionTimeout: { name: 'Task timeout', pos: -1 }, taskMaxRetries: { name: 'Task retries', pos: -1 }, + appId: { name: 'App id', pos: -1 }, appName: { name: 'App name', pos: -1 }, isPartialReload: { name: 'Partial reload', pos: -1 }, isManuallyTriggered: { name: 'Manually triggered', pos: -1 }, + taskStatus: { name: 'Task status', pos: -1 }, taskStarted: { name: 'Task started', pos: -1 }, taskEnded: { name: 'Task ended', pos: -1 }, taskDuration: { name: 'Task duration', pos: -1 }, taskExecutionNode: { name: 'Task executedon node', pos: -1 }, + + extPgmPath: { name: 'Ext program path', pos: -1 }, + extPgmParam: { name: 'Ext program parameters', pos: -1 }, + taskTags: { name: 'Tags', pos: -1 }, taskCustomProperties: { name: 'Custom properties', pos: -1 }, + eventCounter: { name: 'Event counter', pos: -1 }, eventType: { name: 'Event type', pos: -1 }, eventName: { name: 'Event name', pos: -1 }, @@ -158,6 +165,10 @@ function getTaskColumnPosFromHeaderRow(headerRow) { taskFileColumnHeaders.taskEnded.pos = headerRow.findIndex((item) => item === taskFileColumnHeaders.taskEnded.name); taskFileColumnHeaders.taskDuration.pos = headerRow.findIndex((item) => item === taskFileColumnHeaders.taskDuration.name); taskFileColumnHeaders.taskExecutionNode.pos = headerRow.findIndex((item) => item === taskFileColumnHeaders.taskExecutionNode.name); + + taskFileColumnHeaders.extPgmPath.pos = headerRow.findIndex((item) => item === taskFileColumnHeaders.extPgmPath.name); + taskFileColumnHeaders.extPgmParam.pos = headerRow.findIndex((item) => item === taskFileColumnHeaders.extPgmParam.name); + taskFileColumnHeaders.taskTags.pos = headerRow.findIndex((item) => item === taskFileColumnHeaders.taskTags.name); taskFileColumnHeaders.taskCustomProperties.pos = headerRow.findIndex( (item) => item === taskFileColumnHeaders.taskCustomProperties.name From 986beff4f4aeda78d99091c5b3fe6f5fe68bf0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Sun, 12 Nov 2023 17:22:34 +0000 Subject: [PATCH 3/4] #167 wip --- .vscode/launch.json | 144 ++++++++++++++--------------- src/lib/cmd/gettask.js | 25 ++++-- src/lib/task/class_alltasks.js | 160 +++++++++++++++++++++++---------- src/lib/task/class_task.js | 69 ++++++++++---- src/lib/util/lookups.js | 2 + 5 files changed, 262 insertions(+), 138 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2c98751..7489d52 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -58,44 +58,44 @@ // ------------------------------------ // Import tasks from Excel file // ------------------------------------ - "args": [ - "task-import", - "--auth-type", - "cert", - "--host", - "192.168.100.109", - "--auth-cert-file", - "./cert/client.pem", - "--auth-cert-key-file", - "./cert/client_key.pem", - "--auth-user-dir", - "LAB", - "--auth-user-id", - "goran", + // "args": [ + // "task-import", + // "--auth-type", + // "cert", + // "--host", + // "192.168.100.109", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", - "--file-type", - "excel", + // "--file-type", + // "excel", - "--file-name", - "testdata/tasks.xlsx", - "--sheet-name", - "5" + // "--file-name", + // "testdata/tasks.xlsx", + // "--sheet-name", + // "7" - // "--import-app", - // "--import-app-sheet-name", - // "App import", + // // "--import-app", + // // "--import-app-sheet-name", + // // "App import", - // "--qvf-overwrite", - // "no", + // // "--qvf-overwrite", + // // "no", - // "--limit-import-count", - // "2", + // // "--limit-import-count", + // // "2", - // "--sleep-app-upload", - // "500", + // // "--sleep-app-upload", + // // "500", - // "--dry-run" - ] + // // "--dry-run" + // ] // ------------------------------------ // Import tasks from CSV file @@ -121,7 +121,8 @@ // "--file-name", // // "tasks2source.csv", // // "task-chain.csv", - // "testdata/reload-tasks.csv", + // // "testdata/reload-tasks.csv", + // "./tasks5.csv", // // "--qvf-overwrite", // // "no", @@ -318,45 +319,46 @@ // "comprule" // ] - // ------------------------------------ - // Get reload tasks as CSV/Excel/JSON file - // ------------------------------------ - // "args": [ - // "task-get", - // "--auth-type", - // "cert", - // "--host", - // "192.168.100.109", - // // "--auth-cert-file", - // // "./cert/client.pem", - // // "--auth-cert-key-file", - // // "./cert/client_key.pem", - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", - // // "--task-id", - // // "82bc3e66-c899-4e44-b52f-552145da5ee0", - // // "82bc3e66-c899-4e44-b52f-552145da5ee1", - // // "5748afa9-3abe-43ab-bb1f-127c48ced075", - // // "5520e710-91ad-41d2-aeb6-434cafbf366b", - // // "--task-tag", - // // "Ctrl-Q demo", - // // "Butler 5.0 demo", - - // "--output-format", - // "table", - // "--output-dest", - // "file", - // "--output-file-name", - // // "tasks.xlsx", - // "tasks.csv", - // // "tasks2.json", - // // "tasks2.xlsx", - // "--output-file-format", - // // "excel", - // // "json", - // "csv", + ------------------------------------ + Get reload tasks as CSV/Excel/JSON file + ------------------------------------ + "args": [ + "task-get", + "--auth-type", + "cert", + "--host", + "192.168.100.109", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + "--auth-user-dir", + "LAB", + "--auth-user-id", + "goran", + // "--task-id", + // "82bc3e66-c899-4e44-b52f-552145da5ee0", + // "82bc3e66-c899-4e44-b52f-552145da5ee1", + // "5748afa9-3abe-43ab-bb1f-127c48ced075", + // "5520e710-91ad-41d2-aeb6-434cafbf366b", + // "--task-tag", + // "Ctrl-Q demo", + // "Butler 5.0 demo", + + "--output-format", + "table", + "--output-dest", + "file", + "--output-file-name", + // "tasks.xlsx", + // "tasks.csv", + "tasks_all.csv", + // "tasks2.json", + // "tasks2.xlsx", + "--output-file-format", + // "excel", + // "json", + "csv", // "--output-file-name", // // "reload-tasks.xlsx", diff --git a/src/lib/cmd/gettask.js b/src/lib/cmd/gettask.js index 1ef8380..04f6c22 100644 --- a/src/lib/cmd/gettask.js +++ b/src/lib/cmd/gettask.js @@ -437,10 +437,25 @@ const getTask = async (options) => { taskTable = taskTable.concat([row]); // Find all triggers for this task - const schemaEventsForThisTask = schemaEventList.filter((item) => item.schemaEvent?.reloadTask?.id === task.taskId); - const compositeEventsForThisTask = compositeEventList.filter( - (item) => item.compositeEvent?.reloadTask?.id === task.taskId - ); + const schemaEventsForThisTask = schemaEventList.filter((item) => { + if (item.schemaEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.schemaEvent?.externalProgramTask?.id === task.taskId) { + return true; + } + return false; + }); + + const compositeEventsForThisTask = compositeEventList.filter((item) => { + if (item.compositeEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.compositeEvent?.externalProgramTask?.id === task.taskId) { + return true; + } + return false; + }); // Write schema events to table if (columnBlockShow.schematrigger) { @@ -690,7 +705,7 @@ const getTask = async (options) => { } if (columnBlockShow.extprogram) { - headerRow = headerRow.concat(['Path', 'Parameters']); + headerRow = headerRow.concat(['Ext program path', 'Ext program parameters']); } if (columnBlockShow.lastexecution) { diff --git a/src/lib/task/class_alltasks.js b/src/lib/task/class_alltasks.js index c10ed1a..eddbbd8 100644 --- a/src/lib/task/class_alltasks.js +++ b/src/lib/task/class_alltasks.js @@ -272,6 +272,7 @@ class QlikSenseTasks { // Get schema events for this task, storing the info using the same structure as returned from QRS API currentTask.schemaEvents = this.parseSchemaEvents({ + taskType: 'reload', taskRows: param.taskRows, taskFileColumnHeaders: param.taskFileColumnHeaders, taskCounter: param.taskCounter, @@ -411,6 +412,7 @@ class QlikSenseTasks { // Get schema events for this task, storing the info using the same structure as returned from QRS API currentTask.schemaEvents = this.parseSchemaEvents({ + taskType: 'external program', taskRows: param.taskRows, taskFileColumnHeaders: param.taskFileColumnHeaders, taskCounter: param.taskCounter, @@ -435,6 +437,7 @@ class QlikSenseTasks { // Function to get schema events for a specific task // Parameters: + // - taskType: Type of task. Possible values: "reload", "external program" // - taskRows: Array of rows associated with the task. All rows associated with the task are passed to this function // - taskFileColumnHeaders: Object containing info about which column contains what data // - taskCounter: Counter for the current task @@ -443,7 +446,7 @@ class QlikSenseTasks { // - nodesWithEvents: Set of nodes that have associated events parseSchemaEvents(param) { // Get schema events for this task, storing the info using the same structure as returned from QRS API - const prelAchemaEvents = []; + const prelSchemaEvents = []; const schemaEventRows = param.taskRows.filter( (item) => @@ -472,12 +475,22 @@ class QlikSenseTasks { schemaFilterDescription: [schemaEventRow[param.taskFileColumnHeaders.schemaFilterDescription.pos]], incrementDescription: schemaEventRow[param.taskFileColumnHeaders.schemaIncrementDescription.pos], incrementOption: mapIncrementOption.get(schemaEventRow[param.taskFileColumnHeaders.schemaIncrementOption.pos]), - reloadTask: { - id: param.fakeTaskId, - }, schemaPath: 'SchemaEvent', }; + if (param.taskType === 'reload') { + schemaEvent.reloadTask = { + id: param.fakeTaskId, + }; + } else if (param.taskType === 'external program') { + schemaEvent.externalProgramTask = { + id: param.fakeTaskId, + }; + } else { + logger.error(`(${param.taskCounter}) PARSE TASKS FROM FILE: Incorrect task type "${param.taskType}". Exiting.`); + process.exit(1); + } + this.qlikSenseSchemaEvents.addSchemaEvent(schemaEvent); // Add schema event to network representation of tasks @@ -496,24 +509,39 @@ class QlikSenseTasks { completeSchemaEvent: schemaEvent, }); - this.taskNetwork.edges.push({ - from: nodeId, - to: schemaEvent.reloadTask.id, - }); + // Add edge from schema trigger node to current task, taking into account task type + if (param.taskType === 'reload') { + this.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.reloadTask.id, + }); - // Keep a note that this node has associated events - param.nodesWithEvents.add(schemaEvent.reloadTask.id); + // Keep a note that this node has associated events + param.nodesWithEvents.add(schemaEvent.reloadTask.id); - // Add this schema event to the current task - // Remove reference to task ID first though - delete schemaEvent.reloadTask.id; - delete schemaEvent.reloadTask; + // Remove reference to task ID + delete schemaEvent.reloadTask.id; + delete schemaEvent.reloadTask; + } else if (param.taskType === 'external program') { + this.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.externalProgramTask.id, + }); - prelAchemaEvents.push(schemaEvent); + // Keep a note that this node has associated events + param.nodesWithEvents.add(schemaEvent.externalProgramTask.id); + + // Remove reference to task ID + delete schemaEvent.externalProgramTask.id; + delete schemaEvent.externalProgramTask; + } + + // Add this schema event to the current task + prelSchemaEvents.push(schemaEvent); } } - return prelAchemaEvents; + return prelSchemaEvents; } // Function to get composite events for a specific task @@ -613,7 +641,7 @@ class QlikSenseTasks { } else { // The task pointed to by the composite event rule does not exist logger.error( - `(${param.taskCounter}) PARSE TASKS FROM FILE: Task "${ + `(${param.taskCounter}) PARSE COMPOSITE EVENT RULE FROM FILE: Task "${ rule[param.taskFileColumnHeaders.ruleTaskId.pos] }" does not exist. Exiting.` ); @@ -828,6 +856,12 @@ class QlikSenseTasks { return -1; } + // Is first column empty? + if (item[taskFileColumnHeaders.taskCounter.pos] === undefined) { + // Empty task counter column + return -1; + } + if (item[taskFileColumnHeaders.taskCounter.pos] === taskFileColumnHeaders.taskCounter.name) { // This is the header row return -1; @@ -1112,10 +1146,10 @@ class QlikSenseTasks { // Set task ID for the composite event itself, i.e. which task is the event associated with (i.e. the downstream task) // Handle different task types differently - if (item.compositeEvent.reloadTask.id) { + if (item.compositeEvent?.reloadTask?.id) { // Reload task a.compositeEvent.reloadTask.id = this.taskIdMap.get(item.compositeEvent.reloadTask.id); - } else if (item.compositeEvent.externalProgramTask.id) { + } else if (item.compositeEvent?.externalProgramTask?.id) { // External program task a.compositeEvent.externalProgramTask.id = this.taskIdMap.get(item.compositeEvent.externalProgramTask.id); } @@ -1128,6 +1162,14 @@ class QlikSenseTasks { // Get triggering/upstream task id const id = this.taskIdMap.get(b.task.id); + + // If id is not found in the mapping table, it means that the task + // referenced by the rule (i.e. the upstream teask) is neither a task + // that existed before this execution of Ctrl-Q, nor a task that was + // created during this execution of Ctrl-Q. + // This is an error - the task ID should exist. + // Most likely the error is caused by an invalid value in the "Rule task id" + // column in the source file. if (id !== undefined && validate(id) === true) { // Determine what kind of task this is. Options are: // - reload @@ -1157,23 +1199,31 @@ class QlikSenseTasks { // Use mapTaskType to get the string variant of the task type. Convert to lower case. const taskTypeString = mapTaskType.get(taskType).trim().toLowerCase(); + // Ensure we got a valid task type + if (!['reload', 'externalprogram'].includes(taskTypeString)) { + logger.error( + `PREPARING COMPOSITE EVENT: Invalid task type "${taskTypeString}" for upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` + ); + process.exit(1); + } + if (taskTypeString === 'reload') { b.reloadTask = { id }; } else if (taskTypeString === 'externalprogram') { b.externalProgramTask = { id }; } - } else if (this.options.dryRun === false || this.options.dryRun === undefined) { + } else if (id === undefined) { + // (this.options.dryRun === false || this.options.dryRun === undefined) { logger.error( - `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.reloadTask.id}" in rule for composite event "${a.compositeEvent.name}" ` + `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` ); - b.reloadTask.id = null; + process.exit(1); } return b; }); return a; }); - // Loop over all composite events in the source file, create missing ones where needed logger.info('-------------------------------------------------------------------'); logger.info('Creating composite events for the just created tasks...'); @@ -1233,9 +1283,17 @@ class QlikSenseTasks { .then((result) => { if (result.status === 201) { const response = JSON.parse(result.data); - logger.info( - `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.reloadTask.id}. Result: ${result.status}/${result.statusText}.` - ); + + if (response?.reloadTask) { + logger.info( + `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.reloadTask.id}. Result: ${result.status}/${result.statusText}.` + ); + } else if (response?.externalProgramTask) { + logger.info( + `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.externalProgramTask.id}. Result: ${result.status}/${result.statusText}.` + ); + } + resolve(response.id); } else { reject(); @@ -1325,26 +1383,27 @@ class QlikSenseTasks { // Build a body for the API call const body = { - name: newTask.name, - taskType: 1, - enabled: newTask.enabled, - taskSessionTimeout: newTask.taskSessionTimeout, - maxRetries: newTask.maxRetries, - path: newTask.path, - parameters: newTask.parameters, - tags: newTask.tags, - customProperties: newTask.customProperties, - schemaPath: 'ExternalProgramTask', - // schemaPath: 'ExternalProgramTask', + task: { + name: newTask.name, + taskType: 1, + enabled: newTask.enabled, + taskSessionTimeout: newTask.taskSessionTimeout, + maxRetries: newTask.maxRetries, + path: newTask.path, + parameters: newTask.parameters, + tags: newTask.tags, + customProperties: newTask.customProperties, + schemaPath: 'ExternalProgramTask', + }, + schemaEvents: newTask.schemaEvents, }; - // schemaEvents: newTask.schemaEvents, // Save task to QSEoW const axiosConfig = setupQRSConnection(this.options, { method: 'post', fileCert: this.fileCert, fileCertKey: this.fileCertKey, - path: '/qrs/externalprogramtask', + path: '/qrs/externalprogramtask/create', body, }); @@ -2041,7 +2100,7 @@ class QlikSenseTasks { // Add edges from upstream tasks to the new meta node // eslint-disable-next-line no-restricted-syntax for (const rule of compositeEvent.compositeEvent.compositeRules) { - if (validate(rule.reloadTask.id)) { + if (validate(rule?.reloadTask?.id)) { // Upstream task is a reload task logger.debug( `Composite event "${compositeEvent.compositeEvent.name}" is triggered by reload task with ID=${rule.reloadTask.id}.` @@ -2057,14 +2116,14 @@ class QlikSenseTasks { completeCompositeEvent: compositeEvent.compositeEvent, rule, }); - } else if (validate(rule.externalProgramTask.id)) { + } else if (validate(rule?.externalProgramTask?.id)) { // Upstream task is an external program task logger.debug( `Composite event "${compositeEvent.compositeEvent.name}" is triggered by external program task with ID=${rule.externalProgramTask.id}.` ); this.taskNetwork.edges.push({ - from: rule.reloadTask.id, + from: rule.externalProgramTask.id, fromTaskType: 'ExternalProgram', to: nodeId, toTaskType: 'Composite', @@ -2201,7 +2260,7 @@ class QlikSenseTasks { ); this.taskNetwork.edges.push({ - from: rule.reloadTask.id, + from: rule.externalProgramTask.id, fromTaskType: 'ExternalProgram', to: nodeId, toTaskType: 'Composite', @@ -2222,10 +2281,17 @@ class QlikSenseTasks { `Added edge from new meta composite event node "${nodeId}" to reload task ID=${compositeEvent.compositeEvent?.reloadTask?.id}.` ); - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.compositeEvent.reloadTask.id, - }); + if (compositeEvent.compositeEvent?.reloadTask) { + this.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.reloadTask.id, + }); + } else if (compositeEvent.compositeEvent?.externalProgramTask) { + this.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.externalProgramTask.id, + }); + } } } } diff --git a/src/lib/task/class_task.js b/src/lib/task/class_task.js index fdb57ca..f3aed7d 100644 --- a/src/lib/task/class_task.js +++ b/src/lib/task/class_task.js @@ -20,15 +20,23 @@ class QlikSenseTask { // Data in the "task" parameter was loaded from a Qlik Sense (QSEoW) server if (task.schemaPath === 'ReloadTask') { this.sourceType = 'from_qseow'; - this.taskId = task.id; + + if (task.id) { + this.taskId = task.id; + } if (anonymizeTaskNames === true) { this.taskName = randomWords({ min: 2, max: 5, join: ' ' }); - this.appName = randomWords({ min: 2, max: 5, join: ' ' }); + if (task.app.name) { + this.appName = randomWords({ min: 2, max: 5, join: ' ' }); + } } else { this.taskName = task.name; - this.appName = task.app.name; + if (task.app.name) { + this.appName = task.app.name; + } } + this.taskEnabled = task.enabled; this.appId = task.app.id; this.appPublished = task.app.published; @@ -37,6 +45,7 @@ class QlikSenseTask { this.taskSessionTimeout = task.taskSessionTimeout; this.isPartialReload = task.isPartialReload; this.isManuallyTriggered = task.isManuallyTriggered; + this.taskLastExecutionStartTimestamp = task.operational.lastExecutionResult.startTime === '1753-01-01T00:00:00.000Z' ? '' @@ -49,6 +58,7 @@ class QlikSenseTask { this.taskLastExecutionExecutingNodeName = task.operational.lastExecutionResult.executingNodeName; this.taskNextExecutionTimestamp = task.operational.nextExecution === '1753-01-01T00:00:00.000Z' ? '' : task.operational.nextExecution; + this.taskTags = task.tags; this.taskTagsFriendly = task.tags.map((tag) => tag.name); this.taskCustomProperties = task.customProperties; @@ -61,6 +71,7 @@ class QlikSenseTask { } this.completeTaskObject = task; + this.taskType = 0; logger.silly(`Initialised reload task object from QSEoW: ${JSON.stringify(task)}`); } else if (task.schemaPath === 'ExternalProgramTask') { this.sourceType = 'from_qseow'; @@ -77,6 +88,7 @@ class QlikSenseTask { this.taskEnabled = task.enabled; this.taskMaxRetries = task.maxRetries; this.taskSessionTimeout = task.taskSessionTimeout; + this.taskLastExecutionStartTimestamp = task?.operational?.lastExecutionResult?.startTime === '1753-01-01T00:00:00.000Z' ? '' @@ -89,6 +101,7 @@ class QlikSenseTask { this.taskLastExecutionExecutingNodeName = task?.operational?.lastExecutionResult?.executingNodeName; this.taskNextExecutionTimestamp = task?.operational?.nextExecution === '1753-01-01T00:00:00.000Z' ? '' : task?.operational?.nextExecution; + this.taskTags = task.tags; this.taskTagsFriendly = task.tags.map((tag) => tag.name); this.taskCustomProperties = task.customProperties; @@ -101,26 +114,18 @@ class QlikSenseTask { } this.completeTaskObject = task; + this.taskType = 1; logger.silly(`Initialised external program task object from QSEoW: ${JSON.stringify(task)}`); } } else if (source.toLowerCase() === 'from_file') { // Data in the "task" parameter was loaded from a task definition file on disk - this.sourceType = 'from_file'; - if (task.taskType === 0) { - this.taskType = task.taskType; + if (task.schemaPath === 'ReloadTask') { + this.sourceType = 'from_file'; if (task.id) { this.taskId = task.id; } - this.taskName = task.name; - this.taskEnabled = task.enabled; - this.taskSessionTimeout = task.taskSessionTimeout; - this.taskMaxRetries = task.maxRetries; - this.isPartialReload = task.isPartialReload; - this.isManuallyTriggered = task.isManuallyTriggered; - - this.appId = task.app.id; if (anonymizeTaskNames === true) { this.taskName = randomWords({ min: 2, max: 5, join: ' ' }); if (task.app.name) { @@ -133,6 +138,15 @@ class QlikSenseTask { } } + this.taskEnabled = task.enabled; + this.appId = task.app.id; + this.appPublished = task.app.published; + this.appStream = task.app.published ? task.app.stream.name : ''; + this.taskMaxRetries = task.maxRetries; + this.taskSessionTimeout = task.taskSessionTimeout; + this.isPartialReload = task.isPartialReload; + this.isManuallyTriggered = task.isManuallyTriggered; + this.taskTags = task.tags; this.taskTagsFriendly = task.tags.map((tag) => tag.name); this.taskCustomProperties = task.customProperties; @@ -142,8 +156,33 @@ class QlikSenseTask { this.compositeEvents = task.compositeEvents; this.completeTaskObject = task; + this.taskType = 0; + logger.silly(`Initialised task object from file: ${JSON.stringify(task)}`); + } else if (task.schemaPath === 'ExternalProgramTask') { + this.sourceType = 'from_file'; + this.taskId = task.id; + + this.path = task.path; + this.parameters = task.parameters; + + if (anonymizeTaskNames === true) { + this.taskName = randomWords({ min: 2, max: 5, join: ' ' }); + } else { + this.taskName = task.name; + } + this.taskEnabled = task.enabled; + this.taskMaxRetries = task.maxRetries; + this.taskSessionTimeout = task.taskSessionTimeout; + + this.taskTags = task.tags; + this.taskTagsFriendly = task.tags.map((tag) => tag.name); + this.taskCustomProperties = task.customProperties; + this.taskCustomPropertiesFriendly = task.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`); + + this.completeTaskObject = task; + this.taskType = 1; + logger.silly(`Initialised external program task object from file: ${JSON.stringify(task)}`); } - logger.silly(`Initialised task object from file: ${JSON.stringify(task)}`); } } } diff --git a/src/lib/util/lookups.js b/src/lib/util/lookups.js index 878bb90..35cb0cd 100644 --- a/src/lib/util/lookups.js +++ b/src/lib/util/lookups.js @@ -116,6 +116,7 @@ const taskFileColumnHeaders = { eventCreatedDate: { name: 'Event created date', pos: -1 }, eventModifiedDate: { name: 'Event modified date', pos: -1 }, eventModifiedBy: { name: 'Event modified by', pos: -1 }, + schemaIncrementOption: { name: 'Schema increment option', pos: -1 }, schemaIncrementDescription: { name: 'Schema increment description', pos: -1 }, daylightSavingsTime: { name: 'Daylight savings time', pos: -1 }, @@ -123,6 +124,7 @@ const taskFileColumnHeaders = { scheamExpiration: { name: 'Schema expiration', pos: -1 }, schemaFilterDescription: { name: 'Schema filter description', pos: -1 }, schemaTimeZone: { name: 'Schema time zone', pos: -1 }, + timeConstraintSeconds: { name: 'Time contstraint seconds', pos: -1 }, timeConstraintMinutes: { name: 'Time contstraint minutes', pos: -1 }, timeConstraintHours: { name: 'Time contstraint hours', pos: -1 }, From d9db29f96e424e199162d73573459fa0ab857965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 16 Nov 2023 06:02:04 +0100 Subject: [PATCH 4/4] Code cleanup --- .vscode/launch.json | 177 ++++++++++++++++++++------------------ src/ctrl-q.js | 2 +- src/lib/cmd/importtask.js | 55 ++++-------- 3 files changed, 108 insertions(+), 126 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7489d52..d296a5b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "request": "launch", "name": "Launch Program", "program": "${workspaceFolder}/src/ctrl-q.js", - "runtimeVersion": "19", + "runtimeVersion": "20", "cwd": "${workspaceFolder}", // ------------------------------------ @@ -100,38 +100,45 @@ // ------------------------------------ // Import tasks from CSV file // ------------------------------------ - // "args": [ - // "task-import", - // "--auth-type", - // "cert", - // "--host", - // "192.168.100.109", - // // "--auth-cert-file", - // // "./cert/client.pem", - // // "--auth-cert-key-file", - // // "./cert/client_key.pem", - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", + "args": [ + "task-import", + "--auth-type", + "cert", + "--host", + // "192.168.100.109", + "10.211.55.15", - // "--file-type", - // "csv", + "--auth-cert-file", + "../../code/secret/winsrv.local/client.pem", + "--auth-cert-key-file", + "../../code/secret/winsrv.local/client_key.pem", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + "--auth-user-dir", + "winsrv1", + // "LAB", + "--auth-user-id", + "goran", - // "--file-name", - // // "tasks2source.csv", - // // "task-chain.csv", - // // "testdata/reload-tasks.csv", - // "./tasks5.csv", + "--file-type", + "csv", - // // "--qvf-overwrite", - // // "no", + "--file-name", + // "tasks2source.csv", + // "task-chain.csv", + // "testdata/reload-tasks.csv", + "./tasks_all.csv", - // // "--limit-import-count", - // // "2", + // "--qvf-overwrite", + // "no", - // // "--dry-run" - // ] + "--limit-import-count", + "2", + + // "--dry-run" + ] // ------------------------------------ // Export apps to QVF files @@ -319,69 +326,69 @@ // "comprule" // ] - ------------------------------------ - Get reload tasks as CSV/Excel/JSON file - ------------------------------------ - "args": [ - "task-get", - "--auth-type", - "cert", - "--host", - "192.168.100.109", - // "--auth-cert-file", - // "./cert/client.pem", - // "--auth-cert-key-file", - // "./cert/client_key.pem", - "--auth-user-dir", - "LAB", - "--auth-user-id", - "goran", - // "--task-id", - // "82bc3e66-c899-4e44-b52f-552145da5ee0", - // "82bc3e66-c899-4e44-b52f-552145da5ee1", - // "5748afa9-3abe-43ab-bb1f-127c48ced075", - // "5520e710-91ad-41d2-aeb6-434cafbf366b", - // "--task-tag", - // "Ctrl-Q demo", - // "Butler 5.0 demo", - - "--output-format", - "table", - "--output-dest", - "file", - "--output-file-name", - // "tasks.xlsx", - // "tasks.csv", - "tasks_all.csv", - // "tasks2.json", - // "tasks2.xlsx", - "--output-file-format", - // "excel", - // "json", - "csv", + // ------------------------------------ + // Get reload tasks as CSV/Excel/JSON file + // ------------------------------------ + // "args": [ + // "task-get", + // "--auth-type", + // "cert", + // "--host", + // "192.168.100.109", + // // "--auth-cert-file", + // // "./cert/client.pem", + // // "--auth-cert-key-file", + // // "./cert/client_key.pem", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", + // // "--task-id", + // // "82bc3e66-c899-4e44-b52f-552145da5ee0", + // // "82bc3e66-c899-4e44-b52f-552145da5ee1", + // // "5748afa9-3abe-43ab-bb1f-127c48ced075", + // // "5520e710-91ad-41d2-aeb6-434cafbf366b", + // // "--task-tag", + // // "Ctrl-Q demo", + // // "Butler 5.0 demo", + // "--output-format", + // "table", + // "--output-dest", + // "file", // "--output-file-name", - // // "reload-tasks.xlsx", - // "reload-tasks.csv", + // // "tasks.xlsx", + // // "tasks.csv", + // "tasks_all.csv", + // // "tasks2.json", + // // "tasks2.xlsx", + // "--output-file-format", + // // "excel", + // // "json", + // "csv", - // // "--text-color", - // // "no", + // // "--output-file-name", + // // // "reload-tasks.xlsx", + // // "reload-tasks.csv", - // // "--output-file-overwrite", + // // // "--text-color", + // // // "no", - // "--table-details", - // // "common", - // // "lastexecution", - // // "tag", - // // "customproperty", - // // "schematrigger", - // // "compositetrigger", - // // "comptimeconstraint", - // // "comprule", + // // // "--output-file-overwrite", - // // "--log-level", - // // "debug" - // ] + // // "--table-details", + // // // "common", + // // // "lastexecution", + // // // "tag", + // // // "customproperty", + // // // "schematrigger", + // // // "compositetrigger", + // // // "comptimeconstraint", + // // // "comprule", + + // // // "--log-level", + // // // "debug" + // // ] // ------------------------------------ // Create custom property with user activity buckets diff --git a/src/ctrl-q.js b/src/ctrl-q.js index 4f3e584..2f87a19 100644 --- a/src/ctrl-q.js +++ b/src/ctrl-q.js @@ -645,7 +645,7 @@ const program = new Command(); taskImportAssertOptions(options); importTaskFromFile(options); } catch (err) { - logger.error(`IMPORT TASK: ${err}`); + logger.error(`IMPORT TASK 1: ${err}`); } }) .addOption( diff --git a/src/lib/cmd/importtask.js b/src/lib/cmd/importtask.js index 551cfaa..9139707 100644 --- a/src/lib/cmd/importtask.js +++ b/src/lib/cmd/importtask.js @@ -15,65 +15,40 @@ const { getCustomPropertiesFromQseow } = require('../util/customproperties'); const getHeaders = async (options) => { const records = []; + const parser = fs.createReadStream(options.fileName).pipe( parse({ info: true, - to_line: 1, + skip_empty_lines: true, }) ); - parser.on('readable', () => { - let record; - while ((record = parser.read()) !== null) { - // Work with each record - records.push(record); + + // Get the header row + // eslint-disable-next-line no-restricted-syntax + for await (const record of parser) { + if (record.info.lines === 1) { + // Header row + records.push(record.record); } - }); - await finished(parser); + } + return records; }; const processCsvFile = async (options) => { - // First get header row - - // const parser = parse({ - // delimiter: ',', - // info: true, - // // to_line: 1, - // }); - - // const parser = fs.createReadStream(options.fileName).pipe( - // parse({ - // info: true, - // to_line: 1, - // }).pipe(process.stdout) - // ); - const headers = await getHeaders(options); - // const headers = fs.createReadStream(options.fileName).pipe(parser).pipe(process.stdout); - - // const content = await fsp.readFile(options.fileName); - - // Parse the CSV content - // const headers = parse(content, { bom: true, to_line: 1 }); - const headerRow = []; // Push all column headers to array // eslint-disable-next-line no-restricted-syntax - for (const record of headers) { + for (const record of headers[0]) { // Get each column header text - headerRow.push(record.record); + headerRow.push(record); } - // // eslint-disable-next-line no-restricted-syntax - // for await (const record of headers) { - // // for await (const record of parser) { - // // Get each column header text - // headerRow.push(record.record); - // } // Get positions of column headers - const colHeaders = getTaskColumnPosFromHeaderRow(headerRow[0]); + const colHeaders = getTaskColumnPosFromHeaderRow(headerRow); const records = []; const parser = fs.createReadStream(options.fileName).pipe( @@ -439,7 +414,7 @@ const importTaskFromFile = async (options) => { await qlikSenseTasks.init(options, importedApps); const taskList = await qlikSenseTasks.getTaskModelFromFile(tasksFromFile, tagsExisting, cpExisting); } catch (err) { - logger.error(`IMPORT TASK: ${err.stack}`); + logger.error(`IMPORT TASK 2: ${err.stack}`); } };