diff --git a/.gitignore b/.gitignore index e6d5f7e8..dd11e956 100644 --- a/.gitignore +++ b/.gitignore @@ -133,25 +133,23 @@ dist # TernJS port file .tern-port + +# Project specific files tasktree.json tasktable.csv task-chain.csv .vscode/launch.json -.vscode/launch.json +sea-prep.blob build.cjs +build-sea.sh +build ctrl-q a.json a1.csv -build-sea.sh -build certtest.js logcertfile -sea-prep.blob -.vscode/launch.json -.vscode/launch.json certificate.p12 -.vscode/launch.json ctrl-q.exe -.vscode/launch.json .test.env -.vscode/launch.json +src/lib/task/parse_reload_task copy.js +testdata/~$tasks.xlsx diff --git a/.vscode/launch.json b/.vscode/launch.json index fa811d95..8efe21f3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,9 +9,8 @@ "request": "launch", "name": "Launch Program", "program": "${workspaceFolder}/src/ctrl-q.js", - "runtimeVersion": "20", + "runtimeVersion": "23", "cwd": "${workspaceFolder}", - // ------------------------------------ // Set custom property of reload task // ------------------------------------ @@ -19,28 +18,23 @@ // "task-custom-property-set", // "--host", // "192.168.100.109", - // "--port", // // "4747", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // // "--auth-cert-file", // // "./cert/client.pem", // // "--auth-cert-key-file", // // "./cert/client_key.pem", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // "--task-id", // "82bc3e66-c899-4e44-b52f-552145da5ee0", // "5748afa9-3abe-43ab-bb1f-127c48ced075", @@ -51,27 +45,21 @@ // // "Test data", // // "api1", // "apiCreaeted", - // "--custom-property-name", // "Department", // // "Department2", // // "DemoApp", // // "ReportMonth", - // "--custom-property-value", // // "2023 Jan", // "Finance", // "Sales", - // "--update-mode", // // "append", // "replace", - // "--overwrite", - // "--dry-run" // ] - // ------------------------------------ // Import tasks from Excel file // ------------------------------------ @@ -79,52 +67,40 @@ // "task-import", // "--host", // "192.168.100.109", - // "--port", // // "4747", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // // "--auth-cert-file", // // "./cert/client.pem", // // "--auth-cert-key-file", // // "./cert/client_key.pem", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // "--file-type", // "excel", - // "--file-name", // "testdata/tasks.xlsx", // "--sheet-name", // "1" - // // "--import-app", // // "--import-app-sheet-name", // // "App import", - // // "--qvf-overwrite", // // "no", - // // "--limit-import-count", // // "2", - // // "--sleep-app-upload", // // "500", - // // "--dry-run" // ] - // ------------------------------------ // Import tasks from CSV file // ------------------------------------ @@ -132,7 +108,6 @@ // "task-import", // "--auth-type", // "cert", - // // p2w1 // "--host", // "192.168.100.109", @@ -146,7 +121,6 @@ // "LAB", // "--auth-user-id", // "goran", - // // Parallels // // "--host", // // "10.211.55.15", @@ -158,94 +132,71 @@ // // "winsrv1", // // "--auth-user-id", // // "goran", - // "--file-type", // "csv", - // "--file-name", // // "tasks2source.csv", // // "task-chain.csv", // // "testdata/reload-tasks.csv", // // "./tasks_all.csv", // "./testdata/tasks-1.csv" - // // "--qvf-overwrite", // // "no", - // // "--limit-import-count", // // "2", - // // "--dry-run" // ] - // ------------------------------------ // Export apps to QVF files // ------------------------------------ - "args": [ - "app-export", - "--host", - "pro2-win1.lab.ptarmiganlabs.net", - // "192.168.100.109", - - "--port", - "4242", - // "443", - - // "--virtual-proxy", - // "jwt", - // "--auth-type", - // "jwt", - // "--auth-jwt", - // "", - - "--auth-cert-file", - "./cert/client.pem", - "--auth-cert-key-file", - "./cert/client_key.pem", - - "--auth-user-dir", - "LAB", - "--auth-user-id", - "goran", - - "--app-tag", - "apiCreated", - "Ctrl-Q import", - - "--app-id", - "eb3ab049-d007-43d3-93da-5962f9208c65", - "2933711d-6638-41d4-a2d2-6dd2d965208b", - - "--app-published", - - "--exclude-app-data", - // "false", - "true", - - "--qvf-name-format", - // "export-time", - "app-name", - "export-date", - - "--qvf-name-separator", - "__", - - "--output-dir", - "qvfs", - - // "--limit-export-count", - // "2", - - "--sleep-app-export", - "500", - - "--qvf-overwrite", - - // "--dry-run" - - "--metadata-file-create" - ] - + // "args": [ + // "app-export", + // "--host", + // "pro2-win1.lab.ptarmiganlabs.net", + // // "192.168.100.109", + // "--port", + // "4242", + // // "443", + // // "--virtual-proxy", + // // "jwt", + // // "--auth-type", + // // "jwt", + // // "--auth-jwt", + // // "", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", + // "--app-tag", + // "apiCreated", + // "Ctrl-Q import", + // "--app-id", + // "eb3ab049-d007-43d3-93da-5962f9208c65", + // "2933711d-6638-41d4-a2d2-6dd2d965208b", + // "--app-published", + // "--exclude-app-data", + // // "false", + // "true", + // "--qvf-name-format", + // // "export-time", + // "app-name", + // "export-date", + // "--qvf-name-separator", + // "__", + // "--output-dir", + // "qvfs", + // // "--limit-export-count", + // // "2", + // "--sleep-app-export", + // "500", + // "--qvf-overwrite", + // // "--dry-run" + // "--metadata-file-create" + // ] // ------------------------------------ // Import apps from Excel file // ------------------------------------ @@ -253,93 +204,81 @@ // "app-import", // "--host", // "192.168.100.109", - // "--port", // // "4747", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // // "--auth-cert-file", // // "./cert/client.pem", // // "--auth-cert-key-file", // // "./cert/client_key.pem", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // // "--file-type", // // "excel", - // "--file-name", // "testdata/tasks.xlsx", // "--sheet-name", // "App import", - // // "--limit-import-count", // // "2", - // "--sleep-app-upload", // "500" - // // "--dry-run" // ] - // ------------------------------------ // Get task tree // ------------------------------------ // "args": [ - // "task-get", - // // "--auth-type", - // // "cert", - // "--host", - // "192.168.100.109", - // // "192.168.100.116", - - // // "--auth-cert-file", - // // "./cert/pve-winsrv19-1/192.168.100.116/client.pem", - // // "--auth-cert-key-file", - // // "./cert/pve-winsrv19-1/192.168.100.116/client_key.pem", - - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", - // // "--task-id", - // // "82bc3e66-c899-4e44-b52f-552145da5ee0", - // // "--task-tag", - // // "Test data", - // "--output-format", - // // "table", - // "tree", - - // "--tree-icons" - - // // "--tree-details", - // // "taskid", - // // "appname", - - // // "--task-type", - // // "reload", - // // "ext-program" - - // // "--output-dest", - // // "screen", - // // "file", - // // "--output-file-name", - // // "tasks.json", - // // "--output-file-format", - // // "json", - - // // "--text-color", - // // "no", + // "qseow", + // "task-get", + // "--host", + // "192.168.100.109", + // // "192.168.100.116", + // "--secure", + // "false", + // "--auth-type", + // "cert", + // "--auth-cert-file", + // "/Users/goran/code/secret/pro2win1-nopwd/client.pem", + // "--auth-cert-key-file", + // "/Users/goran/code/secret/pro2win1-nopwd/client_key.pem", + // "--auth-root-cert-file", + // "/Users/goran/code/secret/pro2win1-nopwd/root.pem", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", + // // "--task-id", + // // "82bc3e66-c899-4e44-b52f-552145da5ee0", + // // "--task-tag", + // // "Test data", + // "--output-format", + // // "table", + // "tree", + // "--tree-icons" + // // "--tree-details", + // // "taskid", + // // "appname", + // // "--task-type", + // // "reload", + // // "ext-program" + // // "--output-dest", + // // "screen", + // // "file", + // // "--output-file-name", + // // "tasks.json", + // // "--output-file-format", + // // "json", + // // "--text-color", + // // "no", // ] // ------------------------------------ @@ -349,34 +288,28 @@ // "task-get", // "--host", // "192.168.100.109", - // "--port", // // "4747", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // // "--auth-cert-file", // // "../../code/secret/pro2win1-nopwd/client.pem", // // // "./cert/client.pem", // // "--auth-cert-key-file", // // "../../code/secret/pro2win1-nopwd/client_key.pem", // // // "./cert/client_key.pem", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // // "--task-type", // // "reload", // // "ext-program", - // // "--task-id", // // "afc250bc-28c3-49a2-8d63-80966749abe3", // // "5748afa9-3abe-43ab-bb1f-127c48ced075", @@ -384,7 +317,6 @@ // "--output-format", // "table", // // "tree", - // "--output-dest", // "screen", // // "--output-file-name", @@ -394,10 +326,8 @@ // // "csv", // // "json", // // "excel", - // // "--text-color", // // "no", - // "--table-details", // "common", // // "lastexecution", @@ -406,7 +336,6 @@ // // "schematrigger", // // "compositetrigger", // ] - // ------------------------------------ // Get tasks as CSV/Excel/JSON file // ------------------------------------ @@ -432,7 +361,6 @@ // // "--task-tag", // // "Ctrl-Q demo", // // "Butler 5.0 demo", - // "--output-format", // "table", // "--output-dest", @@ -447,16 +375,12 @@ // // "excel", // // "json", // "csv", - // // "--output-file-name", // // // "reload-tasks.xlsx", // // "reload-tasks.csv", - // // // "--text-color", // // // "no", - // // // "--output-file-overwrite", - // // "--table-details", // // // "common", // // // "lastexecution", @@ -466,11 +390,9 @@ // // // "compositetrigger", // // // "comptimeconstraint", // // // "comprule", - // // // "--log-level", // // // "debug" // // ] - // ------------------------------------ // Create custom property with user activity buckets // ------------------------------------ @@ -499,7 +421,6 @@ // "14", // "7", // ] - // ------------------------------------ // Import master items from Excel file, using column names // ------------------------------------ @@ -509,38 +430,31 @@ // "cert", // "--host", // "192.168.100.109", - // "--port", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // // "--auth-cert-file", // // "./cert/client.pem", // // "--auth-cert-key-file", // // "./cert/client_key.pem", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // "--app-id", // "2933711d-6638-41d4-a2d2-6dd2d965208b", // Ctrl-Q CLI app // // "a3e0f5d2-000a-464f-998d-33d333b175d7", // Thumbnail demo app - // "--file-type", // "excel", // "--file", // "./testdata/ctrl-q-master-items.xlsx", // "--sheet", // "Sales", - // "--col-ref-by", // "name", // "--col-item-type", @@ -559,17 +473,14 @@ // "Color", // "--col-master-item-per-value-color", // "Per value color", - // "--sleep-between-imports", // // "1000", // "0", - // // "--limit-import-count", // // "5", // "--log-level", // "info" // ] - // ------------------------------------ // Get measure // ------------------------------------ @@ -577,17 +488,14 @@ // "master-item-measure-get", // "--host", // "192.168.100.109", - // "--port", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // "--app-id", // "a3e0f5d2-000a-464f-998d-33d333b175d7", // // "2933711d-6638-41d4-a2d2-6dd2d965208b", @@ -597,13 +505,11 @@ // // "e392f53f-17b5-4ede-8fd1-f64b4fae630f", // "--output-format", // "table", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran" // ] - // ------------------------------------ // Delete master measure // ------------------------------------ @@ -611,21 +517,17 @@ // "master-item-measure-delete", // "--host", // "192.168.100.109", - // "--port", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // "--app-id", // "2933711d-6638-41d4-a2d2-6dd2d965208b", // "--auth-user-dir", - // "LAB", // "--auth-user-id", // "goran", @@ -636,7 +538,6 @@ // // "# No. of Contacts (LY)", // // "--delete-all" // ] - // ------------------------------------ // Get dimension // ------------------------------------ @@ -644,30 +545,24 @@ // "master-item-dim-get", // "--host", // "192.168.100.109", - // "--port", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // "--app-id", // // "a3e0f5d2-000a-464f-998d-33d333b175d7", // "2933711d-6638-41d4-a2d2-6dd2d965208b", - // "--output-format", // "table", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran" // ] - // ------------------------------------ // Delete dimension // ------------------------------------ @@ -675,32 +570,26 @@ // "master-item-dim-delete", // "--host", // "192.168.100.109", - // "--port", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // "--app-id", // "2933711d-6638-41d4-a2d2-6dd2d965208b", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // "--id-type", // "name", // "--master-item", // "Dim2", // "Dim3" // ] - // ------------------------------------ // Get variables // ------------------------------------ @@ -709,15 +598,12 @@ // "--host", // // "192.168.100.109", // "pro2-win1.lab.ptarmiganlabs.net", - // // "--qrs-port", // // "4242", // // "443", - // // "--engine-port", // // // "4747", // // "443", - // // "--virtual-proxy", // // "jwt", // "--auth-type", @@ -725,19 +611,16 @@ // "cert", // // "--auth-jwt", // // "", - // "--app-id", // "a3e0f5d2-000a-464f-998d-33d333b175d7", // // "2933711d-6638-41d4-a2d2-6dd2d965208b", // "--app-tag", // "Ctrl-Q variable 1", // "Ctrl-Q variable 2", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // // "--id-type", // // "id", // // "--variable", @@ -751,14 +634,11 @@ // "TimestampFormat", // "var1", // "vVar1", - // "--output-format", // "table" - // // "--log-level", // // "info" // ] - // ------------------------------------ // Delete variables // ------------------------------------ @@ -766,41 +646,34 @@ // "variable-delete", // "--host", // "192.168.100.109", - // "--qrs-port", // // "4242", // "443", - // "--engine-port", // // "4747", // "443", - // "--virtual-proxy", // "jwt", // "--auth-type", // "jwt", // "--auth-jwt", // "", - // "--app-id", // // "a3e0f5d2-000a-464f-998d-33d333b175d7", // "2933711d-6638-41d4-a2d2-6dd2d965208b", // // "--app-tag", // // "Ctrl-Q variable 1", // // "Ctrl-Q variable 2", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // // "--id-type", // // "id", // // "--variable", // // "2a70741b-6fdc-4252-a30d-7a2b2b4403a7", // // "2a70741b-6fdc-4252-a30d-7a2b2b4403a6", // // "2a70741b-6fdc-4252-a30d-7a2b2b4403a8", - // // "--id-type", // // "name", // // "--variable", @@ -811,7 +684,6 @@ // "--delete-all", // "--dry-run", // ] - // ------------------------------------ // Get all bookmarks // ------------------------------------ @@ -819,10 +691,8 @@ // "bookmark-get", // "--host", // "192.168.100.109", - // // "--port", // // "4747", - // "--port", // "443", // "--virtual-proxy", @@ -831,18 +701,15 @@ // "jwt", // "--auth-jwt", // "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VySWQiOiJnb3JhbiIsInVzZXJEaXJlY3RvcnkiOiJMQUIiLCJuYW1lIjoiR8O2cmFuIFNhbmRlciIsImVtYWlsIjoiZ29yYW5AcHRhcm1pZ2FubGFicy5jb20iLCJncm91cCI6WyJncm91cDEiLCJncm91cCAyIl0sImlhdCI6MTY4NDUyNTQ3MCwiZXhwIjoxOTk5ODg1NDcwLCJhdWQiOiJhYmMxMjMifQ.iiCr2obzBF_ixs9KAcSdR4813Dc_8UBknYGA4Jug-exwzw9YN-l9L1luZ_d0b6NBRGRcj3up__xR6X97CMeeNznD3TU_HFgRMcji7xjzyr2GHQWLlNWU3_doAKcUQtqQdN77rJSQ4ZMvdMV1J8-c60Yl8H3Altz6gfBBDmNjkH5XnBfIqVi5IKuMtFI9ehC5wM1aI83Fb8Dt0pq03UNdLlypYV4UQg0UCNhdYgPRhhoBFr-9Mbwc1onOwC9i9YYYyXIhRZaBVDq1pvXizSTwcDhmSg_FBl1GTQnlx9Kx0AeBiPqrpkdLvfzph4uxLNgp-yOliWBynMRF340pyyPQqC3PKNm_CkVzJ3YKA3rZM6KApGj0kbm0I5wF4SX9XRQTe8OGa4jiyaGWYtO_ymFpoTWhYtXTOJnXhQ24-BgVkWBEiKUNw7G-G5Y4JbSwlGG8nP17v9Knb--K_VAQQjdoe9bA7l8D4U9bvpOsUMMUgSr2cMJuU2VG1SVVWLQ6eCfpi-IHYgCosajEZCxxHqBND8TutXoxsNmHQLXYmKIhZWggKHtM7Gy2VEiJj5wnfh6ueLjWdZ3gIMt1KNnylkS_Bz03eT5R_0i3KuzjpdCdingXLybIWiWWVaIc9x_8ankhLGUaHamBDno_gsTegNxIsDIL5182VgZwMrjW92L-cdc", - // "--app-id", // // "2933711d-6638-41d4-a2d2-6dd2d965208b", // "449f2186-0e86-4e19-b46f-c4c23212d731", // App does not exist // // "449f2186-0e86-4e19-b46f-c4c23212d730", // No bookmarks app // // "e5d051f0-34c6-4f47-9bc8-7dfabf784f18", // Has 2 bookmarks - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // "--id-type", // "id", // // "--bookmark", @@ -851,7 +718,6 @@ // "--output-format", // "table" // ] - // ------------------------------------ // Get sessions // ------------------------------------ @@ -869,31 +735,25 @@ // // "--virtual-proxy", // // "jwt", // // "", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // // "--host-proxy", // // "pro2-win1.lab.ptarmiganlabs.net", // // "pro2-win2.lab.ptarmiganlabs.net", - // // "--session-virtual-proxy", // // "dev1", // // "finance", // // "", - // "--output-format", // // "table", // "json", - // "--sort-by", // "prefix" // // "proxyhost", // // "proxyname", // ] - // ------------------------------------ // Delete sessions // ------------------------------------ @@ -911,26 +771,21 @@ // "--virtual-proxy", // "finance", // // "", - // "--host-proxy", // "pro2-win1.lab.ptarmiganlabs.net", // // "pro2-win2.lab.ptarmiganlabs.net", - // // "--session-id", // // "d6e457ca-37c9-4f80-91c2-c63d919c97f9", // // "d6e457ca-37c9-4f80-91c2-c63d919c97f8", // // "41904757-d2f5-4f6b-bfd6-3c0e125d3e1c", - // //"--session-virtual-proxy", // // "", // //"finance", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran" // ] - // ------------------------------------ // Get measure by id // ------------------------------------ @@ -954,7 +809,6 @@ // "--output-format", // "table" // ] - // ------------------------------------ // Get measure by name // ------------------------------------ @@ -978,7 +832,6 @@ // "--output-format", // "table" // ] - // ------------------------------------ // Delete measure by id // ------------------------------------ @@ -998,7 +851,6 @@ // "77ead64e-8bf3-4cd4-bc97-ae45b7521081", // "7dde35ea-4cd7-43f5-a83e-40bb0d45339b" // ] - // ------------------------------------ // Delete measure by name // ------------------------------------ @@ -1018,40 +870,65 @@ // "# No. of Contacts", // // "# No. of Contacts (LY)" // ] - // ------------------------------------ // Scramble field // ------------------------------------ // "args": [ - // "field-scramble", - // "--host", - // "192.168.100.109", - - // "--port", - // // "4747", - // "443", + // "qseow", + // "field-scramble", + // "--host", + // // "192.168.100.109", + // "pro2-win1.lab.ptarmiganlabs.net", + // "--qrs-port", + // "4242", - // "--virtual-proxy", - // "jwt", - // "--auth-type", - // "jwt", - // "--auth-jwt", - // "", + // // JWT auth + // // "--virtual-proxy", + // // "jwt", + // // "--auth-type", + // // "jwt", + // // "--auth-jwt", + // // "", + // // "--engine-port", + // // "443", - // "--app-id", - // "2933711d-6638-41d4-a2d2-6dd2d965208b", + // // Cert auth + // "--engine-port", + // "4747", + // "--auth-type", + // "cert", - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", + // "--auth-cert-file", + // "../secret/pro2win1-nopwd/client.pem", + // "--auth-cert-key-file", + // "../secret/pro2win1-nopwd/client_key.pem", + // "--auth-root-cert-file", + // "../secret/pro2win1-nopwd/root.pem", - // "--field-name", - // "Dim4", - // "Dim2", + // "--auth-user-dir", + // "LAB", + // "--auth-user-id", + // "goran", - // "--new-app-name", - // "'New scrambled app'" + // "--app-id", + // "2933711d-6638-41d4-a2d2-6dd2d965208b", + // // "b091456e-145f-474a-ae1e-8afe01b3ebad", + + // "--field-name", + // "Dim4", + // "Dim2", + // "--new-app-name", + // "_New-scrambled-app-2", + + // "--new-app-cmd", + // // "publish", + // // "--new-app-cmd-name", + // // "Ctrl-Q demo apps" + // "replace", + // "--new-app-cmd-id", + // "4dcee765-4f10-4351-a78a-333d4a0a7790" + + // // "--force" // ] // ------------------------------------ @@ -1062,46 +939,39 @@ // "--host", // "192.168.100.109", // // "pro2-win1.lab.ptarmiganlabs.net", - // // "--port", // // "4747", // // "443", - // // "--virtual-proxy", // // "jwt", - // // "--auth-type", // // "jwt", // // "cert", // // "--auth-jwt", // // "", - // "--app-id", // "deba4bcf-47e4-472e-97b2-4fe8d6498e11", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // "--log-level", // "info" // ] // ------------------------------------ - // Connection test + // QSEoW Connection test // ------------------------------------ // "args": [ + // "qseow", // "connection-test", // "--host", // // "192.168.100.109", // "pro2-win1.lab.ptarmiganlabs.net", // // "--port", // // "443", - // // "--virtual-proxy", // // "/jwt", - // "--auth-type", // // "jwt", // "cert", @@ -1115,57 +985,57 @@ // "./cert/root.pem", // // "--auth-jwt", // // "....", - // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", - // // "--log-level", // // "info" // ] + // // ------------------------------------ - // Network chart visualization + // QS Cloud connection test // ------------------------------------ - // "args": [ - // "task-vis", - // "--host", - // "192.168.100.109", - // // "--port", - // // "443", - - // "--vis-host", - // "192.168.100.143", - // "--vis-port", - // "3000", - - // // "--virtual-proxy", - // // "/jwt", + // "args": ["qscloud", "connection-test", "--tenant-host", "plabs.eu.qlikcloud.com", "--auth-type", "apikey", "--apikey", ""] - // "--auth-type", - // // "jwt", - // "cert", - // // "--host", - // // "192.168.100.109", - // // "--auth-cert-file", - // // "../../code/secret/pro2win1-nopwd/client.pem", - // // "--auth-cert-key-file", - // // "../../code/secret/pro2win1-nopwd/client_key.pem", - // // "./cert/client.pem", - // // "./cert/client_key.pem", - // // "--auth-jwt", - // // "....", - - // "--auth-user-dir", - // "LAB", - // "--auth-user-id", - // "goran", - - // "--log-level", - // "info" - // // "verbose" - // ] + // ------------------------------------ + // Network chart visualization + // ------------------------------------ + "args": [ + "qseow", + "task-vis", + "--host", + // "192.168.100.109", + "pro2-win1.lab.ptarmiganlabs.net", + // "--port", + // "443", + "--vis-host", + "192.168.1.210", + // "192.168.100.143", + "--vis-port", + "3000", + // "--virtual-proxy", + // "/jwt", + "--auth-type", + // "jwt", + "cert", + "--auth-cert-file", + "/Users/goran/code/secret/pro2win1-nopwd/client.pem", + "--auth-cert-key-file", + "/Users/goran/code/secret/pro2win1-nopwd/client_key.pem", + "--auth-root-cert-file", + "/Users/goran/code/secret/pro2win1-nopwd/root.pem", + // "--auth-jwt", + // "....", + "--auth-user-dir", + "LAB", + "--auth-user-id", + "goran" + // "--log-level", + // "info" + // "verbose" + ] } ] } diff --git a/src/__tests__/cmd/qseow/task_get_cert.test.js b/src/__tests__/cmd/qseow/task_get_cert.test.js index 8e3b7910..3416d2f0 100644 --- a/src/__tests__/cmd/qseow/task_get_cert.test.js +++ b/src/__tests__/cmd/qseow/task_get_cert.test.js @@ -3,6 +3,8 @@ import { jest, test, expect, describe } from '@jest/globals'; import fs from 'node:fs'; import path from 'node:path'; import { getTask } from '../../../lib/cmd/qseow/gettask.js'; +import { getTaskAssertOptions } from '../../../lib/util/qseow/assert-options.js'; +import { logger } from '../../../globals.js'; const options = { logLevel: process.env.CTRL_Q_LOG_LEVEL || 'info', @@ -25,6 +27,8 @@ const defaultTestTimeout = process.env.CTRL_Q_TEST_TIMEOUT || 600000; // 10 minu console.log(`Jest timeout: ${defaultTestTimeout}`); jest.setTimeout(defaultTestTimeout); +const validTaskIdDoesNotExist = '123e4567-e89b-12d3-a456-426614174000'; + test('get tasks (verify parameters) ', async () => { expect(options.authCertFile).not.toHaveLength(0); expect(options.authCertKeyFile).not.toHaveLength(0); @@ -65,12 +69,101 @@ describe('get tasks as table (cert auth)', () => { // Test suite for task tree describe('get tasks as tree (cert auth)', () => { - test('get tasks as tree on screen, no detail columns, colored text', async () => { + test('get tasks as tree on screen, valid task ID that does not exist in Sense', async () => { + // Test assertion: + options.taskId = [validTaskIdDoesNotExist]; + options.outputFormat = 'tree'; + options.outputDest = 'screen'; + + // Remove taskType option if it exists + if (options.taskType) { + delete options.taskType; + } + + // Mock logger.warn to capture console output + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => {}); + + // Should succeed, but with warning in console + const result = await getTaskAssertOptions(options); + expect(result).toBe(true); + + // Was there a warning logged by the Winston logger? + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Task with ID')); + + // Restore the original implementation + warnSpy.mockRestore(); + }); + + test('get tasks as tree on screen, task tag that does not exist in Sense', async () => { + // Test assertion: + options.taskTag = ['tag_does_not_exist']; + options.outputFormat = 'tree'; + options.outputDest = 'screen'; + + // Remove taskType option if it exists + if (options.taskType) { + delete options.taskType; + } + + // Mock logger.warn to capture console output + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => {}); + + // Should succeed, but with warning in console + const result = await getTaskAssertOptions(options); + expect(result).toBe(true); + + // Was there a warning logged by the Winston logger? + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Tag does not exist in the Qlik Sense environment')); + + // Restore the original implementation + warnSpy.mockRestore(); + }); + + test('get all tasks as tree on screen, no detail columns, colored text', async () => { + options.outputFormat = 'tree'; + options.outputDest = 'screen'; + options.treeDetails = ''; + options.treeIcons = true; + options.textColor = 'yes'; + + const result = await getTask(options); + expect(result).toBe(true); + }); + + test('get some tasks (filtered by task id) as tree on screen, no detail columns, colored text', async () => { + options.outputFormat = 'tree'; + options.outputDest = 'screen'; + options.treeDetails = ''; + options.treeIcons = true; + options.textColor = 'yes'; + options.taskId = ['4174b1ec-0fd1-4cbe-8d47-0afe545d69bd']; + + const result = await getTask(options); + expect(result).toBe(true); + }); + + test('get some tasks (filtered by task tag) as tree on screen, no detail columns, colored text', async () => { + options.outputFormat = 'tree'; + options.outputDest = 'screen'; + options.treeDetails = ''; + options.treeIcons = true; + options.textColor = 'yes'; + options.taskTag = ['Test data']; + + const result = await getTask(options); + expect(result).toBe(true); + }); + + test('get some tasks (filtered by task id and tag) as tree on screen, no detail columns, colored text', async () => { options.outputFormat = 'tree'; options.outputDest = 'screen'; options.treeDetails = ''; options.treeIcons = true; options.textColor = 'yes'; + options.taskTag = ['Test data']; + options.taskId = ['4174b1ec-0fd1-4cbe-8d47-0afe545d69bd']; const result = await getTask(options); expect(result).toBe(true); diff --git a/src/lib/cli/qseow-get-task.js b/src/lib/cli/qseow-get-task.js index 786625d7..e763b547 100644 --- a/src/lib/cli/qseow-get-task.js +++ b/src/lib/cli/qseow-get-task.js @@ -18,7 +18,7 @@ export function setupGetTaskCommand(qseow) { } await qseowSharedParamAssertOptions(newOptions); - getTaskAssertOptions(newOptions); + await getTaskAssertOptions(newOptions); // If --output-format=table and --task-type is not specified, default to ['reload', 'ext-program'] if (newOptions.outputFormat === 'table' && !newOptions.taskType) { @@ -83,16 +83,8 @@ export function setupGetTaskCommand(qseow) { .addOption( new Option('--task-type ', 'type of tasks to include').choices(['reload', 'ext-program']).env('CTRLQ_TASK_TYPE') ) - .addOption( - new Option('--task-id ', 'use task IDs to select which tasks to retrieve. Only allowed when --output-format=table').env( - 'CTRLQ_TASK_ID' - ) - ) - .addOption( - new Option('--task-tag ', 'use tags to select which tasks to retrieve. Only allowed when --output-format=table').env( - 'CTRLQ_TASK_TAG' - ) - ) + .addOption(new Option('--task-id ', 'use task IDs to select which tasks to retrieve.').env('CTRLQ_TASK_ID')) + .addOption(new Option('--task-tag ', 'use tags to select which tasks to retrieve.').env('CTRLQ_TASK_TAG')) .addOption( new Option('--output-format ', 'output format').choices(['table', 'tree']).default('tree').env('CTRLQ_OUTPUT_FORMAT') ) diff --git a/src/lib/cli/qseow-visualise-task.js b/src/lib/cli/qseow-visualise-task.js index fd6ad5f4..e07253af 100644 --- a/src/lib/cli/qseow-visualise-task.js +++ b/src/lib/cli/qseow-visualise-task.js @@ -1,7 +1,7 @@ import { Option } from 'commander'; import { catchLog } from '../util/log.js'; -import { qseowSharedParamAssertOptions } from '../util/qseow/assert-options.js'; +import { qseowSharedParamAssertOptions, visTaskAssertOptions } from '../util/qseow/assert-options.js'; import { visTask } from '../cmd/qseow/vistask.js'; export function setupQseowVisualiseTaskCommand(qseow) { @@ -10,6 +10,7 @@ export function setupQseowVisualiseTaskCommand(qseow) { .description('visualise task network') .action(async (options) => { await qseowSharedParamAssertOptions(options); + await visTaskAssertOptions(options); await visTask(options); }) @@ -65,6 +66,21 @@ export function setupQseowVisualiseTaskCommand(qseow) { .addOption( new Option('--auth-jwt ', 'JSON Web Token (JWT) to use for authentication with Qlik Sense server').env('CTRLQ_AUTH_JWT') ) + .addOption( + new Option('--task-type ', 'type of tasks to include').choices(['reload', 'ext-program']).env('CTRLQ_TASK_TYPE') + ) + .addOption(new Option('--task-id ', 'task(s) in task chains to include in the network graph.').env('CTRLQ_TASK_ID')) + .addOption(new Option('--task-tag ', 'task tag(s) to include in the network graph.').env('CTRLQ_TASK_TAG')) + .addOption( + new Option('--app-id ', 'app(s) associated with the tasks that should be included in the network graph.').env( + 'CTRLQ_APP_ID' + ) + ) + .addOption( + new Option('--app-tag ', 'app tag(s) associated with the tasks that should be included in the network graph.').env( + 'CTRLQ_APP_TAG' + ) + ) .addOption(new Option('--vis-host ', 'host for visualisation server').default('localhost').env('CTRLQ_VIS_HOST')) .addOption(new Option('--vis-port ', 'port for visualisation server').default('3000').env('CTRLQ_VIS_PORT')); } diff --git a/src/lib/cmd/qseow/gettask copy.js b/src/lib/cmd/qseow/gettask copy.js new file mode 100644 index 00000000..86d9d281 --- /dev/null +++ b/src/lib/cmd/qseow/gettask copy.js @@ -0,0 +1,772 @@ +import tree from 'text-treeview'; +import { table } from 'table'; +import { promises as Fs } from 'node:fs'; +import xlsx from 'node-xlsx'; +import { stringify } from 'csv-stringify'; +import yesno from 'yesno'; +import { logger, setLoggingLevel, isSea, execPath, verifyFileSystemExists } from '../../../globals.js'; +import { QlikSenseTasks } from '../../task/class_alltasks.js'; +import { mapEventType, mapIncrementOption, mapDaylightSavingTime, mapRuleState } from '../../util/qseow/lookups.js'; +import { getTagsFromQseow } from '../../util/qseow/tag.js'; +import { catchLog } from '../../util/log.js'; + +const consoleTableConfig = { + border: { + topBody: `─`, + topJoin: `┬`, + topLeft: `┌`, + topRight: `┐`, + + bottomBody: `─`, + bottomJoin: `┴`, + bottomLeft: `└`, + bottomRight: `┘`, + + bodyLeft: `│`, + bodyRight: `│`, + bodyJoin: `│`, + + joinBody: `─`, + joinLeft: `├`, + joinRight: `┤`, + joinJoin: `┼`, + }, + columns: { + // 3: { width: 40 }, + // 4: { width: 40 }, + // 5: { width: 40 }, + // 6: { width: 40 }, + // 9: { width: 40 }, + }, +}; + +// Only keep "text" and "children" properties +function cleanupTaskTree(taskTree) { + taskTree.forEach((element) => { + for (const prop in element) { + if (prop !== 'text' && prop !== 'children') { + delete element[prop]; + } else if (typeof element[prop] === 'object') { + cleanupTaskTree(element[prop]); + } + } + }); +} + +// Used to sort task trees +function compareTree(a, b) { + if (a.text < b.text) { + return -1; + } + if (a.text > b.text) { + return 1; + } + return 0; +} + +// Used to sort task tables +function compareTable(a, b) { + if (`${a.completeTaskObject.schemaPath}|${a.taskName}` < `${b.completeTaskObject.schemaPath}|${b.taskName}`) { + return -1; + } + if (`${a.completeTaskObject.schemaPath}|${a.taskName}` > `${b.completeTaskObject.schemaPath}|${b.taskName}`) { + return 1; + } + // if (a.taskName < b.taskName) { + // return -1; + // } + // if (a.taskName > b.taskName) { + // return 1; + // } + return 0; +} + +// CLI command: qseow get-task +// Options are assumed to be verified before calling this function +export async function getTask(options) { + try { + // Set log level + setLoggingLevel(options.logLevel); + + logger.verbose(`Ctrl-Q was started as a stand-alone binary: ${isSea}`); + logger.verbose(`Ctrl-Q was started from ${execPath}`); + + logger.verbose('Get tasks'); + logger.debug(`Options: ${JSON.stringify(options, null, 2)}`); + + let returnValue = false; + + // Get all tags + const tags = await getTagsFromQseow(options); + + // Get reload and external program tasks + const qlikSenseTasks = new QlikSenseTasks(); + await qlikSenseTasks.init(options); + const res1 = await qlikSenseTasks.getTaskModelFromQseow(); + if (!res1) { + logger.error('Failed to get task model from QSEoW'); + return false; + } + + logger.info(''); + logger.info(`Parsing ${qlikSenseTasks.taskNetwork.nodes.length} tasks in task model...`); + + // What should we do with the retrieved task data? + if (options.outputFormat === 'tree') { + const taskModel = qlikSenseTasks.taskNetwork; + let taskTree = []; + + // Get all tasks that have a schedule associated with them + // Schedules are represented by "meta nodes" that are linked to the task node in Ctrl-Q's internal data model + // There is one meta-node per schema trigger, meaning that a task with several schema triggers will have several top-level meta nodes. + // We only want the task to show up once in the tree, so we have do de-duplicate the top level task nodes. + const topLevelTasksWithSchemaTriggers = taskModel.nodes.filter((node) => { + if (node.metaNode && node.metaNodeType === 'schedule') { + return true; + } + // Exclude all non-meta nodes + return false; + }); + + // Remove all duplicates from the topLevelTasksWithSchemaTriggers array. + // Use completeSchemaEvent.reloadTask.id as the key to determine if the task is a duplicate or not. + const topLevelTasksWithSchemaTriggersUnique = topLevelTasksWithSchemaTriggers.filter((task, index, self) => { + // Handle reload tasks, external program tasks + if (task.completeSchemaEvent.reloadTask) { + return ( + index === self.findIndex((t) => t.completeSchemaEvent?.reloadTask?.id === task.completeSchemaEvent.reloadTask.id) + ); + } + if (task.completeSchemaEvent.externalProgramTask) { + return ( + index === + self.findIndex( + (t) => t.completeSchemaEvent?.externalProgramTask?.id === task.completeSchemaEvent.externalProgramTask.id + ) + ); + } + return false; + }); + + // Sort the array alfabetically, using the task name as the key + // Task name is found in either completeSchemaEvent.externalProgramTask.name or completeSchemaEvent.reloadTask.name + topLevelTasksWithSchemaTriggersUnique.sort((a, b) => { + if (a.completeSchemaEvent.reloadTask) { + if (b.completeSchemaEvent.reloadTask) { + if (a.completeSchemaEvent.reloadTask.name < b.completeSchemaEvent.reloadTask.name) { + return -1; + } + if (a.completeSchemaEvent.reloadTask.name > b.completeSchemaEvent.reloadTask.name) { + return 1; + } + } else if (b.completeSchemaEvent.externalProgramTask) { + if (a.completeSchemaEvent.reloadTask.name < b.completeSchemaEvent.externalProgramTask.name) { + return -1; + } + if (a.completeSchemaEvent.reloadTask.name > b.completeSchemaEvent.externalProgramTask.name) { + return 1; + } + } + } + if (a.completeSchemaEvent.externalProgramTask) { + if (b.completeSchemaEvent.externalProgramTask) { + if (a.completeSchemaEvent.externalProgramTask.name < b.completeSchemaEvent.externalProgramTask.name) { + return -1; + } + if (a.completeSchemaEvent.externalProgramTask.name > b.completeSchemaEvent.externalProgramTask.name) { + return 1; + } + } else if (b.completeSchemaEvent.reloadTask) { + if (a.completeSchemaEvent.externalProgramTask.name < b.completeSchemaEvent.reloadTask.name) { + return -1; + } + if (a.completeSchemaEvent.externalProgramTask.name > b.completeSchemaEvent.reloadTask.name) { + return 1; + } + } + } + return 0; + }); + + for (const task of topLevelTasksWithSchemaTriggersUnique) { + if (task.metaNode && task.metaNodeType === 'schedule') { + const subTree = qlikSenseTasks.getTaskSubTree(task, 0, null); + subTree[0].isTopLevelNode = true; + subTree[0].isScheduled = true; + taskTree = taskTree.concat(subTree); + } + } + + // Add new top level node with clock/scheduler emoji, if tree icons are enabled + if (options.treeIcons) { + taskTree = [{ text: '⏰ --==| Scheduled tasks |==--', children: taskTree }]; + } else { + taskTree = [{ text: '--==| Scheduled tasks |==--', children: taskTree }]; + } + + // Add unscheduled tasks that are also top level tasks. + const unscheduledTasks = qlikSenseTasks.taskNetwork.nodes.filter((node) => { + if (!node.metaNode && node.isTopLevelNode) { + const a = !taskTree.some((el) => { + const b = el.taskId === node.id; + return b; + }); + return a; + } + // Don't include meta nodes + return false; + }); + + // Sort unscheduled tasks alfabetically + // Use taskName as the key + unscheduledTasks.sort((a, b) => { + if (a.taskName < b.taskName) { + return -1; + } + if (a.taskName > b.taskName) { + return 1; + } + return 0; + }); + + for (const task of unscheduledTasks) { + const subTree = qlikSenseTasks.getTaskSubTree(task, 0, null); + subTree[0].isTopLevelNode = true; + subTree[0].isScheduled = false; + taskTree = taskTree.concat(subTree); + } + + // Output task tree to correct destination + if (options.outputDest === 'screen') { + logger.info(``); + logger.info(`# top-level rows in tree: ${taskTree.length}`); + logger.info(`\n${tree(taskTree)}`); + returnValue = true; + } else if (options.outputDest === 'file') { + logger.verbose(`Writing task tree to disk file "${options.outputFileName}"`); + let buffer; + if (options.outputFileFormat === 'json') { + // Only keep the text and children properties of the tree object + cleanupTaskTree(taskTree); + + // Format JSON nicely + buffer = JSON.stringify(taskTree, null, 4); + } else { + logger.error(`Output file format "${options.outputFileFormat}" not supported for task trees. Exiting.`); + process.exit(1); + } + + // Check if file exists + if ((await verifyFileSystemExists(options.outputFileName, true)) === false) { + // File doesn't exist + } else if (!options.outputFileOverwrite) { + // Target file exist. Ask if user wants to overwrite + logger.info(); + const ok = await yesno({ + question: ` Destination file "${options.outputFileName}" exists. Do you want to overwrite it? (y/n)`, + }); + logger.info(); + if (ok === false) { + logger.info('❌ Not overwriting existing output file. Exiting.'); + process.exit(1); + } + } else if (options.outputFileOverwrite) { + // File exists and force overwrite is set + logger.info(`❗️ Existing output file will be replaced.`); + } + logger.info(`✅ Writing task tree to disk file "${options.outputFileName}".`); + await Fs.writeFile(options.outputFileName, buffer); + returnValue = true; + } + } else if (options.outputFormat === 'table') { + const { tasks } = qlikSenseTasks.taskNetwork; + const { schemaEventList } = qlikSenseTasks.qlikSenseSchemaEvents; + const { compositeEventList } = qlikSenseTasks.qlikSenseCompositeEvents; + + let taskTable = []; + let taskCount = 1; + + // Sort tasks, group by task type + tasks.sort(compareTable); + + // Determine which column blocks should be included in table + const columnBlockShow = { + common: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'common')) + ), + lastexecution: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'lastexecution')) + ), + tag: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'tag')) + ), + customproperty: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'customproperty')) + ), + schematrigger: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails?.find((item) => item === 'schematrigger')) + ), + compositetrigger: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails?.find((item) => item === 'compositetrigger')) + ), + }; + + for (const task of tasks) { + if ( + (options.taskType?.find((item) => item === 'reload') && task.completeTaskObject.schemaPath === 'ReloadTask') || + (options.taskType?.find((item) => item === 'ext-program') && + task.completeTaskObject.schemaPath === 'ExternalProgramTask') + ) { + let row = []; + let tmpRow = []; + let eventCount = 1; + + // Get icon for task status + let taskStatus = ''; + if (task.taskLastStatus) { + if (task.taskLastStatus === 'FinishedSuccess') { + taskStatus = `✅ ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'FinishedFail') { + taskStatus = `❌ ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'Skipped') { + taskStatus = `🚫 ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'Aborted') { + taskStatus = `🛑 ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'Never started') { + taskStatus = `💤 ${task.taskLastStatus}`; + } else { + taskStatus = `❔ ${task.taskLastStatus}`; + } + } + + if (task.completeTaskObject.schemaPath === 'ReloadTask') { + row = [taskCount, 'Reload']; + } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { + row = [taskCount, 'External program']; + } + + if (columnBlockShow.common) { + tmpRow = [ + task.taskName, + task.taskId, + task.taskEnabled, + task.taskSessionTimeout, + task.taskMaxRetries, + task.appId ? task.appId : '', + task.isPartialReload ? task.isPartialReload : '', + task.isManuallyTriggered ? task.isManuallyTriggered : '', + task.path ? task.path : '', + task.parameters ? task.parameters : '', + ]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.lastexecution) { + tmpRow = [ + taskStatus, + task.taskLastExecutionStartTimestamp, + task.taskLastExecutionStopTimestamp, + task.taskLastExecutionDuration, + task.taskLastExecutionExecutingNodeName, + ]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.tag) { + tmpRow = [task.taskTags.map((item) => item.name).join(' / ')]; + row = row.concat(tmpRow[0]); + } + + if (columnBlockShow.customproperty) { + tmpRow = [task.taskCustomProperties.map((item) => `${item.definition.name}=${item.value}`).join(' / ')]; + row = row.concat(tmpRow[0]); + } + + // If complete details are requested, or if schema or composite columns are requested, add empty columns for general event info + if ( + options.tableDetails === true || + options.tableDetails === '' || + columnBlockShow.schematrigger || + columnBlockShow.compositetrigger + ) { + tmpRow = Array(7).fill(''); + row = row.concat(tmpRow); + } + + // If complete details are requested, or if schema columns are requested, add empty columns for schema event info + if (columnBlockShow.schematrigger) { + tmpRow = Array(7).fill(''); + row = row.concat(tmpRow); + } + + // If complete details are requested, or if composite columns are requested, add empty columns for composite event info + if (columnBlockShow.compositetrigger) { + tmpRow = Array(8).fill(''); + row = row.concat(tmpRow); + } + + // Add main task info to table + taskTable = taskTable.concat([row]); + + // Find all schema events for this task + const schemaEventsForThisTask = schemaEventList.filter((item) => { + if (item.schemaEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.schemaEvent?.externalProgramTask?.id === task.taskId) { + return true; + } + return false; + }); + + // Find all composite events for this task + const compositeEventsForThisTask = compositeEventList.filter((item) => { + if (item.compositeEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.compositeEvent?.externalProgramTask?.id === task.taskId) { + return true; + } + return false; + }); + + // Write schema events to table + if (columnBlockShow.schematrigger) { + for (const event of schemaEventsForThisTask) { + row = [taskCount, '']; + + if (columnBlockShow.common) { + tmpRow = [...Array(10).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.lastexecution) { + tmpRow = [...Array(5).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.tag) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.customproperty) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); + } + + // Include general event columns if schema or composite columns should be shown + tmpRow = [eventCount, mapEventType.get(event.schemaEvent.eventType)]; + row = row.concat(tmpRow); + + tmpRow = [ + event.schemaEvent.name, + event.schemaEvent.enabled, + event.schemaEvent.createdDate, + event.schemaEvent.modifiedDate, + event.schemaEvent.modifiedByUserName, + + mapIncrementOption.get(event.schemaEvent.incrementOption), + event.schemaEvent.incrementDescription, + mapDaylightSavingTime.get(event.schemaEvent.daylightSavingTime), + event.schemaEvent.startDate, + event.schemaEvent.expirationDate, + event.schemaEvent.schemaFilterDescription[0], + event.schemaEvent.timeZone, + ]; + row = row.concat(tmpRow); + + if (columnBlockShow.compositetrigger) { + tmpRow = Array(8).fill(''); + row = row.concat(tmpRow); + } + + taskTable = taskTable.concat([row]); + + eventCount += 1; + } + } + + // Write composite events to table + if (columnBlockShow.compositetrigger) { + for (const event of compositeEventsForThisTask) { + row = [taskCount, '']; + + if (columnBlockShow.common) { + tmpRow = [...Array(10).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.lastexecution) { + tmpRow = [...Array(5).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.tag) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.customproperty) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); + } + + // Include general event columns + tmpRow = [eventCount, mapEventType.get(event.compositeEvent.eventType)]; + row = row.concat(tmpRow); + + if (columnBlockShow.compositetrigger) { + tmpRow = [ + event.compositeEvent.name, + event.compositeEvent.enabled, + event.compositeEvent.createdDate, + event.compositeEvent.modifiedDate, + event.compositeEvent.modifiedByUserName, + ]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.schematrigger) { + tmpRow = [...Array(7).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.compositetrigger) { + // Composite task time constraints + tmpRow = [ + event.compositeEvent.timeConstraint.seconds, + event.compositeEvent.timeConstraint.minutes, + event.compositeEvent.timeConstraint.hours, + event.compositeEvent.timeConstraint.days, + '', + '', + '', + '', + ]; + row = row.concat(tmpRow); + } + + taskTable = taskTable.concat([row]); + + // Add all composite rules to table + let ruleCount = 1; + + for (const rule of event.compositeEvent.compositeRules) { + row = [taskCount, '']; + + if (columnBlockShow.common) { + tmpRow = [...Array(10).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.lastexecution) { + tmpRow = [...Array(5).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.tag) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.customproperty) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); + } + + // Include general event columns + tmpRow = [eventCount, '', '', '', '', '', '']; + row = row.concat(tmpRow); + + if (columnBlockShow.schematrigger) { + // Add empty columns for schema event info + tmpRow = [...Array(7).fill('')]; + row = row.concat(tmpRow); + } + + // Is it a reload task or external program task? + if (rule.reloadTask) { + tmpRow = [ + '', + '', + '', + '', + ruleCount, + mapRuleState.get(rule.ruleState), + rule.reloadTask.name, + rule.reloadTask.id, + ]; + } else if (rule.externalProgramTask) { + tmpRow = [ + '', + '', + '', + '', + ruleCount, + mapRuleState.get(rule.ruleState), + rule.externalProgramTask.name, + rule.externalProgramTask.id, + ]; + } + + row = row.concat(tmpRow); + + taskTable = taskTable.concat([row]); + + ruleCount += 1; + } + + eventCount += 1; + } + } + + taskCount += 1; + } else { + logger.debug(`Skipped task "${task.taskName}" due to incorrect task type`); + } + } + + // Add column headers + let headerRow = ['Task counter', 'Task type']; + + if (columnBlockShow.common) { + headerRow = headerRow.concat([ + 'Task name', + 'Task id', + 'Task enabled', + 'Task timeout', + 'Task retries', + 'App id', + 'Partial reload', + 'Manually triggered', + 'Ext program path', + 'Ext program parameters', + ]); + } + + if (columnBlockShow.lastexecution) { + headerRow = headerRow.concat(['Task status', 'Task started', 'Task ended', 'Task duration', 'Task executedon node']); + } + + if (columnBlockShow.tag) { + headerRow = headerRow.concat(['Tags']); + } + + if (columnBlockShow.customproperty) { + headerRow = headerRow.concat(['Custom properties']); + } + + if (columnBlockShow.schematrigger || columnBlockShow.compositetrigger) { + headerRow = headerRow.concat([ + 'Event counter', + 'Event type', + 'Event name', + 'Event enabled', + 'Event created date', + 'Event modified date', + 'Event modified by', + ]); + } + + if (columnBlockShow.schematrigger) { + headerRow = headerRow.concat([ + 'Schema increment option', + 'Schema increment description', + 'Daylight savings time', + 'Schema start', + 'Schema expiration', + 'Schema filter description', + 'Schema time zone', + ]); + } + + if (columnBlockShow.compositetrigger) { + headerRow = headerRow.concat([ + 'Time contstraint seconds', + 'Time contstraint minutes', + 'Time contstraint hours', + 'Time contstraint days', + 'Rule counter', + 'Rule state', + 'Rule task name', + 'Rule task id', + ]); + } + + consoleTableConfig.header = { + alignment: 'left', + content: `# reload tasks: ${taskTable.filter((task) => task[1] === 'Reload').length}, # external program tasks: ${ + taskTable.filter((task) => task[1] === 'External program').length + }, # rows in table: ${taskTable.length}`, + }; + + taskTable.unshift(headerRow); + + if (options.outputDest === 'screen') { + logger.info(`# rows in table: ${taskTable.length - 1}`); + logger.info(`# reload tasks in table: ${taskTable.filter((task) => task[1] === 'Reload').length}`); + logger.info(`# external program tasks in table: ${taskTable.filter((task) => task[1] === 'External program').length}`); + logger.info(`\n${table(taskTable, consoleTableConfig)}`); + returnValue = true; + } else if (options.outputDest === 'file') { + logger.verbose(`Writing task table to disk file "${options.outputFileName}"`); + let buffer; + if (options.outputFileFormat === 'excel') { + // Save to Excel file + buffer = xlsx.build([{ name: 'Ctrl-Q task export', data: taskTable }]); + } else if (options.outputFileFormat === 'csv') { + // Remove newlines in column names + taskTable[0] = taskTable[0].map((item) => item.replace('\n', ' ')); + + // Create CSV string + buffer = stringify(taskTable); + } else if (options.outputFileFormat === 'json') { + // Remove newlines in column names + taskTable[0] = taskTable[0].map((item) => item.replace('\n', ' ')); + + // Format JSON nicely + buffer = JSON.stringify(taskTable, null, 4); + } + + // Check if file exists + if ((await verifyFileSystemExists(options.outputFileName, true)) === false) { + // File doesn't exist + } else if (!options.outputFileOverwrite) { + // Target file exist. Ask if user wants to overwrite + logger.info(); + const ok = await yesno({ + question: ` Destination file "${options.outputFileName}" exists. Do you want to overwrite it? (y/n)`, + }); + logger.info(); + if (ok === false) { + logger.info('❌ Not overwriting existing output file. Exiting.'); + process.exit(1); + } + } else if (options.outputFileOverwrite) { + // File exists and force overwrite is set + logger.info(`❗️ Existing output file will be replaced.`); + } + logger.info(`✅ Writing task table to disk file "${options.outputFileName}".`); + await Fs.writeFile(options.outputFileName, buffer); + returnValue = true; + } + } + return returnValue; + } catch (err) { + catchLog('Get task', err); + return false; + } +} diff --git a/src/lib/cmd/qseow/gettask.js b/src/lib/cmd/qseow/gettask.js index 86d9d281..3e60848b 100644 --- a/src/lib/cmd/qseow/gettask.js +++ b/src/lib/cmd/qseow/gettask.js @@ -40,7 +40,11 @@ const consoleTableConfig = { }, }; -// Only keep "text" and "children" properties +/** + * Recursively remove all properties from a task tree except for 'text' and 'children'. + * + * @param {Object[]} taskTree - The task tree to clean up. + */ function cleanupTaskTree(taskTree) { taskTree.forEach((element) => { for (const prop in element) { @@ -53,7 +57,12 @@ function cleanupTaskTree(taskTree) { }); } -// Used to sort task trees +/** + * Used to sort task trees + * @param {object} a - first task to compare + * @param {object} b - second task to compare + * @returns {number} -1 if a comes before b, 1 if b comes before a, 0 if order is the same + */ function compareTree(a, b) { if (a.text < b.text) { return -1; @@ -64,7 +73,12 @@ function compareTree(a, b) { return 0; } -// Used to sort task tables +/** + * Used to sort task tables + * @param {object} a - first task to compare + * @param {object} b - second task to compare + * @returns {number} -1 if a comes before b, 1 if b comes before a, 0 if order is the same + */ function compareTable(a, b) { if (`${a.completeTaskObject.schemaPath}|${a.taskName}` < `${b.completeTaskObject.schemaPath}|${b.taskName}`) { return -1; @@ -81,692 +95,800 @@ function compareTable(a, b) { return 0; } -// CLI command: qseow get-task -// Options are assumed to be verified before calling this function +/** + * CLI command: qseow get-task + * Options are assumed to be verified before calling this function + * @param {object} options - CLI options + * @returns {Promise} True if successful, false otherwise + */ export async function getTask(options) { - try { - // Set log level - setLoggingLevel(options.logLevel); + // Set log level + setLoggingLevel(options.logLevel); - logger.verbose(`Ctrl-Q was started as a stand-alone binary: ${isSea}`); - logger.verbose(`Ctrl-Q was started from ${execPath}`); + logger.verbose(`Ctrl-Q was started as a stand-alone binary: ${isSea}`); + logger.verbose(`Ctrl-Q was started from ${execPath}`); - logger.verbose('Get tasks'); - logger.debug(`Options: ${JSON.stringify(options, null, 2)}`); + logger.verbose('Get tasks'); + logger.debug(`Options: ${JSON.stringify(options, null, 2)}`); - let returnValue = false; + // Get all tags + const tags = await getTags(options); - // Get all tags - const tags = await getTagsFromQseow(options); + // Get reload and external program tasks + const qlikSenseTasks = await getTaskModelFromQseow(options); - // Get reload and external program tasks - const qlikSenseTasks = new QlikSenseTasks(); - await qlikSenseTasks.init(options); - const res1 = await qlikSenseTasks.getTaskModelFromQseow(); - if (!res1) { - logger.error('Failed to get task model from QSEoW'); - return false; - } + // What should we do with the retrieved task data? + // return outputTaskData(options, qlikSenseTasks, tags); + let returnValue = false; + if (options.outputFormat === 'tree') { + returnValue = await parseTree(options, qlikSenseTasks); + } else if (options.outputFormat === 'table') { + returnValue = await parseTable(options, qlikSenseTasks, tags); + } - logger.info(''); - logger.info(`Parsing ${qlikSenseTasks.taskNetwork.nodes.length} tasks in task model...`); + return returnValue; +} - // What should we do with the retrieved task data? - if (options.outputFormat === 'tree') { - const taskModel = qlikSenseTasks.taskNetwork; - let taskTree = []; +/** + * Get all tags from QSEoW + * @param {object} options - CLI options + * @returns {Promise>} Array of tag names + */ +async function getTags(options) { + logger.debug('Getting tags from QSEoW'); + const tags = await getTagsFromQseow(options); + return tags; +} - // Get all tasks that have a schedule associated with them - // Schedules are represented by "meta nodes" that are linked to the task node in Ctrl-Q's internal data model - // There is one meta-node per schema trigger, meaning that a task with several schema triggers will have several top-level meta nodes. - // We only want the task to show up once in the tree, so we have do de-duplicate the top level task nodes. - const topLevelTasksWithSchemaTriggers = taskModel.nodes.filter((node) => { - if (node.metaNode && node.metaNodeType === 'schedule') { - return true; - } - // Exclude all non-meta nodes - return false; - }); +/** + * Get the task model from QSEoW. + * @param {object} options - CLI options + * @returns {Promise} The task model + */ +async function getTaskModelFromQseow(options) { + logger.debug('Getting task model from QSEoW'); + const qlikSenseTasks = new QlikSenseTasks(); + await qlikSenseTasks.init(options); + const res1 = await qlikSenseTasks.getTaskModelFromQseow(); + if (!res1) { + logger.error('Failed to get task model from QSEoW'); + return false; + } - // Remove all duplicates from the topLevelTasksWithSchemaTriggers array. - // Use completeSchemaEvent.reloadTask.id as the key to determine if the task is a duplicate or not. - const topLevelTasksWithSchemaTriggersUnique = topLevelTasksWithSchemaTriggers.filter((task, index, self) => { - // Handle reload tasks, external program tasks - if (task.completeSchemaEvent.reloadTask) { - return ( - index === self.findIndex((t) => t.completeSchemaEvent?.reloadTask?.id === task.completeSchemaEvent.reloadTask.id) - ); - } - if (task.completeSchemaEvent.externalProgramTask) { - return ( - index === - self.findIndex( - (t) => t.completeSchemaEvent?.externalProgramTask?.id === task.completeSchemaEvent.externalProgramTask.id - ) - ); - } - return false; - }); + logger.info(''); + logger.info(`Parsing ${qlikSenseTasks.taskNetwork.nodes.length} tasks in task model...`); - // Sort the array alfabetically, using the task name as the key - // Task name is found in either completeSchemaEvent.externalProgramTask.name or completeSchemaEvent.reloadTask.name - topLevelTasksWithSchemaTriggersUnique.sort((a, b) => { - if (a.completeSchemaEvent.reloadTask) { - if (b.completeSchemaEvent.reloadTask) { - if (a.completeSchemaEvent.reloadTask.name < b.completeSchemaEvent.reloadTask.name) { - return -1; - } - if (a.completeSchemaEvent.reloadTask.name > b.completeSchemaEvent.reloadTask.name) { - return 1; - } - } else if (b.completeSchemaEvent.externalProgramTask) { - if (a.completeSchemaEvent.reloadTask.name < b.completeSchemaEvent.externalProgramTask.name) { - return -1; - } - if (a.completeSchemaEvent.reloadTask.name > b.completeSchemaEvent.externalProgramTask.name) { - return 1; - } - } - } - if (a.completeSchemaEvent.externalProgramTask) { - if (b.completeSchemaEvent.externalProgramTask) { - if (a.completeSchemaEvent.externalProgramTask.name < b.completeSchemaEvent.externalProgramTask.name) { - return -1; - } - if (a.completeSchemaEvent.externalProgramTask.name > b.completeSchemaEvent.externalProgramTask.name) { - return 1; - } - } else if (b.completeSchemaEvent.reloadTask) { - if (a.completeSchemaEvent.externalProgramTask.name < b.completeSchemaEvent.reloadTask.name) { - return -1; - } - if (a.completeSchemaEvent.externalProgramTask.name > b.completeSchemaEvent.reloadTask.name) { - return 1; - } - } - } - return 0; - }); + return qlikSenseTasks; +} - for (const task of topLevelTasksWithSchemaTriggersUnique) { - if (task.metaNode && task.metaNodeType === 'schedule') { - const subTree = qlikSenseTasks.getTaskSubTree(task, 0, null); - subTree[0].isTopLevelNode = true; - subTree[0].isScheduled = true; - taskTree = taskTree.concat(subTree); +/** + * CLI command: qseow get-task + * Parse the task data into a table format (tab separated values) + * and print it to the console or write it to a file. + * @param {object} options - CLI options + * @param {object} qlikSenseTasks - Qlik Sense task model + * @param {Array} tags - Array of tags + * @returns {Promise} True if successful, false otherwise + */ +async function parseTable(options, qlikSenseTasks, tags) { + let returnValue = false; + + const { tasks } = qlikSenseTasks.taskNetwork; + const { schemaEventList } = qlikSenseTasks.qlikSenseSchemaEvents; + const { compositeEventList } = qlikSenseTasks.qlikSenseCompositeEvents; + + let taskTable = []; + let taskCount = 1; + + // Sort tasks, group by task type + tasks.sort(compareTable); + + // Determine which column blocks should be included in table + const columnBlockShow = { + common: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'common')) + ), + lastexecution: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'lastexecution')) + ), + tag: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'tag')) + ), + customproperty: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'customproperty')) + ), + schematrigger: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails?.find((item) => item === 'schematrigger')) + ), + compositetrigger: !!( + options.tableDetails === true || + options.tableDetails === '' || + (typeof options.tableDetails === 'object' && options.tableDetails?.find((item) => item === 'compositetrigger')) + ), + }; + + for (const task of tasks) { + if ( + (options.taskType?.find((item) => item === 'reload') && task.completeTaskObject.schemaPath === 'ReloadTask') || + (options.taskType?.find((item) => item === 'ext-program') && task.completeTaskObject.schemaPath === 'ExternalProgramTask') + ) { + let row = []; + let tmpRow = []; + let eventCount = 1; + + // Get icon for task status + let taskStatus = ''; + if (task.taskLastStatus) { + if (task.taskLastStatus === 'FinishedSuccess') { + taskStatus = `✅ ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'FinishedFail') { + taskStatus = `❌ ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'Skipped') { + taskStatus = `🚫 ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'Aborted') { + taskStatus = `🛑 ${task.taskLastStatus}`; + } else if (task.taskLastStatus === 'Never started') { + taskStatus = `💤 ${task.taskLastStatus}`; + } else { + taskStatus = `❔ ${task.taskLastStatus}`; } } - // Add new top level node with clock/scheduler emoji, if tree icons are enabled - if (options.treeIcons) { - taskTree = [{ text: '⏰ --==| Scheduled tasks |==--', children: taskTree }]; - } else { - taskTree = [{ text: '--==| Scheduled tasks |==--', children: taskTree }]; + if (task.completeTaskObject.schemaPath === 'ReloadTask') { + row = [taskCount, 'Reload']; + } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { + row = [taskCount, 'External program']; + } + + if (columnBlockShow.common) { + tmpRow = [ + task.taskName, + task.taskId, + task.taskEnabled, + task.taskSessionTimeout, + task.taskMaxRetries, + task.appId ? task.appId : '', + task.isPartialReload ? task.isPartialReload : '', + task.isManuallyTriggered ? task.isManuallyTriggered : '', + task.path ? task.path : '', + task.parameters ? task.parameters : '', + ]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.lastexecution) { + tmpRow = [ + taskStatus, + task.taskLastExecutionStartTimestamp, + task.taskLastExecutionStopTimestamp, + task.taskLastExecutionDuration, + task.taskLastExecutionExecutingNodeName, + ]; + row = row.concat(tmpRow); } - // Add unscheduled tasks that are also top level tasks. - const unscheduledTasks = qlikSenseTasks.taskNetwork.nodes.filter((node) => { - if (!node.metaNode && node.isTopLevelNode) { - const a = !taskTree.some((el) => { - const b = el.taskId === node.id; - return b; - }); - return a; + if (columnBlockShow.tag) { + tmpRow = [task.taskTags.map((item) => item.name).join(' / ')]; + row = row.concat(tmpRow[0]); + } + + if (columnBlockShow.customproperty) { + tmpRow = [task.taskCustomProperties.map((item) => `${item.definition.name}=${item.value}`).join(' / ')]; + row = row.concat(tmpRow[0]); + } + + // If complete details are requested, or if schema or composite columns are requested, add empty columns for general event info + if ( + options.tableDetails === true || + options.tableDetails === '' || + columnBlockShow.schematrigger || + columnBlockShow.compositetrigger + ) { + tmpRow = Array(7).fill(''); + row = row.concat(tmpRow); + } + + // If complete details are requested, or if schema columns are requested, add empty columns for schema event info + if (columnBlockShow.schematrigger) { + tmpRow = Array(7).fill(''); + row = row.concat(tmpRow); + } + + // If complete details are requested, or if composite columns are requested, add empty columns for composite event info + if (columnBlockShow.compositetrigger) { + tmpRow = Array(8).fill(''); + row = row.concat(tmpRow); + } + + // Add main task info to table + taskTable = taskTable.concat([row]); + + // Find all schema events for this task + const schemaEventsForThisTask = schemaEventList.filter((item) => { + if (item.schemaEvent?.reloadTask?.id === task.taskId) { + return true; + } + if (item.schemaEvent?.externalProgramTask?.id === task.taskId) { + return true; } - // Don't include meta nodes return false; }); - // Sort unscheduled tasks alfabetically - // Use taskName as the key - unscheduledTasks.sort((a, b) => { - if (a.taskName < b.taskName) { - return -1; + // Find all composite events for this task + const compositeEventsForThisTask = compositeEventList.filter((item) => { + if (item.compositeEvent?.reloadTask?.id === task.taskId) { + return true; } - if (a.taskName > b.taskName) { - return 1; + if (item.compositeEvent?.externalProgramTask?.id === task.taskId) { + return true; } - return 0; + return false; }); - for (const task of unscheduledTasks) { - const subTree = qlikSenseTasks.getTaskSubTree(task, 0, null); - subTree[0].isTopLevelNode = true; - subTree[0].isScheduled = false; - taskTree = taskTree.concat(subTree); - } + // Write schema events to table + if (columnBlockShow.schematrigger) { + for (const event of schemaEventsForThisTask) { + row = [taskCount, '']; - // Output task tree to correct destination - if (options.outputDest === 'screen') { - logger.info(``); - logger.info(`# top-level rows in tree: ${taskTree.length}`); - logger.info(`\n${tree(taskTree)}`); - returnValue = true; - } else if (options.outputDest === 'file') { - logger.verbose(`Writing task tree to disk file "${options.outputFileName}"`); - let buffer; - if (options.outputFileFormat === 'json') { - // Only keep the text and children properties of the tree object - cleanupTaskTree(taskTree); - - // Format JSON nicely - buffer = JSON.stringify(taskTree, null, 4); - } else { - logger.error(`Output file format "${options.outputFileFormat}" not supported for task trees. Exiting.`); - process.exit(1); - } + if (columnBlockShow.common) { + tmpRow = [...Array(10).fill('')]; + row = row.concat(tmpRow); + } + + if (columnBlockShow.lastexecution) { + tmpRow = [...Array(5).fill('')]; + row = row.concat(tmpRow); + } - // Check if file exists - if ((await verifyFileSystemExists(options.outputFileName, true)) === false) { - // File doesn't exist - } else if (!options.outputFileOverwrite) { - // Target file exist. Ask if user wants to overwrite - logger.info(); - const ok = await yesno({ - question: ` Destination file "${options.outputFileName}" exists. Do you want to overwrite it? (y/n)`, - }); - logger.info(); - if (ok === false) { - logger.info('❌ Not overwriting existing output file. Exiting.'); - process.exit(1); + if (columnBlockShow.tag) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); } - } else if (options.outputFileOverwrite) { - // File exists and force overwrite is set - logger.info(`❗️ Existing output file will be replaced.`); - } - logger.info(`✅ Writing task tree to disk file "${options.outputFileName}".`); - await Fs.writeFile(options.outputFileName, buffer); - returnValue = true; - } - } else if (options.outputFormat === 'table') { - const { tasks } = qlikSenseTasks.taskNetwork; - const { schemaEventList } = qlikSenseTasks.qlikSenseSchemaEvents; - const { compositeEventList } = qlikSenseTasks.qlikSenseCompositeEvents; - - let taskTable = []; - let taskCount = 1; - - // Sort tasks, group by task type - tasks.sort(compareTable); - - // Determine which column blocks should be included in table - const columnBlockShow = { - common: !!( - options.tableDetails === true || - options.tableDetails === '' || - (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'common')) - ), - lastexecution: !!( - options.tableDetails === true || - options.tableDetails === '' || - (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'lastexecution')) - ), - tag: !!( - options.tableDetails === true || - options.tableDetails === '' || - (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'tag')) - ), - customproperty: !!( - options.tableDetails === true || - options.tableDetails === '' || - (typeof options.tableDetails === 'object' && options.tableDetails.find((item) => item === 'customproperty')) - ), - schematrigger: !!( - options.tableDetails === true || - options.tableDetails === '' || - (typeof options.tableDetails === 'object' && options.tableDetails?.find((item) => item === 'schematrigger')) - ), - compositetrigger: !!( - options.tableDetails === true || - options.tableDetails === '' || - (typeof options.tableDetails === 'object' && options.tableDetails?.find((item) => item === 'compositetrigger')) - ), - }; - - for (const task of tasks) { - if ( - (options.taskType?.find((item) => item === 'reload') && task.completeTaskObject.schemaPath === 'ReloadTask') || - (options.taskType?.find((item) => item === 'ext-program') && - task.completeTaskObject.schemaPath === 'ExternalProgramTask') - ) { - let row = []; - let tmpRow = []; - let eventCount = 1; - - // Get icon for task status - let taskStatus = ''; - if (task.taskLastStatus) { - if (task.taskLastStatus === 'FinishedSuccess') { - taskStatus = `✅ ${task.taskLastStatus}`; - } else if (task.taskLastStatus === 'FinishedFail') { - taskStatus = `❌ ${task.taskLastStatus}`; - } else if (task.taskLastStatus === 'Skipped') { - taskStatus = `🚫 ${task.taskLastStatus}`; - } else if (task.taskLastStatus === 'Aborted') { - taskStatus = `🛑 ${task.taskLastStatus}`; - } else if (task.taskLastStatus === 'Never started') { - taskStatus = `💤 ${task.taskLastStatus}`; - } else { - taskStatus = `❔ ${task.taskLastStatus}`; - } + + if (columnBlockShow.customproperty) { + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); } - if (task.completeTaskObject.schemaPath === 'ReloadTask') { - row = [taskCount, 'Reload']; - } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { - row = [taskCount, 'External program']; + // Include general event columns if schema or composite columns should be shown + tmpRow = [eventCount, mapEventType.get(event.schemaEvent.eventType)]; + row = row.concat(tmpRow); + + tmpRow = [ + event.schemaEvent.name, + event.schemaEvent.enabled, + event.schemaEvent.createdDate, + event.schemaEvent.modifiedDate, + event.schemaEvent.modifiedByUserName, + + mapIncrementOption.get(event.schemaEvent.incrementOption), + event.schemaEvent.incrementDescription, + mapDaylightSavingTime.get(event.schemaEvent.daylightSavingTime), + event.schemaEvent.startDate, + event.schemaEvent.expirationDate, + event.schemaEvent.schemaFilterDescription[0], + event.schemaEvent.timeZone, + ]; + row = row.concat(tmpRow); + + if (columnBlockShow.compositetrigger) { + tmpRow = Array(8).fill(''); + row = row.concat(tmpRow); } + taskTable = taskTable.concat([row]); + + eventCount += 1; + } + } + + // Write composite events to table + if (columnBlockShow.compositetrigger) { + for (const event of compositeEventsForThisTask) { + row = [taskCount, '']; + if (columnBlockShow.common) { - tmpRow = [ - task.taskName, - task.taskId, - task.taskEnabled, - task.taskSessionTimeout, - task.taskMaxRetries, - task.appId ? task.appId : '', - task.isPartialReload ? task.isPartialReload : '', - task.isManuallyTriggered ? task.isManuallyTriggered : '', - task.path ? task.path : '', - task.parameters ? task.parameters : '', - ]; + tmpRow = [...Array(10).fill('')]; row = row.concat(tmpRow); } if (columnBlockShow.lastexecution) { - tmpRow = [ - taskStatus, - task.taskLastExecutionStartTimestamp, - task.taskLastExecutionStopTimestamp, - task.taskLastExecutionDuration, - task.taskLastExecutionExecutingNodeName, - ]; + tmpRow = [...Array(5).fill('')]; row = row.concat(tmpRow); } if (columnBlockShow.tag) { - tmpRow = [task.taskTags.map((item) => item.name).join(' / ')]; - row = row.concat(tmpRow[0]); + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); } if (columnBlockShow.customproperty) { - tmpRow = [task.taskCustomProperties.map((item) => `${item.definition.name}=${item.value}`).join(' / ')]; - row = row.concat(tmpRow[0]); + tmpRow = [...Array(1).fill('')]; + row = row.concat(tmpRow); } - // If complete details are requested, or if schema or composite columns are requested, add empty columns for general event info - if ( - options.tableDetails === true || - options.tableDetails === '' || - columnBlockShow.schematrigger || - columnBlockShow.compositetrigger - ) { - tmpRow = Array(7).fill(''); + // Include general event columns + tmpRow = [eventCount, mapEventType.get(event.compositeEvent.eventType)]; + row = row.concat(tmpRow); + + if (columnBlockShow.compositetrigger) { + tmpRow = [ + event.compositeEvent.name, + event.compositeEvent.enabled, + event.compositeEvent.createdDate, + event.compositeEvent.modifiedDate, + event.compositeEvent.modifiedByUserName, + ]; row = row.concat(tmpRow); } - // If complete details are requested, or if schema columns are requested, add empty columns for schema event info if (columnBlockShow.schematrigger) { - tmpRow = Array(7).fill(''); + tmpRow = [...Array(7).fill('')]; row = row.concat(tmpRow); } - // If complete details are requested, or if composite columns are requested, add empty columns for composite event info if (columnBlockShow.compositetrigger) { - tmpRow = Array(8).fill(''); + // Composite task time constraints + tmpRow = [ + event.compositeEvent.timeConstraint.seconds, + event.compositeEvent.timeConstraint.minutes, + event.compositeEvent.timeConstraint.hours, + event.compositeEvent.timeConstraint.days, + '', + '', + '', + '', + ]; row = row.concat(tmpRow); } - // Add main task info to table taskTable = taskTable.concat([row]); - // Find all schema events for this task - const schemaEventsForThisTask = schemaEventList.filter((item) => { - if (item.schemaEvent?.reloadTask?.id === task.taskId) { - return true; - } - if (item.schemaEvent?.externalProgramTask?.id === task.taskId) { - return true; - } - return false; - }); + // Add all composite rules to table + let ruleCount = 1; + + for (const rule of event.compositeEvent.compositeRules) { + row = [taskCount, '']; - // Find all composite events for this task - const compositeEventsForThisTask = compositeEventList.filter((item) => { - if (item.compositeEvent?.reloadTask?.id === task.taskId) { - return true; + if (columnBlockShow.common) { + tmpRow = [...Array(10).fill('')]; + row = row.concat(tmpRow); } - if (item.compositeEvent?.externalProgramTask?.id === task.taskId) { - return true; + + if (columnBlockShow.lastexecution) { + tmpRow = [...Array(5).fill('')]; + row = row.concat(tmpRow); } - return false; - }); - // Write schema events to table - if (columnBlockShow.schematrigger) { - for (const event of schemaEventsForThisTask) { - row = [taskCount, '']; - - if (columnBlockShow.common) { - tmpRow = [...Array(10).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.lastexecution) { - tmpRow = [...Array(5).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.tag) { - tmpRow = [...Array(1).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.customproperty) { - tmpRow = [...Array(1).fill('')]; - row = row.concat(tmpRow); - } - - // Include general event columns if schema or composite columns should be shown - tmpRow = [eventCount, mapEventType.get(event.schemaEvent.eventType)]; + if (columnBlockShow.tag) { + tmpRow = [...Array(1).fill('')]; row = row.concat(tmpRow); + } - tmpRow = [ - event.schemaEvent.name, - event.schemaEvent.enabled, - event.schemaEvent.createdDate, - event.schemaEvent.modifiedDate, - event.schemaEvent.modifiedByUserName, - - mapIncrementOption.get(event.schemaEvent.incrementOption), - event.schemaEvent.incrementDescription, - mapDaylightSavingTime.get(event.schemaEvent.daylightSavingTime), - event.schemaEvent.startDate, - event.schemaEvent.expirationDate, - event.schemaEvent.schemaFilterDescription[0], - event.schemaEvent.timeZone, - ]; + if (columnBlockShow.customproperty) { + tmpRow = [...Array(1).fill('')]; row = row.concat(tmpRow); + } - if (columnBlockShow.compositetrigger) { - tmpRow = Array(8).fill(''); - row = row.concat(tmpRow); - } + // Include general event columns + tmpRow = [eventCount, '', '', '', '', '', '']; + row = row.concat(tmpRow); - taskTable = taskTable.concat([row]); + if (columnBlockShow.schematrigger) { + // Add empty columns for schema event info + tmpRow = [...Array(7).fill('')]; + row = row.concat(tmpRow); + } - eventCount += 1; + // Is it a reload task or external program task? + if (rule.reloadTask) { + tmpRow = [ + '', + '', + '', + '', + ruleCount, + mapRuleState.get(rule.ruleState), + rule.reloadTask.name, + rule.reloadTask.id, + ]; + } else if (rule.externalProgramTask) { + tmpRow = [ + '', + '', + '', + '', + ruleCount, + mapRuleState.get(rule.ruleState), + rule.externalProgramTask.name, + rule.externalProgramTask.id, + ]; } - } - // Write composite events to table - if (columnBlockShow.compositetrigger) { - for (const event of compositeEventsForThisTask) { - row = [taskCount, '']; - - if (columnBlockShow.common) { - tmpRow = [...Array(10).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.lastexecution) { - tmpRow = [...Array(5).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.tag) { - tmpRow = [...Array(1).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.customproperty) { - tmpRow = [...Array(1).fill('')]; - row = row.concat(tmpRow); - } - - // Include general event columns - tmpRow = [eventCount, mapEventType.get(event.compositeEvent.eventType)]; - row = row.concat(tmpRow); + row = row.concat(tmpRow); - if (columnBlockShow.compositetrigger) { - tmpRow = [ - event.compositeEvent.name, - event.compositeEvent.enabled, - event.compositeEvent.createdDate, - event.compositeEvent.modifiedDate, - event.compositeEvent.modifiedByUserName, - ]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.schematrigger) { - tmpRow = [...Array(7).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.compositetrigger) { - // Composite task time constraints - tmpRow = [ - event.compositeEvent.timeConstraint.seconds, - event.compositeEvent.timeConstraint.minutes, - event.compositeEvent.timeConstraint.hours, - event.compositeEvent.timeConstraint.days, - '', - '', - '', - '', - ]; - row = row.concat(tmpRow); - } - - taskTable = taskTable.concat([row]); - - // Add all composite rules to table - let ruleCount = 1; - - for (const rule of event.compositeEvent.compositeRules) { - row = [taskCount, '']; - - if (columnBlockShow.common) { - tmpRow = [...Array(10).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.lastexecution) { - tmpRow = [...Array(5).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.tag) { - tmpRow = [...Array(1).fill('')]; - row = row.concat(tmpRow); - } - - if (columnBlockShow.customproperty) { - tmpRow = [...Array(1).fill('')]; - row = row.concat(tmpRow); - } - - // Include general event columns - tmpRow = [eventCount, '', '', '', '', '', '']; - row = row.concat(tmpRow); - - if (columnBlockShow.schematrigger) { - // Add empty columns for schema event info - tmpRow = [...Array(7).fill('')]; - row = row.concat(tmpRow); - } - - // Is it a reload task or external program task? - if (rule.reloadTask) { - tmpRow = [ - '', - '', - '', - '', - ruleCount, - mapRuleState.get(rule.ruleState), - rule.reloadTask.name, - rule.reloadTask.id, - ]; - } else if (rule.externalProgramTask) { - tmpRow = [ - '', - '', - '', - '', - ruleCount, - mapRuleState.get(rule.ruleState), - rule.externalProgramTask.name, - rule.externalProgramTask.id, - ]; - } - - row = row.concat(tmpRow); - - taskTable = taskTable.concat([row]); - - ruleCount += 1; - } - - eventCount += 1; - } + taskTable = taskTable.concat([row]); + + ruleCount += 1; } - taskCount += 1; - } else { - logger.debug(`Skipped task "${task.taskName}" due to incorrect task type`); + eventCount += 1; } } - // Add column headers - let headerRow = ['Task counter', 'Task type']; + taskCount += 1; + } else { + logger.debug(`Skipped task "${task.taskName}" due to incorrect task type`); + } + } - if (columnBlockShow.common) { - headerRow = headerRow.concat([ - 'Task name', - 'Task id', - 'Task enabled', - 'Task timeout', - 'Task retries', - 'App id', - 'Partial reload', - 'Manually triggered', - 'Ext program path', - 'Ext program parameters', - ]); - } + // Add column headers + let headerRow = ['Task counter', 'Task type']; + + if (columnBlockShow.common) { + headerRow = headerRow.concat([ + 'Task name', + 'Task id', + 'Task enabled', + 'Task timeout', + 'Task retries', + 'App id', + 'Partial reload', + 'Manually triggered', + 'Ext program path', + 'Ext program parameters', + ]); + } - if (columnBlockShow.lastexecution) { - headerRow = headerRow.concat(['Task status', 'Task started', 'Task ended', 'Task duration', 'Task executedon node']); - } + if (columnBlockShow.lastexecution) { + headerRow = headerRow.concat(['Task status', 'Task started', 'Task ended', 'Task duration', 'Task executedon node']); + } - if (columnBlockShow.tag) { - headerRow = headerRow.concat(['Tags']); - } + if (columnBlockShow.tag) { + headerRow = headerRow.concat(['Tags']); + } - if (columnBlockShow.customproperty) { - headerRow = headerRow.concat(['Custom properties']); - } + if (columnBlockShow.customproperty) { + headerRow = headerRow.concat(['Custom properties']); + } - if (columnBlockShow.schematrigger || columnBlockShow.compositetrigger) { - headerRow = headerRow.concat([ - 'Event counter', - 'Event type', - 'Event name', - 'Event enabled', - 'Event created date', - 'Event modified date', - 'Event modified by', - ]); - } + if (columnBlockShow.schematrigger || columnBlockShow.compositetrigger) { + headerRow = headerRow.concat([ + 'Event counter', + 'Event type', + 'Event name', + 'Event enabled', + 'Event created date', + 'Event modified date', + 'Event modified by', + ]); + } - if (columnBlockShow.schematrigger) { - headerRow = headerRow.concat([ - 'Schema increment option', - 'Schema increment description', - 'Daylight savings time', - 'Schema start', - 'Schema expiration', - 'Schema filter description', - 'Schema time zone', - ]); - } + if (columnBlockShow.schematrigger) { + headerRow = headerRow.concat([ + 'Schema increment option', + 'Schema increment description', + 'Daylight savings time', + 'Schema start', + 'Schema expiration', + 'Schema filter description', + 'Schema time zone', + ]); + } - if (columnBlockShow.compositetrigger) { - headerRow = headerRow.concat([ - 'Time contstraint seconds', - 'Time contstraint minutes', - 'Time contstraint hours', - 'Time contstraint days', - 'Rule counter', - 'Rule state', - 'Rule task name', - 'Rule task id', - ]); + if (columnBlockShow.compositetrigger) { + headerRow = headerRow.concat([ + 'Time contstraint seconds', + 'Time contstraint minutes', + 'Time contstraint hours', + 'Time contstraint days', + 'Rule counter', + 'Rule state', + 'Rule task name', + 'Rule task id', + ]); + } + + consoleTableConfig.header = { + alignment: 'left', + content: `# reload tasks: ${taskTable.filter((task) => task[1] === 'Reload').length}, # external program tasks: ${taskTable.filter((task) => task[1] === 'External program').length}, # rows in table: ${taskTable.length}`, + }; + + taskTable.unshift(headerRow); + + if (options.outputDest === 'screen') { + logger.info(`# rows in table: ${taskTable.length - 1}`); + logger.info(`# reload tasks in table: ${taskTable.filter((task) => task[1] === 'Reload').length}`); + logger.info(`# external program tasks in table: ${taskTable.filter((task) => task[1] === 'External program').length}`); + logger.info(`\n${table(taskTable, consoleTableConfig)}`); + returnValue = true; + } else if (options.outputDest === 'file') { + logger.verbose(`Writing task table to disk file "${options.outputFileName}"`); + let buffer; + if (options.outputFileFormat === 'excel') { + // Save to Excel file + buffer = xlsx.build([{ name: 'Ctrl-Q task export', data: taskTable }]); + } else if (options.outputFileFormat === 'csv') { + // Remove newlines in column names + taskTable[0] = taskTable[0].map((item) => item.replace('\n', ' ')); + + // Create CSV string + buffer = stringify(taskTable); + } else if (options.outputFileFormat === 'json') { + // Remove newlines in column names + taskTable[0] = taskTable[0].map((item) => item.replace('\n', ' ')); + + // Format JSON nicely + buffer = JSON.stringify(taskTable, null, 4); + } + + // Check if file exists + if ((await verifyFileSystemExists(options.outputFileName, true)) === false) { + // File doesn't exist + } else if (!options.outputFileOverwrite) { + // Target file exist. Ask if user wants to overwrite + logger.info(); + const ok = await yesno({ + question: ` Destination file "${options.outputFileName}" exists. Do you want to overwrite it? (y/n)`, + }); + logger.info(); + if (ok === false) { + logger.info('❌ Not overwriting existing output file. Exiting.'); + process.exit(1); } + } else if (options.outputFileOverwrite) { + // File exists and force overwrite is set + logger.info(`❗️ Existing output file will be replaced.`); + } + logger.info(`✅ Writing task table to disk file "${options.outputFileName}".`); + await Fs.writeFile(options.outputFileName, buffer); + returnValue = true; + } + return returnValue; +} - consoleTableConfig.header = { - alignment: 'left', - content: `# reload tasks: ${taskTable.filter((task) => task[1] === 'Reload').length}, # external program tasks: ${ - taskTable.filter((task) => task[1] === 'External program').length - }, # rows in table: ${taskTable.length}`, - }; - - taskTable.unshift(headerRow); - - if (options.outputDest === 'screen') { - logger.info(`# rows in table: ${taskTable.length - 1}`); - logger.info(`# reload tasks in table: ${taskTable.filter((task) => task[1] === 'Reload').length}`); - logger.info(`# external program tasks in table: ${taskTable.filter((task) => task[1] === 'External program').length}`); - logger.info(`\n${table(taskTable, consoleTableConfig)}`); - returnValue = true; - } else if (options.outputDest === 'file') { - logger.verbose(`Writing task table to disk file "${options.outputFileName}"`); - let buffer; - if (options.outputFileFormat === 'excel') { - // Save to Excel file - buffer = xlsx.build([{ name: 'Ctrl-Q task export', data: taskTable }]); - } else if (options.outputFileFormat === 'csv') { - // Remove newlines in column names - taskTable[0] = taskTable[0].map((item) => item.replace('\n', ' ')); - - // Create CSV string - buffer = stringify(taskTable); - } else if (options.outputFileFormat === 'json') { - // Remove newlines in column names - taskTable[0] = taskTable[0].map((item) => item.replace('\n', ' ')); - - // Format JSON nicely - buffer = JSON.stringify(taskTable, null, 4); - } +/** + * Parse and generate a task tree based on the provided options and Qlik Sense tasks. + * + * This function extracts tasks with associated schedules (schema triggers) and constructs + * a task tree. It ensures that each task appears only once in the tree by de-duplicating + * tasks based on their unique identifiers. The function also handles sorting tasks + * alphabetically by their names and adds unscheduled top-level tasks to the tree. + * + * The resulting task tree can be output to either the screen or a file, with support + * for JSON format. + * + * @param {object} options - CLI options that control the output and formatting. + * @param {QlikSenseTasks} qlikSenseTasks - An instance of QlikSenseTasks containing task data. + * @returns {Promise} - Returns true if the task tree was successfully generated + * and output, false otherwise. + */ + +async function parseTree(options, qlikSenseTasks) { + let returnValue = false; + + // Array to keep track of which nodes in task model to visualize + const nodesToVisualize = []; + + const taskModel = qlikSenseTasks.taskNetwork; + let taskTree = []; + + // If no task id or tag filters specified, visualize all nodes in task model + if (!options.taskId && !options.taskTag) { + // No task id filters specified + // Visualize all nodes in task model + logger.verbose('No task id or tag filters specified. Visualizing all nodes in task model.'); + + nodesToVisualize.push(...taskModel.nodes); + } else { + // Task id filters specified. + // Get all task chains the tasks are part of, + // then get the root nodes of each chain. They will be the starting points for the task tree. + + // Array to keep track of root nodes of task chains + const rootNodes = await qlikSenseTasks.getRootNodesFromFilter(); + + // Set nodesToVisualize to root nodes, if there are any + if (rootNodes) { + nodesToVisualize.push(...rootNodes); + } + } - // Check if file exists - if ((await verifyFileSystemExists(options.outputFileName, true)) === false) { - // File doesn't exist - } else if (!options.outputFileOverwrite) { - // Target file exist. Ask if user wants to overwrite - logger.info(); - const ok = await yesno({ - question: ` Destination file "${options.outputFileName}" exists. Do you want to overwrite it? (y/n)`, - }); - logger.info(); - if (ok === false) { - logger.info('❌ Not overwriting existing output file. Exiting.'); - process.exit(1); - } - } else if (options.outputFileOverwrite) { - // File exists and force overwrite is set - logger.info(`❗️ Existing output file will be replaced.`); + // De-duplicate nodesToVisualize array, using id as the key + const nodesToVisualizeUnique = nodesToVisualize.filter((node, index, self) => { + return index === self.findIndex((n) => n.id === node.id); + }); + + // Get all tasks that have a schedule associated with them + // Schedules are represented by "meta nodes" that are linked to the task node in Ctrl-Q's internal data model + // There is one meta-node per schema trigger, meaning that a task with several schema triggers will have several top-level meta nodes. + // We only want the task to show up once in the tree, so we have do de-duplicate the top level task nodes. + const topLevelTasksWithSchemaTriggers = nodesToVisualizeUnique.filter((node) => { + if (node.metaNode && node.metaNodeType === 'schedule') { + return true; + } + // Exclude all non-meta nodes + return false; + }); + + // Remove all duplicates from the topLevelTasksWithSchemaTriggers array. + // Use completeSchemaEvent.reloadTask.id as the key to determine if the task is a duplicate or not. + const topLevelTasksWithSchemaTriggersUnique = topLevelTasksWithSchemaTriggers.filter((task, index, self) => { + // Handle reload tasks, external program tasks + if (task.completeSchemaEvent.reloadTask) { + return index === self.findIndex((t) => t.completeSchemaEvent?.reloadTask?.id === task.completeSchemaEvent.reloadTask.id); + } + if (task.completeSchemaEvent.externalProgramTask) { + return ( + index === + self.findIndex((t) => t.completeSchemaEvent?.externalProgramTask?.id === task.completeSchemaEvent.externalProgramTask.id) + ); + } + return false; + }); + + // Sort the array alfabetically, using the task name as the key + // Task name is found in either completeSchemaEvent.externalProgramTask.name or completeSchemaEvent.reloadTask.name + topLevelTasksWithSchemaTriggersUnique.sort((a, b) => { + if (a.completeSchemaEvent.reloadTask) { + if (b.completeSchemaEvent.reloadTask) { + if (a.completeSchemaEvent.reloadTask.name < b.completeSchemaEvent.reloadTask.name) { + return -1; } - logger.info(`✅ Writing task table to disk file "${options.outputFileName}".`); - await Fs.writeFile(options.outputFileName, buffer); - returnValue = true; + if (a.completeSchemaEvent.reloadTask.name > b.completeSchemaEvent.reloadTask.name) { + return 1; + } + } else if (b.completeSchemaEvent.externalProgramTask) { + if (a.completeSchemaEvent.reloadTask.name < b.completeSchemaEvent.externalProgramTask.name) { + return -1; + } + if (a.completeSchemaEvent.reloadTask.name > b.completeSchemaEvent.externalProgramTask.name) { + return 1; + } + } + } + if (a.completeSchemaEvent.externalProgramTask) { + if (b.completeSchemaEvent.externalProgramTask) { + if (a.completeSchemaEvent.externalProgramTask.name < b.completeSchemaEvent.externalProgramTask.name) { + return -1; + } + if (a.completeSchemaEvent.externalProgramTask.name > b.completeSchemaEvent.externalProgramTask.name) { + return 1; + } + } else if (b.completeSchemaEvent.reloadTask) { + if (a.completeSchemaEvent.externalProgramTask.name < b.completeSchemaEvent.reloadTask.name) { + return -1; + } + if (a.completeSchemaEvent.externalProgramTask.name > b.completeSchemaEvent.reloadTask.name) { + return 1; + } + } + } + return 0; + }); + + for (const task of topLevelTasksWithSchemaTriggersUnique) { + if (task.metaNode && task.metaNodeType === 'schedule') { + const subTree = await qlikSenseTasks.getTaskSubTree(task, 0, null); + // Ensure subTree is not empty or null + if (!subTree || subTree.length === 0) { + logger.error(`Failed to get sub tree for task "${task.taskName}"`); + return false; } + + subTree[0].isTopLevelNode = true; + subTree[0].isScheduled = true; + taskTree = taskTree.concat(subTree); + } + } + + // Add new top level node with clock/scheduler emoji, if tree icons are enabled + if (options.treeIcons) { + taskTree = [{ text: '⏰ --==| Scheduled tasks |==--', children: taskTree, isTreeLabel: true }]; + } else { + taskTree = [{ text: '--==| Scheduled tasks |==--', children: taskTree, isTreeLabel: true }]; + } + + // Add unscheduled tasks that are also top level tasks. + const unscheduledTasks = nodesToVisualizeUnique.filter((node) => { + if (!node.metaNode && node.isTopLevelNode) { + const a = !taskTree.some((el) => { + const b = el.taskId === node.id; + return b; + }); + return a; } - return returnValue; - } catch (err) { - catchLog('Get task', err); + // Don't include meta nodes return false; + }); + + // Sort unscheduled tasks alfabetically + // Use taskName as the key + unscheduledTasks.sort((a, b) => { + if (a.taskName < b.taskName) { + return -1; + } + if (a.taskName > b.taskName) { + return 1; + } + return 0; + }); + + for (const task of unscheduledTasks) { + const subTree = await qlikSenseTasks.getTaskSubTree(task, 0, null); + // Ensure subTree is not empty or null + if (!subTree || subTree.length === 0) { + logger.error(`Failed to get sub tree for task "${task.taskName}"`); + return false; + } + + subTree[0].isTopLevelNode = true; + subTree[0].isScheduled = false; + taskTree = taskTree.concat(subTree); + } + + // Output task tree to correct destination + if (options.outputDest === 'screen') { + logger.info(``); + // Calculate number of top-level nodes in tree. This is the sum of: + // - For all nodes where isTreeLabel is true, count number of children + // - Number of root nodes where isTreeLabel is false or undefined + let topLevelNodeCount = 0; + for (const node of taskTree) { + if (node.isTreeLabel) { + topLevelNodeCount += node.children.length; + } else { + topLevelNodeCount += 1; + } + } + logger.info(`# top-level rows in tree: ${topLevelNodeCount}`); + logger.info(`\n${tree(taskTree)}`); + returnValue = true; + } else if (options.outputDest === 'file') { + logger.verbose(`Writing task tree to disk file "${options.outputFileName}"`); + let buffer; + if (options.outputFileFormat === 'json') { + // Only keep the text and children properties of the tree object + cleanupTaskTree(taskTree); + + // Format JSON nicely + buffer = JSON.stringify(taskTree, null, 4); + } else { + logger.error(`Output file format "${options.outputFileFormat}" not supported for task trees. Exiting.`); + process.exit(1); + } + + // Check if file exists + if ((await verifyFileSystemExists(options.outputFileName, true)) === false) { + // File doesn't exist + } else if (!options.outputFileOverwrite) { + // Target file exist. Ask if user wants to overwrite + logger.info(); + const ok = await yesno({ + question: ` Destination file "${options.outputFileName}" exists. Do you want to overwrite it? (y/n)`, + }); + logger.info(); + if (ok === false) { + logger.info('❌ Not overwriting existing output file. Exiting.'); + process.exit(1); + } + } else if (options.outputFileOverwrite) { + // File exists and force overwrite is set + logger.info(`❗️ Existing output file will be replaced.`); + } + logger.info(`✅ Writing task tree to disk file "${options.outputFileName}".`); + await Fs.writeFile(options.outputFileName, buffer); + returnValue = true; } + return returnValue; } diff --git a/src/lib/cmd/qseow/vistask.js b/src/lib/cmd/qseow/vistask.js index 3e8d425a..47962212 100644 --- a/src/lib/cmd/qseow/vistask.js +++ b/src/lib/cmd/qseow/vistask.js @@ -7,6 +7,8 @@ import sea from 'node:sea'; import { appVersion, logger, setLoggingLevel, isSea, execPath, verifyFileSystemExists, verifySeaAssetExists } from '../../../globals.js'; import { QlikSenseTasks } from '../../task/class_alltasks.js'; +import { findCircularTaskChains } from '../../task/find_circular_task_chain.js'; +import { catchLog } from '../../util/log.js'; // js: 'application/javascript', const MIME_TYPES = { @@ -240,18 +242,46 @@ const prepareFile = async (url) => { const template = handlebars.compile(file, { noEscape: true }); + // Debug logging of task network, which consists of three properties: + // 1. nodes: Array of nodes + // 2. edges: Array of edges + // 3. tasks: Array of tasks + logger.debug(`Tasks found: ${taskNetwork?.tasks?.length}`); + for (const task of taskNetwork.tasks) { + // Log task type + if (task.metaNode === false) { + logger.debug(`Task: [${task.id}] - "${task.taskName}"`); + } else { + logger.debug(`Meta node: [${task.id}], Meta node type: ${task.metaNodeType}`); + } + } + + // Log nodes + logger.debug(`Nodes found: ${taskNetwork?.nodes?.length}`); + for (const node of taskNetwork.nodes) { + if (node.metaNode === true) { + logger.debug(`Meta node: [${node.id}] - "${node.label}"`); + } else { + logger.debug(`Task node: [${node.id}] - "${node.label}"`); + } + } + + // Log edges + logger.debug(`Edges found: ${taskNetwork?.edges?.length}`); + for (const edge of taskNetwork.edges) { + logger.debug(`Edge: ${JSON.stringify(edge)}`); + } + // Get task network model const taskModel = taskNetwork; // Add schema nodes const nodes = taskModel.nodes.filter((node) => node.metaNode === true); - // let nodes = taskModel.nodes.filter((node) => node.metaNodeType === 'schedule'); let nodesNetwork = nodes.map((node) => { const newNode = {}; if (node.metaNodeType === 'schedule') { newNode.id = node.id; newNode.label = node.label; - // newNode.title = node.label; newNode.title = `Schema trigger
Name: ${node.label}
Enabled: ${ node.enabled }
Schema: ${getSchemaText( @@ -265,10 +295,10 @@ const prepareFile = async (url) => { node.completeSchemaEvent.operational.timesTriggered }`; newNode.shape = 'triangle'; - // newNode.icon = { face: 'fontawesome', code: '\uf017' }; newNode.color = node.enabled ? '#FFA807' : '#BCB9BF'; // Needed to distinguish real tasks from meta tasks in the network diagram newNode.isReloadTask = false; + newNode.nodeType = 'scheduleTrigger'; } else if (node.metaNodeType === 'composite') { newNode.id = node.id; newNode.label = node.label; @@ -277,10 +307,10 @@ const prepareFile = async (url) => { newNode.color = '#FFA807'; // Needed to distinguish real tasks from meta tasks in the network diagram newNode.isReloadTask = false; + newNode.nodeType = 'compositeTrigger'; } else { logger.error(`Huh? That's an unknown meta node type: ${node.metaNodeType}`); } - // task.color = task.schemaEvent.enabled ? '#FFA807' : '#BCB9BF'; return newNode; }); @@ -298,17 +328,18 @@ const prepareFile = async (url) => { // Reload task newNode.title = `Reload task
Name: ${node.taskName}
Task ID: ${node.taskId}
Enabled: ${node.taskEnabled}
App: ${node.appName}
Last exec status: ${node.taskLastStatus}
Last exec start: ${node.taskLastExecutionStartTimestamp}
Last exec stop: ${node.taskLastExecutionStopTimestamp}`; newNode.shape = 'box'; + newNode.nodeType = 'reloadTask'; } else if (node.taskType === 1) { // External program task newNode.title = `Ext. program task
Name: ${node.taskName}
Task ID: ${node.taskId}
Enabled: ${node.taskEnabled}
Last exec status: ${node.taskLastStatus}
Last exec start: ${node.taskLastExecutionStartTimestamp}
Last exec stop: ${node.taskLastExecutionStopTimestamp}`; newNode.shape = 'ellipse'; + newNode.nodeType = 'externalProgramTask'; } // Needed to distinguish real tasks from meta tasks in the network diagram newNode.isReloadTask = true; - // newNode.color = node.taskEnabled ? '#FFA807' : '#BCB9BF'; - if (node.taskLastStatus === 'NeverStarted') { + if (node.taskLastStatus === 'NeverStarted' || node.taskLastStatus === '?') { newNode.color = '#999'; } else if (node.taskLastStatus === 'Triggered' || node.taskLastStatus === 'Queued') { newNode.color = '#6cf'; @@ -326,7 +357,7 @@ const prepareFile = async (url) => { } else if (node.taskLastStatus === 'FinishedSuccess') { newNode.color = '#21ff06'; } else if (node.taskLastStatus === 'Skipped') { - newNode.color = '#6cf'; + newNode.color = '#ffcc00'; } newNode.taskLastStatus = node.taskLastStatus; return newNode; @@ -407,7 +438,7 @@ const requestHandler = async (req, res) => { } }; -// Set up http server for serviing html pages with the task visualization +// Set up http server for serving html pages with the task visualization const startHttpServer = async (options) => { const server = http.createServer(requestHandler); @@ -419,6 +450,12 @@ const startHttpServer = async (options) => { }); }; +/** + * Start an HTTP server for visualizing QSEoW tasks as a network diagram. + * + * @param {Object} options - Options for the visTask function. + * @returns {Promise} - A promise that resolves to true if the server was started successfully, false otherwise. + */ export async function visTask(options) { // Set log level setLoggingLevel(options.logLevel); @@ -492,23 +529,120 @@ export async function visTask(options) { logger.error('Failed to get task model from QSEoW'); return false; } - taskNetwork = qlikSenseTasks.taskNetwork; + + // Filter tasks based on CLI options. Two possible + // 1. If no filters specified, show all tasks. + // 2. At least one filter specified. + // - If --task-id specified + // - Get root task(s) for each specified task id + // - Include in network diagram all tasks that are children of the root tasks + // - If --task-tag specified + // - Get all tasks that have the specified tag(s) + // - Get root task(s) for each task that has the specified tag(s)§ + // - If --app-id specified + // - Get all tasks that are associated with the specified app id(s) + // - Get root task(s) for each task that is associated with the specified app id(s) + // - Include in network diagram all tasks that are children of the root tasks + // - If --app-tag specified + // - Get all apps that are associated with the specified app tag(s) + // - Get all tasks that are associated with the apps that have the specified app tag(s) + // - Get root task(s) for each task that is associated with the apps that have the specified app tag(s) + // - Include in network diagram all tasks that are children of the root tasks + // + // Filters above are additive, i.e. all tasks that match any of the filters are included in the network diagram. + // Make sure to de-duplicate root tasks. + + // If no task id or tag filters specified, visualize all nodes in task model + if (!options.taskId && !options.taskTag) { + // No task id filters specified + // Visualize all nodes in task model + logger.verbose('No task id or tag filters specified. Visualizing all nodes in task model.'); + + taskNetwork = qlikSenseTasks.taskNetwork; + } else { + // Task id filters specified. + // Get all task chains the tasks are part of, + // then get the rMeta nodeoot nodes of each chain. They will be the starting points for the task tree. + + // Array to keep track of root nodes of task chains + const rootNodes = await qlikSenseTasks.getRootNodesFromFilter(); + + // List root nodes to console + logger.verbose(`${rootNodes.length} root nodes sent to visualizer:`); + rootNodes.forEach((node) => { + // Meta node? + if (node.metaNode === true) { + // Reload task? + if (node.taskType === 'reloadTask') { + logger.verbose( + `Meta node: metanode type=${node.metaNodeType} id=[${node.id}] task type=${node.taskType} task name="${node.completeSchemaEvent.reloadTask.name}"` + ); + } + } else { + logger.verbose(`Root node: [${node.id}] "${node.taskName}"`); + } + }); + + // Get all nodes that are children of the root nodes + const { nodes, edges, tasks } = await qlikSenseTasks.getNodesAndEdgesFromRootNodes(rootNodes); + + taskNetwork = { nodes, edges, tasks }; + } + + // Look for circular task chains in the task network + logger.info(''); + logger.info('Looking for circular task chains in the task network'); + + try { + const circularTaskChains = findCircularTaskChains(taskNetwork, logger); + + // Errros? + if (circularTaskChains === false) { + return false; + } + + // De-duplicate circular task chains (where fromTask.id and toTask.id matches in two different chains). + const deduplicatedCircularTaskChain = circularTaskChains.filter((chain, index, self) => { + return self.findIndex((c) => c.fromTask.id === chain.fromTask.id && c.toTask.id === chain.toTask.id) === index; + }); + + // Log circular task chains, if any were found. + if (deduplicatedCircularTaskChain?.length > 0) { + logger.warn(''); + logger.warn(`Found ${deduplicatedCircularTaskChain.length} circular task chains in task model`); + for (const chain of deduplicatedCircularTaskChain) { + logger.warn(`Circular task chain:`); + + logger.warn(` From task : [${chain.fromTask.id}] "${chain.fromTask.taskName}"`); + logger.warn(` To task : [${chain.toTask.id}] "${chain.toTask.taskName}"`); + } + } else { + logger.info('No circular task chains found in task model'); + } + } catch (error) { + catchLog('FIND CIRCULAR TASK CHAINS', error); + return false; + } // Add additional values to Handlebars template templateData.visTaskHost = options.visHost; templateData.visTaskPort = options.visPort; // Get reload task count, i.e. tasks where taskType === 0 - templateData.reloadTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 0).length; + // templateData.reloadTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 0).length; + templateData.reloadTaskCount = taskNetwork.tasks.filter((task) => task.taskType === 0).length; // Get external program task count, i.e. tasks where taskType === 1 - templateData.externalProgramTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 1).length; + // templateData.externalProgramTaskCount = qlikSenseTasks.taskList.filter((task) => task.taskType === 1).length; + templateData.externalProgramTaskCount = taskNetwork.tasks.filter((task) => task.taskType === 1).length; // Get schema trigger count - templateData.schemaTriggerCount = qlikSenseTasks.qlikSenseSchemaEvents.schemaEventList.length; + // Count taskNetwork.nodes events where metaNodeType === 'schedule' + templateData.schemaTriggerCount = taskNetwork.nodes.filter((node) => node.metaNodeType === 'schedule').length; // Get composite trigger count - templateData.compositeTaskCount = qlikSenseTasks.qlikSenseCompositeEvents.compositeEventList.length; + // Count taskNetwork.nodes events where metaNodeType === 'composite' + templateData.compositeTaskCount = taskNetwork.nodes.filter((node) => node.metaNodeType === 'composite').length; startHttpServer(optionsNew); return true; diff --git a/src/lib/task/class_alltasks.js b/src/lib/task/class_alltasks.js index f9df8d75..5a507449 100644 --- a/src/lib/task/class_alltasks.js +++ b/src/lib/task/class_alltasks.js @@ -1,26 +1,24 @@ import axios from 'axios'; -import { v4 as uuidv4, validate } from 'uuid'; + import { logger } from '../../globals.js'; import { setupQrsConnection } from '../util/qseow/qrs.js'; - -import { - mapTaskType, - mapDaylightSavingTime, - mapEventType, - mapIncrementOption, - mapRuleState, - getTaskColumnPosFromHeaderRow, -} from '../util/qseow/lookups.js'; - import { QlikSenseTask } from './class_task.js'; import { QlikSenseSchemaEvents } from './class_allschemaevents.js'; import { QlikSenseCompositeEvents } from './class_allcompositeevents.js'; -import { getTagIdByName } from '../util/qseow/tag.js'; -import { getCustomPropertyIdByName } from '../util/qseow/customproperties.js'; -import { getAppById } from '../util/qseow/app.js'; -import { taskExistById, getTaskById } from '../util/qseow/task.js'; import { catchLog } from '../util/log.js'; import { getCertFilePaths } from '../util/qseow/cert.js'; +import { extParseReloadTask } from './parse_reload_task.js'; +import { extParseExternalProgramTask } from './parse_ext_program_task.js'; +import { extGetTaskModelFromQseow } from './get_task_model_from_qseow.js'; +import { extGetTasksFromQseow } from './get_tasks_from_qseow.js'; +import { extGetTaskSubGraph } from './get_task_sub_graph.js'; +import { extGetTaskSubTree } from './get_task_sub_tree.js'; +import { extGetTaskSubTable } from './get_task_sub_table.js'; +import { extParseSchemaEvents } from './parse_schema_events.js'; +import { extGetTaskModelFromFile } from './get_task_model_from_file.js'; +import { extSaveTaskModelToQseow } from './save_task_model_to_qseow.js'; +import { extGetRootNodesFromFilter } from './get_root_nodes_from_filter.js'; +import { extFindRootNodes } from './find_root_nodes.js'; export class QlikSenseTasks { constructor() { @@ -38,8 +36,11 @@ export class QlikSenseTasks { // Map that will map fake task IDs (used in source file) with real task IDs after tasks have been created in Sense this.taskIdMap = new Map(); - // Data structure to keep track of which up-tree nodes a node is connected to in a task tree - this.taskTreeCyclicVisited = new Set(); + // Data structure to keep track of which up-tree nodes a node is connected to in a task tree or network + this.taskCyclicStack = new Set(); + + // Data structure to keep track of which nodes have been visited when looking for root nodes + this.nodeRootCyclicVisited = new Set(); if (options.authType === 'cert') { // Get certificate paths @@ -60,17 +61,6 @@ export class QlikSenseTasks { } } - // Function to determine if a task tree is cyclic - isTaskTreeCyclic(task) { - if (this.taskTreeCyclicVisited.has(task)) { - return true; - } - - this.taskTreeCyclicVisited.add(task); - - return false; - } - getTask(taskId) { if (taskId === undefined || taskId === null) { return false; @@ -93,6 +83,27 @@ export class QlikSenseTasks { this.taskList.push(newTask); } + /** + * Recursively find root nodes in a node tree. + * Each node may have one or more upstream nodes. + * When there are no more upstream nodes, the node is a root node. + * + * Upstream nodes are found by following the edges in the task network. + * edge.to === node.id, then look at edge.from to find the upstream node id. + * + * After the final call to this function, the rootNodes array will contain all root nodes. + * There may be duplicates in the array, as the function does not check for duplicates. + * De-deplication should be done after the function has been called. + * + * @param {object} node - Node to start the search from. + * @returns {Array} Array of found root nodes. + */ + findRootNodes(node) { + const result = extFindRootNodes(this, node, logger); + + return result; + } + // Function to parse the rows associated with a specific reload task in the source file // Properties in the param object: // - taskRows: Array of rows associated with the task. All rows associated with the task are passed to this function @@ -109,206 +120,8 @@ export class QlikSenseTasks { // - currentTask: Object containing task data // - taskCreationOption: Task creation option. Possible values: "if-exists-add-another", "if-exists-update-existing" async parseReloadTask(param) { - let currentTask = null; - let taskCreationOption; - - // Create task object using same structure as results from QRS API - - // Determine if the task is associated with an app that existed before Ctrl-Q was started, or - // an app that's been imported as part of this Ctrl-Q execution. - // Possible values for the app ID column: - // - newapp- (app has been imported as part of this Ctrl-Q execution) - // - A real, existing app ID. I.e. the app existed before Ctrl-Q was started. - const appIdRaw = param.taskRows[0][param.taskFileColumnHeaders.appId.pos].trim(); - let appId; - - if (appIdRaw.substring(0, 7).toLowerCase() === 'newapp-') { - // App ID starts with "newapp-". This means the app been imported as part of this Ctrl-Q session - // No guarantee that it is the case though. Maybe no apps were imported, or maybe the app specified for this very task was not imported - - // Have ANY apps been imported? - if (!this.importedApps) { - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: No apps have been imported, but app "${param.taskRows[0][ - param.taskFileColumnHeaders.appId.pos - ].trim()}" has been specified in the task definition file. Exiting.` - ); - process.exit(1); - } - - // Has this specific app been imported? - if (!this.importedApps.appIdMap.has(appIdRaw.toLowerCase())) { - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: App "${param.taskRows[0][ - param.taskFileColumnHeaders.appId.pos - ].trim()}" has not been imported, but has been specified in the task definition file. Exiting.` - ); - process.exit(1); - } - - appId = this.importedApps.appIdMap.get(appIdRaw.toLowerCase()); - - // Ensure the app exists - // Reasons for the app not existing could be: - // - The app was imported but has since been deleted or replaced. This could happen if the app-import step has several - // apps that are published-replaced or deleted-published to the same stream. In that case only the last published app will be present - - if (appId === undefined) { - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Cannot figure out which Sense app "${param.taskRows[0][ - param.taskFileColumnHeaders.appId.pos - ].trim()}" belongs to. App with ID "${appIdRaw}" not found.` - ); - - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: This could be because the app was imported but has since been deleted or replaced, for example during app publishing. Don't know how to proceed, exiting.` - ); - - process.exit(1); - } - - const app = await getAppById(appId, param?.options); - - if (!app) { - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: App with ID "${appId}" not found. This could be because the app was imported but has since been deleted or replaced, for example during app publishing. Don't know how to proceed, exiting.` - ); - process.exit(1); - } - } else if (validate(appIdRaw)) { - // App ID is a proper UUID. We don't know if the app actually exists though. - - const app = await getAppById(appIdRaw, param?.options); - - if (!app) { - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: App with ID "${appIdRaw}" not found. This could be because the app was imported but has since been deleted or replaced, for example during app publishing. Don't know how to proceed, exiting.` - ); - process.exit(1); - } - - appId = appIdRaw; - } else { - logger.error(`(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Incorrect app ID "${appIdRaw}". Exiting.`); - process.exit(1); - } - - if (param.taskFileColumnHeaders.importOptions.pos === 999) { - // No task creation options column in the file - // Use the default task creation option - taskCreationOption = 'if-exists-update-existing'; - } else { - // Task creation options column exists in the file - // Use the value from the file - taskCreationOption = param.taskRows[0][param.taskFileColumnHeaders.importOptions.pos]; - } - - // Ensure task creation option is valid. Allow empty option - if ( - taskCreationOption && - taskCreationOption.trim() !== '' && - !['if-exists-add-another', 'if-exists-update-existing'].includes(taskCreationOption) - ) { - logger.error( - `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.` - ); - process.exit(1); - } - - currentTask = { - id: param.taskRows[0][param.taskFileColumnHeaders.taskId.pos], - name: param.taskRows[0][param.taskFileColumnHeaders.taskName.pos], - taskType: mapTaskType.get(param.taskRows[0][param.taskFileColumnHeaders.taskType.pos]), - enabled: param.taskRows[0][param.taskFileColumnHeaders.taskEnabled.pos], - taskSessionTimeout: param.taskRows[0][param.taskFileColumnHeaders.taskSessionTimeout.pos], - maxRetries: param.taskRows[0][param.taskFileColumnHeaders.taskMaxRetries.pos], - isManuallyTriggered: param.taskRows[0][param.taskFileColumnHeaders.isManuallyTriggered.pos], - isPartialReload: param.taskRows[0][param.taskFileColumnHeaders.isPartialReload.pos], - app: { - id: appId, - // name: taskData[0][taskFileColumnHeaders.appName.pos], - }, - tags: [], - customProperties: [], - schemaPath: 'ReloadTask', - schemaEvents: [], - compositeEvents: [], - prelCompositeEvents: [], - }; - - // Add tags to task object - if (param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos]) { - const tmpTags = param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos] - .split('/') - .filter((item) => item.trim().length !== 0) - .map((item) => item.trim()); - - for (const item of tmpTags) { - const tagId = await getTagIdByName(item, param.tagsExisting); - currentTask.tags.push({ - id: tagId, - name: item, - }); - } - } - - // Add custom properties to task object - if (param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos]) { - const tmpCustomProperties = param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos] - .split('/') - .filter((item) => item.trim().length !== 0) - .map((cp) => cp.trim()); - - for (const item of tmpCustomProperties) { - const tmpCustomProperty = item - .split('=') - .filter((item2) => item2.trim().length !== 0) - .map((cp) => cp.trim()); - - // Do we have two items in the array? First is the custom property name, second is the value - if (tmpCustomProperty?.length === 2) { - const customPropertyId = getCustomPropertyIdByName('ReloadTask', tmpCustomProperty[0], param.cpExisting); - - // If previous call returned false, it means the custom property does not exist in Sense - // or cannot be used with this task type. In that case, skip it. - if (customPropertyId) { - currentTask.customProperties.push({ - definition: { - id: customPropertyId, - name: tmpCustomProperty[0].trim(), - }, - value: tmpCustomProperty[1].trim(), - }); - } - } - } - } - - // Get schema events for this task, storing the info using the same structure as returned from QRS API - currentTask.schemaEvents = this.parseSchemaEvents({ - taskType: 'reload', - taskRows: param.taskRows, - taskFileColumnHeaders: param.taskFileColumnHeaders, - taskCounter: param.taskCounter, - currentTask, - fakeTaskId: param.fakeTaskId, - nodesWithEvents: param.nodesWithEvents, - options: param?.options, - }); - - // Get composite events for this task - currentTask.prelCompositeEvents = await this.parseCompositeEvents({ - taskType: 'reload', - taskRows: param.taskRows, - taskFileColumnHeaders: param.taskFileColumnHeaders, - taskCounter: param.taskCounter, - currentTask, - fakeTaskId: param.fakeTaskId, - nodesWithEvents: param.nodesWithEvents, - options: param?.options, - }); - - return { currentTask, taskCreationOption }; + const result = await extParseReloadTask(this, param, logger); + return result; } // Function to parse the rows associated with a specific external program task in the source file @@ -327,127 +140,8 @@ export class QlikSenseTasks { // - currentTask: Object containing task data // - taskCreationOption: Task creation option. Possible values: "if-exists-add-another", "if-exists-update-existing" async parseExternalProgramTask(param) { - let currentTask = null; - let taskCreationOption; - - // Create task object using same structure as results from QRS API - - // Get task import options - if (param.taskFileColumnHeaders.importOptions.pos === 999) { - // No task creation options column in the file - // Use the default task creation option - taskCreationOption = 'if-exists-update-existing'; - } else { - // Task creation options column exists in the file - // Use the value from the file - taskCreationOption = param.taskRows[0][param.taskFileColumnHeaders.importOptions.pos]; - } - - // Ensure task creation option is valid. Allow empty option - if ( - taskCreationOption && - taskCreationOption.trim() !== '' && - !['if-exists-add-another', 'if-exists-update-existing'].includes(taskCreationOption) - ) { - logger.error( - `(${param.taskCounter}) PARSE EXTERNAL PROGRAM TASK FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.` - ); - process.exit(1); - } - - currentTask = { - id: param.taskRows[0][param.taskFileColumnHeaders.taskId.pos], - name: param.taskRows[0][param.taskFileColumnHeaders.taskName.pos], - taskType: mapTaskType.get(param.taskRows[0][param.taskFileColumnHeaders.taskType.pos]), - enabled: param.taskRows[0][param.taskFileColumnHeaders.taskEnabled.pos], - taskSessionTimeout: param.taskRows[0][param.taskFileColumnHeaders.taskSessionTimeout.pos], - maxRetries: param.taskRows[0][param.taskFileColumnHeaders.taskMaxRetries.pos], - - path: param.taskRows[0][param.taskFileColumnHeaders.extPgmPath.pos], - parameters: param.taskRows[0][param.taskFileColumnHeaders.extPgmParam.pos], - - tags: [], - customProperties: [], - - schemaPath: 'ExternalProgramTask', - schemaEvents: [], - compositeEvents: [], - prelCompositeEvents: [], - }; - - // Add tags to task object - if (param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos]) { - const tmpTags = param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos] - .split('/') - .filter((item) => item.trim().length !== 0) - .map((item) => item.trim()); - - for (const item of tmpTags) { - const tagId = await getTagIdByName(item, param.tagsExisting); - currentTask.tags.push({ - id: tagId, - name: item, - }); - } - } - - // Add custom properties to task object - if (param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos]) { - const tmpCustomProperties = param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos] - .split('/') - .filter((item) => item.trim().length !== 0) - .map((cp) => cp.trim()); - - for (const item of tmpCustomProperties) { - const tmpCustomProperty = item - .split('=') - .filter((item2) => item2.trim().length !== 0) - .map((cp) => cp.trim()); - - // Do we have two items in the array? First is the custom property name, second is the value - if (tmpCustomProperty?.length === 2) { - const customPropertyId = getCustomPropertyIdByName('ExternalProgramTask', tmpCustomProperty[0], param.cpExisting); - - // If previous call returned false, it means the custom property does not exist in Sense - // or cannot be used with this task type. In that case, skip it. - if (customPropertyId) { - currentTask.customProperties.push({ - definition: { - id: customPropertyId, - name: tmpCustomProperty[0].trim(), - }, - value: tmpCustomProperty[1].trim(), - }); - } - } - } - } - - // Get schema events for this task, storing the info using the same structure as returned from QRS API - currentTask.schemaEvents = this.parseSchemaEvents({ - taskType: 'external program', - taskRows: param.taskRows, - taskFileColumnHeaders: param.taskFileColumnHeaders, - taskCounter: param.taskCounter, - currentTask, - fakeTaskId: param.fakeTaskId, - nodesWithEvents: param.nodesWithEvents, - options: param?.options, - }); - - // Get composite events for this task - currentTask.prelCompositeEvents = await this.parseCompositeEvents({ - taskType: 'external program', - taskRows: param.taskRows, - taskFileColumnHeaders: param.taskFileColumnHeaders, - taskCounter: param.taskCounter, - currentTask, - fakeTaskId: param.fakeTaskId, - nodesWithEvents: param.nodesWithEvents, - options: param?.options, - }); - - return { currentTask, taskCreationOption }; + const result = await extParseExternalProgramTask(this, param, logger); + return result; } // Function to get schema events for a specific task @@ -461,102 +155,8 @@ export class QlikSenseTasks { // - nodesWithEvents: Set of nodes that have associated events // - options: CLI options parseSchemaEvents(param) { - // Get schema events for this task, storing the info using the same structure as returned from QRS API - const prelSchemaEvents = []; - - const schemaEventRows = param.taskRows.filter( - (item) => - item[param.taskFileColumnHeaders.eventType.pos] && - item[param.taskFileColumnHeaders.eventType.pos].trim().toLowerCase() === 'schema' - ); - if (!schemaEventRows || schemaEventRows?.length === 0) { - logger.verbose(`(${param.taskCounter}) PARSE SCHEMA EVENT: No schema events for task "${param.currentTask.name}"`); - } else { - logger.verbose( - `(${param.taskCounter}) PARSE SCHEMA EVENT: ${schemaEventRows.length} schema event(s) for task "${param.currentTask.name}"` - ); - - // Add schema edges and start/trigger nodes - for (const schemaEventRow of schemaEventRows) { - // Create object using same format that Sense uses for schema events - const schemaEvent = { - enabled: schemaEventRow[param.taskFileColumnHeaders.eventEnabled.pos], - eventType: mapEventType.get(schemaEventRow[param.taskFileColumnHeaders.eventType.pos]), - name: schemaEventRow[param.taskFileColumnHeaders.eventName.pos], - daylightSavingTime: mapDaylightSavingTime.get(schemaEventRow[param.taskFileColumnHeaders.daylightSavingsTime.pos]), - timeZone: schemaEventRow[param.taskFileColumnHeaders.schemaTimeZone.pos], - startDate: schemaEventRow[param.taskFileColumnHeaders.schemaStart.pos], - expirationDate: schemaEventRow[param.taskFileColumnHeaders.scheamExpiration.pos], - schemaFilterDescription: [schemaEventRow[param.taskFileColumnHeaders.schemaFilterDescription.pos]], - incrementDescription: schemaEventRow[param.taskFileColumnHeaders.schemaIncrementDescription.pos], - incrementOption: mapIncrementOption.get(schemaEventRow[param.taskFileColumnHeaders.schemaIncrementOption.pos]), - schemaPath: 'SchemaEvent', - }; - - if (param.taskType === 'reload') { - schemaEvent.reloadTask = { - id: param.fakeTaskId, - }; - } else if (param.taskType === 'external program') { - schemaEvent.externalProgramTask = { - id: param.fakeTaskId, - }; - } else { - logger.error(`(${param.taskCounter}) PARSE SCHEMA EVENT: Incorrect task type "${param.taskType}". Exiting.`); - process.exit(1); - } - - this.qlikSenseSchemaEvents.addSchemaEvent(schemaEvent); - - // Add schema event to network representation of tasks - // Create an id for this node - const nodeId = `schema-event-${uuidv4()}`; - - // Add schema trigger nodes. These represent the implicit starting nodes that a schema event really are - this.taskNetwork.nodes.push({ - id: nodeId, - metaNodeType: 'schedule', // Meta nodes are not Sense tasks, but rather nodes representing task-like properties (e.g. a starting point for a reload chain) - metaNode: true, - isTopLevelNode: true, - label: schemaEvent.name, - enabled: schemaEvent.enabled, - - completeSchemaEvent: schemaEvent, - }); - - // Add edge from schema trigger node to current task, taking into account task type - if (param.taskType === 'reload') { - this.taskNetwork.edges.push({ - from: nodeId, - to: schemaEvent.reloadTask.id, - }); - - // Keep a note that this node has associated events - param.nodesWithEvents.add(schemaEvent.reloadTask.id); - - // Remove reference to task ID - delete schemaEvent.reloadTask.id; - delete schemaEvent.reloadTask; - } else if (param.taskType === 'external program') { - this.taskNetwork.edges.push({ - from: nodeId, - to: schemaEvent.externalProgramTask.id, - }); - - // Keep a note that this node has associated events - param.nodesWithEvents.add(schemaEvent.externalProgramTask.id); - - // Remove reference to task ID - delete schemaEvent.externalProgramTask.id; - delete schemaEvent.externalProgramTask; - } - - // Add this schema event to the current task - prelSchemaEvents.push(schemaEvent); - } - } - - return prelSchemaEvents; + const result = extParseSchemaEvents(this, param, logger); + return result; } // Function to get composite events for a specific task @@ -572,260 +172,8 @@ export class QlikSenseTasks { // - nodesWithEvents: Set of nodes that have associated events // - options: CLI options async parseCompositeEvents(param) { - // Get all composite events for this task - // - // Composite events - // - Consists of one main row defining the event, followed by one or more rows defining the composite event rules. - // - The main row is followed by one or more rows defining the composite event rules - // - All rows associated with a composite event share the same value in the "Event counter" column - // - Each composite event rule row has a unique value in the "Rule counter" column - const prelCompositeEvents = []; - - // Get all "main rows" of all composite events in this task - const compositeEventRows = param.taskRows.filter( - (item) => - item[param.taskFileColumnHeaders.eventType.pos] && - item[param.taskFileColumnHeaders.eventType.pos].trim().toLowerCase() === 'composite' - ); - if (!compositeEventRows || compositeEventRows?.length === 0) { - logger.verbose(`(${param.taskCounter}) PARSE COMPOSITE EVENT: No composite events for task "${param.currentTask.name}"`); - } else { - logger.verbose( - `(${param.taskCounter}) PARSE COMPOSITE EVENT: ${compositeEventRows.length} composite event(s) for task "${param.currentTask.name}"` - ); - - // Loop over all composite events, adding them and their event rules - for (const compositeEventRow of compositeEventRows) { - // Get value in "Event counter" column for this composite event, then get array of all associated event rules - const compositeEventCounter = compositeEventRow[param.taskFileColumnHeaders.eventCounter.pos]; - const compositeEventRules = param.taskRows.filter( - (item) => - item[param.taskFileColumnHeaders.eventCounter.pos] === compositeEventCounter && - item[param.taskFileColumnHeaders.ruleCounter.pos] > 0 - ); - - // Create an object using same format that the Sense API uses for composite events - // Add task type specific properties in later step - const compositeEvent = { - timeConstraint: { - days: compositeEventRow[param.taskFileColumnHeaders.timeConstraintDays.pos], - hours: compositeEventRow[param.taskFileColumnHeaders.timeConstraintHours.pos], - minutes: compositeEventRow[param.taskFileColumnHeaders.timeConstraintMinutes.pos], - seconds: compositeEventRow[param.taskFileColumnHeaders.timeConstraintSeconds.pos], - }, - compositeRules: [], - name: compositeEventRow[param.taskFileColumnHeaders.eventName.pos], - enabled: compositeEventRow[param.taskFileColumnHeaders.eventEnabled.pos], - eventType: mapEventType.get(compositeEventRow[param.taskFileColumnHeaders.eventType.pos]), - schemaPath: 'CompositeEvent', - }; - - if (param.taskType === 'reload') { - compositeEvent.reloadTask = { - id: param.fakeTaskId, - }; - } else if (param.taskType === 'external program') { - compositeEvent.externalProgramTask = { - id: param.fakeTaskId, - }; - } else { - logger.error(`(${param.taskCounter}) PARSE COMPOSITE EVENT: Incorrect task type "${param.taskType}". Exiting.`); - process.exit(1); - } - - // Add rules - for (const rule of compositeEventRules) { - // Does the upstream task pointed to by the composite rule exist? - // If it *does* exist it means it's a real, existing task in QSEoW that should be used. - // If it is not a valid guid or does not exist, it's (best case) a referefence to some other task in the task definitions file. - // If the task pointed to by the rule doesn't exist in Sense and doesn't point to some other task in the file, an error should be shown. - if (validate(rule[param.taskFileColumnHeaders.ruleTaskId.pos])) { - // The rule points to an valid UUID. It should exist, otherwise it's an error - - const taskExists = await taskExistById(rule[param.taskFileColumnHeaders.ruleTaskId.pos], this.options); - - if (taskExists) { - // Add task ID to mapping table that will be used later when building the composite event data structures - // In this case we're adding a task ID that maps to itself, indicating that it's a task that already exists in QSEoW. - this.taskIdMap.set( - rule[param.taskFileColumnHeaders.ruleTaskId.pos], - rule[param.taskFileColumnHeaders.ruleTaskId.pos] - ); - } else { - // The task pointed to by the composite event rule does not exist - logger.error( - `(${param.taskCounter}) PARSE COMPOSITE EVENT: Task "${ - rule[param.taskFileColumnHeaders.ruleTaskId.pos] - }" does not exist. Exiting.` - ); - process.exit(1); - } - } else { - logger.verbose( - `(${param.taskCounter}) PARSE COMPOSITE EVENT: "${ - rule[param.taskFileColumnHeaders.ruleTaskId.pos] - }" is not a valid UUID` - ); - } - - // Save composite event rule. - // Also add the upstream task id to the correct property in the rule object, depending on task type - - let upstreamTask; - let upstreamTaskExistence; - // First get upstream task type - // Two options: - // 1. The rule's task ID is a valid GUID. Get the associated task's metadata from Sense, if the task exists - // 2. The rule's task ID is not a valid GUID. It's a reference to a task that is created during this execution of Ctrl-Q. - if (!validate(rule[param.taskFileColumnHeaders.ruleTaskId.pos])) { - // The rule's task ID is not a valid GUID. It's a reference to a task that is created during this execution of Ctrl-Q. - // Add the task ID to the mapping table, indicating that it's a task that is created during this execution of Ctrl-Q. - - // // Check if the task ID already exists in the mapping table - // if (this.taskIdMap.has(rule[param.taskFileColumnHeaders.ruleTaskId.pos])) { - // // The task ID already exists in the mapping table. This means that the task has already been created during this execution of Ctrl-Q. - // // This is not allowed. The task ID must be unique. - // logger.error( - // `(${param.taskCounter}) PARSE TASKS FROM FILE: Task ID "${ - // rule[param.taskFileColumnHeaders.ruleTaskId.pos] - // }" already exists in mapping table. This is not allowed. Exiting.` - // ); - // process.exit(1); - // } - - // // Add task ID to mapping table - // this.taskIdMap.set(rule[param.taskFileColumnHeaders.ruleTaskId.pos], `fake-task-${uuidv4()}`); - - upstreamTaskExistence = 'exists-in-source-file'; - } else { - upstreamTask = await getTaskById(rule[param.taskFileColumnHeaders.ruleTaskId.pos], param?.options); - - // Save upstream task in shared task list - this.compositeEventUpstreamTask.push(upstreamTask); - - upstreamTaskExistence = 'exists-in-sense'; - } - - if (upstreamTaskExistence === 'exists-in-source-file') { - // Upstream task is a task that is created during this execution of Ctrl-Q - // We don't yet know what task ID it will get in Sense, so we'll have to find this when creating composite events later - compositeEvent.compositeRules.push({ - upstreamTaskExistence, - ruleState: mapRuleState.get(rule[param.taskFileColumnHeaders.ruleState.pos]), - task: { - id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], - }, - }); - } else if (mapTaskType.get(upstreamTask.taskType).toLowerCase() === 'reload') { - // Upstream task is a reload task - compositeEvent.compositeRules.push({ - upstreamTaskExistence, - ruleState: mapRuleState.get(rule[param.taskFileColumnHeaders.ruleState.pos]), - task: { - id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], - }, - reloadTask: { - id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], - }, - }); - } else if (mapTaskType.get(upstreamTask.taskType).toLowerCase() === 'externalprogram') { - // Upstream task is an external program task - compositeEvent.compositeRules.push({ - upstreamTaskExistence, - ruleState: mapRuleState.get(rule[param.taskFileColumnHeaders.ruleState.pos]), - task: { - id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], - }, - externalProgramTask: { - id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], - }, - }); - } - } - - this.qlikSenseCompositeEvents.addCompositeEvent(compositeEvent); - - // Add composite event to network representation of tasks - if (compositeEvent.compositeRules.length === 1) { - // This trigger has exactly ONE upstream task - // For triggers with >1 upstream task we want an extra meta node to represent the waiting of all upstream tasks to finish - - if (param.taskType === 'reload') { - // Add edge from upstream task to current task, taking into account task type - this.taskNetwork.edges.push({ - from: compositeEvent.compositeRules[0].task.id, - to: compositeEvent.reloadTask.id, - - completeCompositeEvent: compositeEvent, - rule: compositeEvent.compositeRules, - // color: compositeEvent.enabled ? '#9FC2F7' : '#949298', - // color: edgeColor, - // dashes: compositeEvent.enabled ? false : [15, 15], - // title: compositeEvent.name + '
' + 'asdasd', - // label: compositeEvent.name, - }); - - // Keep a note that this node has associated events - param.nodesWithEvents.add(compositeEvent.compositeRules[0].task.id); - param.nodesWithEvents.add(compositeEvent.reloadTask.id); - } else if (param.taskType === 'external program') { - // Add edge from upstream task to current task, taking into account task type - this.taskNetwork.edges.push({ - from: compositeEvent.compositeRules[0].task.id, - to: compositeEvent.externalProgramTask.id, - - completeCompositeEvent: compositeEvent, - rule: compositeEvent.compositeRules, - }); - // Keep a note that this node has associated events - param.nodesWithEvents.add(compositeEvent.compositeRules[0].task.id); - param.nodesWithEvents.add(compositeEvent.externalProgramTask.id); - } - } else { - // There are more than one task involved in triggering a downstream task. - // Insert a proxy node that represents a Qlik Sense composite event - - const nodeId = `composite-event-${uuidv4()}`; - this.taskNetwork.nodes.push({ - id: nodeId, - label: '', - enabled: true, - metaNodeType: 'composite', - metaNode: true, - }); - param.nodesWithEvents.add(nodeId); - - // Add edges from upstream tasks to the new meta node - for (const rule of compositeEvent.compositeRules) { - this.taskNetwork.edges.push({ - from: rule.task.id, - to: nodeId, - - completeCompositeEvent: compositeEvent, - rule, - }); - } - - // Add edge from new meta node to current node, taking into account task type - if (param.taskType === 'reload') { - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.reloadTask.id, - }); - } else if (param.taskType === 'external program') { - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.externalProgramTask.id, - }); - } - } - - // Add this composite event to the current task - prelCompositeEvents.push(compositeEvent); - } - } - - return prelCompositeEvents; + const result = extParseCompositeEvents(this, param, logger); + return result; } // Function to read task definitions from disk file (CSV or Excel) @@ -835,522 +183,53 @@ export class QlikSenseTasks { // - cpExisting: Array of existing custom properties in QSEoW // - options: Options object passed on the command line async getTaskModelFromFile(tasksFromFile, tagsExisting, cpExisting, options) { - return new Promise(async (resolve, reject) => { - try { - logger.debug('PARSE TASKS FROM FILE: Starting get tasks from data in file'); - - this.clear(); - - // Figure out which data is in which column - const taskFileColumnHeaders = getTaskColumnPosFromHeaderRow(tasksFromFile.data[0]); - - // Assign values to columns that were not part of earlier versions of the task file - // This is to make sure that the code below does not break when reading older versions of the task file - if (taskFileColumnHeaders.importOptions.pos === -1) { - // No import options column in file. Add a fake one to make sure code below does not break - logger.debug('PARSE TASKS FROM FILE: No import options column in file. Adding fake one'); - taskFileColumnHeaders.importOptions.pos = 999; - } - - // We now have all info about the task. Store it to the internal task data structure - this.taskNetwork = { nodes: [], edges: [], tasks: [] }; - const nodesWithEvents = new Set(); - - // Find max task counter = number of tasks to be imported - // This can be tricky for Excel files, as these can contain empty rows that are still saved in the file on disk. - // Somehow we need to determine which rows should be treated as input data. - // The criteria is that the "Task counter" column should either be a heading (i.e. "Task counter") or a number. - const taskImportCount = Math.max( - ...tasksFromFile.data.map((item) => { - if (item.length === 0) { - // Empty row - return -1; - } - - // Is first column empty? - if (item[taskFileColumnHeaders.taskCounter.pos] === undefined) { - // Empty task counter column - return -1; - } - - if (item[taskFileColumnHeaders.taskCounter.pos] === taskFileColumnHeaders.taskCounter.name) { - // This is the header row - return -1; - } - - // When reading from CSV files all columns will be strings. - let taskNum; - if (this.options.fileType === 'csv') { - taskNum = item[taskFileColumnHeaders.taskCounter.pos]; - } - if (this.options.fileType === 'excel') { - taskNum = item[taskFileColumnHeaders.taskCounter.pos]; - } - return taskNum; - }) - ); - - logger.info('-------------------------------------------------------------------'); - logger.info('Creating tasks...'); - - // Loop over all tasks in source file - for (let taskCounter = 1; taskCounter <= taskImportCount; taskCounter += 1) { - // Get all rows associated with this task - // One row will contain task data, other rows will contain event data associated with the task. - const taskRows = tasksFromFile.data.filter((item) => item[taskFileColumnHeaders.taskCounter.pos] === taskCounter); - logger.debug( - `(${taskCounter}) PARSE TASKS FROM FILE: Processing task #${taskCounter} of ${taskImportCount}. Data being used:\n${JSON.stringify( - taskRows, - null, - 2 - )}` - ); - - // Verify that first row contains task data. Following rows should contain event data associated with the task. - // Valid task types are: - // - Reload - // - External program - if ( - !taskRows[0][taskFileColumnHeaders.taskType.pos] || - !['reload', 'external program'].includes(taskRows[0][taskFileColumnHeaders.taskType.pos].trim().toLowerCase()) - ) { - logger.error( - `(${taskCounter}) PARSE TASKS FROM FILE: Incorrect task type "${ - taskRows[0][taskFileColumnHeaders.taskType.pos] - }". Exiting.` - ); - process.exit(1); - } - - // Handle each task type separately - // Get task type (lower case) from first row - const taskType = taskRows[0][taskFileColumnHeaders.taskType.pos].trim().toLowerCase(); - - // Reload task - if (taskType === 'reload') { - // Create a fake ID for this task. Used to associate task with schema/composite events - const fakeTaskId = `reload-task-${uuidv4()}`; - - const res = await this.parseReloadTask({ - taskRows, - taskFileColumnHeaders, - taskCounter, - tagsExisting, - cpExisting, - fakeTaskId, - nodesWithEvents, - options, - }); - - // Add reload task as node in task network - // NB: A top level node is defined as: - // 1. A task whose taskID does not show up in the "to" field of any edge. - - this.taskNetwork.nodes.push({ - id: res.currentTask.id, - metaNode: false, - isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === res.currentTask.id), - label: res.currentTask.name, - enabled: res.currentTask.enabled, - - completeTaskObject: res.currentTask, - - // Tabulator columns - taskId: res.currentTask.id, - taskName: res.currentTask.name, - taskEnabled: res.currentTask.enabled, - appId: res.currentTask.app.id, - appName: 'N/A', - appPublished: 'N/A', - appStream: 'N/A', - taskMaxRetries: res.currentTask.maxRetries, - taskLastExecutionStartTimestamp: 'N/A', - taskLastExecutionStopTimestamp: 'N/A', - taskLastExecutionDuration: 'N/A', - taskLastExecutionExecutingNodeName: 'N/A', - taskNextExecutionTimestamp: 'N/A', - taskLastStatus: 'N/A', - taskTags: res.currentTask.tags.map((tag) => tag.name), - taskCustomProperties: res.currentTask.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), - }); - - // We now have a basic task object including tags and custom properties. - // Schema events are included but composite events are only partially there, as they may need - // IDs of tasks that have not yet been created. - // Still, store all info for composite events and then do another loop where those events are created for - // tasks for which they are defined. - // - // The strategey is to first create all tasks, then add composite events. - // Only then can we be sure all composite events refer to existing tasks. - - // Create new or update existing reload task in QSEoW - // Should we create a new task? - // Controlled by option --update-mode - if (this.options.updateMode === 'create') { - // Create new task - if (this.options.dryRun === false || this.options.dryRun === undefined) { - const newTaskId = await this.createReloadTaskInQseow(res.currentTask, taskCounter); - logger.info( - `(${taskCounter}) Created new reload task "${res.currentTask.name}", new task id: ${newTaskId}.` - ); - - // Add mapping between fake task ID used when creating task network and actual, newly created task ID - this.taskIdMap.set(fakeTaskId, newTaskId); - - // Add mapping between fake task ID specified in source file and actual, newly created task ID - if (res.currentTask.id) { - this.taskIdMap.set(res.currentTask.id, newTaskId); - } - - res.currentTask.idRef = res.currentTask.id; - res.currentTask.id = newTaskId; - - await this.addTask('from_file', res.currentTask, false); - } else { - logger.info(`(${taskCounter}) DRY RUN: Creating reload task in QSEoW "${res.currentTask.name}"`); - } - } else if (this.options.updateMode === 'update-if-exists') { - // Update existing task - // TODO - // // Verify task ID is a valid UUID - // // If it's not a valid UUID, the ID specified in the source file will be treated as a task name - // if (!validate(res.currentTask.id)) { - // // eslint-disable-next-line no-await-in-loop - // const task = await getTaskByName(res.currentTask.id); - // if (task) { - // // eslint-disable-next-line no-await-in-loop - // await this.updateReloadTaskInQseow(res.currentTask, taskCounter); - // } else { - // throw new Error( - // `Task "${res.currentTask.id}" does not exist in QSEoW and cannot be updated. ` + - // 'Please specify a valid task ID or task name in the source file.' - // ); - // } - // } else { - // // Verify task ID exists in QSEoW - // // eslint-disable-next-line no-await-in-loop - // const taskExists = await getTaskById(res.currentTask.id); - // if (!taskExists) { - // throw new Error( - // `Task "${res.currentTask.id}" does not exist in QSEoW and cannot be updated. ` + - // 'Please specify a valid task ID or task name in the source file.' - // ); - // } else { - // // eslint-disable-next-line no-await-in-loop - // await this.updateReloadTaskInQseow(res.currentTask, taskCounter); - // logger.info( - // `(${taskCounter}) Updated existing task "${res.currentTask.name}", task id: ${res.currentTask.id}.` - // ); - // } - // } - // logger.info( - // `(${taskCounter}) Updated existing task "${res.currentTask.name}", task id: ${res.currentTask.id}.` - // ); - } else { - // Invalid combination of import options - throw new Error( - `Invalid task update mode. Valid values are "create" and "update-if-exists". You specified "${this.options.updateMode}".` - ); - } - } else if (taskType === 'external program') { - // External program task - - // Create a fake ID for this task. Used to associate task with schema/composite events - const fakeTaskId = `ext-pgm-task-${uuidv4()}`; - - const res = await this.parseExternalProgramTask({ - taskRows, - taskFileColumnHeaders, - taskCounter, - tagsExisting, - cpExisting, - fakeTaskId, - nodesWithEvents, - options, - }); - - // Add external program task as node in task network - // NB: A top level node is defined as: - // 1. A task whose taskID does not show up in the "to" field of any edge. - - this.taskNetwork.nodes.push({ - id: res.currentTask.id, - metaNode: false, - isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === res.currentTask.id), - label: res.currentTask.name, - enabled: res.currentTask.enabled, - - completeTaskObject: res.currentTask, - - taskTags: res.currentTask.tags.map((tag) => tag.name), - taskCustomProperties: res.currentTask.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), - }); - - // We now have a basic task object including tags and custom properties. - // Schema events are included but composite events are only partially there, as they may need - // IDs of tasks that have not yet been created. - // Still, store all info for composite events and then do another loop where those events are created for - // tasks for which they are defined. - // - // The strategey is to first create all tasks, then add composite events. - // Only then can we be sure all composite events refer to existing tasks. - - // Create new or update existing external program task in QSEoW - // Should we create a new task? - // Controlled by option --update-mode - if (this.options.updateMode === 'create') { - // Create new task - if (this.options.dryRun === false || this.options.dryRun === undefined) { - const newTaskId = await this.createExternalProgramTaskInQseow(res.currentTask, taskCounter); - logger.info( - `(${taskCounter}) Created new external program task "${res.currentTask.name}", new task id: ${newTaskId}.` - ); - - // Add mapping between fake task ID used when creating task network and actual, newly created task ID - this.taskIdMap.set(fakeTaskId, newTaskId); - - // Add mapping between fake task ID specified in source file and actual, newly created task ID - if (res.currentTask.id) { - this.taskIdMap.set(res.currentTask.id, newTaskId); - } - - res.currentTask.idRef = res.currentTask.id; - res.currentTask.id = newTaskId; - - await this.addTask('from_file', res.currentTask, false); - } else { - logger.info(`(${taskCounter}) DRY RUN: Creating external program task in QSEoW "${res.currentTask.name}"`); - } - } else if (this.options.updateMode === 'update-if-exists') { - // Update existing task - // TODO - // // Verify task ID is a valid UUID - // // If it's not a valid UUID, the ID specified in the source file will be treated as a task name - // if (!validate(res.currentTask.id)) { - // // eslint-disable-next-line no-await-in-loop - // const task = await - } else { - // Invalid combination of import options - throw new Error( - `Invalid task update mode. Valid values are "create" and "update-if-exists". You specified "${this.options.updateMode}".` - ); - } - } - } - - // At this point all tasks have been created or updated in Sense. - // The tasks' associated composite events have been parsed and are stored in the qlikSenseCompositeEvents object. - // Time now to create the composite events in Sense. - - // Make sure all composite tasks contain real, valid task UUIDs pointing to previously existing or newly created tasks. - // Get task IDs for upstream tasks that composite events are connected to via their respective rules. - // Some rules will point to upstream tasks that are created during this execution of Ctrl-Q, other upstream tasks existed before this execution of Ctrl-Q. - // Use the task ID map to get the correct task ID for each upstream task. - this.qlikSenseCompositeEvents.compositeEventList.map((item) => { - const a = item; - - // Set task ID for the composite event itself, i.e. which task is the event associated with (i.e. the downstream task) - // Handle different task types differently - if (item.compositeEvent?.reloadTask?.id) { - // Reload task - a.compositeEvent.reloadTask.id = this.taskIdMap.get(item.compositeEvent.reloadTask.id); - } else if (item.compositeEvent?.externalProgramTask?.id) { - // External program task - a.compositeEvent.externalProgramTask.id = this.taskIdMap.get(item.compositeEvent.externalProgramTask.id); - } - - // For this composite event, set the correct task id for each each rule. - // Different properties are used for reload tasks, external program tasks etc. - // Some rules may be pointing to newly created tasks. These can be looked up in the taskIdMap. - a.compositeEvent.compositeRules.map((item2) => { - const b = item2; - - // Get triggering/upstream task id - const id = this.taskIdMap.get(b.task.id); - - // If id is not found in the mapping table, it means that the task - // referenced by the rule (i.e. the upstream teask) is neither a task - // that existed before this execution of Ctrl-Q, nor a task that was - // created during this execution of Ctrl-Q. - // This is an error - the task ID should exist. - // Most likely the error is caused by an invalid value in the "Rule task id" - // column in the source file. - if (id !== undefined && validate(id) === true) { - // Determine what kind of task this is. Options are: - // - reload - // - external program - // - // Also need to know if the task is a new task created during this execution of Ctrl-Q or if it's an existing task in Sense. - let taskType; - if (b.upstreamTaskExistence === 'exists-in-source-file') { - const task = this.taskList.find((item3) => item3.taskId === id); - taskType = task.taskType; - // const { taskType } = this.taskNetwork.nodes.find((node) => node.id === id).completeTaskObject; - } else if (b.upstreamTaskExistence === 'exists-in-sense') { - const task = this.compositeEventUpstreamTask.find((item4) => item4.id === b.task.id); - - // Ensure we got a task back - if (!task) { - logger.error( - `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". This is an error - that task ID should exist. Existing.` - ); - process.exit(1); - } - - taskType = task.taskType; - } - - // Use mapTaskType to get the string variant of the task type. Convert to lower case. - const taskTypeString = mapTaskType.get(taskType).trim().toLowerCase(); - - // Ensure we got a valid task type - if (!['reload', 'externalprogram'].includes(taskTypeString)) { - logger.error( - `PREPARING COMPOSITE EVENT: Invalid task type "${taskTypeString}" for upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` - ); - process.exit(1); - } - - if (taskTypeString === 'reload') { - b.reloadTask = { id }; - } else if (taskTypeString === 'externalprogram') { - b.externalProgramTask = { id }; - } - } else if (id === undefined) { - // (this.options.dryRun === false || this.options.dryRun === undefined) { - logger.error( - `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` - ); - process.exit(1); - } - return b; - }); - return a; - }); - - logger.info('-------------------------------------------------------------------'); - logger.info('Creating composite events for the just created tasks...'); - - for (const { compositeEvent } of this.qlikSenseCompositeEvents.compositeEventList) { - if (this.options.dryRun === false || this.options.dryRun === undefined) { - await this.createCompositeEventInQseow(compositeEvent); - } else { - logger.info(`DRY RUN: Creating composite event "${compositeEvent.name}"`); - } - } - - // Add tasks to network array in plain, non-hierarchical format - this.taskNetwork.tasks = this.taskList; - - resolve(this.taskList); - } catch (err) { - catchLog('PARSE TASKS FROM FILE 1', err); - - if (err?.response?.status) { - logger.error(`Received error ${err.response?.status}/${err.response?.statusText} from QRS API`); - } - if (err?.response?.data) { - logger.error(`Error message from QRS API: ${err.response.data}`); - } - if (err?.config?.data) { - logger.error(`Data sent to Sense: ${JSON.stringify(JSON.parse(err.config.data), null, 2)}}`); - } - reject(err); - } - }); + const result = extGetTaskModelFromFile(this, tasksFromFile, tagsExisting, cpExisting, options, logger); + return result; } - createCompositeEventInQseow(newCompositeEvent) { + // Function to create new reload task in QSEoW + createReloadTaskInQseow(newTask, taskCounter) { return new Promise(async (resolve, reject) => { try { - logger.debug('CREATE COMPOSITE EVENT IN QSEOW: Starting'); + logger.debug(`(${taskCounter}) CREATE RELOAD TASK IN QSEOW: Starting`); // Build a body for the API call - const body = newCompositeEvent; + const body = { + task: { + app: { + id: newTask.app.id, + }, + name: newTask.name, + isManuallyTriggered: newTask.isManuallyTriggered, + isPartialReload: newTask.isPartialReload, + taskType: 0, + enabled: newTask.enabled, + taskSessionTimeout: newTask.taskSessionTimeout, + maxRetries: newTask.maxRetries, + tags: newTask.tags, + customProperties: newTask.customProperties, + schemaPath: 'ReloadTask', + }, + schemaEvents: newTask.schemaEvents, + }; // Save task to QSEoW const axiosConfig = setupQrsConnection(this.options, { method: 'post', - path: '/qrs/compositeevent', + path: '/qrs/reloadtask/create', body, }); - logger.debug(`/qrs/compositevent body: ${JSON.stringify(body, null, 2)}`); - axios .request(axiosConfig) .then((result) => { - if (result.status === 201) { - const response = JSON.parse(result.data); + const response = JSON.parse(result.data); - if (response?.reloadTask) { - logger.info( - `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.reloadTask.id}. Result: ${result.status}/${result.statusText}.` - ); - } else if (response?.externalProgramTask) { - logger.info( - `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.externalProgramTask.id}. Result: ${result.status}/${result.statusText}.` - ); - } + logger.debug( + `(${taskCounter}) CREATE RELOAD TASK IN QSEOW: "${newTask.name}", new task id: ${response.id}. Result: ${result.status}/${result.statusText}.` + ); - resolve(response.id); - } else { - reject(); - } - }) - .catch((err) => { - catchLog('CREATE COMPOSITE EVENT IN QSEOW 1', err); - }); - } catch (err) { - catchLog('CREATE COMPOSITE EVENT IN QSEOW 2', err); - reject(err); - } - }); - } - - // Function to create new reload task in QSEoW - createReloadTaskInQseow(newTask, taskCounter) { - return new Promise(async (resolve, reject) => { - try { - logger.debug(`(${taskCounter}) CREATE RELOAD TASK IN QSEOW: Starting`); - - // Build a body for the API call - const body = { - task: { - app: { - id: newTask.app.id, - }, - name: newTask.name, - isManuallyTriggered: newTask.isManuallyTriggered, - isPartialReload: newTask.isPartialReload, - taskType: 0, - enabled: newTask.enabled, - taskSessionTimeout: newTask.taskSessionTimeout, - maxRetries: newTask.maxRetries, - tags: newTask.tags, - customProperties: newTask.customProperties, - schemaPath: 'ReloadTask', - }, - schemaEvents: newTask.schemaEvents, - }; - - // Save task to QSEoW - const axiosConfig = setupQrsConnection(this.options, { - method: 'post', - path: '/qrs/reloadtask/create', - body, - }); - - axios - .request(axiosConfig) - .then((result) => { - const response = JSON.parse(result.data); - - logger.debug( - `(${taskCounter}) CREATE RELOAD TASK IN QSEOW: "${newTask.name}", new task id: ${response.id}. Result: ${result.status}/${result.statusText}.` - ); - - if (result.status === 201) { + if (result.status === 201) { resolve(response.id); } else { reject(); @@ -1367,614 +246,29 @@ export class QlikSenseTasks { }); } - // Function to create new external program task in QSEoW - // Parameters: - // - newTask: Object containing task data - // - taskCounter: Task counter, unique for each task in the source file - createExternalProgramTaskInQseow(newTask, taskCounter) { - return new Promise(async (resolve, reject) => { - try { - logger.debug(`(${taskCounter}) CREATE EXTERNAL PROGRAM TASK IN QSEOW: Starting`); - - // Build a body for the API call - const body = { - task: { - name: newTask.name, - taskType: 1, - enabled: newTask.enabled, - taskSessionTimeout: newTask.taskSessionTimeout, - maxRetries: newTask.maxRetries, - path: newTask.path, - parameters: newTask.parameters, - tags: newTask.tags, - customProperties: newTask.customProperties, - schemaPath: 'ExternalProgramTask', - }, - schemaEvents: newTask.schemaEvents, - }; - - // Save task to QSEoW - const axiosConfig = setupQrsConnection(this.options, { - method: 'post', - path: '/qrs/externalprogramtask/create', - body, - }); - - axios - .request(axiosConfig) - .then((result) => { - const response = JSON.parse(result.data); - - logger.debug( - `(${taskCounter}) CREATE EXTERNAL PROGRAM TASK IN QSEOW: "${newTask.name}", new task id: ${response.id}. Result: ${result.status}/${result.statusText}.` - ); - - if (result.status === 201) { - resolve(response.id); - } else { - reject(); - } - }) - .catch((err) => { - catchLog('CREATE EXTERNAL PROGRAM TASK IN QSEOW 1', err); - reject(err); - }); - } catch (err) { - catchLog('CREATE EXTERNAL PROGRAM TASK IN QSEOW 2', err); - reject(err); - } - }); - } - saveTaskModelToQseow() { - return new Promise(async (resolve, reject) => { - try { - logger.debug('SAVE TASKS TO QSEOW: Starting save tasks to QSEoW'); - - for (const task of this.taskList) { - await new Promise((resolve2, reject2) => { - // Build a body for the API call - const body = { - task: { - app: { - id: task.appId, - }, - name: task.taskName, - isManuallyTriggered: task.isManuallyTriggered, - isPartialReload: task.isPartialReload, - taskType: task.taskType, - enabled: task.taskEnabled, - taskSessionTimeout: task.taskSessionTimeout, - maxRetries: task.taskMaxRetries, - tags: task.taskTags, - customProperties: task.taskCustomProperties, - schemaPath: 'ReloadTask', - }, - schemaEvents: task.schemaEvents, - compositeEvents: task.compositeEvents, - }; - - // Save task to QSEoW - const axiosConfig = setupQrsConnection(this.options, { - method: 'post', - path: '/qrs/reloadtask/create', - body, - }); - - try { - axios.request(axiosConfig).then((result) => { - logger.info( - `SAVE TASK TO QSEOW: Task name: "${task.taskName}", Result: ${result.status}/${result.statusText}` - ); - if (result.status === 201) { - resolve2(); - } else { - reject2(); - } - }); - } catch (err) { - catchLog('SAVE TASK TO QSEOW 1', err); - reject2(); - } - }); - logger.debug(`SAVE TASK TO QSEOW: Done saving task "${task.taskName}"`); - } - resolve(); - } catch (err) { - catchLog('SAVE TASK TO QSEOW 2', err); - reject(err); - } - }); + const result = extSaveTaskModelToQseow(this, logger); + return result; } async getTasksFromQseow() { - return new Promise(async (resolve, reject) => { - // try { - logger.debug('GET TASKS FROM QSEOW: Starting get reload tasks from QSEoW'); - - let filter = ''; - - // Should we get all tasks? - if (this.options.getAllTasks === true) { - // No task filters specified - filter = ''; - } else if (this.options.outputFormat !== 'tree') { - // Are there any task filters specified? - // If so, build a query string - - // Don't add task id and tag filtering if the output is a task tree - - // Add task id(s) to query string - if (this.options.taskId && this.options?.taskId.length >= 1) { - // At least one task ID specified - // Add first task ID - filter += encodeURIComponent(`(id eq ${this.options.taskId[0]}`); - } - if (this.options.taskId && this.options?.taskId.length >= 2) { - // Add remaining task IDs, if any - for (let i = 1; i < this.options.taskId.length; i += 1) { - filter += encodeURIComponent(` or id eq ${this.options.taskId[i]}`); - } - } - - // Add closing parenthesis - if (this.options.taskId && this.options?.taskId.length >= 1) { - filter += encodeURIComponent(')'); - } - logger.debug(`GET TASKS FROM QSEOW: QRS query filter (incl ids): ${filter}`); - - // Add task tag(s) to query string - if (this.options.taskTag && this.options?.taskTag.length >= 1) { - // At least one task ID specified - if (filter.length >= 1) { - // We've previously added some task ids - // Add first task tag - filter += encodeURIComponent(` or (tags.name eq '${this.options.taskTag[0]}'`); - } else { - // No task ids added yet - // Add first task tag - filter += encodeURIComponent(`(tags.name eq '${this.options.taskTag[0]}'`); - } - } - if (this.options.taskTag && this.options?.taskTag.length >= 2) { - // Add remaining task tags, if any - for (let i = 1; i < this.options.taskTag.length; i += 1) { - filter += encodeURIComponent(` or tags.name eq '${this.options.taskTag[i]}'`); - } - } - - // Add closing parenthesis - if (this.options.taskTag && this.options?.taskTag.length >= 1) { - filter += encodeURIComponent(')'); - } - } - - logger.debug(`GET TASKS FROM QSEOW: QRS query filter (incl ids, tags): ${filter}`); - - let axiosConfig; - let tasks = []; - let result; - - try { - // Get reload tasks - if (filter === '') { - axiosConfig = setupQrsConnection(this.options, { - method: 'get', - path: '/qrs/reloadtask/full', - }); - } else { - axiosConfig = setupQrsConnection(this.options, { - method: 'get', - path: '/qrs/reloadtask/full', - queryParameters: [{ name: 'filter', value: filter }], - }); - } - - result = await axios.request(axiosConfig); - logger.debug(`GET RELOAD TASK: Result=result.status`); - - tasks = tasks.concat(JSON.parse(result.data)); - logger.verbose(`GET RELOAD TASK: # tasks: ${tasks.length}`); - } catch (err) { - catchLog('GET TASKS FROM QSEOW 1', err); - reject(err); - } - try { - // Get external program tasks - if (filter === '') { - axiosConfig = setupQrsConnection(this.options, { - method: 'get', - path: '/qrs/externalprogramtask/full', - }); - } else { - axiosConfig = setupQrsConnection(this.options, { - method: 'get', - path: '/qrs/externalprogramtask/full', - queryParameters: [{ name: 'filter', value: filter }], - }); - } - - result = await axios.request(axiosConfig); - logger.debug(`GET EXT PROGRAM TASK: Result=result.status`); - - tasks = tasks.concat(JSON.parse(result.data)); - logger.verbose(`GET EXT PROGRAM TASK: # tasks: ${tasks.length}`); - } catch (err) { - catchLog('GET EXTERNAL PROGRAM TASKS FROM QSEOW 1', err); - reject(err); - } - - // TODO - // Determine whether task name anonymisation should be done - const anonymizeTaskNames = false; - - this.clear(); - for (let i = 0; i < tasks.length; i += 1) { - if (tasks[i].schemaPath === 'ReloadTask' || tasks[i].schemaPath === 'ExternalProgramTask') { - this.addTask('from_qseow', tasks[i], anonymizeTaskNames); - } - } - resolve(this.taskList); - }); + const result = extGetTasksFromQseow(this, logger); + return result; } - getTaskSubTree(task, parentTreeLevel, parentTask) { - try { - const self = this; - - if (!task || !task?.id) { - logger.debug('Task parameter empty or does not include a task ID'); - } - - // Were we called from top-level? - if (parentTreeLevel === 0) { - // Set up new data structure for detecting cicrular task trees - this.taskTreeCyclicVisited = new Set(); - } - - const newTreeLevel = parentTreeLevel + 1; - let subTree = []; - - logger.debug( - `GET TASK SUBTREE: Meta node type: ${task.metaNodeType}, task type: ${task.taskType}, tree level: ${newTreeLevel}, task name: ${task.taskName}` - ); - - // Does this node (=task) have any downstream connections? - const downstreamTasks = self.taskNetwork.edges.filter((edge) => edge.from === task.id); - - let kids = []; - const validDownstreamTasks = []; - for (const downstreamTask of downstreamTasks) { - logger.debug( - `GET TASK SUBTREE: Processing downstream task: ${downstreamTask.to}. Current/source task: ${downstreamTask.from}` - ); - if (downstreamTask.to !== undefined) { - // Get downstream task object - const tmp = self.taskNetwork.nodes.find((el) => el.id === downstreamTask.to); - - if (!tmp) { - logger.warn( - `Downstream task "${downstreamTask.to}" in task tree not found. Current/source task: ${downstreamTask.from}` - ); - kids = [ - { - id: task.id, - }, - ]; - } else { - // Keep track of this downstream task - validDownstreamTasks.push({ sourceTask: task, downstreamTask: tmp }); - - // Don't check for cyclic task relationships yet, as that could trigger if two or more sibling tasks are triggered from the same source task. - } - } - } - - // Now that all downstream tasks have been retrieved, we can check if there are any general issues with those tasks - // Examples are cyclic task tree relationships, multiple downstream tasks with the same ID etc. - - // Check for downstream tasks with the same ID and same relationship with parent task (e.g. on-success or on-failure) - // downstreamTasks is an array of all downstream tasks from the current task. Properties are (the ones relevant here) - // - from: Source task/node ID - // - fromTaskType: Source task/node type. "Reload" or "ExternalProgram" - // - to: Destination task/node ID - // - toTaskType: Destination task/node type. "Reload", "ExternalProgram" or "Composite" - // - rule: Array of rules for the relationship between source and destination task. "on-success", "on-failure" etc. Properties for each object are - // - id: Rule ID - // - ruleState: Rule state/type. 1 = TaskSuccessful, 2 = TaskFail. mapRuleState.get(ruleState) gives the string representation of the rule state, given the number. - - // Check if there are multiple downstream tasks with the same ID and same relationship with the parent task. - // The relationship is the same if rule.ruleState is the same for two downstream tasks with the same ID. - // If there are, log a warning. - const duplicateDownstreamTasks = []; - for (const downstreamTask of downstreamTasks) { - // Are there any rules? - // downstreamTask.rule is an array of rules. Properties are - // - id: Rule ID - // - ruleState: Rule state/type. 1 = TaskSuccessful, 2 = TaskFail. mapRuleState.get(ruleState) gives the string representation of the rule state, given the number. - if (downstreamTask.rule) { - // Filter out downstream tasks with the same ID and the same rule state - const tmp = downstreamTasks.filter((el) => { - const sameDest = el.to === downstreamTask.to; - - // Same rule state? - // el.rule can be either an array or an object. If it's an object, convert it to an array. - if (!Array.isArray(el.rule)) { - el.rule = [el.rule]; - } - - // Is one of the rule's ruleState properties the same as one or more of downstreamTask.rule[].ruleState? - const sameRuleState = el.rule.some((rule) => { - return downstreamTask.rule.some((rule2) => { - return rule.ruleState === rule2.ruleState; - }); - }); - - return sameDest && sameRuleState; - }); - - if (tmp.length > 1) { - // Look up current and downstream task objects - const currentTask = self.taskNetwork.nodes.find((el) => el.id === task.id); - const downstreamTask = self.taskNetwork.nodes.find((el) => el.id === tmp[0].to); - - // Get the rule state that is shared between the downstream tasks and the parent task - const ruleState = mapRuleState.get(tmp[0].rule[0].ruleState); - - // Log warning unless this parent/child relationship is already in the list of duplicate downstream tasks - if ( - !duplicateDownstreamTasks.some( - (el) => el[0].to === tmp[0].to && el[0].rule[0].ruleState === tmp[0].rule[0].ruleState - ) - ) { - logger.warn( - `Multiple downstream tasks (${tmp.length}) with the same ID and the same trigger relationship "${ruleState}" with the parent task.` - ); - logger.warn(` Parent task : ${currentTask.completeTaskObject.name}`); - logger.warn(` Downstream task : ${downstreamTask.completeTaskObject.name}`); - } - - duplicateDownstreamTasks.push(tmp); - } - } - } - - // Check if there are any cyclic task tree relationships - // If there are none, we can add the downstream tasks to the tree - // First make sure all downstream task IDs are unique. Remove duplicates. - const uniqueDownstreamTasks = Array.from(new Set(validDownstreamTasks.map((a) => a.downstreamTask.id))).map((id) => { - return validDownstreamTasks.find((a) => a.downstreamTask.id === id); - }); - - for (const validDownstreamTask of uniqueDownstreamTasks) { - if (this.isTaskTreeCyclic(validDownstreamTask.downstreamTask)) { - if (parentTask) { - logger.warn(`Cyclic dependency detected in task tree. Won't go deeper.`); - logger.warn(` From task : ${validDownstreamTask.sourceTask.taskName}`); - logger.warn(` To task : ${validDownstreamTask.downstreamTask.taskName}`); - - // Add node indicating cyclic dependency - kids = kids.concat([ - { - id: task.id, - text: ` ==> !!! Cyclic dependency detected from task "${validDownstreamTask.sourceTask.taskName}" to "${validDownstreamTask.downstreamTask.taskName}"`, - }, - ]); - } else { - logger.warn(`Cyclic dependency detected in task tree. No parent task detected. Won't go deeper.`); - } - } else { - const tmp3 = self.getTaskSubTree(validDownstreamTask.downstreamTask, newTreeLevel, validDownstreamTask.sourceTask); - kids = kids.concat(tmp3); - } - } - - // Only push real Sense tasks to the tree (don't include meta nodes) - if (!task.metaNodeType) { - if (kids && kids.length > 0) { - subTree = { - id: task.id, - children: kids, - }; - } else { - subTree = { - id: task.id, - }; - } - - if (this.options.treeIcons) { - if (task.taskLastStatus === 'FinishedSuccess') { - subTree.text = `✅ ${task.taskName}`; - // subTree.text = this.options.textColor ? `✅ \x1b[0m${task.taskName}\x1b[0m` : `✅ ${task.taskName}`; - } else if (task.taskLastStatus === 'FinishedFail') { - subTree.text = `❌ ${task.taskName}`; - } else if (task.taskLastStatus === 'Skipped') { - subTree.text = `🚫 ${task.taskName}`; - } else if (task.taskLastStatus === 'Aborted') { - subTree.text = `🛑 ${task.taskName}`; - } else if (task.taskLastStatus === 'Never started') { - subTree.text = `💤 ${task.taskName}`; - } else { - subTree.text = `❔ ${task.taskName}`; - } - } else { - subTree.text = task.taskName; - } - - if (this.options.treeDetails === true) { - // All task details should be included - if (task.completeTaskObject.schemaPath === 'ReloadTask') { - if (this.options.textColor === 'yes') { - subTree.text += ` \x1b[2mTask id: \x1b[3m${task.id}\x1b[0;2m, Last start/stop: \x1b[3m${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}\x1b[0;2m, Next start: \x1b[3m${task.taskNextExecutionTimestamp}\x1b[0;2m, App name: \x1b[3m${task.appName}\x1b[0;2m, App stream: \x1b[3m${task.appStream}\x1b[0;2m\x1b[0m`; - } else { - subTree.text += ` Task id: ${task.id}, Last start/stop: ${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}, Next start: ${task.taskNextExecutionTimestamp}, App name: ${task.appName}, App stream: ${task.appStream}`; - } - } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { - if (this.options.textColor === 'yes') { - subTree.text += ` \x1b[2m--EXTERNAL PROGRAM--Task id: \x1b[3m${task.id}\x1b[0;2m, Last start/stop: \x1b[3m${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}\x1b[0;2m, Next start: \x1b[3m${task.taskNextExecutionTimestamp}\x1b[0;2m, Path: \x1b[3m${task.path}\x1b[0;2m, Parameters: \x1b[3m${task.parameters}\x1b[0;2m\x1b[0m`; - } else { - subTree.text += `--EXTERNAL PROGRAM--Task id: ${task.id}, Last start/stop: ${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}, Next start: ${task.taskNextExecutionTimestamp}, path: ${task.path}, Parameters: ${task.oarameters}`; - } - } - } else if (this.options.treeDetails) { - // Some task details should be included - if (this.options.treeDetails.find((item) => item === 'taskid')) { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, Task id: \x1b[3m${task.id}\x1b[0;2m\x1b[0m` - : `, Task id: ${task.id}`; - } - if (this.options.treeDetails.find((item) => item === 'laststart')) { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, Last start: \x1b[3m${task.taskLastExecutionStartTimestamp}\x1b[0;2m\x1b[0m` - : `, Last start: ${task.taskLastExecutionStartTimestamp}`; - } - if (this.options.treeDetails.find((item) => item === 'laststop')) { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, Last stop: \x1b[3m${task.taskLastExecutionStopTimestamp}\x1b[0;2m\x1b[0m` - : `, Last stop: ${task.taskLastExecutionStopTimestamp}`; - } - if (this.options.treeDetails.find((item) => item === 'nextstart')) { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, Next start: \x1b[3m${task.taskNextExecutionTimestamp}\x1b[0;2m\x1b[0m` - : `, Next start: ${task.taskNextExecutionTimestamp}`; - } - if (this.options.treeDetails.find((item) => item === 'appname')) { - if (task.completeTaskObject.schemaPath === 'ReloadTask') { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, App name: \x1b[3m${task.appName}\x1b[0;2m\x1b[0m` - : `, App name: ${task.appName}`; - } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, Path: \x1b[3m${task.path}\x1b[0;2m\x1b[0m` - : `, Path: ${task.path}`; - } - } - if (this.options.treeDetails.find((item) => item === 'appstream')) { - if (task.completeTaskObject.schemaPath === 'ReloadTask') { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, App stream: \x1b[3m${task.appStream}\x1b[0;2m\x1b[0m` - : `, App stream: ${task.appStream}`; - } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { - subTree.text += - this.options.textColor === 'yes' - ? `\x1b[2m, Parameters: \x1b[3m${task.parameters}\x1b[0;2m\x1b[0m` - : `, Parameters: ${task.parameters}`; - } - } - } - - // Tabulator columns - subTree.taskId = task.taskId; - subTree.taskName = task.taskName; - subTree.taskEnabled = task.taskEnabled; - subTree.appId = task.appId; - subTree.appName = task.appName; - subTree.appPublished = task.appPublished; - subTree.appStream = task.appStream; - subTree.taskMaxRetries = task.taskMaxRetries; - subTree.taskLastExecutionStartTimestamp = task.taskLastExecutionStartTimestamp; - subTree.taskLastExecutionStopTimestamp = task.taskLastExecutionStopTimestamp; - subTree.taskLastExecutionDuration = task.taskLastExecutionDuration; - subTree.taskLastExecutionExecutingNodeName = task.taskLastExecutionExecutingNodeName; - subTree.taskNextExecutionTimestamp = task.taskNextExecutionTimestamp; - subTree.taskLastStatus = task.taskLastStatus; - subTree.taskTags = task.completeTaskObject.tags.map((tag) => tag.name); - subTree.taskCustomProperties = task.completeTaskObject.customProperties.map((el) => `${el.definition.name}=${el.value}`); - subTree.completeTaskObject = task.completeTaskObject; - - if (newTreeLevel === 1) { - subTree = [subTree]; - } - } else { - subTree = kids; - } + async getTaskSubGraph(node, parentTreeLevel, parentNode) { + const result = extGetTaskSubGraph(this, node, parentTreeLevel, parentNode, logger); + return result; + } - return subTree; - // console.log('subTree: ' + JSON.stringify(subTree)); - } catch (err) { - catchLog('GET TASK SUBTREE (tree)', err); - return false; - } + async getTaskSubTree(task, parentTreeLevel, parentTask) { + const result = extGetTaskSubTree(this, task, parentTreeLevel, parentTask, logger); + return result; } getTaskSubTable(task, parentTreeLevel) { - try { - const self = this; - - const newTreeLevel = parentTreeLevel + 1; - let subTree = []; - - // Debug - // logger.debug(`GET TASK SUBTABLE: Tree level: ${newTreeLevel}, task name: ${task.taskName}`); - - // Does this node (=task) have any downstream connections? - const downstreamTasks = self.taskNetwork.edges.filter((edge) => edge.from === task.id); - // console.log('downStreamTasks 1: ' + JSON.stringify(downstreamTasks)); - let kids = []; - for (const downstreamTask of downstreamTasks) { - if (downstreamTask.to !== undefined) { - // Get downstream task object - const tmp = self.taskNetwork.nodes.find((el) => el.id === downstreamTask.to); - const tmp3 = self.getTaskSubTable(tmp, newTreeLevel); - kids = kids.concat([tmp3]); - } - } - - // Only push real Sense tasks to the tree (don't include meta nodes) - if (!task.metaNodeType) { - if (kids && kids.length > 0) { - subTree = { - id: task.id, - children: kids, - }; - } else { - subTree = { - id: task.id, - }; - } - - subTree.text = task.taskName; - - // Tabulator columns - subTree.taskId = task.taskId; - subTree.taskName = task.taskName; - subTree.taskEnabled = task.taskEnabled; - subTree.appId = task.appId; - subTree.appName = task.appName; - subTree.appPublished = task.appPublished; - subTree.appStream = task.appStream; - subTree.taskMaxRetries = task.taskMaxRetries; - subTree.taskLastExecutionStartTimestamp = task.taskLastExecutionStartTimestamp; - subTree.taskLastExecutionStopTimestamp = task.taskLastExecutionStopTimestamp; - subTree.taskLastExecutionDuration = task.taskLastExecutionDuration; - subTree.taskLastExecutionExecutingNodeName = task.taskLastExecutionExecutingNodeName; - subTree.taskNextExecutionTimestamp = task.taskNextExecutionTimestamp; - subTree.taskLastStatus = task.taskLastStatus; - subTree.taskTags = task.completeTaskObject.tags.map((tag) => tag.name); - subTree.taskCustomProperties = task.completeTaskObject.customProperties.map((el) => `${el.definition.name}=${el.value}`); - subTree.completeTaskObject = task.completeTaskObject; - - if (newTreeLevel <= 2) { - subTree = kids.concat([[newTreeLevel, task.taskName, task.taskId, task.taskEnabled]]); - } else { - subTree = kids.concat([[newTreeLevel, '--'.repeat(newTreeLevel - 2) + task.taskName, task.taskId, task.taskEnabled]]); - } - } else { - subTree = kids; - } - - return subTree; - } catch (err) { - catchLog('GET TASK SUBTABLE (table)', err); - return null; - } + const result = extGetTaskSubTable(this, task, parentTreeLevel, logger); + return result; } getTableTaskTable() { @@ -1995,466 +289,96 @@ export class QlikSenseTasks { } async getTaskModelFromQseow() { - logger.debug('GET TASK: Getting task model from QSEoW'); - - // Get all tasks from QSEoW - try { - logger.verbose(`Getting tasks from QSEoW...`); - await this.getTasksFromQseow(); - } catch (err) { - catchLog('GET TASK MODEL FROM QSEOW 1', err); - return false; - } - - // Get all schema events from QSEoW - try { - logger.verbose(`Getting schema events from QSEoW...`); - const result1 = await this.qlikSenseSchemaEvents.getSchemaEventsFromQseow(); - - logger.silly(`Schema events from QSEoW: ${JSON.stringify(result1, null, 2)}`); - } catch (err) { - catchLog('GET TASK MODEL FROM QSEOW 2', err); - return false; - } - - // Get all composite events from QSEoW - try { - logger.verbose(`Getting composite events from QSEoW...`); - const result2 = await this.qlikSenseCompositeEvents.getCompositeEventsFromQseow(); - - logger.silly(`Composite events from QSEoW: ${JSON.stringify(result2, null, 2)}`); - } catch (err) { - catchLog('GET TASK MODEL FROM QSEOW 3', err); - return false; - } - - logger.verbose('GET TASK MODEL FROM QSEOW: Done getting task model from QSEoW'); - - // Get all top level apps, i.e apps that aren't triggered by any other apps succeeding or failing. - // They might have scheduled triggers though. - this.taskNetwork = { nodes: [], edges: [], tasks: [] }; - const nodesWithEvents = new Set(); - - // We already have all tasks in plain, non-hierarchical format - this.taskNetwork.tasks = this.taskList; - - // Add schema edges and start/trigger nodes - logger.verbose('GET TASK MODEL FROM QSEOW: Adding schema edges and start/trigger nodes to internal task model'); - for (const schemaEvent of this.qlikSenseSchemaEvents.schemaEventList) { - logger.silly(`Schema event contents: ${JSON.stringify(schemaEvent, null, 2)}`); - // Schedule is associated with a reload task - if (schemaEvent.schemaEvent.reloadTask !== null) { - logger.debug( - `Processing schema event "${schemaEvent?.schemaEvent?.name}" for reload task "${schemaEvent?.schemaEvent?.reloadTask?.name}" (${schemaEvent?.schemaEvent?.reloadTask?.id})` - ); - - // Add schema trigger nodes. These represent the implicit starting nodes that a schema event really are - const nodeId = `node-${uuidv4()}`; - this.taskNetwork.nodes.push({ - id: nodeId, - metaNodeType: 'schedule', // Meta nodes are not Sense tasks, but rather nodes representing task-like properties (e.g. a starting point for a reload chain) - metaNode: true, - isTopLevelNode: true, - label: schemaEvent.schemaEvent.name, - enabled: schemaEvent.schemaEvent.enabled, - taskType: 'reloadTask', - - completeSchemaEvent: schemaEvent.schemaEvent, - }); - - this.taskNetwork.edges.push({ - from: nodeId, - to: schemaEvent.schemaEvent.reloadTask.id, - }); - - // Keep a note that this node has associated events - nodesWithEvents.add(schemaEvent.schemaEvent.reloadTask.id); - } else if (schemaEvent.schemaEvent.externalProgramTask !== null) { - // Schedule is associated with an external program task - logger.debug( - `Processing schema event "${schemaEvent?.schemaEvent?.name}" for external program task "${schemaEvent?.schemaEvent?.reloadTask?.name}" (${schemaEvent?.schemaEvent?.externalProgramTask?.id})` - ); - - // Add schema trigger nodes. These represent the implicit starting nodes that a schema event really are - const nodeId = `node-${uuidv4()}`; - this.taskNetwork.nodes.push({ - id: nodeId, - metaNodeType: 'schedule', // Meta nodes are not Sense tasks, but rather nodes representing task-like properties (e.g. a starting point for a reload chain) - metaNode: true, - isTopLevelNode: true, - label: schemaEvent.schemaEvent.name, - enabled: schemaEvent.schemaEvent.enabled, - taskType: 'externalProgramTask', - - completeSchemaEvent: schemaEvent.schemaEvent, - }); - - this.taskNetwork.edges.push({ - from: nodeId, - to: schemaEvent.schemaEvent.externalProgramTask.id, - }); - - // Keep a note that this node has associated events - nodesWithEvents.add(schemaEvent.schemaEvent.externalProgramTask.id); - } - } - - // Add composite events - logger.verbose('GET TASK MODEL FROM QSEOW: Adding composite events to internal task model'); - for (const compositeEvent of this.qlikSenseCompositeEvents.compositeEventList) { - logger.silly(`Composite event contents: ${JSON.stringify(compositeEvent, null, 2)}`); - - if (compositeEvent?.compositeEvent?.reloadTask) { - logger.debug( - `Processing composite event "${compositeEvent?.compositeEvent?.name}" for reload task "${compositeEvent?.compositeEvent?.reloadTask?.name}" (${compositeEvent?.compositeEvent?.reloadTask?.id})` - ); - } else if (compositeEvent?.compositeEvent?.externalProgramTask) { - logger.debug( - `Processing composite event "${compositeEvent?.compositeEvent?.name}" for external program task "${compositeEvent?.compositeEvent?.externalProgramTask?.name}" (${compositeEvent?.compositeEvent?.externalProgramTask?.id})` - ); - } else if (compositeEvent?.compositeEvent?.userSyncTask) { - logger.debug( - `Processing composite event "${compositeEvent?.compositeEvent?.name}" for user sync task "${compositeEvent?.compositeEvent?.userSyncTask?.name}" (${compositeEvent?.compositeEvent?.userSyncTask?.id})` - ); - } - - if (compositeEvent.compositeEvent.reloadTask !== null) { - // Current composite event triggers a reload task - logger.debug( - `Current composite event triggers a reload task. Processing composite event "${compositeEvent?.compositeEvent?.name}" for reload task "${compositeEvent?.compositeEvent?.reloadTask?.name}" (${compositeEvent?.compositeEvent?.reloadTask?.id})` - ); - - if (compositeEvent.compositeEvent.reloadTask.id === undefined || compositeEvent.compositeEvent.reloadTask.id === null) { - logger.warn(`Composite event "${compositeEvent.compositeEvent.name}" has no reload task ID associated with it.`); - } else if (compositeEvent.compositeEvent.compositeRules.length === 1) { - // This trigger has exactly ONE upstream task - // For triggers with >1 upstream task we want an extra meta node to represent the waiting of all upstream tasks to finish - if (validate(compositeEvent.compositeEvent.compositeRules[0]?.reloadTask?.id)) { - logger.verbose( - `Composite event "${compositeEvent.compositeEvent.name}" has a reload task triggered by reload task with ID=${compositeEvent.compositeEvent.compositeRules[0].reloadTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: compositeEvent.compositeEvent.compositeRules[0].reloadTask.id, - fromTaskType: 'Reload', - - to: compositeEvent.compositeEvent.reloadTask.id, - toTaskType: 'Reload', - - completeCompositeEvent: compositeEvent.compositeEvent, - rule: compositeEvent.compositeEvent.compositeRules, - // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', - // color: edgeColor, - // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], - // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', - // label: compositeEvent.compositeEvent.name, - }); - - // Keep a note that this node has associated events - nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].reloadTask.id); - nodesWithEvents.add(compositeEvent.compositeEvent.reloadTask.id); - } else if (validate(compositeEvent.compositeEvent.compositeRules[0]?.externalProgramTask?.id)) { - logger.verbose( - `Composite event "${compositeEvent?.compositeEvent?.name}" has a reload task triggered by external program task with ID=${compositeEvent.compositeEvent.compositeRules[0]?.externalProgramTask?.id}.` - ); - - this.taskNetwork.edges.push({ - from: compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id, - fromTaskType: 'ExternalProgram', - - to: compositeEvent.compositeEvent.reloadTask.id, - toTaskType: 'Reload', - - completeCompositeEvent: compositeEvent.compositeEvent, - rule: compositeEvent.compositeEvent.compositeRules, - // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', - // color: edgeColor, - // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], - // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', - // label: compositeEvent.compositeEvent.name, - }); - - // Keep a note that this node has associated events - nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id); - nodesWithEvents.add(compositeEvent.compositeEvent.reloadTask.id); - } - } else { - // There are more than one task involved in triggering a downstream task. - // Insert a proxy node that represents a Qlik Sense composite event - logger.verbose( - `Composite event "${compositeEvent?.compositeEvent?.name}" is triggerer by ${compositeEvent?.compositeEvent?.compositeRules.length} upstream tasks.` - ); - - // TODO - const nodeId = `node-${uuidv4()}`; - this.taskNetwork.nodes.push({ - id: nodeId, - label: compositeEvent.compositeEvent.name, - enabled: true, - metaNodeType: 'composite', - metaNode: true, - }); - nodesWithEvents.add(nodeId); - - // Add edges from upstream tasks to the new meta node - for (const rule of compositeEvent.compositeEvent.compositeRules) { - if (validate(rule?.reloadTask?.id)) { - // Upstream task is a reload task - logger.debug( - `Composite event "${compositeEvent.compositeEvent.name}" is triggered by reload task with ID=${rule.reloadTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: rule.reloadTask.id, - fromTaskType: 'Reload', - to: nodeId, - toTaskType: 'Composite', - - // TODO Correct? Or should it be at next edges.push? - completeCompositeEvent: compositeEvent.compositeEvent, - rule, - }); - } else if (validate(rule?.externalProgramTask?.id)) { - // Upstream task is an external program task - logger.debug( - `Composite event "${compositeEvent.compositeEvent.name}" is triggered by external program task with ID=${rule.externalProgramTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: rule.externalProgramTask.id, - fromTaskType: 'ExternalProgram', - to: nodeId, - toTaskType: 'Composite', - - // TODO Correct? Or should it be at next edges.push? - completeCompositeEvent: compositeEvent.compositeEvent, - rule, - }); - } else { - logger.warn( - `Upstream task for composite event "${compositeEvent.compositeEvent.name}" is not of a supported task type (reload task, external program task).` - ); - } - } - - // Add edge from new meta node to current node - logger.debug( - `Added edge from composite event meta node "${nodeId}" to reload task ID=${compositeEvent.compositeEvent.reloadTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.compositeEvent.reloadTask.id, - }); - } - } else if (compositeEvent.compositeEvent.externalProgramTask !== null) { - // Current composite event triggers an external program task - logger.debug( - `Composite event "${compositeEvent.compositeEvent.name}" triggers an external program task "${compositeEvent.compositeEvent.externalProgramTask.name}" (${compositeEvent.compositeEvent.externalProgramTask.id}).` - ); - - if ( - compositeEvent.compositeEvent.externalProgramTask.id === undefined || - compositeEvent.compositeEvent.externalProgramTask.id === null - ) { - logger.warn( - `Composite event "${compositeEvent.compositeEvent.name}" has no external program task ID associated with it.` - ); - } else if (compositeEvent.compositeEvent.compositeRules.length === 1) { - // This trigger has exactly ONE upstream task - // For triggers with >1 upstream task we want an extra meta node to represent the waiting of all upstream tasks to finish - logger.verbose(`Composite event "${compositeEvent.compositeEvent.name}" has exactly one upstream task.`); - - if (validate(compositeEvent.compositeEvent.compositeRules[0]?.reloadTask?.id)) { - logger.verbose( - `Composite event "${compositeEvent?.compositeEvent?.name}" has an external program task triggered by reload task with ID=${compositeEvent.compositeEvent.compositeRules[0]?.reloadTask?.id}.` - ); - - this.taskNetwork.edges.push({ - from: compositeEvent.compositeEvent.compositeRules[0].reloadTask.id, - fromTaskType: 'Reload', - - to: compositeEvent.compositeEvent.externalProgramTask.id, - toTaskType: 'ExternalProgram', - - completeCompositeEvent: compositeEvent.compositeEvent, - rule: compositeEvent.compositeEvent.compositeRules, - // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', - // color: edgeColor, - // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], - // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', - // label: compositeEvent.compositeEvent.name, - }); - // Keep a note that this node has associated events - nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].reloadTask.id); - nodesWithEvents.add(compositeEvent.compositeEvent.externalProgramTask.id); - } else if (validate(compositeEvent.compositeEvent.compositeRules[0]?.externalProgramTask?.id)) { - logger.verbose( - `Composite event "${compositeEvent.compositeEvent.name}" has an external program task triggered by external program task with ID=${compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id, - fromTaskType: 'ExternalProgram', - - to: compositeEvent.compositeEvent.externalProgramTask.id, - toTaskType: 'ExternalProgram', - - completeCompositeEvent: compositeEvent.compositeEvent, - rule: compositeEvent.compositeEvent.compositeRules, - // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', - // color: edgeColor, - // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], - // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', - // label: compositeEvent.compositeEvent.name, - }); - - // Keep a note that this node has associated events - nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id); - nodesWithEvents.add(compositeEvent.compositeEvent.externalProgramTask.id); - } else { - logger.warn(`Composite event "${compositeEvent.compositeEvent.name}" is triggered by an unsupported task type.`); - } - } else { - // There are more than one task involved in triggering a downstream task. - // Insert a proxy node that represents a Qlik Sense composite event - logger.verbose( - `Composite event "${compositeEvent?.compositeEvent?.name}" has ${compositeEvent?.compositeEvent?.compositeRules.length} upstream tasks.` - ); - - const nodeId = `node-${uuidv4()}`; - this.taskNetwork.nodes.push({ - id: nodeId, - label: compositeEvent.compositeEvent.name, - enabled: true, - metaNodeType: 'composite', - metaNode: true, - }); - nodesWithEvents.add(nodeId); - - // Add edges from upstream tasks to the new meta node - for (const rule of compositeEvent.compositeEvent.compositeRules) { - if (validate(rule?.reloadTask?.id)) { - // Upstream task is a reload task - logger.debug( - `Upstream task for composite event "${compositeEvent.compositeEvent.name}" is a reload task with ID=${rule.reloadTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: rule.reloadTask.id, - fromTaskType: 'Reload', - to: nodeId, - toTaskType: 'Composite', - - // TODO Correct? Or should it be at next edges.push? - completeCompositeEvent: compositeEvent.compositeEvent, - rule, - }); - } else if (validate(rule?.externalProgramTask?.id)) { - // Upstream task is an external program task - logger.debug( - `Upstream task for composite event "${compositeEvent.compositeEvent.name}" is an external program task with ID=${rule.externalProgramTask.id}.` - ); - - this.taskNetwork.edges.push({ - from: rule.externalProgramTask.id, - fromTaskType: 'ExternalProgram', - to: nodeId, - toTaskType: 'Composite', + const result = await extGetTaskModelFromQseow(this, logger); + return result; + } - // TODO Correct? Or should it be at next edges.push? - completeCompositeEvent: compositeEvent.compositeEvent, - rule, - }); - } else { - logger.warn( - `Upstream task for composite event "${compositeEvent?.compositeEvent?.name}" is not of a supported task type (reload task, external program task).` - ); - } - } + /** + * Returns an array of root nodes in the task network. + * + * Method: + * 1. Use various task and app filters specified in CLI options to get a list of tasks to consider. + * 2. For each task, check if it has any upstream dependencies. + * 3. If it does not have any upstream dependencies, it is a root node. + * 4. If it has upstream dependencies, it is not a root node. Continue to the next upstream node. + * 5. Repeat until all upstream nodes have been checked. + * 6. Return an array of root nodes. + * + * Root nodes are tasks or metatasks that have no upstream dependencies. + * In other words, they are not triggered by any other tasks. + * + * @param {Array} tasks List of tasks to consider + * @param {Array} apps List of apps to consider + * @returns {Promise>} An array of root nodes in the task network, or false if an error occurred + */ + async getRootNodesFromFilter() { + const result = await extGetRootNodesFromFilter(this, logger); + return result; + } - // Add edge from new meta node to current node - logger.debug( - `Added edge from new meta composite event node "${nodeId}" to reload task ID=${compositeEvent.compositeEvent?.reloadTask?.id}.` - ); + /** + * Extract nodes and edges starting from the provided root nodes. + * This function processes the root nodes to identify all connected nodes and edges, + * effectively building a subgraph starting from these root nodes. + * + * While doing this, make sure to + * - De-duplicate nodes where applicable. Node id is unique, but may appear at different places in the task network. + * - Keep track of edges between nodes and store them in edgesToVisualize + * - Store nodes in nodesToVisualize + * - Detect cyclic dependencies. Log warning if detected. + * - Detect identical, duplicate edges between nodes. Log warning if detected. + * + * @param {Array} rootNodes - An array of root node objects from which the extraction begins. + * @returns {Promise} - A promise that resolves to an object containing the nodes and edges. + * Object properties: + * - nodes: Array of nodes, + * - edges: Array of edges + */ + async getNodesAndEdgesFromRootNodes(rootNodes) { + // De-duplicate root nodes + const uniqueRootNodes = rootNodes.filter((node, index, self) => { + return index === self.findIndex((t) => t.id === node.id); + }); - if (compositeEvent.compositeEvent?.reloadTask) { - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.compositeEvent.reloadTask.id, - }); - } else if (compositeEvent.compositeEvent?.externalProgramTask) { - this.taskNetwork.edges.push({ - from: nodeId, - to: compositeEvent.compositeEvent.externalProgramTask.id, - }); - } - } - } + // Initialize arrays to store nodes and edges + let nodesFound = []; + let edgesFound = []; + let tasksFound = []; + + // Extract nodes and edges from root nodes + for (const rootNode of uniqueRootNodes) { + const subGraph = await this.getTaskSubGraph(rootNode, 0, null); + // Ensure subgraph is not empty + if (!subGraph) { + logger.verbose(`No subgraph found for root node ${rootNode.id}.`); + continue; + } + nodesFound.push(...subGraph.nodes); + edgesFound.push(...subGraph.edges); + tasksFound.push(...subGraph.tasks); } - // Add all Sense tasks as nodes in task network - // NB: A top level node is defined as: - // 1. A task whose taskID does not show up in the "to" field of any edge. - - for (const node of this.taskList) { - if (node.completeTaskObject.schemaPath === 'ReloadTask') { - this.taskNetwork.nodes.push({ - id: node.taskId, - metaNode: false, - isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === node.taskId), - label: node.taskName, - enabled: node.taskEnabled, - - completeTaskObject: node.completeTaskObject, - - // Tabulator columns - taskId: node.taskId, - taskName: node.taskName, - taskEnabled: node.taskEnabled, - appId: node.appId, - appName: node.appName, - appPublished: node.appPublished, - appStream: node.appStream, - taskMaxRetries: node.taskMaxRetries, - taskLastExecutionStartTimestamp: node.taskLastExecutionStartTimestamp, - taskLastExecutionStopTimestamp: node.taskLastExecutionStopTimestamp, - taskLastExecutionDuration: node.taskLastExecutionDuration, - taskLastExecutionExecutingNodeName: node.taskLastExecutionExecutingNodeName, - taskNextExecutionTimestamp: node.taskNextExecutionTimestamp, - taskLastStatus: node.taskLastStatus, - taskTags: node.completeTaskObject.tags.map((tag) => tag.name), - taskCustomProperties: node.completeTaskObject.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), - }); - } else if (node.completeTaskObject.schemaPath === 'ExternalProgramTask') { - this.taskNetwork.nodes.push({ - id: node.taskId, - metaNode: false, - isTopLevelNode: !this.taskNetwork.edges.find((edge) => edge.to === node.taskId), - label: node.taskName, - enabled: node.taskEnabled, + // De-duplicate nodes using node id + nodesFound = nodesFound.filter((node, index, self) => { + return index === self.findIndex((t) => t.id === node.id); + }); - completeTaskObject: node.completeTaskObject, + // De-duplicate tasks using task id + tasksFound = tasksFound.filter((task, index, self) => { + return ( + index === + self.findIndex((t) => { + return t.taskId === task.taskId; + }) + ); + }); - // Tabulator columns - taskId: node.taskId, - taskName: node.taskName, - taskEnabled: node.taskEnabled, + // De-duplicate edges. + // Edges are identical if they connect the same two nodes, i.e. have the same from and to properties. + edgesFound = edgesFound.filter((edge, index, self) => { + return index === self.findIndex((t) => t.from === edge.from && t.to === edge.to); + }); - path: node.path, - parameters: node.parameters, - taskMaxRetries: node.taskMaxRetries, - taskLastExecutionStartTimestamp: node.taskLastExecutionStartTimestamp, - taskLastExecutionStopTimestamp: node.taskLastExecutionStopTimestamp, - taskLastExecutionDuration: node.taskLastExecutionDuration, - taskLastExecutionExecutingNodeName: node.taskLastExecutionExecutingNodeName, - taskNextExecutionTimestamp: node.taskNextExecutionTimestamp, - taskLastStatus: node.taskLastStatus, - taskTags: node.completeTaskObject.tags.map((tag) => tag.name), - taskCustomProperties: node.completeTaskObject.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), - }); - } - } - return this.taskNetwork; + return { nodes: nodesFound, edges: edgesFound, tasks: tasksFound }; } } diff --git a/src/lib/task/find_circular_task_chain.js b/src/lib/task/find_circular_task_chain.js new file mode 100644 index 00000000..e36e4379 --- /dev/null +++ b/src/lib/task/find_circular_task_chain.js @@ -0,0 +1,125 @@ +import { catchLog } from '../util/log.js'; + +/** + * Recursively detect circular task chains starting from a given task node. + * For the provided node, get all downstream nodes and associated edges, then recursively investigate each of those downstream nodes. + * Examine the network top down, from current node to downstream node(s). + * + * @param {object} taskNetwork - Complete task network object with properties: + * - nodes: An array of task nodes. + * - edges: An array of task edges. + * - tasks: An array of task objects. + * @param {object} node - Task node object to start the search from. + * @param {Set} visitedNodes - Set of node IDs that have been visited. + * @param {object} logger - Logger object for logging information. + * + * @returns {Array} An array of objects, each with properties (which are all objects): + * - fromTask: The source node where the circular dependency was detected. + * - toTask: The target node where the circular dependency was detected. + * - edge: The edge connecting the two tasks. + * + * Returns false if something went wrong. + */ +function recursiveFindCircularTaskChain(taskNetwork, node, visitedNodes, logger) { + try { + const circularTaskChain = []; + + // Get downstream nodes. Downstream nodes are identified by node.id === edge.from. + // The downstream node id is then found in edge.to. + const downstreamEdges = taskNetwork.edges.filter((edge) => { + return edge.from === node.id; + }); + + // Any downstream edges found? + if (downstreamEdges?.length > 0) { + for (const downstreamEdge of downstreamEdges) { + // Get downstream node + const downstreamNode = taskNetwork.nodes.find((n) => n.id === downstreamEdge.to); + // If the task network is correctly defined in nodes and edges, the downstream node should always be found. + // If not, there is an error in the task network definition. + if (downstreamNode === undefined) { + logger.error(`DOWNSTREAM NODE NOT FOUND: ${downstreamEdge.to}`); + continue; + } + + // Check if the downstream node has been visited before. + // If so, a circular dependency has been detected. + if (visitedNodes.has(downstreamNode.id)) { + circularTaskChain.push({ fromTask: node, toTask: downstreamNode, edge: downstreamEdge }); + return circularTaskChain; + } + + // Add downstream node to visited nodes. + // visitedNodes.add(downstreamNode.id); + visitedNodes.add(node.id); + + // Recursively investigate downstream node. + const result = recursiveFindCircularTaskChain(taskNetwork, downstreamNode, visitedNodes, logger); + + if (result?.length > 0) { + circularTaskChain.push(...result); + } + } + } + + return circularTaskChain; + } catch (err) { + catchLog('FIND CIRCULAR TASK CHAINS', err); + return false; + } +} + +/** + * Detect circular task chains within the task network. + * + * @param {object} taskNetwork - Task network object with properties: + * - nodes: An array of task nodes. + * - edges: An array of task edges. + * - tasks: An array of task objects. + * @param {object} logger - Logger object for logging information. + * @returns {Array} An array of objects representing circular task chains, each with properties: + * - fromTask: The source node where the circular dependency was detected. + * - toTask: The target node where the circular dependency was detected. + * - edge: The edge connecting the two tasks. + * + * Returns false if something went wrong. + */ +export function findCircularTaskChains(taskNetwork, logger) { + const circularTaskChains = []; + + try { + // Get all root nodes in task network. + // Root nodes are nodes meeting either of the following criteria: + // - node's isTopLevelNode property is true. + const rootNodes = taskNetwork.nodes.filter((node) => node.isTopLevelNode); + + if (!rootNodes) { + logger.error('Could not find root nodes in task model'); + return false; + } + + // Log number of root nodes found. + logger.info(`Found ${rootNodes.length} root nodes in task model`); + + // For each root node, find circular task chains. + // Add all found circular task chains to the circularTaskChains array + for (const rootNode of rootNodes) { + // Returns array of zero of more objects describing circular task chains, or false if something went wrong. + const result = recursiveFindCircularTaskChain(taskNetwork, rootNode, new Set(), logger); + + if (result) { + circularTaskChains.push(...result); + } else { + logger.error('Error when looking for circular task chains in task model'); + return false; + } + } + + return circularTaskChains; + } catch (err) { + catchLog('FIND CIRCULAR TASK CHAINS', err); + return false; + } + + return circularTaskChains; +} diff --git a/src/lib/task/find_root_nodes.js b/src/lib/task/find_root_nodes.js new file mode 100644 index 00000000..de4c8fcf --- /dev/null +++ b/src/lib/task/find_root_nodes.js @@ -0,0 +1,47 @@ +import { catchLog } from '../util/log.js'; + +export function extFindRootNodes(_, node, logger, visitedNodes = new Set()) { + const rootNodes = []; + + try { + if (visitedNodes.has(node.id)) { + logger.verbose(`Circular dependency detected when looking for root nodes: [${node.id}] "${node.taskName}"`); + return rootNodes; + } + visitedNodes.add(node.id); + + if (node.isTopLevelNode) { + rootNodes.push(node); + } else { + // This node is not a root node. + // Investigate upstream nodes. + const upstreamEdges = _.taskNetwork.edges.filter((edge) => edge.to === node.id); + + for (const upstreamEdge of upstreamEdges) { + const upstreamNode = _.taskNetwork.nodes.find((n) => n.id === upstreamEdge.from); + // If the task network is correctly defined in nodes and edges, the upstream node should always be found. + // If not, there is an error in the task network definition. + if (upstreamNode === undefined) { + logger.error(`UPSTREAM NODE NOT FOUND: ${upstreamEdge.from}`); + continue; + } + + // Is the upstream node a root node? + // If so add it to the rootNodes array and don't go further up the tree. + if (upstreamNode.isTopLevelNode) { + rootNodes.push(upstreamNode); + continue; + } + + const result = extFindRootNodes(_, upstreamNode, logger, visitedNodes); + if (result.length > 0) { + rootNodes.push(...result); + } + } + } + } catch (err) { + catchLog('FIND ROOT NODES', err); + } + + return rootNodes; +} diff --git a/src/lib/task/get_root_nodes_from_filter.js b/src/lib/task/get_root_nodes_from_filter.js new file mode 100644 index 00000000..db582c69 --- /dev/null +++ b/src/lib/task/get_root_nodes_from_filter.js @@ -0,0 +1,160 @@ +export async function extGetRootNodesFromFilter(_, logger) { + // Ensure task network and options are available + if (!_.taskNetwork) { + logger.error('getRootNodesFromFilter: Task network not available. Exiting.'); + return false; + } + + if (!_.options) { + logger.error('getRootNodesFromFilter: Options not available. Exiting.'); + return false; + } + + let rootNodes = []; + const nodesToVisualize = []; + + // Helper function to find root nodes with circular dependency detection + function findRootNodesWithCircularCheck(node, visitedNodes = new Set()) { + if (visitedNodes.has(node.id)) { + logger.verbose(`Circular dependency detected when looking for root nodes: [${node.id}] "${node.taskName}"`); + return []; + } + visitedNodes.add(node.id); + return _.findRootNodes(node, visitedNodes); + } + + // Start by checking if any task id filters are specified + if (_.options.taskId) { + // _.options.taskId is an array of task ids + // Get all matching tasks in task model + logger.verbose(`Task id filters specified: ${_.options.taskId}`); + + const nodesFiltered = _.taskNetwork.nodes.filter((node) => { + if (_.options.taskId.includes(node.id)) { + return true; + } else { + return false; + } + }); + + // Method: + // 1. For each node in nodesFiltered, find its root node. + try { + // Did task filters result in any actual tasks/nodes? + if (nodesFiltered.length > 0) { + for (const node of nodesFiltered) { + // node can be isolated, i.e. not part of a chain, or part of a chain + // If isolated, it is by definition a root node + // If part of a chain, it may or may not be a root node + + // Method to find root node: + // 1. Check if node is a top level/root node. isTopLevelNode property is true for root nodes. + // 2. Check if node has any upstream nodes. + // 1. Recursively investigate upstream nodes until a root node is found. + // 3. Save all found root nodes. + + // Is the node a root node? + if (node.isTopLevelNode) { + // Add the node to rootNodes + rootNodes.push(node); + } else { + const tmpRootNodes = findRootNodesWithCircularCheck(node); + rootNodes.push(...tmpRootNodes); + } + } + + // Set nodesToVisualize to root nodes + nodesToVisualize.push(...rootNodes); + + logger.verbose(`Found ${rootNodes.length} root nodes in task model via task id filter`); + // Log root node type, id and if available name + rootNodes.forEach((node) => { + if (node.taskName) { + logger.debug(`Root task: [${node.id}] - "${node.taskName}"`); + } else if (node.metaNodeType) { + logger.debug(`Root meta task: [${node.id}] - "${node.metaNodeType}"`); + } + }); + } else { + logger.warn('No tasks found matching the specified task id(s)/tag(s). Exiting.'); + return false; + } + } catch (error) { + console.error(error); + console.error('Error in parseTree()'); + } + } + + // Any task tag filters specified? + if (_.options.taskTag) { + // Get all matching tasks in task model + logger.verbose(`Task tag filters specified: ${_.options.taskTag}`); + + rootNodes = []; // Reset rootNodes array + + const nodesFiltered = _.taskNetwork.nodes.filter((node) => { + // Are there any tags in this node? + if (!node.taskTags) { + return false; + } + + if (node.taskTags.some((tag) => _.options.taskTag.includes(tag))) { + return true; + } else { + return false; + } + }); + + // Method: + // 1. For each node in nodesFiltered, find its root node. + try { + // Did task filters result in any actual tasks/nodes? + if (nodesFiltered.length > 0) { + for (const node of nodesFiltered) { + // node can be isolated, i.e. not part of a chain, or part of a chain + // If isolated, it is by definition a root node + // If part of a chain, it may or may not be a root node + + // Method to find root node: + // 1. Check if node is a top level/root node. isTopLevelNode property is true for root nodes. + // 2. Check if node has any upstream nodes. + // 1. Recursively investigate upstream nodes until a root node is found. + // 3. Save all found root nodes. + + // Is the node a root node? + if (node.isTopLevelNode) { + // Add the node to rootNodes + rootNodes.push(node); + } else { + const tmpRootNodes = findRootNodesWithCircularCheck(node); + rootNodes.push(...tmpRootNodes); + } + } + + logger.verbose(`Found ${rootNodes.length} root nodes in task model via task tag filter`); + // Log root node type, id and if available name + rootNodes.forEach((node) => { + if (node.taskName) { + logger.debug(`Root task: [${node.id}] - "${node.taskName}"`); + } else if (node.metaNodeType) { + logger.debug(`Root meta task: [${node.id}] - "${node.metaNodeType}"`); + } + }); + } else { + logger.warn('No tasks found matching the specified task id(s)/tag(s). Exiting.'); + return false; + } + } catch (error) { + console.error(error); + console.error('Error in parseTree()'); + } + } + + // De-duplicate root nodes + rootNodes = rootNodes.filter((node, index, self) => { + return index === self.findIndex((t) => t.id === node.id); + }); + + logger.debug(`getRootNodesFromFilter done.`); + return rootNodes; +} diff --git a/src/lib/task/get_task_model_from_file.js b/src/lib/task/get_task_model_from_file.js new file mode 100644 index 00000000..d255b545 --- /dev/null +++ b/src/lib/task/get_task_model_from_file.js @@ -0,0 +1,426 @@ +import { mapTaskType } from '../util/qseow/lookups.js'; +import { catchLog } from '../util/log.js'; + +export async function extGetTaskModelFromFile(_, tasksFromFile, tagsExisting, cpExisting, options, logger) { + return new Promise(async (resolve, reject) => { + try { + logger.debug('PARSE TASKS FROM FILE: Starting get tasks from data in file'); + + _.clear(); + + // Figure out which data is in which column + const taskFileColumnHeaders = getTaskColumnPosFromHeaderRow(tasksFromFile.data[0]); + + // Assign values to columns that were not part of earlier versions of the task file + // This is to make sure that the code below does not break when reading older versions of the task file + if (taskFileColumnHeaders.importOptions.pos === -1) { + // No import options column in file. Add a fake one to make sure code below does not break + logger.debug('PARSE TASKS FROM FILE: No import options column in file. Adding fake one'); + taskFileColumnHeaders.importOptions.pos = 999; + } + + // We now have all info about the task. Store it to the internal task data structure + _.taskNetwork = { nodes: [], edges: [], tasks: [] }; + const nodesWithEvents = new Set(); + + // Find max task counter = number of tasks to be imported + // This can be tricky for Excel files, as these can contain empty rows that are still saved in the file on disk. + // Somehow we need to determine which rows should be treated as input data. + // The criteria is that the "Task counter" column should either be a heading (i.e. "Task counter") or a number. + const taskImportCount = Math.max( + ...tasksFromFile.data.map((item) => { + if (item.length === 0) { + // Empty row + return -1; + } + + // Is first column empty? + if (item[taskFileColumnHeaders.taskCounter.pos] === undefined) { + // Empty task counter column + return -1; + } + + if (item[taskFileColumnHeaders.taskCounter.pos] === taskFileColumnHeaders.taskCounter.name) { + // This is the header row + return -1; + } + + // When reading from CSV files all columns will be strings. + let taskNum; + if (_.options.fileType === 'csv') { + taskNum = item[taskFileColumnHeaders.taskCounter.pos]; + } + if (_.options.fileType === 'excel') { + taskNum = item[taskFileColumnHeaders.taskCounter.pos]; + } + return taskNum; + }) + ); + + logger.info('-------------------------------------------------------------------'); + logger.info('Creating tasks...'); + + // Loop over all tasks in source file + for (let taskCounter = 1; taskCounter <= taskImportCount; taskCounter += 1) { + // Get all rows associated with this task + // One row will contain task data, other rows will contain event data associated with the task. + const taskRows = tasksFromFile.data.filter((item) => item[taskFileColumnHeaders.taskCounter.pos] === taskCounter); + logger.debug( + `(${taskCounter}) PARSE TASKS FROM FILE: Processing task #${taskCounter} of ${taskImportCount}. Data being used:\n${JSON.stringify( + taskRows, + null, + 2 + )}` + ); + + // Verify that first row contains task data. Following rows should contain event data associated with the task. + // Valid task types are: + // - Reload + // - External program + if ( + !taskRows[0][taskFileColumnHeaders.taskType.pos] || + !['reload', 'external program'].includes(taskRows[0][taskFileColumnHeaders.taskType.pos].trim().toLowerCase()) + ) { + logger.error( + `(${taskCounter}) PARSE TASKS FROM FILE: Incorrect task type "${ + taskRows[0][taskFileColumnHeaders.taskType.pos] + }". Exiting.` + ); + process.exit(1); + } + + // Handle each task type separately + // Get task type (lower case) from first row + const taskType = taskRows[0][taskFileColumnHeaders.taskType.pos].trim().toLowerCase(); + + // Reload task + if (taskType === 'reload') { + // Create a fake ID for this task. Used to associate task with schema/composite events + const fakeTaskId = `reload-task-${uuidv4()}`; + + const res = await _.parseReloadTask({ + taskRows, + taskFileColumnHeaders, + taskCounter, + tagsExisting, + cpExisting, + fakeTaskId, + nodesWithEvents, + options, + }); + + // Add reload task as node in task network + // NB: A top level node is defined as: + // 1. A task whose taskID does not show up in the "to" field of any edge. + + _.taskNetwork.nodes.push({ + id: res.currentTask.id, + metaNode: false, + isTopLevelNode: !_.taskNetwork.edges.find((edge) => edge.to === res.currentTask.id), + label: res.currentTask.name, + enabled: res.currentTask.enabled, + + completeTaskObject: res.currentTask, + + // Tabulator columns + taskId: res.currentTask.id, + taskName: res.currentTask.name, + taskEnabled: res.currentTask.enabled, + appId: res.currentTask.app.id, + appName: 'N/A', + appPublished: 'N/A', + appStream: 'N/A', + taskMaxRetries: res.currentTask.maxRetries, + taskLastExecutionStartTimestamp: 'N/A', + taskLastExecutionStopTimestamp: 'N/A', + taskLastExecutionDuration: 'N/A', + taskLastExecutionExecutingNodeName: 'N/A', + taskNextExecutionTimestamp: 'N/A', + taskLastStatus: 'N/A', + taskTags: res.currentTask.tags.map((tag) => tag.name), + taskCustomProperties: res.currentTask.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), + }); + + // We now have a basic task object including tags and custom properties. + // Schema events are included but composite events are only partially there, as they may need + // IDs of tasks that have not yet been created. + // Still, store all info for composite events and then do another loop where those events are created for + // tasks for which they are defined. + // + // The strategey is to first create all tasks, then add composite events. + // Only then can we be sure all composite events refer to existing tasks. + + // Create new or update existing reload task in QSEoW + // Should we create a new task? + // Controlled by option --update-mode + if (_.options.updateMode === 'create') { + // Create new task + if (_.options.dryRun === false || _.options.dryRun === undefined) { + const newTaskId = await createReloadTaskInQseow(res.currentTask, taskCounter, _.options); + logger.info(`(${taskCounter}) Created new reload task "${res.currentTask.name}", new task id: ${newTaskId}.`); + + // Add mapping between fake task ID used when creating task network and actual, newly created task ID + _.taskIdMap.set(fakeTaskId, newTaskId); + + // Add mapping between fake task ID specified in source file and actual, newly created task ID + if (res.currentTask.id) { + _.taskIdMap.set(res.currentTask.id, newTaskId); + } + + res.currentTask.idRef = res.currentTask.id; + res.currentTask.id = newTaskId; + + await _.addTask('from_file', res.currentTask, false); + } else { + logger.info(`(${taskCounter}) DRY RUN: Creating reload task in QSEoW "${res.currentTask.name}"`); + } + } else if (_.options.updateMode === 'update-if-exists') { + // Update existing task + // TODO + // // Verify task ID is a valid UUID + // // If it's not a valid UUID, the ID specified in the source file will be treated as a task name + // if (!validate(res.currentTask.id)) { + // // eslint-disable-next-line no-await-in-loop + // const task = await getTaskByName(res.currentTask.id); + // if (task) { + // // eslint-disable-next-line no-await-in-loop + // await _.updateReloadTaskInQseow(res.currentTask, taskCounter); + // } else { + // throw new Error( + // `Task "${res.currentTask.id}" does not exist in QSEoW and cannot be updated. ` + + // 'Please specify a valid task ID or task name in the source file.' + // ); + // } + // } else { + // // Verify task ID exists in QSEoW + // // eslint-disable-next-line no-await-in-loop + // const taskExists = await getTaskById(res.currentTask.id); + // if (!taskExists) { + // throw new Error( + // `Task "${res.currentTask.id}" does not exist in QSEoW and cannot be updated. ` + + // 'Please specify a valid task ID or task name in the source file.' + // ); + // } else { + // // eslint-disable-next-line no-await-in-loop + // await _.updateReloadTaskInQseow(res.currentTask, taskCounter); + // logger.info( + // `(${taskCounter}) Updated existing task "${res.currentTask.name}", task id: ${res.currentTask.id}.` + // ); + // } + // } + // logger.info( + // `(${taskCounter}) Updated existing task "${res.currentTask.name}", task id: ${res.currentTask.id}.` + // ); + } else { + // Invalid combination of import options + throw new Error( + `Invalid task update mode. Valid values are "create" and "update-if-exists". You specified "${_.options.updateMode}".` + ); + } + } else if (taskType === 'external program') { + // External program task + + // Create a fake ID for this task. Used to associate task with schema/composite events + const fakeTaskId = `ext-pgm-task-${uuidv4()}`; + + const res = await _.parseExternalProgramTask({ + taskRows, + taskFileColumnHeaders, + taskCounter, + tagsExisting, + cpExisting, + fakeTaskId, + nodesWithEvents, + options, + }); + + // Add external program task as node in task network + // NB: A top level node is defined as: + // 1. A task whose taskID does not show up in the "to" field of any edge. + + _.taskNetwork.nodes.push({ + id: res.currentTask.id, + metaNode: false, + isTopLevelNode: !_.taskNetwork.edges.find((edge) => edge.to === res.currentTask.id), + label: res.currentTask.name, + enabled: res.currentTask.enabled, + + completeTaskObject: res.currentTask, + + taskTags: res.currentTask.tags.map((tag) => tag.name), + taskCustomProperties: res.currentTask.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), + }); + + // We now have a basic task object including tags and custom properties. + // Schema events are included but composite events are only partially there, as they may need + // IDs of tasks that have not yet been created. + // Still, store all info for composite events and then do another loop where those events are created for + // tasks for which they are defined. + // + // The strategey is to first create all tasks, then add composite events. + // Only then can we be sure all composite events refer to existing tasks. + + // Create new or update existing external program task in QSEoW + // Should we create a new task? + // Controlled by option --update-mode + if (_.options.updateMode === 'create') { + // Create new task + if (_.options.dryRun === false || _.options.dryRun === undefined) { + const newTaskId = await createExternalProgramTaskInQseow(res.currentTask, taskCounter, _.options); + logger.info( + `(${taskCounter}) Created new external program task "${res.currentTask.name}", new task id: ${newTaskId}.` + ); + + // Add mapping between fake task ID used when creating task network and actual, newly created task ID + _.taskIdMap.set(fakeTaskId, newTaskId); + + // Add mapping between fake task ID specified in source file and actual, newly created task ID + if (res.currentTask.id) { + _.taskIdMap.set(res.currentTask.id, newTaskId); + } + + res.currentTask.idRef = res.currentTask.id; + res.currentTask.id = newTaskId; + + await _.addTask('from_file', res.currentTask, false); + } else { + logger.info(`(${taskCounter}) DRY RUN: Creating external program task in QSEoW "${res.currentTask.name}"`); + } + } else if (_.options.updateMode === 'update-if-exists') { + // Update existing task + // TODO + // // Verify task ID is a valid UUID + // // If it's not a valid UUID, the ID specified in the source file will be treated as a task name + // if (!validate(res.currentTask.id)) { + // // eslint-disable-next-line no-await-in-loop + // const task = await + } else { + // Invalid combination of import options + throw new Error( + `Invalid task update mode. Valid values are "create" and "update-if-exists". You specified "${_.options.updateMode}".` + ); + } + } + } + + // At this point all tasks have been created or updated in Sense. + // The tasks' associated composite events have been parsed and are stored in the qlikSenseCompositeEvents object. + // Time now to create the composite events in Sense. + + // Make sure all composite tasks contain real, valid task UUIDs pointing to previously existing or newly created tasks. + // Get task IDs for upstream tasks that composite events are connected to via their respective rules. + // Some rules will point to upstream tasks that are created during this execution of Ctrl-Q, other upstream tasks existed before this execution of Ctrl-Q. + // Use the task ID map to get the correct task ID for each upstream task. + _.qlikSenseCompositeEvents.compositeEventList.map((item) => { + const a = item; + + // Set task ID for the composite event itself, i.e. which task is the event associated with (i.e. the downstream task) + // Handle different task types differently + if (item.compositeEvent?.reloadTask?.id) { + // Reload task + a.compositeEvent.reloadTask.id = _.taskIdMap.get(item.compositeEvent.reloadTask.id); + } else if (item.compositeEvent?.externalProgramTask?.id) { + // External program task + a.compositeEvent.externalProgramTask.id = _.taskIdMap.get(item.compositeEvent.externalProgramTask.id); + } + + // For this composite event, set the correct task id for each each rule. + // Different properties are used for reload tasks, external program tasks etc. + // Some rules may be pointing to newly created tasks. These can be looked up in the taskIdMap. + a.compositeEvent.compositeRules.map((item2) => { + const b = item2; + + // Get triggering/upstream task id + const id = _.taskIdMap.get(b.task.id); + + // If id is not found in the mapping table, it means that the task + // referenced by the rule (i.e. the upstream teask) is neither a task + // that existed before this execution of Ctrl-Q, nor a task that was + // created during this execution of Ctrl-Q. + // This is an error - the task ID should exist. + // Most likely the error is caused by an invalid value in the "Rule task id" + // column in the source file. + if (id !== undefined && validate(id) === true) { + // Determine what kind of task this is. Options are: + // - reload + // - external program + // + // Also need to know if the task is a new task created during this execution of Ctrl-Q or if it's an existing task in Sense. + let taskType; + if (b.upstreamTaskExistence === 'exists-in-source-file') { + const task = _.taskList.find((item3) => item3.taskId === id); + taskType = task.taskType; + // const { taskType } = _.taskNetwork.nodes.find((node) => node.id === id).completeTaskObject; + } else if (b.upstreamTaskExistence === 'exists-in-sense') { + const task = _.compositeEventUpstreamTask.find((item4) => item4.id === b.task.id); + + // Ensure we got a task back + if (!task) { + logger.error( + `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". This is an error - that task ID should exist. Existing.` + ); + process.exit(1); + } + + taskType = task.taskType; + } + + // Use mapTaskType to get the string variant of the task type. Convert to lower case. + const taskTypeString = mapTaskType.get(taskType).trim().toLowerCase(); + + // Ensure we got a valid task type + if (!['reload', 'externalprogram'].includes(taskTypeString)) { + logger.error( + `PREPARING COMPOSITE EVENT: Invalid task type "${taskTypeString}" for upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` + ); + process.exit(1); + } + + if (taskTypeString === 'reload') { + b.reloadTask = { id }; + } else if (taskTypeString === 'externalprogram') { + b.externalProgramTask = { id }; + } + } else if (id === undefined) { + // (_.options.dryRun === false || _.options.dryRun === undefined) { + logger.error( + `PREPARING COMPOSITE EVENT: Invalid upstream task ID "${b.task.id}" in rule for composite event "${a.compositeEvent.name}". Exiting.` + ); + process.exit(1); + } + return b; + }); + return a; + }); + + logger.info('-------------------------------------------------------------------'); + logger.info('Creating composite events for the just created tasks...'); + + for (const { compositeEvent } of _.qlikSenseCompositeEvents.compositeEventList) { + if (_.options.dryRun === false || _.options.dryRun === undefined) { + await createCompositeEventInQseow(compositeEvent, _.options); + } else { + logger.info(`DRY RUN: Creating composite event "${compositeEvent.name}"`); + } + } + + // Add tasks to network array in plain, non-hierarchical format + _.taskNetwork.tasks = _.taskList; + + resolve(_.taskList); + } catch (err) { + catchLog('PARSE TASKS FROM FILE 1', err); + + if (err?.response?.status) { + logger.error(`Received error ${err.response?.status}/${err.response?.statusText} from QRS API`); + } + if (err?.response?.data) { + logger.error(`Error message from QRS API: ${err.response.data}`); + } + if (err?.config?.data) { + logger.error(`Data sent to Sense: ${JSON.stringify(JSON.parse(err.config.data), null, 2)}}`); + } + reject(err); + } + }); +} diff --git a/src/lib/task/get_task_model_from_qseow.js b/src/lib/task/get_task_model_from_qseow.js new file mode 100644 index 00000000..f3a1988e --- /dev/null +++ b/src/lib/task/get_task_model_from_qseow.js @@ -0,0 +1,465 @@ +import { v4 as uuidv4, validate } from 'uuid'; + +import { catchLog } from '../util/log.js'; + +export async function extGetTaskModelFromQseow(_, logger) { + logger.debug('GET TASK: Getting task model from QSEoW'); + + // Get all tasks from QSEoW + try { + logger.verbose(`Getting tasks from QSEoW...`); + await _.getTasksFromQseow(); + } catch (err) { + catchLog('GET TASK MODEL FROM QSEOW 1', err); + return false; + } + + // Get all schema events from QSEoW + try { + logger.verbose(`Getting schema events from QSEoW...`); + const result1 = await _.qlikSenseSchemaEvents.getSchemaEventsFromQseow(); + + logger.silly(`Schema events from QSEoW: ${JSON.stringify(result1, null, 2)}`); + } catch (err) { + catchLog('GET TASK MODEL FROM QSEOW 2', err); + return false; + } + + // Get all composite events from QSEoW + try { + logger.verbose(`Getting composite events from QSEoW...`); + const result2 = await _.qlikSenseCompositeEvents.getCompositeEventsFromQseow(); + + logger.silly(`Composite events from QSEoW: ${JSON.stringify(result2, null, 2)}`); + } catch (err) { + catchLog('GET TASK MODEL FROM QSEOW 3', err); + return false; + } + + logger.verbose('GET TASK MODEL FROM QSEOW: Done getting task model from QSEoW'); + + // Get all top level apps, i.e apps that aren't triggered by any other apps succeeding or failing. + // They might have scheduled triggers though. + _.taskNetwork = { nodes: [], edges: [], tasks: [] }; + const nodesWithEvents = new Set(); + + // We already have all tasks in plain, non-hierarchical format + _.taskNetwork.tasks = _.taskList; + + // Add schema edges and start/trigger nodes + logger.verbose('GET TASK MODEL FROM QSEOW: Adding schema edges and start/trigger nodes to internal task model'); + for (const schemaEvent of _.qlikSenseSchemaEvents.schemaEventList) { + logger.silly(`Schema event contents: ${JSON.stringify(schemaEvent, null, 2)}`); + // Schedule is associated with a reload task + if (schemaEvent.schemaEvent.reloadTask !== null) { + logger.debug( + `Processing schema event "${schemaEvent?.schemaEvent?.name}" for reload task "${schemaEvent?.schemaEvent?.reloadTask?.name}" (${schemaEvent?.schemaEvent?.reloadTask?.id})` + ); + + // Add schema trigger nodes. These represent the implicit starting nodes that a schema event really are + const nodeId = `node-${uuidv4()}`; + _.taskNetwork.nodes.push({ + id: nodeId, + metaNodeType: 'schedule', // Meta nodes are not Sense tasks, but rather nodes representing task-like properties (e.g. a starting point for a reload chain) + metaNode: true, + isTopLevelNode: true, + label: schemaEvent.schemaEvent.name, + enabled: schemaEvent.schemaEvent.enabled, + taskType: 'reloadTask', + + completeSchemaEvent: schemaEvent.schemaEvent, + }); + + _.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.schemaEvent.reloadTask.id, + }); + + // Keep a note that this node has associated events + nodesWithEvents.add(schemaEvent.schemaEvent.reloadTask.id); + } else if (schemaEvent.schemaEvent.externalProgramTask !== null) { + // Schedule is associated with an external program task + logger.debug( + `Processing schema event "${schemaEvent?.schemaEvent?.name}" for external program task "${schemaEvent?.schemaEvent?.reloadTask?.name}" (${schemaEvent?.schemaEvent?.externalProgramTask?.id})` + ); + + // Add schema trigger nodes. These represent the implicit starting nodes that a schema event really are + const nodeId = `node-${uuidv4()}`; + _.taskNetwork.nodes.push({ + id: nodeId, + metaNodeType: 'schedule', // Meta nodes are not Sense tasks, but rather nodes representing task-like properties (e.g. a starting point for a reload chain) + metaNode: true, + isTopLevelNode: true, + label: schemaEvent.schemaEvent.name, + enabled: schemaEvent.schemaEvent.enabled, + taskType: 'externalProgramTask', + + completeSchemaEvent: schemaEvent.schemaEvent, + }); + + _.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.schemaEvent.externalProgramTask.id, + }); + + // Keep a note that this node has associated events + nodesWithEvents.add(schemaEvent.schemaEvent.externalProgramTask.id); + } + } + + // Add composite events + logger.verbose('GET TASK MODEL FROM QSEOW: Adding composite events to internal task model'); + for (const compositeEvent of _.qlikSenseCompositeEvents.compositeEventList) { + logger.silly(`Composite event contents: ${JSON.stringify(compositeEvent, null, 2)}`); + + if (compositeEvent?.compositeEvent?.reloadTask) { + logger.debug( + `Processing composite event "${compositeEvent?.compositeEvent?.name}" for reload task "${compositeEvent?.compositeEvent?.reloadTask?.name}" (${compositeEvent?.compositeEvent?.reloadTask?.id})` + ); + } else if (compositeEvent?.compositeEvent?.externalProgramTask) { + logger.debug( + `Processing composite event "${compositeEvent?.compositeEvent?.name}" for external program task "${compositeEvent?.compositeEvent?.externalProgramTask?.name}" (${compositeEvent?.compositeEvent?.externalProgramTask?.id})` + ); + } else if (compositeEvent?.compositeEvent?.userSyncTask) { + logger.debug( + `Processing composite event "${compositeEvent?.compositeEvent?.name}" for user sync task "${compositeEvent?.compositeEvent?.userSyncTask?.name}" (${compositeEvent?.compositeEvent?.userSyncTask?.id})` + ); + } + + if (compositeEvent.compositeEvent.reloadTask !== null) { + // Current composite event triggers a reload task + logger.debug( + `Current composite event triggers a reload task. Processing composite event "${compositeEvent?.compositeEvent?.name}" for reload task "${compositeEvent?.compositeEvent?.reloadTask?.name}" (${compositeEvent?.compositeEvent?.reloadTask?.id})` + ); + + if (compositeEvent.compositeEvent.reloadTask.id === undefined || compositeEvent.compositeEvent.reloadTask.id === null) { + logger.warn(`Composite event "${compositeEvent.compositeEvent.name}" has no reload task ID associated with it.`); + } else if (compositeEvent.compositeEvent.compositeRules.length === 1) { + // This trigger has exactly ONE upstream task + // For triggers with >1 upstream task we want an extra meta node to represent the waiting of all upstream tasks to finish + if (validate(compositeEvent.compositeEvent.compositeRules[0]?.reloadTask?.id)) { + logger.debug( + `Composite event "${compositeEvent.compositeEvent.name}" has a reload task triggered by reload task with ID=${compositeEvent.compositeEvent.compositeRules[0].reloadTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: compositeEvent.compositeEvent.compositeRules[0].reloadTask.id, + fromTaskType: 'Reload', + + to: compositeEvent.compositeEvent.reloadTask.id, + toTaskType: 'Reload', + + completeCompositeEvent: compositeEvent.compositeEvent, + rule: compositeEvent.compositeEvent.compositeRules, + // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', + // color: edgeColor, + // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], + // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', + // label: compositeEvent.compositeEvent.name, + }); + + // Keep a note that this node has associated events + nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].reloadTask.id); + nodesWithEvents.add(compositeEvent.compositeEvent.reloadTask.id); + } else if (validate(compositeEvent.compositeEvent.compositeRules[0]?.externalProgramTask?.id)) { + logger.debug( + `Composite event "${compositeEvent?.compositeEvent?.name}" has a reload task triggered by external program task with ID=${compositeEvent.compositeEvent.compositeRules[0]?.externalProgramTask?.id}.` + ); + + _.taskNetwork.edges.push({ + from: compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id, + fromTaskType: 'ExternalProgram', + + to: compositeEvent.compositeEvent.reloadTask.id, + toTaskType: 'Reload', + + completeCompositeEvent: compositeEvent.compositeEvent, + rule: compositeEvent.compositeEvent.compositeRules, + // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', + // color: edgeColor, + // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], + // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', + // label: compositeEvent.compositeEvent.name, + }); + + // Keep a note that this node has associated events + nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id); + nodesWithEvents.add(compositeEvent.compositeEvent.reloadTask.id); + } + } else { + // There are more than one task involved in triggering a downstream task. + // Insert a proxy node that represents a Qlik Sense composite event + logger.debug( + `Composite event "${compositeEvent?.compositeEvent?.name}" is triggerer by ${compositeEvent?.compositeEvent?.compositeRules.length} upstream tasks.` + ); + + // TODO + const nodeId = `node-${uuidv4()}`; + _.taskNetwork.nodes.push({ + id: nodeId, + label: compositeEvent.compositeEvent.name, + enabled: true, + metaNodeType: 'composite', + metaNode: true, + }); + nodesWithEvents.add(nodeId); + + // Add edges from upstream tasks to the new meta node + for (const rule of compositeEvent.compositeEvent.compositeRules) { + if (validate(rule?.reloadTask?.id)) { + // Upstream task is a reload task + logger.debug( + `Composite event "${compositeEvent.compositeEvent.name}" is triggered by reload task with ID=${rule.reloadTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: rule.reloadTask.id, + fromTaskType: 'Reload', + to: nodeId, + toTaskType: 'Composite', + + // TODO Correct? Or should it be at next edges.push? + completeCompositeEvent: compositeEvent.compositeEvent, + rule, + }); + } else if (validate(rule?.externalProgramTask?.id)) { + // Upstream task is an external program task + logger.debug( + `Composite event "${compositeEvent.compositeEvent.name}" is triggered by external program task with ID=${rule.externalProgramTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: rule.externalProgramTask.id, + fromTaskType: 'ExternalProgram', + to: nodeId, + toTaskType: 'Composite', + + // TODO Correct? Or should it be at next edges.push? + completeCompositeEvent: compositeEvent.compositeEvent, + rule, + }); + } else { + logger.warn( + `Upstream task for composite event "${compositeEvent.compositeEvent.name}" is not of a supported task type (reload task, external program task).` + ); + } + } + + // Add edge from new meta node to current node + logger.debug( + `Added edge from composite event meta node "${nodeId}" to reload task ID=${compositeEvent.compositeEvent.reloadTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.reloadTask.id, + }); + } + } else if (compositeEvent.compositeEvent.externalProgramTask !== null) { + // Current composite event triggers an external program task + logger.debug( + `Composite event "${compositeEvent.compositeEvent.name}" triggers an external program task "${compositeEvent.compositeEvent.externalProgramTask.name}" (${compositeEvent.compositeEvent.externalProgramTask.id}).` + ); + + if ( + compositeEvent.compositeEvent.externalProgramTask.id === undefined || + compositeEvent.compositeEvent.externalProgramTask.id === null + ) { + logger.warn(`Composite event "${compositeEvent.compositeEvent.name}" has no external program task ID associated with it.`); + } else if (compositeEvent.compositeEvent.compositeRules.length === 1) { + // This trigger has exactly ONE upstream task + // For triggers with >1 upstream task we want an extra meta node to represent the waiting of all upstream tasks to finish + logger.debug(`Composite event "${compositeEvent.compositeEvent.name}" has exactly one upstream task.`); + + if (validate(compositeEvent.compositeEvent.compositeRules[0]?.reloadTask?.id)) { + logger.debug( + `Composite event "${compositeEvent?.compositeEvent?.name}" has an external program task triggered by reload task with ID=${compositeEvent.compositeEvent.compositeRules[0]?.reloadTask?.id}.` + ); + + _.taskNetwork.edges.push({ + from: compositeEvent.compositeEvent.compositeRules[0].reloadTask.id, + fromTaskType: 'Reload', + + to: compositeEvent.compositeEvent.externalProgramTask.id, + toTaskType: 'ExternalProgram', + + completeCompositeEvent: compositeEvent.compositeEvent, + rule: compositeEvent.compositeEvent.compositeRules, + // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', + // color: edgeColor, + // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], + // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', + // label: compositeEvent.compositeEvent.name, + }); + // Keep a note that this node has associated events + nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].reloadTask.id); + nodesWithEvents.add(compositeEvent.compositeEvent.externalProgramTask.id); + } else if (validate(compositeEvent.compositeEvent.compositeRules[0]?.externalProgramTask?.id)) { + logger.debug( + `Composite event "${compositeEvent.compositeEvent.name}" has an external program task triggered by external program task with ID=${compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id, + fromTaskType: 'ExternalProgram', + + to: compositeEvent.compositeEvent.externalProgramTask.id, + toTaskType: 'ExternalProgram', + + completeCompositeEvent: compositeEvent.compositeEvent, + rule: compositeEvent.compositeEvent.compositeRules, + // color: compositeEvent.compositeEvent.enabled ? '#9FC2F7' : '#949298', + // color: edgeColor, + // dashes: compositeEvent.compositeEvent.enabled ? false : [15, 15], + // title: compositeEvent.compositeEvent.name + '
' + 'asdasd', + // label: compositeEvent.compositeEvent.name, + }); + + // Keep a note that this node has associated events + nodesWithEvents.add(compositeEvent.compositeEvent.compositeRules[0].externalProgramTask.id); + nodesWithEvents.add(compositeEvent.compositeEvent.externalProgramTask.id); + } else { + logger.warn(`Composite event "${compositeEvent.compositeEvent.name}" is triggered by an unsupported task type.`); + } + } else { + // There are more than one task involved in triggering a downstream task. + // Insert a proxy node that represents a Qlik Sense composite event + logger.verbose( + `Composite event "${compositeEvent?.compositeEvent?.name}" has ${compositeEvent?.compositeEvent?.compositeRules.length} upstream tasks.` + ); + + const nodeId = `node-${uuidv4()}`; + _.taskNetwork.nodes.push({ + id: nodeId, + label: compositeEvent.compositeEvent.name, + enabled: true, + metaNodeType: 'composite', + metaNode: true, + }); + nodesWithEvents.add(nodeId); + + // Add edges from upstream tasks to the new meta node + for (const rule of compositeEvent.compositeEvent.compositeRules) { + if (validate(rule?.reloadTask?.id)) { + // Upstream task is a reload task + logger.debug( + `Upstream task for composite event "${compositeEvent.compositeEvent.name}" is a reload task with ID=${rule.reloadTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: rule.reloadTask.id, + fromTaskType: 'Reload', + to: nodeId, + toTaskType: 'Composite', + + // TODO Correct? Or should it be at next edges.push? + completeCompositeEvent: compositeEvent.compositeEvent, + rule, + }); + } else if (validate(rule?.externalProgramTask?.id)) { + // Upstream task is an external program task + logger.debug( + `Upstream task for composite event "${compositeEvent.compositeEvent.name}" is an external program task with ID=${rule.externalProgramTask.id}.` + ); + + _.taskNetwork.edges.push({ + from: rule.externalProgramTask.id, + fromTaskType: 'ExternalProgram', + to: nodeId, + toTaskType: 'Composite', + + // TODO Correct? Or should it be at next edges.push? + completeCompositeEvent: compositeEvent.compositeEvent, + rule, + }); + } else { + logger.warn( + `Upstream task for composite event "${compositeEvent?.compositeEvent?.name}" is not of a supported task type (reload task, external program task).` + ); + } + } + + // Add edge from new meta node to current node + logger.debug( + `Added edge from new meta composite event node "${nodeId}" to reload task ID=${compositeEvent.compositeEvent?.reloadTask?.id}.` + ); + + if (compositeEvent.compositeEvent?.reloadTask) { + _.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.reloadTask.id, + }); + } else if (compositeEvent.compositeEvent?.externalProgramTask) { + _.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.compositeEvent.externalProgramTask.id, + }); + } + } + } + } + + // Add all Sense tasks as nodes in task network + // NB: A top level node is defined as: + // 1. A task whose taskID does not show up in the "to" field of any edge. + + for (const node of _.taskList) { + if (node.completeTaskObject.schemaPath === 'ReloadTask') { + _.taskNetwork.nodes.push({ + id: node.taskId, + metaNode: false, + isTopLevelNode: !_.taskNetwork.edges.find((edge) => edge.to === node.taskId), + label: node.taskName, + enabled: node.taskEnabled, + + completeTaskObject: node.completeTaskObject, + + // Tabulator columns + taskId: node.taskId, + taskName: node.taskName, + taskEnabled: node.taskEnabled, + appId: node.appId, + appName: node.appName, + appPublished: node.appPublished, + appStream: node.appStream, + taskMaxRetries: node.taskMaxRetries, + taskLastExecutionStartTimestamp: node.taskLastExecutionStartTimestamp, + taskLastExecutionStopTimestamp: node.taskLastExecutionStopTimestamp, + taskLastExecutionDuration: node.taskLastExecutionDuration, + taskLastExecutionExecutingNodeName: node.taskLastExecutionExecutingNodeName, + taskNextExecutionTimestamp: node.taskNextExecutionTimestamp, + taskLastStatus: node.taskLastStatus, + taskTags: node.completeTaskObject.tags.map((tag) => tag.name), + taskCustomProperties: node.completeTaskObject.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), + }); + } else if (node.completeTaskObject.schemaPath === 'ExternalProgramTask') { + _.taskNetwork.nodes.push({ + id: node.taskId, + metaNode: false, + isTopLevelNode: !_.taskNetwork.edges.find((edge) => edge.to === node.taskId), + label: node.taskName, + enabled: node.taskEnabled, + + completeTaskObject: node.completeTaskObject, + + // Tabulator columns + taskId: node.taskId, + taskName: node.taskName, + taskEnabled: node.taskEnabled, + + path: node.path, + parameters: node.parameters, + taskMaxRetries: node.taskMaxRetries, + taskLastExecutionStartTimestamp: node.taskLastExecutionStartTimestamp, + taskLastExecutionStopTimestamp: node.taskLastExecutionStopTimestamp, + taskLastExecutionDuration: node.taskLastExecutionDuration, + taskLastExecutionExecutingNodeName: node.taskLastExecutionExecutingNodeName, + taskNextExecutionTimestamp: node.taskNextExecutionTimestamp, + taskLastStatus: node.taskLastStatus, + taskTags: node.completeTaskObject.tags.map((tag) => tag.name), + taskCustomProperties: node.completeTaskObject.customProperties.map((cp) => `${cp.definition.name}=${cp.value}`), + }); + } + } + return _.taskNetwork; +} diff --git a/src/lib/task/get_task_sub_graph.js b/src/lib/task/get_task_sub_graph.js new file mode 100644 index 00000000..fe5ab759 --- /dev/null +++ b/src/lib/task/get_task_sub_graph.js @@ -0,0 +1,229 @@ +import { catchLog } from '../util/log.js'; +import { mapTaskType, mapRuleState } from '../util/qseow/lookups.js'; + +/** + * Function to get a subgraph of a task network/graph. + * + * @param {object} _ - QlikSenseTasks object. Corresponds to the 'this' keyword in a class. + * @param {object} node - Node object + * @param {number} parentTreeLevel - Tree level of the parent task. 0 = top level + * @param {object} parentNode - Parent node object + * @param {object} logger - Logger object + * + * @returns {object} Object with three properties: nodes, edges and tasks. + * - nodes is an array of node objects in the subgraph + * - edges is an array of edge objects in the subgraph + * - tasks is an array of task objects in the subgraph + */ +export function extGetTaskSubGraph(_, node, parentTreeLevel, parentNode, logger) { + try { + if (!node || !node?.id) { + logger.debug('Node parameter empty or does not include a node ID'); + } + + let subGraphNodes = []; + let subGraphEdges = []; + let subGraphTasks = []; + + // Were we called from top-level? + if (parentTreeLevel === 0) { + // Set up new data structure for detecting cicrular task chains + _.taskCyclicStack = new Set(); + } + + const newTreeLevel = parentTreeLevel + 1; + + if (node.metaNode === true) { + // Meta node (e.g. schedule, composite task) + logger.debug(`GET TASK SUBGRAPH: Meta node type: ${node.metaNodeType}, tree level: ${newTreeLevel}, node id: ${node.id}`); + + // Add meta node to subGraphNodes + subGraphNodes.push(node); + } else { + // Get task associated with node, using node/task ID as key (if it's a regular task node, not a meta node) + const task = _.taskNetwork.tasks.find((el) => el.taskId === node.id); + if (!task || task === undefined) { + 0; + logger.warn(`Task not found for node ID: ${node.id}`); + } else { + subGraphNodes.push(node); + subGraphTasks.push(task); + // Not sure yet if we should add the edge to subGraphEdges, there may or may not be downstream node(s) + // Thus wait until we know if there are downstream nodes before adding the edge + } + + logger.debug( + `GET TASK SUBGRAPH: Task type: ${mapTaskType.get(task.taskType)}, tree level: ${newTreeLevel}, task id: ${task.taskId}, task name: ${task.taskName}` + ); + } + + // Does this node have any downstream connections? + const edgesToDownstreamNodes = _.taskNetwork.edges.filter((edge) => edge.from === node.id); + + // Keep track of downstream nodes and their associated edge from the current node + // Array of objects with properties + // - sourceNode: Current node object + // - downstreamNode: Downstream node object + // - edge: Edge object between sourceNode and edgeDownstreamNode + const validDownstreamNodes = []; + for (const edgeToDownstreamNode of edgesToDownstreamNodes) { + logger.debug( + `GET TASK SUBGRAPH: Processing downstream node: ${edgeToDownstreamNode.to}. Current/source node: ${edgeToDownstreamNode.from}` + ); + if (edgeToDownstreamNode.to !== undefined) { + // Get downstream node object + const downstreamNode = _.taskNetwork.nodes.find((el) => el.id === edgeToDownstreamNode.to); + + if (!downstreamNode) { + logger.warn( + `Downstream node "${edgeToDownstreamNode.to}" in task network not found. Current/source node: ${edgeToDownstreamNode.from}` + ); + } else { + // Keep track of this downstream node and the associated edge + validDownstreamNodes.push({ sourceNode: node, downstreamNode: downstreamNode, edge: edgeToDownstreamNode }); + + // Don't check for cyclic task relationships yet, as that could trigger if two or more sibling tasks are triggered from the same source task. + } + } + } + + // Now that all downstream nodes have been retrieved, we can check if there are any general issues with those nodes. + // Examples are cyclic task tree relationships, multiple downstream tasks with the same ID etc. + + // Check for downstream nodes with the same ID and same relationship with parent node (e.g. on-success or on-failure) + // edgesToDownstreamNodes is an array of all downstream nodes from the current node. Properties are (the ones relevant here) + // - from: Source node ID + // - fromTaskType: Source node type. "Reload" or "ExternalProgram" + // - to: Destination node ID + // - toTaskType: Destination node type. "Reload", "ExternalProgram" or "Composite" + // - rule: Array of rules for the relationship between source and destination node. "on-success", "on-failure" etc. Properties for each object are + // - id: Rule ID + // - ruleState: Rule state/type. 1 = TaskSuccessful, 2 = TaskFail. mapRuleState.get(ruleState) gives the string representation of the rule state, given the number. + + // Check if there are multiple downstream nodes with the same ID and same relationship with the parent node. + // The relationship is the same if rule.ruleState is the same for two downstream nodes with the same ID. + // If there are, log a warning. + const duplicateDownstreamNodes = []; + for (const edgeToDownstreamNode of edgesToDownstreamNodes) { + // Are there any rules? + // edgeToDownstreamNode.rule is an array of rules. Properties are + // - id: Rule ID + // - ruleState: Rule state/type. 1 = TaskSuccessful, 2 = TaskFail. mapRuleState.get(ruleState) gives the string representation of the rule state, given the number. + if (edgeToDownstreamNode.rule) { + // Filter out downstream nodes with the same ID and the same rule state + const tmp = edgesToDownstreamNodes.filter((el) => { + const sameDest = el.to === edgeToDownstreamNode.to; + + // Same rule state? + // el.rule can be either an array or an object. If it's an object, convert it to an array. + if (!Array.isArray(el.rule)) { + el.rule = [el.rule]; + } + + // Is one of the rule's ruleState properties the same as one or more of edgeToDownstreamNode.rule[].ruleState? + const sameRuleState = el.rule.some((rule) => { + return edgeToDownstreamNode.rule.some((rule2) => { + return rule.ruleState === rule2.ruleState; + }); + }); + + return sameDest && sameRuleState; + }); + + if (tmp.length > 1) { + // Look up current and downstream node objects + const currentNode = _.taskNetwork.nodes.find((el) => el.id === edgeToDownstreamNode.from); + const edgeDownstreamNode = _.taskNetwork.nodes.find((el) => el.id === tmp[0].to); + + // Get the rule state that is shared between the downstream tasks and the parent task + const ruleState = mapRuleState.get(tmp[0].rule[0].ruleState); + + // Log warning unless this parent/child relationship is already in the list of duplicate downstream tasks + if ( + !duplicateDownstreamNodes.some( + (el) => el[0].to === tmp[0].to && el[0].rule[0].ruleState === tmp[0].rule[0].ruleState + ) + ) { + logger.warn( + `Multiple downstream nodes (${tmp.length}) with the same ID and the same trigger relationship "${ruleState}" with the parent node.` + ); + logger.warn(` Parent node : ${currentNode.completeTaskObject.name}`); + logger.warn(` Downstream node : ${edgeDownstreamNode.completeTaskObject.name}`); + } + + duplicateDownstreamNodes.push(tmp); + } + } + } + + // Check if there are any cyclic node relationships + // If there are none, we can add the downstream node to the graph + + // First make sure all downstream node IDs are unique. Remove duplicates. + const uniqueDownstreamNodes = Array.from(new Set(validDownstreamNodes.map((a) => a.downstreamNode.id))).map((id) => { + return validDownstreamNodes.find((a) => a.downstreamNode.id === id); + }); + + for (const uniqueDownstreamNode of uniqueDownstreamNodes) { + if (_.taskCyclicStack.has(uniqueDownstreamNode.downstreamNode.id)) { + // Cyclic dependency detected + if (parentNode) { + // Log verbose info + logger.info(`Cyclic dependency detected in task network. Won't go deeper.`); + logger.verbose(` From task : [${uniqueDownstreamNode.sourceNode.id}] "${uniqueDownstreamNode.sourceNode.taskName}"`); + logger.verbose( + ` To task : [${uniqueDownstreamNode.downstreamNode.id}] "${uniqueDownstreamNode.downstreamNode.taskName}"` + ); + + // Add edge, but don't add downstream node, and don't go deeper as this is a cyclic dependency + subGraphEdges.push(uniqueDownstreamNode.edge); + } else { + // Log warning when there is no parent task (should not happen?) + logger.warn(`Cyclic dependency detected in task tree. No parent task detected. Won't go deeper.`); + } + } else { + // No cyclic dependency detected + + // Add downstream node to stack + // uniqueDownstreamNode is an object with properties + // - sourceNode: Current node object + // - downstreamNode: Downstream node object + // - edge: Edge object between sourceNode and edgeDownstreamNode + _.taskCyclicStack.add(uniqueDownstreamNode.downstreamNode.id); + + // // Add node to subGraphNodes + // subGraphNodes.push(uniqueDownstreamNode.downstreamNode); + + // Add edge to downstream node to subGraphEdges + subGraphEdges.push(uniqueDownstreamNode.edge); + + // // Add task to subGraphTasks + // subGraphTasks.push(task); + + // Examine downstream node + const tmp3 = extGetTaskSubGraph(_, uniqueDownstreamNode.downstreamNode, newTreeLevel, node, logger); + + // Remove downstream node from stack + _.taskCyclicStack.delete(uniqueDownstreamNode.downstreamNode.id); + + // Add tmp3.nodes to subGraphNodes + subGraphNodes = subGraphNodes.concat(...tmp3.nodes); + + // Add tmp3.edges to subGraphEdges + subGraphEdges = subGraphEdges.concat(...tmp3.edges); + + // Add tmp3.tasks to subGraphTasks + subGraphTasks = subGraphTasks.concat(...tmp3.tasks); + } + } + + return { + nodes: subGraphNodes, + edges: subGraphEdges, + tasks: subGraphTasks, + }; + } catch (err) { + catchLog('GET TASK SUBGRAPH', err); + return false; + } +} diff --git a/src/lib/task/get_task_sub_table.js b/src/lib/task/get_task_sub_table.js new file mode 100644 index 00000000..e9649533 --- /dev/null +++ b/src/lib/task/get_task_sub_table.js @@ -0,0 +1,74 @@ +import { catchLog } from '../util/log.js'; + +export function extGetTaskSubTable(_, task, parentTreeLevel, logger) { + try { + const self = _; + + const newTreeLevel = parentTreeLevel + 1; + let subTree = []; + + // Debug + // logger.debug(`GET TASK SUBTABLE: Tree level: ${newTreeLevel}, task name: ${task.taskName}`); + + // Does this node (=task) have any downstream connections? + const downstreamTasks = self.taskNetwork.edges.filter((edge) => edge.from === task.id); + // console.log('downStreamTasks 1: ' + JSON.stringify(downstreamTasks)); + let kids = []; + for (const downstreamTask of downstreamTasks) { + if (downstreamTask.to !== undefined) { + // Get downstream task object + const tmp = self.taskNetwork.nodes.find((el) => el.id === downstreamTask.to); + const tmp3 = self.getTaskSubTable(tmp, newTreeLevel); + kids = kids.concat([tmp3]); + } + } + + // Only push real Sense tasks to the tree (don't include meta nodes) + if (!task.metaNodeType) { + if (kids && kids.length > 0) { + subTree = { + id: task.id, + children: kids, + }; + } else { + subTree = { + id: task.id, + }; + } + + subTree.text = task.taskName; + + // Tabulator columns + subTree.taskId = task.taskId; + subTree.taskName = task.taskName; + subTree.taskEnabled = task.taskEnabled; + subTree.appId = task.appId; + subTree.appName = task.appName; + subTree.appPublished = task.appPublished; + subTree.appStream = task.appStream; + subTree.taskMaxRetries = task.taskMaxRetries; + subTree.taskLastExecutionStartTimestamp = task.taskLastExecutionStartTimestamp; + subTree.taskLastExecutionStopTimestamp = task.taskLastExecutionStopTimestamp; + subTree.taskLastExecutionDuration = task.taskLastExecutionDuration; + subTree.taskLastExecutionExecutingNodeName = task.taskLastExecutionExecutingNodeName; + subTree.taskNextExecutionTimestamp = task.taskNextExecutionTimestamp; + subTree.taskLastStatus = task.taskLastStatus; + subTree.taskTags = task.completeTaskObject.tags.map((tag) => tag.name); + subTree.taskCustomProperties = task.completeTaskObject.customProperties.map((el) => `${el.definition.name}=${el.value}`); + subTree.completeTaskObject = task.completeTaskObject; + + if (newTreeLevel <= 2) { + subTree = kids.concat([[newTreeLevel, task.taskName, task.taskId, task.taskEnabled]]); + } else { + subTree = kids.concat([[newTreeLevel, '--'.repeat(newTreeLevel - 2) + task.taskName, task.taskId, task.taskEnabled]]); + } + } else { + subTree = kids; + } + + return subTree; + } catch (err) { + catchLog('GET TASK SUBTABLE (table)', err); + return null; + } +} diff --git a/src/lib/task/get_task_sub_tree.js b/src/lib/task/get_task_sub_tree.js new file mode 100644 index 00000000..eecb290e --- /dev/null +++ b/src/lib/task/get_task_sub_tree.js @@ -0,0 +1,308 @@ +import { catchLog } from '../util/log.js'; +import { mapRuleState } from '../util/qseow/lookups.js'; + +/** + * Function to get a subtree of a task tree from a given task. + * + * @param {object} _ - QlikSenseTasks object. Corresponds to the 'this' keyword in a class. + * @param {object} task - Task object + * @param {number} parentTreeLevel - Tree level of the parent task + * @param {object} parentTask - Parent task object + * @param {object} logger - Logger object + * + * @returns {array} Array of task objects in the subtree + */ +export function extGetTaskSubTree(_, task, parentTreeLevel, parentTask, logger) { + try { + const self = _; + + if (!task || !task?.id) { + logger.debug('Task parameter empty or does not include a task ID'); + } + + // Were we called from top-level? + if (parentTreeLevel === 0) { + // Set up new data structure for detecting cicrular task trees + _.taskCyclicStack = new Set(); + } + + const newTreeLevel = parentTreeLevel + 1; + let subTree = []; + + logger.debug( + `GET TASK SUBTREE: Meta node type: ${task.metaNodeType}, task type: ${task.taskType}, tree level: ${newTreeLevel}, task name: ${task.taskName}` + ); + + // Does this node (=task) have any downstream connections? + const downstreamTasks = self.taskNetwork.edges.filter((edge) => edge.from === task.id); + + let kids = []; + const validDownstreamTasks = []; + for (const downstreamTask of downstreamTasks) { + logger.debug(`GET TASK SUBTREE: Processing downstream task: ${downstreamTask.to}. Current/source task: ${downstreamTask.from}`); + if (downstreamTask.to !== undefined) { + // Get downstream task object + const tmp = self.taskNetwork.nodes.find((el) => el.id === downstreamTask.to); + + if (!tmp) { + logger.warn( + `Downstream task "${downstreamTask.to}" in task tree not found. Current/source task: ${downstreamTask.from}` + ); + kids = [ + { + id: task.id, + }, + ]; + } else { + // Keep track of this downstream task + validDownstreamTasks.push({ sourceTask: task, downstreamTask: tmp }); + + // Don't check for cyclic task relationships yet, as that could trigger if two or more sibling tasks are triggered from the same source task. + } + } + } + + // Now that all downstream tasks have been retrieved, we can check if there are any issues with those tasks. + // Examples are cyclic task tree relationships, multiple downstream tasks with the same ID etc. + + // Check for downstream tasks with the same ID and same relationship with parent task (e.g. on-success or on-failure) + // downstreamTasks is an array of all downstream tasks from the current task. Properties are (the ones relevant here) + // - from: Source task/node ID + // - fromTaskType: Source task/node type. "Reload" or "ExternalProgram" + // - to: Destination task/node ID + // - toTaskType: Destination task/node type. "Reload", "ExternalProgram" or "Composite" + // - rule: Array of rules for the relationship between source and destination task. "on-success", "on-failure" etc. Properties for each object are + // - id: Rule ID + // - ruleState: Rule state/type. 1 = TaskSuccessful, 2 = TaskFail. mapRuleState.get(ruleState) gives the string representation of the rule state, given the number. + + // Check if there are multiple downstream tasks with the same ID and same relationship with the parent task. + // The relationship is the same if rule.ruleState is the same for two downstream tasks with the same ID. + // If there are, log a warning. + const duplicateDownstreamTasks = []; + for (const downstreamTask of downstreamTasks) { + // Are there any rules? + // downstreamTask.rule is an array of rules. Properties are + // - id: Rule ID + // - ruleState: Rule state/type. 1 = TaskSuccessful, 2 = TaskFail. mapRuleState.get(ruleState) gives the string representation of the rule state, given the number. + if (downstreamTask.rule) { + // Filter out downstream tasks with the same ID and the same rule state + const tmp = downstreamTasks.filter((el) => { + const sameDest = el.to === downstreamTask.to; + + // Same rule state? + // el.rule can be either an array or an object. If it's an object, convert it to an array. + if (!Array.isArray(el.rule)) { + el.rule = [el.rule]; + } + + // Is one of the rule's ruleState properties the same as one or more of downstreamTask.rule[].ruleState? + const sameRuleState = el.rule.some((rule) => { + return downstreamTask.rule.some((rule2) => { + return rule.ruleState === rule2.ruleState; + }); + }); + + return sameDest && sameRuleState; + }); + + if (tmp.length > 1) { + // Look up current and downstream task objects + const currentTask = self.taskNetwork.nodes.find((el) => el.id === task.id); + const downstreamTask = self.taskNetwork.nodes.find((el) => el.id === tmp[0].to); + + // Get the rule state that is shared between the downstream tasks and the parent task + const ruleState = mapRuleState.get(tmp[0].rule[0].ruleState); + + // Log warning unless this parent/child relationship is already in the list of duplicate downstream tasks + if ( + !duplicateDownstreamTasks.some( + (el) => el[0].to === tmp[0].to && el[0].rule[0].ruleState === tmp[0].rule[0].ruleState + ) + ) { + logger.warn( + `Multiple downstream tasks (${tmp.length}) with the same ID and the same trigger relationship "${ruleState}" with the parent task.` + ); + logger.warn(` Parent task : ${currentTask.completeTaskObject.name}`); + logger.warn(` Downstream task : ${downstreamTask.completeTaskObject.name}`); + } + + duplicateDownstreamTasks.push(tmp); + } + } + } + + // Check if there are any cyclic task tree relationships + // If there are none, we can add the downstream tasks to the tree + // First make sure all downstream task IDs are unique. Remove duplicates. + const uniqueDownstreamTasks = Array.from(new Set(validDownstreamTasks.map((a) => a.downstreamTask.id))).map((id) => { + return validDownstreamTasks.find((a) => a.downstreamTask.id === id); + }); + + // Now sort the downstream tasks by task name + uniqueDownstreamTasks.sort((a, b) => a.downstreamTask.taskName.localeCompare(b.downstreamTask.taskName)); + + for (const uniqueDownstreamTask of uniqueDownstreamTasks) { + if (_.taskCyclicStack.has(uniqueDownstreamTask.downstreamTask.id)) { + // Cyclic dependency detected + if (parentTask) { + // Log warning + logger.warn(`Cyclic dependency detected in task tree. Won't go deeper.`); + logger.warn(` From task : [${uniqueDownstreamTask.sourceTask.id}] "${uniqueDownstreamTask.sourceTask.taskName}"`); + logger.warn( + ` To task : [${uniqueDownstreamTask.downstreamTask.id}] "${uniqueDownstreamTask.downstreamTask.taskName}"` + ); + + // Add node indicating cyclic dependency + kids = kids.concat([ + { + id: task.id, + text: ` ==> !!! Cyclic dependency detected from task "${uniqueDownstreamTask.sourceTask.taskName}" to "${uniqueDownstreamTask.downstreamTask.taskName}"`, + }, + ]); + } else { + // Log warning when there is no parent task (should not happen?) + logger.warn(`Cyclic dependency detected in task tree. No parent task detected. Won't go deeper.`); + } + } else { + _.taskCyclicStack.add(uniqueDownstreamTask.downstreamTask.id); + const tmp3 = extGetTaskSubTree( + _, + uniqueDownstreamTask.downstreamTask, + newTreeLevel, + uniqueDownstreamTask.sourceTask, + logger + ); + _.taskCyclicStack.delete(uniqueDownstreamTask.downstreamTask.id); + kids = kids.concat(tmp3); + } + } + + // Only push real Sense tasks to the tree (don't include meta nodes) + if (!task.metaNodeType) { + if (kids && kids.length > 0) { + subTree = { + id: task.id, + children: kids, + }; + } else { + subTree = { + id: task.id, + }; + } + + if (_.options.treeIcons) { + if (task.taskLastStatus === 'FinishedSuccess') { + subTree.text = `✅ ${task.taskName}`; + // subTree.text = _.options.textColor ? `✅ \x1b[0m${task.taskName}\x1b[0m` : `✅ ${task.taskName}`; + } else if (task.taskLastStatus === 'FinishedFail') { + subTree.text = `❌ ${task.taskName}`; + } else if (task.taskLastStatus === 'Skipped') { + subTree.text = `🚫 ${task.taskName}`; + } else if (task.taskLastStatus === 'Aborted') { + subTree.text = `🛑 ${task.taskName}`; + } else if (task.taskLastStatus === 'Never started') { + subTree.text = `💤 ${task.taskName}`; + } else { + subTree.text = `❔ ${task.taskName}`; + } + } else { + subTree.text = task.taskName; + } + + if (_.options.treeDetails === true) { + // All task details should be included + if (task.completeTaskObject.schemaPath === 'ReloadTask') { + if (_.options.textColor === 'yes') { + subTree.text += ` \x1b[2mTask id: \x1b[3m${task.id}\x1b[0;2m, Last start/stop: \x1b[3m${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}\x1b[0;2m, Next start: \x1b[3m${task.taskNextExecutionTimestamp}\x1b[0;2m, App name: \x1b[3m${task.appName}\x1b[0;2m, App stream: \x1b[3m${task.appStream}\x1b[0;2m\x1b[0m`; + } else { + subTree.text += ` Task id: ${task.id}, Last start/stop: ${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}, Next start: ${task.taskNextExecutionTimestamp}, App name: ${task.appName}, App stream: ${task.appStream}`; + } + } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { + if (_.options.textColor === 'yes') { + subTree.text += ` \x1b[2m--EXTERNAL PROGRAM--Task id: \x1b[3m${task.id}\x1b[0;2m, Last start/stop: \x1b[3m${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}\x1b[0;2m, Next start: \x1b[3m${task.taskNextExecutionTimestamp}\x1b[0;2m, Path: \x1b[3m${task.path}\x1b[0;2m, Parameters: \x1b[3m${task.parameters}\x1b[0;2m\x1b[0m`; + } else { + subTree.text += `--EXTERNAL PROGRAM--Task id: ${task.id}, Last start/stop: ${task.taskLastExecutionStartTimestamp}/${task.taskLastExecutionStopTimestamp}, Next start: ${task.taskNextExecutionTimestamp}, path: ${task.path}, Parameters: ${task.oarameters}`; + } + } + } else if (_.options.treeDetails) { + // Some task details should be included + if (_.options.treeDetails.find((item) => item === 'taskid')) { + subTree.text += + _.options.textColor === 'yes' ? `\x1b[2m, Task id: \x1b[3m${task.id}\x1b[0;2m\x1b[0m` : `, Task id: ${task.id}`; + } + if (_.options.treeDetails.find((item) => item === 'laststart')) { + subTree.text += + _.options.textColor === 'yes' + ? `\x1b[2m, Last start: \x1b[3m${task.taskLastExecutionStartTimestamp}\x1b[0;2m\x1b[0m` + : `, Last start: ${task.taskLastExecutionStartTimestamp}`; + } + if (_.options.treeDetails.find((item) => item === 'laststop')) { + subTree.text += + _.options.textColor === 'yes' + ? `\x1b[2m, Last stop: \x1b[3m${task.taskLastExecutionStopTimestamp}\x1b[0;2m\x1b[0m` + : `, Last stop: ${task.taskLastExecutionStopTimestamp}`; + } + if (_.options.treeDetails.find((item) => item === 'nextstart')) { + subTree.text += + _.options.textColor === 'yes' + ? `\x1b[2m, Next start: \x1b[3m${task.taskNextExecutionTimestamp}\x1b[0;2m\x1b[0m` + : `, Next start: ${task.taskNextExecutionTimestamp}`; + } + if (_.options.treeDetails.find((item) => item === 'appname')) { + if (task.completeTaskObject.schemaPath === 'ReloadTask') { + subTree.text += + _.options.textColor === 'yes' + ? `\x1b[2m, App name: \x1b[3m${task.appName}\x1b[0;2m\x1b[0m` + : `, App name: ${task.appName}`; + } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { + subTree.text += + _.options.textColor === 'yes' ? `\x1b[2m, Path: \x1b[3m${task.path}\x1b[0;2m\x1b[0m` : `, Path: ${task.path}`; + } + } + if (_.options.treeDetails.find((item) => item === 'appstream')) { + if (task.completeTaskObject.schemaPath === 'ReloadTask') { + subTree.text += + _.options.textColor === 'yes' + ? `\x1b[2m, App stream: \x1b[3m${task.appStream}\x1b[0;2m\x1b[0m` + : `, App stream: ${task.appStream}`; + } else if (task.completeTaskObject.schemaPath === 'ExternalProgramTask') { + subTree.text += + _.options.textColor === 'yes' + ? `\x1b[2m, Parameters: \x1b[3m${task.parameters}\x1b[0;2m\x1b[0m` + : `, Parameters: ${task.parameters}`; + } + } + } + + // Tabulator columns + subTree.taskId = task.taskId; + subTree.taskName = task.taskName; + subTree.taskEnabled = task.taskEnabled; + subTree.appId = task.appId; + subTree.appName = task.appName; + subTree.appPublished = task.appPublished; + subTree.appStream = task.appStream; + subTree.taskMaxRetries = task.taskMaxRetries; + subTree.taskLastExecutionStartTimestamp = task.taskLastExecutionStartTimestamp; + subTree.taskLastExecutionStopTimestamp = task.taskLastExecutionStopTimestamp; + subTree.taskLastExecutionDuration = task.taskLastExecutionDuration; + subTree.taskLastExecutionExecutingNodeName = task.taskLastExecutionExecutingNodeName; + subTree.taskNextExecutionTimestamp = task.taskNextExecutionTimestamp; + subTree.taskLastStatus = task.taskLastStatus; + subTree.taskTags = task.completeTaskObject.tags.map((tag) => tag.name); + subTree.taskCustomProperties = task.completeTaskObject.customProperties.map((el) => `${el.definition.name}=${el.value}`); + subTree.completeTaskObject = task.completeTaskObject; + + if (newTreeLevel === 1) { + subTree = [subTree]; + } + } else { + subTree = kids; + } + + return subTree; + } catch (err) { + catchLog('GET TASK SUBTREE (tree)', err); + return false; + } +} diff --git a/src/lib/task/get_tasks_from_qseow.js b/src/lib/task/get_tasks_from_qseow.js new file mode 100644 index 00000000..632932e9 --- /dev/null +++ b/src/lib/task/get_tasks_from_qseow.js @@ -0,0 +1,138 @@ +import axios from 'axios'; + +import { catchLog } from '../util/log.js'; +import { setupQrsConnection } from '../util/qseow/qrs.js'; + +export async function extGetTasksFromQseow(_, logger) { + return new Promise(async (resolve, reject) => { + // try { + logger.debug('GET TASKS FROM QSEOW: Starting get reload tasks from QSEoW'); + + let filter = ''; + + // Should we get all tasks? + if (_.options.getAllTasks === true) { + // No task filters specified + filter = ''; + } else if (_.options.outputFormat === 'tree') { + // When visualising tasks as a tree, we need to get all tasks + filter = ''; + } else if (_.options.outputFormat !== 'tree') { + // Are there any task filters specified? + // If so, build a query string + + // Don't add task id and tag filtering if the output is a task tree + + // Add task id(s) to query string + if (_.options.taskId && _.options?.taskId.length >= 1) { + // At least one task ID specified + // Add first task ID + filter += encodeURIComponent(`(id eq ${_.options.taskId[0]}`); + } + if (_.options.taskId && _.options?.taskId.length >= 2) { + // Add remaining task IDs, if any + for (let i = 1; i < _.options.taskId.length; i += 1) { + filter += encodeURIComponent(` or id eq ${_.options.taskId[i]}`); + } + } + + // Add closing parenthesis + if (_.options.taskId && _.options?.taskId.length >= 1) { + filter += encodeURIComponent(')'); + } + logger.debug(`GET TASKS FROM QSEOW: QRS query filter (incl ids): ${filter}`); + + // Add task tag(s) to query string + if (_.options.taskTag && _.options?.taskTag.length >= 1) { + // At least one task ID specified + if (filter.length >= 1) { + // We've previously added some task ids + // Add first task tag + filter += encodeURIComponent(` or (tags.name eq '${_.options.taskTag[0]}'`); + } else { + // No task ids added yet + // Add first task tag + filter += encodeURIComponent(`(tags.name eq '${_.options.taskTag[0]}'`); + } + } + if (_.options.taskTag && _.options?.taskTag.length >= 2) { + // Add remaining task tags, if any + for (let i = 1; i < _.options.taskTag.length; i += 1) { + filter += encodeURIComponent(` or tags.name eq '${_.options.taskTag[i]}'`); + } + } + + // Add closing parenthesis + if (_.options.taskTag && _.options?.taskTag.length >= 1) { + filter += encodeURIComponent(')'); + } + } + + logger.debug(`GET TASKS FROM QSEOW: QRS query filter (incl ids, tags): ${filter}`); + + let axiosConfig; + let tasks = []; + let result; + + try { + // Get reload tasks + if (filter === '') { + axiosConfig = setupQrsConnection(_.options, { + method: 'get', + path: '/qrs/reloadtask/full', + }); + } else { + axiosConfig = setupQrsConnection(_.options, { + method: 'get', + path: '/qrs/reloadtask/full', + queryParameters: [{ name: 'filter', value: filter }], + }); + } + + result = await axios.request(axiosConfig); + logger.debug(`GET RELOAD TASK: Result=result.status`); + + tasks = tasks.concat(JSON.parse(result.data)); + logger.verbose(`GET RELOAD TASK: # tasks: ${tasks.length}`); + } catch (err) { + catchLog('GET TASKS FROM QSEOW 1', err); + reject(err); + } + try { + // Get external program tasks + if (filter === '') { + axiosConfig = setupQrsConnection(_.options, { + method: 'get', + path: '/qrs/externalprogramtask/full', + }); + } else { + axiosConfig = setupQrsConnection(_.options, { + method: 'get', + path: '/qrs/externalprogramtask/full', + queryParameters: [{ name: 'filter', value: filter }], + }); + } + + result = await axios.request(axiosConfig); + logger.debug(`GET EXT PROGRAM TASK: Result=result.status`); + + tasks = tasks.concat(JSON.parse(result.data)); + logger.verbose(`GET EXT PROGRAM TASK: # tasks: ${tasks.length}`); + } catch (err) { + catchLog('GET EXTERNAL PROGRAM TASKS FROM QSEOW 1', err); + reject(err); + } + + // TODO + // Determine whether task name anonymisation should be done + const anonymizeTaskNames = false; + + _.clear(); + for (let i = 0; i < tasks.length; i += 1) { + if (tasks[i].schemaPath === 'ReloadTask' || tasks[i].schemaPath === 'ExternalProgramTask') { + _.addTask('from_qseow', tasks[i], anonymizeTaskNames); + } + } + resolve(_.taskList); + }); +} diff --git a/src/lib/task/parse_composite_events.js b/src/lib/task/parse_composite_events.js new file mode 100644 index 00000000..1fd54258 --- /dev/null +++ b/src/lib/task/parse_composite_events.js @@ -0,0 +1,255 @@ +import { mapTaskType, mapRuleState, mapEventType } from '../util/qseow/lookups.js'; + +export async function extParseCompositeEvents(_, param, logger) { + // Get all composite events for this task + // + // Composite events + // - Consists of one main row defining the event, followed by one or more rows defining the composite event rules. + // - The main row is followed by one or more rows defining the composite event rules + // - All rows associated with a composite event share the same value in the "Event counter" column + // - Each composite event rule row has a unique value in the "Rule counter" column + const prelCompositeEvents = []; + + // Get all "main rows" of all composite events in this task + const compositeEventRows = param.taskRows.filter( + (item) => + item[param.taskFileColumnHeaders.eventType.pos] && + item[param.taskFileColumnHeaders.eventType.pos].trim().toLowerCase() === 'composite' + ); + if (!compositeEventRows || compositeEventRows?.length === 0) { + logger.verbose(`(${param.taskCounter}) PARSE COMPOSITE EVENT: No composite events for task "${param.currentTask.name}"`); + } else { + logger.verbose( + `(${param.taskCounter}) PARSE COMPOSITE EVENT: ${compositeEventRows.length} composite event(s) for task "${param.currentTask.name}"` + ); + + // Loop over all composite events, adding them and their event rules + for (const compositeEventRow of compositeEventRows) { + // Get value in "Event counter" column for this composite event, then get array of all associated event rules + const compositeEventCounter = compositeEventRow[param.taskFileColumnHeaders.eventCounter.pos]; + const compositeEventRules = param.taskRows.filter( + (item) => + item[param.taskFileColumnHeaders.eventCounter.pos] === compositeEventCounter && + item[param.taskFileColumnHeaders.ruleCounter.pos] > 0 + ); + + // Create an object using same format that the Sense API uses for composite events + // Add task type specific properties in later step + const compositeEvent = { + timeConstraint: { + days: compositeEventRow[param.taskFileColumnHeaders.timeConstraintDays.pos], + hours: compositeEventRow[param.taskFileColumnHeaders.timeConstraintHours.pos], + minutes: compositeEventRow[param.taskFileColumnHeaders.timeConstraintMinutes.pos], + seconds: compositeEventRow[param.taskFileColumnHeaders.timeConstraintSeconds.pos], + }, + compositeRules: [], + name: compositeEventRow[param.taskFileColumnHeaders.eventName.pos], + enabled: compositeEventRow[param.taskFileColumnHeaders.eventEnabled.pos], + eventType: mapEventType.get(compositeEventRow[param.taskFileColumnHeaders.eventType.pos]), + schemaPath: 'CompositeEvent', + }; + + if (param.taskType === 'reload') { + compositeEvent.reloadTask = { + id: param.fakeTaskId, + }; + } else if (param.taskType === 'external program') { + compositeEvent.externalProgramTask = { + id: param.fakeTaskId, + }; + } else { + logger.error(`(${param.taskCounter}) PARSE COMPOSITE EVENT: Incorrect task type "${param.taskType}". Exiting.`); + process.exit(1); + } + + // Add rules + for (const rule of compositeEventRules) { + // Does the upstream task pointed to by the composite rule exist? + // If it *does* exist it means it's a real, existing task in QSEoW that should be used. + // If it is not a valid guid or does not exist, it's (best case) a referefence to some other task in the task definitions file. + // If the task pointed to by the rule doesn't exist in Sense and doesn't point to some other task in the file, an error should be shown. + if (validate(rule[param.taskFileColumnHeaders.ruleTaskId.pos])) { + // The rule points to an valid UUID. It should exist, otherwise it's an error + + const taskExists = await taskExistById(rule[param.taskFileColumnHeaders.ruleTaskId.pos], _.options); + + if (taskExists) { + // Add task ID to mapping table that will be used later when building the composite event data structures + // In this case we're adding a task ID that maps to itself, indicating that it's a task that already exists in QSEoW. + _.taskIdMap.set(rule[param.taskFileColumnHeaders.ruleTaskId.pos], rule[param.taskFileColumnHeaders.ruleTaskId.pos]); + } else { + // The task pointed to by the composite event rule does not exist + logger.error( + `(${param.taskCounter}) PARSE COMPOSITE EVENT: Task "${ + rule[param.taskFileColumnHeaders.ruleTaskId.pos] + }" does not exist. Exiting.` + ); + process.exit(1); + } + } else { + logger.verbose( + `(${param.taskCounter}) PARSE COMPOSITE EVENT: "${ + rule[param.taskFileColumnHeaders.ruleTaskId.pos] + }" is not a valid UUID` + ); + } + + // Save composite event rule. + // Also add the upstream task id to the correct property in the rule object, depending on task type + + let upstreamTask; + let upstreamTaskExistence; + // First get upstream task type + // Two options: + // 1. The rule's task ID is a valid GUID. Get the associated task's metadata from Sense, if the task exists + // 2. The rule's task ID is not a valid GUID. It's a reference to a task that is created during this execution of Ctrl-Q. + if (!validate(rule[param.taskFileColumnHeaders.ruleTaskId.pos])) { + // The rule's task ID is not a valid GUID. It's a reference to a task that is created during this execution of Ctrl-Q. + // Add the task ID to the mapping table, indicating that it's a task that is created during this execution of Ctrl-Q. + + // // Check if the task ID already exists in the mapping table + // if (_.taskIdMap.has(rule[param.taskFileColumnHeaders.ruleTaskId.pos])) { + // // The task ID already exists in the mapping table. This means that the task has already been created during this execution of Ctrl-Q. + // // This is not allowed. The task ID must be unique. + // logger.error( + // `(${param.taskCounter}) PARSE TASKS FROM FILE: Task ID "${ + // rule[param.taskFileColumnHeaders.ruleTaskId.pos] + // }" already exists in mapping table. This is not allowed. Exiting.` + // ); + // process.exit(1); + // } + + // // Add task ID to mapping table + // _.taskIdMap.set(rule[param.taskFileColumnHeaders.ruleTaskId.pos], `fake-task-${uuidv4()}`); + + upstreamTaskExistence = 'exists-in-source-file'; + } else { + upstreamTask = await getTaskById(rule[param.taskFileColumnHeaders.ruleTaskId.pos], param?.options); + + // Save upstream task in shared task list + _.compositeEventUpstreamTask.push(upstreamTask); + + upstreamTaskExistence = 'exists-in-sense'; + } + + if (upstreamTaskExistence === 'exists-in-source-file') { + // Upstream task is a task that is created during this execution of Ctrl-Q + // We don't yet know what task ID it will get in Sense, so we'll have to find this when creating composite events later + compositeEvent.compositeRules.push({ + upstreamTaskExistence, + ruleState: mapRuleState.get(rule[param.taskFileColumnHeaders.ruleState.pos]), + task: { + id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], + }, + }); + } else if (mapTaskType.get(upstreamTask.taskType).toLowerCase() === 'reload') { + // Upstream task is a reload task + compositeEvent.compositeRules.push({ + upstreamTaskExistence, + ruleState: mapRuleState.get(rule[param.taskFileColumnHeaders.ruleState.pos]), + task: { + id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], + }, + reloadTask: { + id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], + }, + }); + } else if (mapTaskType.get(upstreamTask.taskType).toLowerCase() === 'externalprogram') { + // Upstream task is an external program task + compositeEvent.compositeRules.push({ + upstreamTaskExistence, + ruleState: mapRuleState.get(rule[param.taskFileColumnHeaders.ruleState.pos]), + task: { + id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], + }, + externalProgramTask: { + id: rule[param.taskFileColumnHeaders.ruleTaskId.pos], + }, + }); + } + } + + _.qlikSenseCompositeEvents.addCompositeEvent(compositeEvent); + + // Add composite event to network representation of tasks + if (compositeEvent.compositeRules.length === 1) { + // This trigger has exactly ONE upstream task + // For triggers with >1 upstream task we want an extra meta node to represent the waiting of all upstream tasks to finish + + if (param.taskType === 'reload') { + // Add edge from upstream task to current task, taking into account task type + _.taskNetwork.edges.push({ + from: compositeEvent.compositeRules[0].task.id, + to: compositeEvent.reloadTask.id, + + completeCompositeEvent: compositeEvent, + rule: compositeEvent.compositeRules, + // color: compositeEvent.enabled ? '#9FC2F7' : '#949298', + // color: edgeColor, + // dashes: compositeEvent.enabled ? false : [15, 15], + // title: compositeEvent.name + '
' + 'asdasd', + // label: compositeEvent.name, + }); + + // Keep a note that this node has associated events + param.nodesWithEvents.add(compositeEvent.compositeRules[0].task.id); + param.nodesWithEvents.add(compositeEvent.reloadTask.id); + } else if (param.taskType === 'external program') { + // Add edge from upstream task to current task, taking into account task type + _.taskNetwork.edges.push({ + from: compositeEvent.compositeRules[0].task.id, + to: compositeEvent.externalProgramTask.id, + + completeCompositeEvent: compositeEvent, + rule: compositeEvent.compositeRules, + }); + // Keep a note that this node has associated events + param.nodesWithEvents.add(compositeEvent.compositeRules[0].task.id); + param.nodesWithEvents.add(compositeEvent.externalProgramTask.id); + } + } else { + // There are more than one task involved in triggering a downstream task. + // Insert a proxy node that represents a Qlik Sense composite event + + const nodeId = `composite-event-${uuidv4()}`; + _.taskNetwork.nodes.push({ + id: nodeId, + label: '', + enabled: true, + metaNodeType: 'composite', + metaNode: true, + }); + param.nodesWithEvents.add(nodeId); + + // Add edges from upstream tasks to the new meta node + for (const rule of compositeEvent.compositeRules) { + _.taskNetwork.edges.push({ + from: rule.task.id, + to: nodeId, + + completeCompositeEvent: compositeEvent, + rule, + }); + } + + // Add edge from new meta node to current node, taking into account task type + if (param.taskType === 'reload') { + _.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.reloadTask.id, + }); + } else if (param.taskType === 'external program') { + _.taskNetwork.edges.push({ + from: nodeId, + to: compositeEvent.externalProgramTask.id, + }); + } + } + + // Add this composite event to the current task + prelCompositeEvents.push(compositeEvent); + } + } + + return prelCompositeEvents; +} diff --git a/src/lib/task/parse_ext_program_task.js b/src/lib/task/parse_ext_program_task.js new file mode 100644 index 00000000..0cb8c231 --- /dev/null +++ b/src/lib/task/parse_ext_program_task.js @@ -0,0 +1,127 @@ +import { getTagIdByName } from '../util/qseow/tag.js'; +import { getCustomPropertyIdByName } from '../util/qseow/customproperties.js'; +import { mapTaskType } from '../util/qseow/lookups.js'; + +export async function extParseExternalProgramTask(_, options, logger) { + let currentTask = null; + let taskCreationOption; + + // Create task object using same structure as results from QRS API + + // Get task import options + if (param.taskFileColumnHeaders.importOptions.pos === 999) { + // No task creation options column in the file + // Use the default task creation option + taskCreationOption = 'if-exists-update-existing'; + } else { + // Task creation options column exists in the file + // Use the value from the file + taskCreationOption = param.taskRows[0][param.taskFileColumnHeaders.importOptions.pos]; + } + + // Ensure task creation option is valid. Allow empty option + if ( + taskCreationOption && + taskCreationOption.trim() !== '' && + !['if-exists-add-another', 'if-exists-update-existing'].includes(taskCreationOption) + ) { + logger.error( + `(${param.taskCounter}) PARSE EXTERNAL PROGRAM TASK FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.` + ); + process.exit(1); + } + + currentTask = { + id: param.taskRows[0][param.taskFileColumnHeaders.taskId.pos], + name: param.taskRows[0][param.taskFileColumnHeaders.taskName.pos], + taskType: mapTaskType.get(param.taskRows[0][param.taskFileColumnHeaders.taskType.pos]), + enabled: param.taskRows[0][param.taskFileColumnHeaders.taskEnabled.pos], + taskSessionTimeout: param.taskRows[0][param.taskFileColumnHeaders.taskSessionTimeout.pos], + maxRetries: param.taskRows[0][param.taskFileColumnHeaders.taskMaxRetries.pos], + + path: param.taskRows[0][param.taskFileColumnHeaders.extPgmPath.pos], + parameters: param.taskRows[0][param.taskFileColumnHeaders.extPgmParam.pos], + + tags: [], + customProperties: [], + + schemaPath: 'ExternalProgramTask', + schemaEvents: [], + compositeEvents: [], + prelCompositeEvents: [], + }; + + // Add tags to task object + if (param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos]) { + const tmpTags = param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos] + .split('/') + .filter((item) => item.trim().length !== 0) + .map((item) => item.trim()); + + for (const item of tmpTags) { + const tagId = await getTagIdByName(item, param.tagsExisting); + currentTask.tags.push({ + id: tagId, + name: item, + }); + } + } + + // Add custom properties to task object + if (param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos]) { + const tmpCustomProperties = param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos] + .split('/') + .filter((item) => item.trim().length !== 0) + .map((cp) => cp.trim()); + + for (const item of tmpCustomProperties) { + const tmpCustomProperty = item + .split('=') + .filter((item2) => item2.trim().length !== 0) + .map((cp) => cp.trim()); + + // Do we have two items in the array? First is the custom property name, second is the value + if (tmpCustomProperty?.length === 2) { + const customPropertyId = getCustomPropertyIdByName('ExternalProgramTask', tmpCustomProperty[0], param.cpExisting); + + // If previous call returned false, it means the custom property does not exist in Sense + // or cannot be used with this task type. In that case, skip it. + if (customPropertyId) { + currentTask.customProperties.push({ + definition: { + id: customPropertyId, + name: tmpCustomProperty[0].trim(), + }, + value: tmpCustomProperty[1].trim(), + }); + } + } + } + } + + // Get schema events for this task, storing the info using the same structure as returned from QRS API + currentTask.schemaEvents = _.parseSchemaEvents({ + taskType: 'external program', + taskRows: param.taskRows, + taskFileColumnHeaders: param.taskFileColumnHeaders, + taskCounter: param.taskCounter, + currentTask, + fakeTaskId: param.fakeTaskId, + nodesWithEvents: param.nodesWithEvents, + options: param?.options, + }); + + // Get composite events for this task + currentTask.prelCompositeEvents = await _.parseCompositeEvents({ + taskType: 'external program', + taskRows: param.taskRows, + taskFileColumnHeaders: param.taskFileColumnHeaders, + taskCounter: param.taskCounter, + currentTask, + fakeTaskId: param.fakeTaskId, + nodesWithEvents: param.nodesWithEvents, + options: param?.options, + }); + + return { currentTask, taskCreationOption }; +} diff --git a/src/lib/task/parse_reload_task.js b/src/lib/task/parse_reload_task.js new file mode 100644 index 00000000..5a24f467 --- /dev/null +++ b/src/lib/task/parse_reload_task.js @@ -0,0 +1,207 @@ +import { getTagIdByName } from '../util/qseow/tag.js'; +import { getAppById } from '../util/qseow/app.js'; +import { getCustomPropertyIdByName } from '../util/qseow/customproperties.js'; +import { mapTaskType } from '../util/qseow/lookups.js'; + +export async function extParseReloadTask(_, options, logger) { + let currentTask = null; + let taskCreationOption; + + // Create task object using same structure as results from QRS API + + // Determine if the task is associated with an app that existed before Ctrl-Q was started, or + // an app that's been imported as part of this Ctrl-Q execution. + // Possible values for the app ID column: + // - newapp- (app has been imported as part of this Ctrl-Q execution) + // - A real, existing app ID. I.e. the app existed before Ctrl-Q was started. + const appIdRaw = param.taskRows[0][param.taskFileColumnHeaders.appId.pos].trim(); + let appId; + + if (appIdRaw.substring(0, 7).toLowerCase() === 'newapp-') { + // App ID starts with "newapp-". This means the app been imported as part of this Ctrl-Q session + // No guarantee that it is the case though. Maybe no apps were imported, or maybe the app specified for this very task was not imported + + // Have ANY apps been imported? + if (!_.importedApps) { + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: No apps have been imported, but app "${param.taskRows[0][ + param.taskFileColumnHeaders.appId.pos + ].trim()}" has been specified in the task definition file. Exiting.` + ); + process.exit(1); + } + + // Has this specific app been imported? + if (!_.importedApps.appIdMap.has(appIdRaw.toLowerCase())) { + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: App "${param.taskRows[0][ + param.taskFileColumnHeaders.appId.pos + ].trim()}" has not been imported, but has been specified in the task definition file. Exiting.` + ); + process.exit(1); + } + + appId = _.importedApps.appIdMap.get(appIdRaw.toLowerCase()); + + // Ensure the app exists + // Reasons for the app not existing could be: + // - The app was imported but has since been deleted or replaced. This could happen if the app-import step has several + // apps that are published-replaced or deleted-published to the same stream. In that case only the last published app will be present + + if (appId === undefined) { + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Cannot figure out which Sense app "${param.taskRows[0][ + param.taskFileColumnHeaders.appId.pos + ].trim()}" belongs to. App with ID "${appIdRaw}" not found.` + ); + + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: This could be because the app was imported but has since been deleted or replaced, for example during app publishing. Don't know how to proceed, exiting.` + ); + + process.exit(1); + } + + const app = await getAppById(appId, param?.options); + + if (!app) { + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: App with ID "${appId}" not found. This could be because the app was imported but has since been deleted or replaced, for example during app publishing. Don't know how to proceed, exiting.` + ); + process.exit(1); + } + } else if (validate(appIdRaw)) { + // App ID is a proper UUID. We don't know if the app actually exists though. + + const app = await getAppById(appIdRaw, param?.options); + + if (!app) { + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: App with ID "${appIdRaw}" not found. This could be because the app was imported but has since been deleted or replaced, for example during app publishing. Don't know how to proceed, exiting.` + ); + process.exit(1); + } + + appId = appIdRaw; + } else { + logger.error(`(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Incorrect app ID "${appIdRaw}". Exiting.`); + process.exit(1); + } + + if (param.taskFileColumnHeaders.importOptions.pos === 999) { + // No task creation options column in the file + // Use the default task creation option + taskCreationOption = 'if-exists-update-existing'; + } else { + // Task creation options column exists in the file + // Use the value from the file + taskCreationOption = param.taskRows[0][param.taskFileColumnHeaders.importOptions.pos]; + } + + // Ensure task creation option is valid. Allow empty option + if ( + taskCreationOption && + taskCreationOption.trim() !== '' && + !['if-exists-add-another', 'if-exists-update-existing'].includes(taskCreationOption) + ) { + logger.error( + `(${param.taskCounter}) PARSE RELOAD TASK FROM FILE: Incorrect task creation option "${taskCreationOption}". Exiting.` + ); + process.exit(1); + } + + currentTask = { + id: param.taskRows[0][param.taskFileColumnHeaders.taskId.pos], + name: param.taskRows[0][param.taskFileColumnHeaders.taskName.pos], + taskType: mapTaskType.get(param.taskRows[0][param.taskFileColumnHeaders.taskType.pos]), + enabled: param.taskRows[0][param.taskFileColumnHeaders.taskEnabled.pos], + taskSessionTimeout: param.taskRows[0][param.taskFileColumnHeaders.taskSessionTimeout.pos], + maxRetries: param.taskRows[0][param.taskFileColumnHeaders.taskMaxRetries.pos], + isManuallyTriggered: param.taskRows[0][param.taskFileColumnHeaders.isManuallyTriggered.pos], + isPartialReload: param.taskRows[0][param.taskFileColumnHeaders.isPartialReload.pos], + app: { + id: appId, + // name: taskData[0][taskFileColumnHeaders.appName.pos], + }, + tags: [], + customProperties: [], + schemaPath: 'ReloadTask', + schemaEvents: [], + compositeEvents: [], + prelCompositeEvents: [], + }; + + // Add tags to task object + if (param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos]) { + const tmpTags = param.taskRows[0][param.taskFileColumnHeaders.taskTags.pos] + .split('/') + .filter((item) => item.trim().length !== 0) + .map((item) => item.trim()); + + for (const item of tmpTags) { + const tagId = await getTagIdByName(item, param.tagsExisting); + currentTask.tags.push({ + id: tagId, + name: item, + }); + } + } + + // Add custom properties to task object + if (param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos]) { + const tmpCustomProperties = param.taskRows[0][param.taskFileColumnHeaders.taskCustomProperties.pos] + .split('/') + .filter((item) => item.trim().length !== 0) + .map((cp) => cp.trim()); + + for (const item of tmpCustomProperties) { + const tmpCustomProperty = item + .split('=') + .filter((item2) => item2.trim().length !== 0) + .map((cp) => cp.trim()); + + // Do we have two items in the array? First is the custom property name, second is the value + if (tmpCustomProperty?.length === 2) { + const customPropertyId = getCustomPropertyIdByName('ReloadTask', tmpCustomProperty[0], param.cpExisting); + + // If previous call returned false, it means the custom property does not exist in Sense + // or cannot be used with this task type. In that case, skip it. + if (customPropertyId) { + currentTask.customProperties.push({ + definition: { + id: customPropertyId, + name: tmpCustomProperty[0].trim(), + }, + value: tmpCustomProperty[1].trim(), + }); + } + } + } + } + + // Get schema events for this task, storing the info using the same structure as returned from QRS API + currentTask.schemaEvents = _.parseSchemaEvents({ + taskType: 'reload', + taskRows: param.taskRows, + taskFileColumnHeaders: param.taskFileColumnHeaders, + taskCounter: param.taskCounter, + currentTask, + fakeTaskId: param.fakeTaskId, + nodesWithEvents: param.nodesWithEvents, + options: param?.options, + }); + + // Get composite events for this task + currentTask.prelCompositeEvents = await _.parseCompositeEvents({ + taskType: 'reload', + taskRows: param.taskRows, + taskFileColumnHeaders: param.taskFileColumnHeaders, + taskCounter: param.taskCounter, + currentTask, + fakeTaskId: param.fakeTaskId, + nodesWithEvents: param.nodesWithEvents, + options: param?.options, + }); + + return { currentTask, taskCreationOption }; +} diff --git a/src/lib/task/parse_schema_events.js b/src/lib/task/parse_schema_events.js new file mode 100644 index 00000000..9ab01e69 --- /dev/null +++ b/src/lib/task/parse_schema_events.js @@ -0,0 +1,100 @@ +import { mapIncrementOption, mapEventType, mapDaylightSavingTime } from '../util/qseow/lookups.js'; + +export function extParseSchemaEvents(_, param, logger) { + // Get schema events for this task, storing the info using the same structure as returned from QRS API + const prelSchemaEvents = []; + + const schemaEventRows = param.taskRows.filter( + (item) => + item[param.taskFileColumnHeaders.eventType.pos] && + item[param.taskFileColumnHeaders.eventType.pos].trim().toLowerCase() === 'schema' + ); + if (!schemaEventRows || schemaEventRows?.length === 0) { + logger.verbose(`(${param.taskCounter}) PARSE SCHEMA EVENT: No schema events for task "${param.currentTask.name}"`); + } else { + logger.verbose( + `(${param.taskCounter}) PARSE SCHEMA EVENT: ${schemaEventRows.length} schema event(s) for task "${param.currentTask.name}"` + ); + + // Add schema edges and start/trigger nodes + for (const schemaEventRow of schemaEventRows) { + // Create object using same format that Sense uses for schema events + const schemaEvent = { + enabled: schemaEventRow[param.taskFileColumnHeaders.eventEnabled.pos], + eventType: mapEventType.get(schemaEventRow[param.taskFileColumnHeaders.eventType.pos]), + name: schemaEventRow[param.taskFileColumnHeaders.eventName.pos], + daylightSavingTime: mapDaylightSavingTime.get(schemaEventRow[param.taskFileColumnHeaders.daylightSavingsTime.pos]), + timeZone: schemaEventRow[param.taskFileColumnHeaders.schemaTimeZone.pos], + startDate: schemaEventRow[param.taskFileColumnHeaders.schemaStart.pos], + expirationDate: schemaEventRow[param.taskFileColumnHeaders.scheamExpiration.pos], + schemaFilterDescription: [schemaEventRow[param.taskFileColumnHeaders.schemaFilterDescription.pos]], + incrementDescription: schemaEventRow[param.taskFileColumnHeaders.schemaIncrementDescription.pos], + incrementOption: mapIncrementOption.get(schemaEventRow[param.taskFileColumnHeaders.schemaIncrementOption.pos]), + schemaPath: 'SchemaEvent', + }; + + if (param.taskType === 'reload') { + schemaEvent.reloadTask = { + id: param.fakeTaskId, + }; + } else if (param.taskType === 'external program') { + schemaEvent.externalProgramTask = { + id: param.fakeTaskId, + }; + } else { + logger.error(`(${param.taskCounter}) PARSE SCHEMA EVENT: Incorrect task type "${param.taskType}". Exiting.`); + process.exit(1); + } + + _.qlikSenseSchemaEvents.addSchemaEvent(schemaEvent); + + // Add schema event to network representation of tasks + // Create an id for this node + const nodeId = `schema-event-${uuidv4()}`; + + // Add schema trigger nodes. These represent the implicit starting nodes that a schema event really are + _.taskNetwork.nodes.push({ + id: nodeId, + metaNodeType: 'schedule', // Meta nodes are not Sense tasks, but rather nodes representing task-like properties (e.g. a starting point for a reload chain) + metaNode: true, + isTopLevelNode: true, + label: schemaEvent.name, + enabled: schemaEvent.enabled, + + completeSchemaEvent: schemaEvent, + }); + + // Add edge from schema trigger node to current task, taking into account task type + if (param.taskType === 'reload') { + _.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.reloadTask.id, + }); + + // Keep a note that this node has associated events + param.nodesWithEvents.add(schemaEvent.reloadTask.id); + + // Remove reference to task ID + delete schemaEvent.reloadTask.id; + delete schemaEvent.reloadTask; + } else if (param.taskType === 'external program') { + _.taskNetwork.edges.push({ + from: nodeId, + to: schemaEvent.externalProgramTask.id, + }); + + // Keep a note that this node has associated events + param.nodesWithEvents.add(schemaEvent.externalProgramTask.id); + + // Remove reference to task ID + delete schemaEvent.externalProgramTask.id; + delete schemaEvent.externalProgramTask; + } + + // Add this schema event to the current task + prelSchemaEvents.push(schemaEvent); + } + } + + return prelSchemaEvents; +} diff --git a/src/lib/task/save_task_model_to_qseow.js b/src/lib/task/save_task_model_to_qseow.js new file mode 100644 index 00000000..01c22d6f --- /dev/null +++ b/src/lib/task/save_task_model_to_qseow.js @@ -0,0 +1,60 @@ +import { catchLog } from '../util/log.js'; + +export function extSaveTaskModelToQseow(_, options, logger) { + return new Promise(async (resolve, reject) => { + try { + logger.debug('SAVE TASKS TO QSEOW: Starting save tasks to QSEoW'); + + for (const task of this.taskList) { + await new Promise((resolve2, reject2) => { + // Build a body for the API call + const body = { + task: { + app: { + id: task.appId, + }, + name: task.taskName, + isManuallyTriggered: task.isManuallyTriggered, + isPartialReload: task.isPartialReload, + taskType: task.taskType, + enabled: task.taskEnabled, + taskSessionTimeout: task.taskSessionTimeout, + maxRetries: task.taskMaxRetries, + tags: task.taskTags, + customProperties: task.taskCustomProperties, + schemaPath: 'ReloadTask', + }, + schemaEvents: task.schemaEvents, + compositeEvents: task.compositeEvents, + }; + + // Save task to QSEoW + const axiosConfig = setupQrsConnection(options, { + method: 'post', + path: '/qrs/reloadtask/create', + body, + }); + + try { + axios.request(axiosConfig).then((result) => { + logger.info(`SAVE TASK TO QSEOW: Task name: "${task.taskName}", Result: ${result.status}/${result.statusText}`); + if (result.status === 201) { + resolve2(); + } else { + reject2(); + } + }); + } catch (err) { + catchLog('SAVE TASK TO QSEOW 1', err); + reject2(); + } + }); + logger.debug(`SAVE TASK TO QSEOW: Done saving task "${task.taskName}"`); + } + resolve(); + } catch (err) { + catchLog('SAVE TASK TO QSEOW 2', err); + reject(err); + } + }); +} diff --git a/src/lib/task/task_qrs.js b/src/lib/task/task_qrs.js index b867d205..801de97a 100644 --- a/src/lib/task/task_qrs.js +++ b/src/lib/task/task_qrs.js @@ -1,14 +1,10 @@ import axios from 'axios'; -// const { promises: Fs } = require('fs'); -// const yesno = require('yesno'); import { logger } from '../../globals.js'; import { setupQrsConnection } from '../util/qseow/qrs.js'; import { getCertFilePaths } from '../util/qseow/cert.js'; import { catchLog } from '../util/log.js'; -// const { QlikSenseTasks } = require('./class_alltasks'); -// const { mapEventType, mapIncrementOption, mapDaylightSavingTime, mapRuleState } = require('../util/lookups'); function uniqueByTaskKeys(array, keys) { const filtered = array.filter( diff --git a/src/lib/util/qseow/app.js b/src/lib/util/qseow/app.js index 84e6454f..bac24fdc 100644 --- a/src/lib/util/qseow/app.js +++ b/src/lib/util/qseow/app.js @@ -270,7 +270,20 @@ export async function publishApp(appId, appName, streamId, options) { } } -// Check if an app with a given id exists +/** + * Checks if an app exists in QSEoW by its ID. + * + * Validates the provided app ID and queries the QSEoW to check if an app + * with the specified ID exists. If the app exists and is unique, returns true. + * If the app ID is not valid or if more than one app is found with the same ID, + * an error is logged and false is returned. If an error occurs during the process, + * it is caught, logged, and false is returned. + * + * @param {string} appId - The ID of the app to check. + * @param {object} options - Command line options for QSEoW connection. + * @returns {Promise} - True if the app exists and is unique, false otherwise or if an error occured. + */ + export async function appExistById(appId, options) { try { logger.debug(`Checking if app with id ${appId} exists in QSEoW`); diff --git a/src/lib/util/qseow/assert-options.js b/src/lib/util/qseow/assert-options.js index 605887c4..bb4a50c7 100644 --- a/src/lib/util/qseow/assert-options.js +++ b/src/lib/util/qseow/assert-options.js @@ -3,7 +3,9 @@ import { version as uuidVersion, validate as uuidValidate } from 'uuid'; import { logger, execPath, verifyFileSystemExists } from '../../../globals.js'; import { getCertFilePaths } from '../qseow/cert.js'; import { getStreamById, getStreamByName } from '../qseow/stream.js'; -import { getAppById, getAppByName } from '../qseow/app.js'; +import { getAppById, getAppByName, appExistById } from '../qseow/app.js'; +import { taskExistById } from './task.js'; +import { tagExistByName } from './tag.js'; export const qseowSharedParamAssertOptions = async (options) => { // Ensure that parameters common to all commands are valid @@ -146,23 +148,120 @@ export const getBookmarkAssertOptions = (options) => { // }; -export const getTaskAssertOptions = (options) => { - // ---task-id and --task-tag only allowed for task tables, not trees - if (options.taskId || options.taskTag) { - if (options.outputFormat === 'tree') { - logger.error('Task tree view is not supported when using --task-id or --task-tag. Exiting.'); - process.exit(1); +/** + * Assert options for qseow task-vis command. + * + * @param {object} options - Command line options + * + * @returns {boolean} - True if options are valid, false otherwise. + */ +export async function visTaskAssertOptions(options) { + // Verify all task IDs are valid uuids + // Warn if they do not exist in the Qlik Sense environment + if (options.taskId) { + for (const taskId of options.taskId) { + if (!uuidValidate(taskId)) { + logger.warn(`Invalid format of task ID parameter "${taskId}".`); + return false; + } else { + logger.verbose(`Task id "${taskId}" is a valid uuid version ${uuidVersion(taskId)}`); + } + + // Check if task exists as any task type + // Warn if task with given ID does not exist, but do not return false + const task = await taskExistById(taskId, options); + if (!task) { + logger.warn(`Task with ID ${taskId} does not exist in the Qlik Sense environment.`); + } + } + } + + // Verify all task tags are valid + // Warn if they do not exist in the Qlik Sense environment + if (options.taskTag) { + for (const taskTag of options.taskTag) { + // Check if task tag exists + const tagExists = await tagExistByName(taskTag, options); + if (!tagExists) { + logger.warn(`Tag does not exist in the Qlik Sense environment: "${taskTag}" `); + } } + } + + // Verify all app IDs are valid uuids + // Warn if they do not exist in the Qlik Sense environment + if (options.appId) { + for (const appId of options.appId) { + if (!uuidValidate(appId)) { + logger.warn(`Invalid format of app ID parameter "${appId}".`); + return false; + } else { + logger.verbose(`App id "${appId}" is a valid uuid version ${uuidVersion(appId)}`); + } + + // Check if app exists + // Warn if app with given ID does not exist, but do not return false + const app = await appExistById(appId, options); + if (!app) { + logger.warn(`App with ID "${appId}" does not exist in the Qlik Sense environment.`); + } + } + } - // Verify all task IDs are valid uuids - if (options.taskId) { - for (const taskId of options.taskId) { - if (!uuidValidate(taskId)) { - logger.error(`Invalid format of task ID parameter "${taskId}". Exiting.`); - process.exit(1); - } else { - logger.verbose(`Task id "${taskId}" is a valid uuid version ${uuidVersion(taskId)}`); - } + // Verify all app tags are valid + // Warn if they do not exist in the Qlik Sense environment + if (options.appTag) { + for (const appTag of options.appTag) { + // Check if app tag exists + const tagExists = await tagExistByName(appTag, options); + if (!tagExists) { + logger.warn(`Tag does not exist in the Qlik Sense environment: "${appTag}" `); + } + } + } + + return true; +} + +/** + * Assert options for qseow get-task command. + * + * @param {object} options - CLI options that control the output and formatting. + * + * @returns {boolean} - True if options are valid, false otherwise. + * For fatal errors, the process will exit. + */ +export async function getTaskAssertOptions(options) { + // Verify all task IDs are valid uuids + // Warn if it does not exist (as as any task type, i.e. reload, ext-program, distribution or + // user sync tasks) in the Qlik Sense environment + if (options.taskId) { + for (const taskId of options.taskId) { + if (!uuidValidate(taskId)) { + logger.warn(`Invalid format of task ID parameter "${taskId}".`); + return false; + } else { + logger.verbose(`Task id "${taskId}" is a valid uuid version ${uuidVersion(taskId)}`); + } + + // Check if task exists as any task type + // Warn if task with given ID does not exist, but do not return false + const task = await taskExistById(taskId, options); + if (!task) { + logger.warn(`Task with ID "${taskId}" does not exist in the Qlik Sense environment.`); + } + } + } + + // Verify all task tags are valid + // Warn if they do not exist (as any task type, i.e. reload, ext-program, distribution or + // user sync tasks) in the Qlik Sense environment + if (options.taskTag) { + for (const taskTag of options.taskTag) { + // Check if task tag exists + const tagExists = await tagExistByName(taskTag, options); + if (!tagExists) { + logger.warn(`Tag does not exist in the Qlik Sense environment: "${taskTag}" `); } } } @@ -204,7 +303,9 @@ export const getTaskAssertOptions = (options) => { ); process.exit(1); } -}; + + return true; +} export const setTaskCustomPropertyAssertOptions = (options) => { // diff --git a/src/lib/util/qseow/tag.js b/src/lib/util/qseow/tag.js index 7eeba367..2a7d6a9e 100644 --- a/src/lib/util/qseow/tag.js +++ b/src/lib/util/qseow/tag.js @@ -1,9 +1,46 @@ import axios from 'axios'; -import path from 'node:path'; -import { logger, execPath } from '../../../globals.js'; +import { logger } from '../../../globals.js'; import { setupQrsConnection } from './qrs.js'; import { catchLog } from '../log.js'; +// Check if a tag with a given name exists +export async function tagExistByName(tagName, optionsParam) { + try { + logger.debug(`Checking if tag with name ${tagName} exists`); + + // Did we get any options as parameter? + let options; + if (!optionsParam) { + // Get CLI options + options = getCliOptions(); + } else { + options = optionsParam; + } + + const axiosConfig = setupQrsConnection(options, { + method: 'get', + path: '/qrs/tag/full', + queryParameters: [{ name: 'filter', value: encodeURI(`name eq '${tagName}'`) }], + }); + + const result = await axios.request(axiosConfig); + if (result.status === 200) { + const response = JSON.parse(result.data); + if (response.length === 1) { + logger.debug(`Tag with name ${tagName} exists`); + return true; + } + + logger.debug(`Tag with name ${tagName} does not exist`); + return false; + } + return false; + } catch (err) { + catchLog('TAG EXIST BY NAME', err); + return false; + } +} + export function getTagsFromQseow(options) { return new Promise((resolve, _reject) => { logger.verbose(`Getting tags from QSEoW...`); diff --git a/src/lib/util/qseow/task.js b/src/lib/util/qseow/task.js index ecf6a5b6..c2a2f7ff 100644 --- a/src/lib/util/qseow/task.js +++ b/src/lib/util/qseow/task.js @@ -1,5 +1,4 @@ import axios from 'axios'; -import fs from 'node:fs'; import { validate } from 'uuid'; import { logger, getCliOptions } from '../../../globals.js'; import { setupQrsConnection } from './qrs.js'; @@ -20,8 +19,6 @@ export async function taskExistById(taskId, optionsParam) { options = optionsParam; } - logger.debug(`Auth type: ${options.authType}`); - // Is the task ID a valid GUID? if (!validate(taskId)) { logger.error(`TASK EXIST BY ID: Task ID ${taskId} is not a valid GUID.`); @@ -261,3 +258,169 @@ export async function deleteExternalProgramTaskById(taskId, optionsParam) { return false; } } + +// Function to create new external program task in QSEoW +// Parameters: +// - newTask: Object containing task data +// - taskCounter: Task counter, unique for each task in the source file +export function createExternalProgramTaskInQseow(newTask, taskCounter, options) { + return new Promise(async (resolve, reject) => { + try { + logger.debug(`(${taskCounter}) CREATE EXTERNAL PROGRAM TASK IN QSEOW: Starting`); + + // Build a body for the API call + const body = { + task: { + name: newTask.name, + taskType: 1, + enabled: newTask.enabled, + taskSessionTimeout: newTask.taskSessionTimeout, + maxRetries: newTask.maxRetries, + path: newTask.path, + parameters: newTask.parameters, + tags: newTask.tags, + customProperties: newTask.customProperties, + schemaPath: 'ExternalProgramTask', + }, + schemaEvents: newTask.schemaEvents, + }; + + // Save task to QSEoW + const axiosConfig = setupQrsConnection(options, { + method: 'post', + path: '/qrs/externalprogramtask/create', + body, + }); + + axios + .request(axiosConfig) + .then((result) => { + const response = JSON.parse(result.data); + + logger.debug( + `(${taskCounter}) CREATE EXTERNAL PROGRAM TASK IN QSEOW: "${newTask.name}", new task id: ${response.id}. Result: ${result.status}/${result.statusText}.` + ); + + if (result.status === 201) { + resolve(response.id); + } else { + reject(); + } + }) + .catch((err) => { + catchLog('CREATE EXTERNAL PROGRAM TASK IN QSEOW 1', err); + reject(err); + }); + } catch (err) { + catchLog('CREATE EXTERNAL PROGRAM TASK IN QSEOW 2', err); + reject(err); + } + }); +} + +// Function to create new reload task in QSEoW +export function createReloadTaskInQseow(newTask, taskCounter, options) { + return new Promise(async (resolve, reject) => { + try { + logger.debug(`(${taskCounter}) CREATE RELOAD TASK IN QSEOW: Starting`); + + // Build a body for the API call + const body = { + task: { + app: { + id: newTask.app.id, + }, + name: newTask.name, + isManuallyTriggered: newTask.isManuallyTriggered, + isPartialReload: newTask.isPartialReload, + taskType: 0, + enabled: newTask.enabled, + taskSessionTimeout: newTask.taskSessionTimeout, + maxRetries: newTask.maxRetries, + tags: newTask.tags, + customProperties: newTask.customProperties, + schemaPath: 'ReloadTask', + }, + schemaEvents: newTask.schemaEvents, + }; + + // Save task to QSEoW + const axiosConfig = setupQrsConnection(options, { + method: 'post', + path: '/qrs/reloadtask/create', + body, + }); + + axios + .request(axiosConfig) + .then((result) => { + const response = JSON.parse(result.data); + + logger.debug( + `(${taskCounter}) CREATE RELOAD TASK IN QSEOW: "${newTask.name}", new task id: ${response.id}. Result: ${result.status}/${result.statusText}.` + ); + + if (result.status === 201) { + resolve(response.id); + } else { + reject(); + } + }) + .catch((err) => { + catchLog('CREATE RELOAD TASK IN QSEOW 1', err); + reject(err); + }); + } catch (err) { + catchLog('CREATE RELOAD TASK IN QSEOW 2', err); + reject(err); + } + }); +} + +export function createCompositeEventInQseow(newCompositeEvent, options) { + return new Promise(async (resolve, reject) => { + try { + logger.debug('CREATE COMPOSITE EVENT IN QSEOW: Starting'); + + // Build a body for the API call + const body = newCompositeEvent; + + // Save task to QSEoW + const axiosConfig = setupQrsConnection(options, { + method: 'post', + path: '/qrs/compositeevent', + body, + }); + + logger.debug(`/qrs/compositevent body: ${JSON.stringify(body, null, 2)}`); + + axios + .request(axiosConfig) + .then((result) => { + if (result.status === 201) { + const response = JSON.parse(result.data); + + if (response?.reloadTask) { + logger.info( + `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.reloadTask.id}. Result: ${result.status}/${result.statusText}.` + ); + } else if (response?.externalProgramTask) { + logger.info( + `CREATE COMPOSITE EVENT IN QSEOW: Event name="${newCompositeEvent.name}" for task ID ${response.externalProgramTask.id}. Result: ${result.status}/${result.statusText}.` + ); + } + + resolve(response.id); + } else { + reject(); + } + }) + .catch((err) => { + catchLog('CREATE COMPOSITE EVENT IN QSEOW 1', err); + }); + } catch (err) { + catchLog('CREATE COMPOSITE EVENT IN QSEOW 2', err); + reject(err); + } + }); +} diff --git a/src/static/index.html b/src/static/index.html index c07435d2..b0d74665 100644 --- a/src/static/index.html +++ b/src/static/index.html @@ -200,16 +200,165 @@ #nodeLegend { /* ... existing styles ... */ - top: 100px + top: 100px; width: 450px; } .dot { height: 20px; - width: 42px; + width: 20px; + border-radius: 50%; + display: inline-block; + } + + .triangle { + width: 0; + height: 0; + border-left: 10px solid transparent; + border-right: 10px solid transparent; + border-bottom: 20px solid black; + display: inline-block; + } + .hexagon { + width: 20px; /* Reduce width */ + height: 11.55px; /* Reduce height proportionally */ + background-color: black; + position: relative; + display: inline-block; + transform: rotate(90deg); /* Rotate the hexagon */ + } + .hexagon:before, + .hexagon:after { + content: ""; + position: absolute; + width: 0; + border-left: 10px solid transparent; /* Reduce border size proportionally */ + border-right: 10px solid transparent; /* Reduce border size proportionally */ + } + .hexagon:before { + bottom: 100%; + border-bottom: 5.77px solid black; /* Reduce border size proportionally */ + } + .hexagon:after { + top: 100%; + width: 0; + border-top: 5.77px solid black; /* Reduce border size proportionally */ + } + .ellipse { + width: 30px; + height: 20px; + background-color: black; border-radius: 50%; display: inline-block; } + .rounded-rectangle { + width: 30px; + height: 20px; + background-color: black; + border-radius: 5px; + display: inline-block; + } + .legend-item { + display: flex; + align-items: center; + margin-bottom: 10px; /* Add spacing between legend items */ + } + .legend-item span { + margin-left: 10px; + } + .legend-table { + width: 100%; + border-collapse: collapse; + border: none; /* Hide table borders */ + font-family: Arial, Helvetica, sans-serif; /* Use same font as rest of legend */ + font-size: 14px; /* Use same font size as rest of legend */ + } + .legend-table td { + padding: 5px; + vertical-align: middle; + border: none; /* Hide cell borders */ + } + + #legendTitle { + background-color: #ddd; + padding: 3px; + cursor: move; + width: 100%; + text-align: center; /* Center the header text */ + margin-bottom: 10px; /* Add space after the title */ + } + + .modal { + display: none; + position: fixed; + z-index: 10001; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: auto; + background-color: rgb(0, 0, 0); + background-color: rgba(0, 0, 0, 0.4); + } + + .modal-content { + background-color: #fefefe; + margin: 15% auto; + padding: 20px; + width: 80%; + max-width: 600px; + border-radius: 10px; + box-shadow: 0 5px 15px rgba(0, 0, 0, 0.3); + } + + .close { + color: #aaa; + float: right; + font-size: 28px; + font-weight: bold; + } + + .close:hover, + .close:focus { + color: black; + text-decoration: none; + cursor: pointer; + } + + .modal-header { + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #ddd; + padding-bottom: 10px; + margin-bottom: 20px; + } + + .modal-body { + font-size: 16px; + line-height: 1.5; + } + + .modal-body table { + width: 100%; + border-collapse: collapse; + border: none; + } + + .modal-body th, .modal-body td { + padding: 8px; + text-align: left; + border: none; + } + + .modal-body th { + width: auto; + white-space: nowrap; + } + + .modal-body tr:not(:last-child) { + border-bottom: 1px solid #ddd; + } @@ -272,17 +421,37 @@

Node Legend

-

Successful task

-

Failed task

-

Never executed task

-

Disabled

- - - + +
Successful task
+
Failed or aborted task
+
Aborting/error/reset task
+
Triggered/queued/running task
+
Skipped task
+
Never started task
+
+
Enabled trigger
+
Disabled trigger
+
+

Shapes:

+ + + + + + + + + + + + + + + + + +
Schedule trigger
Composite trigger
External program task
Reload task
+

Ctrl-Q {{appVersion}}

More info at ctrl-q.ptarmiganlabs.com
@@ -290,6 +459,18 @@

Node Legend

Share ideas here

+ +