From 0b2d8b9742743d21586e8cdf479e176ea5c7d042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Sat, 23 Nov 2024 21:12:07 +0100 Subject: [PATCH 1/4] 522 wip --- src/lib/cli/qseow-scramble-field.js | 33 ++++++++- src/lib/cmd/qseow/createdim.js | 2 +- src/lib/cmd/qseow/deletedim.js | 2 +- src/lib/cmd/qseow/deletemeasure.js | 2 +- src/lib/cmd/qseow/getdim.js | 2 +- src/lib/cmd/qseow/scramblefield.js | 107 +++++++++++++++++++++++++-- src/lib/util/qseow/app.js | 77 ++++++++++++++++++- src/lib/util/qseow/assert-options.js | 105 ++++++++++++++++++++++++++ src/lib/util/qseow/stream.js | 48 ++++++++++++ 9 files changed, 364 insertions(+), 14 deletions(-) create mode 100644 src/lib/util/qseow/stream.js diff --git a/src/lib/cli/qseow-scramble-field.js b/src/lib/cli/qseow-scramble-field.js index b3da76f..55de4cf 100644 --- a/src/lib/cli/qseow-scramble-field.js +++ b/src/lib/cli/qseow-scramble-field.js @@ -1,7 +1,7 @@ import { Option } from 'commander'; import { catchLog } from '../util/log.js'; -import { qseowSharedParamAssertOptions } from '../util/qseow/assert-options.js'; +import { qseowSharedParamAssertOptions, qseowScrambleFieldAssertOptions } from '../util/qseow/assert-options.js'; import { scrambleField } from '../cmd/qseow/scramblefield.js'; export function setupQseowScrambleFieldCommand(qseow) { @@ -10,6 +10,7 @@ export function setupQseowScrambleFieldCommand(qseow) { .description('scramble one or more fields in an app. A new app with the scrambled data is created.') .action(async (options) => { await qseowSharedParamAssertOptions(options); + await qseowScrambleFieldAssertOptions(options); scrambleField(options); }) @@ -17,7 +18,8 @@ export function setupQseowScrambleFieldCommand(qseow) { new Option('--log-level ', 'log level').choices(['error', 'warn', 'info', 'verbose', 'debug', 'silly']).default('info') ) .requiredOption('--host ', 'Qlik Sense server IP/FQDN') - .option('--port ', 'Qlik Sense server engine port (usually 4747 for cert auth, 443 for jwt auth)', '4747') + .option('--engine-port ', 'Qlik Sense server engine port (usually 4747 for cert auth, 443 for jwt auth)', '4747') + .option('--qrs-port ', 'Qlik Sense server QRS port (usually 4242 for cert auth, 443 for jwt auth)', '4242') .option('--schema-version ', 'Qlik Sense engine schema version', '12.612.0') .requiredOption('--app-id ', 'Qlik Sense app ID') .requiredOption('--virtual-proxy ', 'Qlik Sense virtual proxy prefix', '') @@ -36,5 +38,30 @@ export function setupQseowScrambleFieldCommand(qseow) { .option('--auth-jwt ', 'JSON Web Token (JWT) to use for authentication with Qlik Sense server') .requiredOption('--field-name ', 'name of field(s) to be scrambled') - .requiredOption('--new-app-name ', 'name of new app that will contain scrambled data'); + .requiredOption('--new-app-name ', 'name of new app that will contain scrambled data') + + .addOption(new Option('--new-app-publish', 'publish scrambled app to a stream')) + .addOption(new Option('--new-app-publish-stream-id ', 'stream ID to publish scrambled app to').default('')) + .addOption(new Option('--new-app-publish-stream-name ', 'stream name to publish scrambled app to').default('')) + + .addOption(new Option('--new-app-publish-replace', 'publish-replace an existing, published app')) + .addOption( + new Option( + '--new-app-publish-replace-app-id ', + 'ID of published app that should be replaced by the new scrambled app' + ).default('') + ) + .addOption( + new Option( + '--new-app-publish-replace-app-name ', + 'Name of published app that should be replaced by the new scrambled app' + ).default('') + ) + + .addOption( + new Option('--new-app-delete-existing-unpublished', 'delete any already existing apps with same name as new scrambled app') + ) + .addOption(new Option('--new-app-delete', 'delete the new scrambled app after all other operations are done')) + + .addOption(new Option('--force', 'force delete and replace operations to proceed without asking for confirmation')); } diff --git a/src/lib/cmd/qseow/createdim.js b/src/lib/cmd/qseow/createdim.js index 52dfd87..90d686d 100644 --- a/src/lib/cmd/qseow/createdim.js +++ b/src/lib/cmd/qseow/createdim.js @@ -247,7 +247,7 @@ export async function createMasterDimension(options) { } if ((await session.close()) === true) { - logger.verbose(`Closed session after managing master items in app ${options.appId} on host ${options.host}`); + logger.verbose(`Closed session after managing master dimension(s) in app ${options.appId} on host ${options.host}`); } else { logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); } diff --git a/src/lib/cmd/qseow/deletedim.js b/src/lib/cmd/qseow/deletedim.js index 3ee687d..d93cb9c 100644 --- a/src/lib/cmd/qseow/deletedim.js +++ b/src/lib/cmd/qseow/deletedim.js @@ -144,7 +144,7 @@ export async function deleteMasterDimension(options) { logger.debug(`Destroyed session object after managing master items in app ${options.appId} on host ${options.host}`); if ((await session.close()) === true) { - logger.verbose(`Closed session after managing master items in app ${options.appId} on host ${options.host}`); + logger.verbose(`Closed session after managing master dimension(s) in app ${options.appId} on host ${options.host}`); } else { logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); } diff --git a/src/lib/cmd/qseow/deletemeasure.js b/src/lib/cmd/qseow/deletemeasure.js index f7a7770..ea15049 100644 --- a/src/lib/cmd/qseow/deletemeasure.js +++ b/src/lib/cmd/qseow/deletemeasure.js @@ -135,7 +135,7 @@ export async function deleteMasterMeasure(options) { logger.debug(`Destroyed session object after managing master items in app ${options.appId} on host ${options.host}`); if ((await session.close()) === true) { - logger.verbose(`Closed session after managing master items in app ${options.appId} on host ${options.host}`); + logger.verbose(`Closed session after managing master measure(s) in app ${options.appId} on host ${options.host}`); } else { logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); } diff --git a/src/lib/cmd/qseow/getdim.js b/src/lib/cmd/qseow/getdim.js index f7b2bed..0aa0343 100644 --- a/src/lib/cmd/qseow/getdim.js +++ b/src/lib/cmd/qseow/getdim.js @@ -257,7 +257,7 @@ export async function getMasterDimension(options) { logger.debug(`Destroyed session object after managing master items in app ${options.appId} on host ${options.host}`); if ((await session.close()) === true) { - logger.verbose(`Closed session after managing master items in app ${options.appId} on host ${options.host}`); + logger.verbose(`Closed session after managing master dimension(s) in app ${options.appId} on host ${options.host}`); } else { logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); } diff --git a/src/lib/cmd/qseow/scramblefield.js b/src/lib/cmd/qseow/scramblefield.js index d1bb6e0..0eae3b2 100644 --- a/src/lib/cmd/qseow/scramblefield.js +++ b/src/lib/cmd/qseow/scramblefield.js @@ -1,7 +1,10 @@ import enigma from 'enigma.js'; +import yesno from 'yesno'; + import { setupEnigmaConnection, addTrafficLogging } from '../../util/qseow/enigma_util.js'; import { logger, setLoggingLevel, isSea, execPath } from '../../../globals.js'; import { catchLog } from '../../util/log.js'; +import { deleteAppById, publishApp } from '../../util/qseow/app.js'; /** * @@ -15,7 +18,6 @@ export async function scrambleField(options) { logger.verbose(`Ctrl-Q was started as a stand-alone binary: ${isSea}`); logger.verbose(`Ctrl-Q was started from ${execPath}`); - logger.info('Scramble field'); logger.debug(`Options: ${JSON.stringify(options, null, 2)}`); // Session ID to use when connecting to the Qlik Sense server @@ -60,15 +62,13 @@ export async function scrambleField(options) { if (options.fieldName.length === 0) { // No fields specified - logger.warn('No fields specified, no scrambling of data will be done'); + logger.warn('No fields specified, no scrambling of data will be done, no new app will be created.'); } else { - // eslint-disable-next-line no-restricted-syntax for (const field of options.fieldName) { // TODO make sure field exists before trying to scramble it // Scramble field try { - // eslint-disable-next-line no-await-in-loop const res = await app.scramble(field); logger.info(`Scrambled field "${field}"`); } catch (err) { @@ -81,10 +81,107 @@ export async function scrambleField(options) { logger.info(`Scrambled data written to new app "${options.newAppName}" with app ID: ${newAppId}`); if ((await session.close()) === true) { - logger.verbose(`Closed session after managing master items in app ${options.appId} on host ${options.host}`); + logger.verbose(`Closed session after scrambling fields in app ${options.appId} on host ${options.host}`); } else { logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); } + + // We now have a new app with scrambled data + // Proceed with other operations on the new app, e.g. publish, publish-replace, delete, etc. + if (options.newAppPublish) { + // Publish the new app to stream specified in options.newAppPublishStreamId or options.newAppPublishStreamName + + // Is stream ID or stream name specified? + let resultPublish; + if (options.newAppPublishStreamId) { + // Publish to stream by stream ID + resultPublish = await publishApp(newAppId, options.newAppName, options.newAppPublishStreamId, options); + } else if (options.newAppPublishStreamName) { + // Publish to stream by stream name + // First look up stream ID by name + // If there are multiple streams with the same name, report error and skip publishing + // If no stream with the specified name is found, report error and skip publishing + // If one stream is found, publish to that stream + const streamArray = await app.getStreamByName(options.newAppPublishStreamName, options); + + if (streamArray.length === 1) { + logger.verbose(`Found stream with name "${options.newAppPublishStreamName}" with ID: ${streamArray[0].id}`); + resultPublish = await publishApp(newAppId, options.newAppName, streamArray[0].id, options); + } else if (streamArray.length > 1) { + logger.error(`More than one stream with name "${options.newAppPublishStreamName}" found. Skipping publish.`); + } else { + logger.error(`No stream with name "${options.newAppPublishStreamName}" found. Skipping publish.`); + } + } + + if (resultPublish) { + logger.info( + `Published new app "${options.newAppName}" with app ID: ${newAppId} to stream "${options.newAppPublishStreamName}"` + ); + } else { + logger.error(`Error publishing new app "${options.newAppName}" with app ID: ${newAppId} to stream.`); + } + } + + if (options.newAppPublishReplace) { + // Publish-replace the new app with an existing published app + // If app ID is specified, use that + // If app name is specified, look up app ID by name + // If no app is found, report error and skip publish-replace + // If more than one app is found, report error and skip publish-replace + // If one app is found, publish-replace + let resultPublishReplace; + if (options.newAppPublishReplaceAppId) { + // Publish-replace by app ID + resultPublishReplace = await replaceApp(newAppId, options.newAppName, options.newAppPublishReplaceAppId, options); + } else if (options.newAppPublishReplaceAppName) { + // Publish-replace by app name + // First look up app ID by name + // If there are multiple apps with the same name, report error and skip publish-replace + // If no app with the specified name is found, report error and skip publish-replace + // If one app is found, publish-replace + const appArray = await app.getAppByName(options.newAppPublishReplaceAppName, options); + + if (appArray.length === 1) { + logger.verbose(`Found app with name "${options.newAppPublishReplaceAppName}" with ID: ${appArray[0].id}`); + resultPublishReplace = await replaceApp(newAppId, options.newAppName, appArray[0].id, options); + } else if (appArray.length > 1) { + logger.error( + `More than one app with name "${options.newAppPublishReplaceAppName}" found. Skipping publish-replace.` + ); + } else { + logger.error(`No app with name "${options.newAppPublishReplaceAppName}" found. Skipping publish-replace.`); + } + } + } + + if (options.newAppDeleteExistingUnpublished) { + // Delete any already existing apps with the same name as the new app + } + + if (options.newAppDelete) { + // Delete the new app after all other operations are done + // Ask user for confirmation unless --force option is set + if (options.force) { + await deleteAppById(newAppId, options); + logger.info(`Deleted new app "${options.newAppName}" with app ID: ${newAppId}`); + } else { + const answer = await yesno({ + question: `Do you want to delete the new app "${options.newAppName}" with app ID: ${newAppId}? (y/n)`, + }); + + if (answer) { + try { + await deleteAppById(newAppId, options); + logger.info(`Deleted new, scrambled app "${options.newAppName}" with app ID: ${newAppId}`); + } catch (err) { + catchLog(`Error deleting new app "${options.newAppName}" with app ID: ${newAppId}`, err); + } + } else { + logger.info(`Did not delete new app "${options.newAppName}" with app ID: ${newAppId}`); + } + } + } } } catch (err) { catchLog('Error in scrambleField', err); diff --git a/src/lib/util/qseow/app.js b/src/lib/util/qseow/app.js index 67b5df8..8389b5f 100644 --- a/src/lib/util/qseow/app.js +++ b/src/lib/util/qseow/app.js @@ -1,5 +1,4 @@ import axios from 'axios'; -import path from 'node:path'; import { validate } from 'uuid'; import { logger, execPath, getCliOptions } from '../../../globals.js'; import { setupQrsConnection } from './qrs.js'; @@ -102,7 +101,7 @@ export async function getAppById(appId, optionsParam) { return false; } - // Should certificates be used for authentication? + // Set up connection to QRS const axiosConfig = setupQrsConnection(options, { method: 'get', path: `/qrs/app/${appId}`, @@ -161,6 +160,80 @@ export async function deleteAppById(appId, options) { } } +// Function to replace app +// If the replaced app is published, only the sheets that were originally published with the app are replaced. +// If the replaced app is not published, the entire app is replaced. +// Parameters: +// - appIdSource: ID of source app +// - appIdTarget: ID of app that will be replaced (=target) app +// - options: Command line options +// +// Returns: +// - true if app was replaced +// - false if app was not replaced +export async function replaceApp(appIdSource, appIdTarget, options) { + try { + logger.debug(`REPLACE APP: Starting replace app id ${appIdTarget} with app id ${appIdSource}`); + + const axiosConfig = setupQrsConnection(options, { + method: 'put', + path: `/qrs/app/${appIdSource}/replace`, + queryParameters: [{ name: 'app', value: appIdTarget }], + }); + + const result = await axios.request(axiosConfig); + logger.debug(`REPLACE APP: Result=${result.status}`); + + if (result.status === 200) { + logger.verbose(`App replaced: ID=${appIdTarget}.`); + return true; + } + + return false; + } catch (err) { + catchLog('REPLACE APP', err); + return false; + } +} + +// Function to publish app +// Parameters: +// - appId: ID of app to publish +// - appName: Name the published app will get in the stream +// - streamId: ID of stream to publish app to +// - options: Command line options +// +// Returns: +// - true if app was published +// - false if app was not published +export async function publishApp(appId, appName, streamId, options) { + try { + logger.debug(`PUBLISH APP: Starting publish app from QSEoW for app id ${appId}`); + + const axiosConfig = setupQrsConnection(options, { + method: 'put', + path: `/qrs/app/${appId}/publish`, + queryParameters: [ + { name: 'stream', value: streamId }, + { name: 'name', value: appName }, + ], + }); + + const result = await axios.request(axiosConfig); + logger.debug(`PUBLISH APP: Result=${result.status}`); + + if (result.status === 200) { + logger.verbose(`App published: ID=${appId}. App name="${appName}"`); + return true; + } + + return false; + } catch (err) { + catchLog('PUBLISH APP', err); + return false; + } +} + // Check if an app with a given id exists export async function appExistById(appId, options) { try { diff --git a/src/lib/util/qseow/assert-options.js b/src/lib/util/qseow/assert-options.js index 089dae0..2940b4b 100644 --- a/src/lib/util/qseow/assert-options.js +++ b/src/lib/util/qseow/assert-options.js @@ -307,3 +307,108 @@ export const userActivityBucketsCustomPropertyAssertOptions = (options) => { process.exit(1); } }; + +export async function qseowScrambleFieldAssertOptions(options) { + // Rules for options: + // - --new-app-publish: Publish the scrambled app to a stream. Optional. + // - If true, --new-app-publish-stream-id and --new-app-publish-stream-name options are used to determine which stream to publish to. Exactly one of those options must be present in this case. + // --new-app-publish-stream-id: Stream to which the scrambled app will be published. Default is ''. + // --new-app-publish-stream-name: Stream to which the scrambled app will be published. Default is ''. If more than one stream matches this name an error is returned. + // --new-app-publish-replace: Do a publish-replace using the scrambled app as source. Optional. + // - If true, The --new-app-publish-replace-app-id and --new-app-publish-replace-app-name options are used to determine which published app should be replaced. Exactly one of those two options must be present in this case. + // - --new-app-publish-replace-app-id: App ID for published app that will be replaced by newly created scrambled app. Default is ''. + // - --new-app-publish-replace-app-name: App name of published app that will be replaced by newly created scrambled app. Default is ''. If more than one published app matches this name an error is returned. + // - --new-app-delete-existing-unpublished: + // - If true, all unpublished apps (irrespective of owner) matching the app name passed in --new-app-name will be deleted before the source app is copied and scrambled. + // - --new-app-delete: Once all other activities are done, delete the newly created scrambled app. + // - --force: Do not ask for acknowledgment before deleting or replacing existing apps. + + // --new-app-publish: Publish the scrambled app to a stream. Optional. + + // Variable to keep track of whether options are valid + let validOptions = true; + + if (options.newAppPublish) { + // Neither of --new-app-publish-stream-id or --new-app-publish-stream-name are non-empty strings, exit + if (options.newAppPublishStreamId === '' && options.newAppPublishStreamName === '') { + logger.error( + 'When --new-app-publish is true, exactly one of --new-app-publish-stream-id or --new-app-publish-stream-name must be present.' + ); + validOptions = false; + } + + // If both --new-app-publish-stream-id and --new-app-publish-stream-name are non-empty strings, exit + if (options.newAppPublishStreamId !== '' && options.newAppPublishStreamName !== '') { + logger.error( + 'When --new-app-publish is true, exactly one of --new-app-publish-stream-id or --new-app-publish-stream-name must be present.' + ); + validOptions = false; + } + + // If --new-app-publish-stream-id is a non-empty string, it must be a valid uuid + if (options.newAppPublishStreamId !== '' && !uuidValidate(options.newAppPublishStreamId)) { + logger.error(`Invalid format of stream ID "${options.newAppPublishStreamId}".`); + validOptions = false; + } + + // If --new-app-publish-stream-name is a non-empty string, it must not contain any special characters + if (options.newAppPublishStreamName !== '' && !/^[a-zA-Z0-9_]+$/.test(options.newAppPublishStreamName)) { + logger.error(`Invalid stream name "${options.newAppPublishStreamName}". Only letters, numbers and underscores are allowed.`); + validOptions = false; + } + + // If --new-app-publish-stream-name is a non-empty string, it must exist in the Qlik Sense environment + if (options.newAppPublishStreamName !== '') { + // TODO: Implement this check + // const stream = await global.getStream(options.newAppPublishStreamName); + // if (stream === null) { + // logger.error(`Stream "${options.newAppPublishStreamName}" does not exist in the Qlik Sense environment.`); + // validOptions = false; + // } + } + } + + // --new-app-publish-replace: Do a publish-replace using the scrambled app as source. Optional. + if (options.newAppPublishReplace) { + // Neither of --new-app-publish-replace-app-id or --new-app-publish-replace-app-name are non-empty strings, exit + if (options.newAppPublishReplaceAppId === '' && options.newAppPublishReplaceAppName === '') { + logger.error( + 'When --new-app-publish-replace is true, exactly one of --new-app-publish-replace-app-id or --new-app-publish-replace-app-name must be present.' + ); + validOptions = false; + } + + // If both --new-app-publish-replace-app-id and --new-app-publish-replace-app-name are non-empty strings, exit + if (options.newAppPublishReplaceAppId !== '' && options.newAppPublishReplaceAppName !== '') { + logger.error( + 'When --new-app-publish-replace is true, exactly one of --new-app-publish-replace-app-id or --new-app-publish-replace-app-name must be present.' + ); + validOptions = false; + } + + // If --new-app-publish-replace-app-id is a non-empty string, it must be a valid uuid + if (options.newAppPublishReplaceAppId !== '' && !uuidValidate(options.newAppPublishReplaceAppId)) { + logger.error(`Invalid format of app ID "${options.newAppPublishReplaceAppId}".`); + validOptions = false; + } + + // If --new-app-publish-replace-app-name is a non-empty string, that app must exist in the Qlik Sense environment and be published + if (options.newAppPublishReplaceAppName !== '') { + // TODO: Implement this check + } + } + + // --new-app-delete-existing-unpublished: If true, all unpublished apps (irrespective of owner) matching the app name passed in --new-app-name will be deleted before the source app is copied and scrambled. + if (options.newAppDeleteExistingUnpublished) { + // --new-app-delete-existing-unpublished is true, but --new-app-name is not a non-empty string + if (options.newAppName === '') { + logger.error('When --new-app-delete-existing-unpublished is true, --new-app-name must be a non-empty string.'); + validOptions = false; + } + } + + if (validOptions === false) { + logger.error('Invalid options, exiting.'); + process.exit(1); + } +} diff --git a/src/lib/util/qseow/stream.js b/src/lib/util/qseow/stream.js new file mode 100644 index 0000000..aec1160 --- /dev/null +++ b/src/lib/util/qseow/stream.js @@ -0,0 +1,48 @@ +import axios from 'axios'; + +import { logger } from '../../../globals.js'; +import { setupQrsConnection } from './qrs.js'; +import { catchLog } from '../log.js'; + +// Function to get stream(s) from QRS, given a stram name +// Parameters: +// - streamName: Name of stream to get +// - options: Command line options +// +// Returns: +// - Array of zero or more stream objects. +// - false if error +export async function getStreamByName(streamName, options) { + try { + logger.debug(`GET STREAM BY NAME: Starting get stream by name from QSEoW for stream name ${streamName}`); + + // Did we get a stream name? + if (!streamName) { + logger.error(`GET STREAM BY NAME: No stream name provided.`); + return false; + } + + // Set up connection to QRS + const axiosConfig = setupQrsConnection(options, { + method: 'get', + path: `/qrs/stream/full`, + queryParameters: [{ name: 'filter', value: `name eq '${streamName}'` }], + }); + + const result = await axios.request(axiosConfig); + logger.debug(`GET STREAM BY NAME: Result=${result.status}`); + + if (result.status === 200) { + const streamArray = JSON.parse(result.data); + logger.debug(`GET STREAM BY NAME: Stream details: ${streamArray}`); + logger.verbose(`Found ${streamArray.length} streams with name ${streamName}`); + + return streamArray; + } + + return false; + } catch (err) { + catchLog('GET STREAM BY NAME', err); + return false; + } +} From 2d34e73a04131a0f8ebec2f5adfd1b715876eda8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 26 Nov 2024 08:04:36 +0100 Subject: [PATCH 2/4] Added unit test for #522 --- src/__tests__/app_scramble_cert.test.js | 350 ++++++++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 src/__tests__/app_scramble_cert.test.js diff --git a/src/__tests__/app_scramble_cert.test.js b/src/__tests__/app_scramble_cert.test.js new file mode 100644 index 0000000..8335c01 --- /dev/null +++ b/src/__tests__/app_scramble_cert.test.js @@ -0,0 +1,350 @@ +import { jest, test, expect, describe } from '@jest/globals'; +import { validate as uuidValidate } from 'uuid'; + +import { scrambleField } from '../lib/cmd/qseow/scramblefield.js'; + +let options = { + logLevel: process.env.CTRL_Q_LOG_LEVEL || 'info', + authType: process.env.CTRL_Q_AUTH_TYPE || 'cert', + authCertFile: process.env.CTRL_Q_AUTH_CERT_FILE || './cert/client.pem', + authCertKeyFile: process.env.CTRL_Q_AUTH_CERT_KEY_FILE || './cert/client_key.pem', + authRootCertFile: process.env.CTRL_Q_AUTH_ROOT_CERT_FILE || './cert/root.pem', + host: process.env.CTRL_Q_HOST || '', + qrsPort: process.env.CTRL_Q_PORT || '4242', + enginePort: process.env.CTRL_Q_ENGINE_PORT || '4747', + schemaVersion: process.env.CTRL_Q_SCHEMA_VERSION || '12.612.0', + virtualProxy: process.env.CTRL_Q_VIRTUAL_PROXY || '', + secure: process.env.CTRL_Q_SECURE || true, + authUserDir: process.env.CTRL_Q_AUTH_USER_DIR || '', + authUserId: process.env.CTRL_Q_AUTH_USER_ID || '', +}; + +const optionsStart = JSON.parse(JSON.stringify(options)); + +const defaultTestTimeout = process.env.CTRL_Q_TEST_TIMEOUT || 600000; // 5 minute default timeout +console.log(`Jest timeout: ${defaultTestTimeout}`); +jest.setTimeout(defaultTestTimeout); + +// Define parameters for coming tests +const existingAppIdExisting1 = '2933711d-6638-41d4-a2d2-6dd2d965208b'; // "Ctrl-Q CLI" +const existingAppIdNonExisting1 = '2933711d-6638-41d4-a2d2-6dd2d965208c'; +const existingAppIInvalidGuid1 = '9f0d0e02-cccc-bbbb-aaaa-3e9a4d0c8a3d'; + +const targetStreamId1 = '9143a1bf-abc3-46f4-8dcb-a1a0ea35860a'; // "Ctrl-Q demo apps" +const targetStreamIdNonExisting1 = '9143a1bf-abc3-46f4-8dcb-a1a0ea35860b'; +const targetStreamIdInvalidGuid1 = '9f0d0e02-cccc-bbbb-aaaa-3e9a4d0c8a3f'; +const targetStreamNameExisting1 = 'Ctrl-Q demo apps'; +const targetStreamNameNonExisting1 = 'Ctrl-Q demo apps - Non-existing'; + +const targetAppName1 = '_Ctrl-Q CLI - Scrambled'; + +const field1 = 'Dim4'; +const field2 = 'Dim2'; + +let createdAppIdUnpublished1; +let createdAppIdPublished1; + +// Scramble fields in existing app, no further action +describe('scramble fields, do not publish (cert auth)', () => { + // Happy path, everything is provided + test('existing source app ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = ''; + options.newAppName = targetAppName1 + '_first'; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe(''); + expect(uuidValidate(result.newAppId)).toBe(true); // App ID should be UUID + expect(result.status).toBe('success'); + + // Save app ID for later use + createdAppIdUnpublished1 = result.newAppId; + }); + + // Scramble app without specifying any fields + test('existing source app ID, no fields specified', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = ''; + options.newAppName = targetAppName1; + options.fieldName = []; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe(''); + expect(result.newAppId).toBeUndefined(); + expect(result.status).toBe('error'); + }); + + // Scramble app without specifying new app name + test('existing source app ID, no new app name specified', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = ''; + options.newAppName = ''; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe(''); + expect(result.newAppId).toBeUndefined(); + expect(result.status).toBe('error'); + }); + + // Scramble app without specifying app ID + test('no source app ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = ''; + options.newAppCmd = ''; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe(''); + expect(result.newAppId).toBeUndefined(); + expect(result.status).toBe('error'); + }); + + // Scramble app ID that does not exist + test('non-existing source app ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdNonExisting1; + options.newAppCmd = ''; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe(''); + expect(result.newAppId).toBeUndefined(); + expect(result.status).toBe('error'); + }); + + // Scramble app ID with invalid GUID + test('source app ID with invalid GUID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIInvalidGuid1; + options.newAppCmd = ''; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe(''); + expect(result.newAppId).toBeUndefined(); + expect(result.status).toBe('error'); + }); +}); + +// Scramble fields in existing app, publish +describe('scramble fields, publish (cert auth)', () => { + // Scramble app and publish to existing stream ID + test('existing source app ID, existing stream ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = 'publish'; + options.newAppCmdId = targetStreamId1; + options.newAppName = targetAppName1 + '_published'; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('publish'); + expect(uuidValidate(result.newAppId)).toBe(true); // App ID should be UUID + expect(result.status).toBe('success'); + + // Save app ID for later use + createdAppIdPublished1 = result.newAppId; + }); + + // Scramble app and publish to non-existing stream ID + test('existing source app ID, non-existing stream ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = 'publish'; + options.newAppCmdId = targetStreamIdNonExisting1; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('publish'); + expect(result.status).toBe('error'); + }); + + // Scramble app and publish to stream with invalid GUID + test('existing source app ID, stream ID with invalid GUID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = 'publish'; + options.newAppCmdId = targetStreamIdInvalidGuid1; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('publish'); + expect(result.status).toBe('error'); + }); + + // Scramble app and publish to existing stream name + test('existing source app ID, existing stream name', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = 'publish'; + options.newAppCmdName = targetStreamNameExisting1; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('publish'); + expect(uuidValidate(result.newAppId)).toBe(true); // App ID should be UUID + expect(result.status).toBe('success'); + }); + + // Scramble app and publish to non-existing stream name + test('existing source app ID, non-existing stream name', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppCmd = 'publish'; + options.newAppCmdName = targetStreamNameNonExisting1; + options.newAppName = targetAppName1; + options.fieldName = [field1, field2]; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('publish'); + expect(result.status).toBe('error'); + }); +}); + +// Scramble fields in existing app, replace +describe('scramble fields, replace unpublished (cert auth)', () => { + // Scramble app and replace existing unpublished app by ID + test('existing source app ID, existing unpublished app ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppName = targetAppName1; + options.newAppCmd = 'replace'; + options.newAppCmdId = createdAppIdUnpublished1; + options.fieldName = [field1, field2]; + options.force = true; + + console.log(options); + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('replace'); + expect(uuidValidate(result.newAppId)).toBe(true); // App ID should be UUID + expect(result.status).toBe('success'); + }); + + // Scramble app and replace existing published app by ID + test('existing source app ID, existing published app ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppName = targetAppName1; + options.newAppCmd = 'replace'; + options.newAppCmdId = createdAppIdPublished1; + options.fieldName = [field1, field2]; + options.force = true; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('replace'); + expect(uuidValidate(result.newAppId)).toBe(true); // App ID should be UUID + expect(result.status).toBe('success'); + }); + + // Scramble app and replace non-existing app ID + test('existing source app ID, non-existing app ID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppName = targetAppName1; + options.newAppCmd = 'replace'; + options.newAppCmdId = existingAppIdNonExisting1; + options.fieldName = [field1, field2]; + options.force = true; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('replace'); + expect(result.status).toBe('error'); + }); + + // Scramble app and replace app with invalid GUID + test('existing source app ID, app ID with invalid GUID', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppName = targetAppName1; + options.newAppCmd = 'replace'; + options.newAppCmdId = existingAppIInvalidGuid1; + options.fieldName = [field1, field2]; + options.force = true; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('replace'); + expect(result.status).toBe('error'); + }); + + // Scramble app and replace unpublish app by name, only one app with this name exists + test('existing source app ID, existing unpublished app by name, only one app with this name exists', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppName = targetAppName1; + options.newAppCmd = 'replace'; + options.newAppCmdName = targetAppName1 + '_first'; + options.fieldName = [field1, field2]; + options.force = true; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('replace'); + expect(uuidValidate(result.newAppId)).toBe(true); // App ID should be UUID + expect(result.status).toBe('success'); + }); + + // Scramble app and replace app by name, no app with this name exists + test('existing source app ID, non-existing app by name', async () => { + // Reset options + options = JSON.parse(JSON.stringify(optionsStart)); + + options.appId = existingAppIdExisting1; + options.newAppName = targetAppName1; + options.newAppCmd = 'replace'; + options.newAppCmdName = targetAppName1 + '_non_existing'; + options.fieldName = [field1, field2]; + options.force = true; + + const result = await scrambleField(options); + expect(result.newAppCmd).toBe('replace'); + expect(result.status).toBe('error'); + }); +}); + +// Scramble fields in existing app, replace published app +describe('scramble fields, replace published (cert auth)', () => {}); + +// Clean up. Delete created apps. From 8cbfe640132627a197b8cc79b50e138c4fab6545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 26 Nov 2024 08:05:29 +0100 Subject: [PATCH 3/4] feat(qseow): Add publish and replace options to field scramble command Implements #522 --- src/lib/cli/qseow-scramble-field.js | 32 ++- src/lib/cmd/qseow/scramblefield.js | 278 ++++++++++++++++++++------- src/lib/util/qseow/app.js | 36 ++++ src/lib/util/qseow/assert-options.js | 95 +++------ src/lib/util/qseow/stream.js | 45 ++++- 5 files changed, 332 insertions(+), 154 deletions(-) diff --git a/src/lib/cli/qseow-scramble-field.js b/src/lib/cli/qseow-scramble-field.js index 55de4cf..4f64107 100644 --- a/src/lib/cli/qseow-scramble-field.js +++ b/src/lib/cli/qseow-scramble-field.js @@ -21,7 +21,6 @@ export function setupQseowScrambleFieldCommand(qseow) { .option('--engine-port ', 'Qlik Sense server engine port (usually 4747 for cert auth, 443 for jwt auth)', '4747') .option('--qrs-port ', 'Qlik Sense server QRS port (usually 4242 for cert auth, 443 for jwt auth)', '4242') .option('--schema-version ', 'Qlik Sense engine schema version', '12.612.0') - .requiredOption('--app-id ', 'Qlik Sense app ID') .requiredOption('--virtual-proxy ', 'Qlik Sense virtual proxy prefix', '') .requiredOption( '--secure ', @@ -37,31 +36,30 @@ export function setupQseowScrambleFieldCommand(qseow) { .option('--auth-root-cert-file ', 'Qlik Sense root certificate file (exported from QMC)', './cert/root.pem') .option('--auth-jwt ', 'JSON Web Token (JWT) to use for authentication with Qlik Sense server') + .requiredOption('--app-id ', 'Qlik Sense app ID to be scrambled') .requiredOption('--field-name ', 'name of field(s) to be scrambled') - .requiredOption('--new-app-name ', 'name of new app that will contain scrambled data') + .requiredOption('--new-app-name ', 'name of new app that will contain scrambled data. Not used if --new-app-cmd=replace') - .addOption(new Option('--new-app-publish', 'publish scrambled app to a stream')) - .addOption(new Option('--new-app-publish-stream-id ', 'stream ID to publish scrambled app to').default('')) - .addOption(new Option('--new-app-publish-stream-name ', 'stream name to publish scrambled app to').default('')) - - .addOption(new Option('--new-app-publish-replace', 'publish-replace an existing, published app')) .addOption( new Option( - '--new-app-publish-replace-app-id ', - 'ID of published app that should be replaced by the new scrambled app' - ).default('') + '--new-app-cmd ', + 'what to do with the new app. If nothing is specified in this option the new app will be placed in My Work. WHen specifying "replace": If the replaced app is published, only the sheets that were originally published with the app are replaced. If the replaced app is not published, the entire app is replaced.' + ) + .choices(['publish', 'replace']) + .default('') ) + .addOption( - new Option( - '--new-app-publish-replace-app-name ', - 'Name of published app that should be replaced by the new scrambled app' - ).default('') + new Option('--new-app-cmd-id ', 'stream/app ID that --new-app-cmd acts on. Cannot be used with --new-app-cmd-name').default( + '' + ) ) - .addOption( - new Option('--new-app-delete-existing-unpublished', 'delete any already existing apps with same name as new scrambled app') + new Option( + '--new-app-cmd-name ', + 'stream/app name that --new-app-cmd acts on. Cannot be used with --new-app-cmd-id' + ).default('') ) - .addOption(new Option('--new-app-delete', 'delete the new scrambled app after all other operations are done')) .addOption(new Option('--force', 'force delete and replace operations to proceed without asking for confirmation')); } diff --git a/src/lib/cmd/qseow/scramblefield.js b/src/lib/cmd/qseow/scramblefield.js index 0eae3b2..9e060a0 100644 --- a/src/lib/cmd/qseow/scramblefield.js +++ b/src/lib/cmd/qseow/scramblefield.js @@ -1,10 +1,12 @@ import enigma from 'enigma.js'; import yesno from 'yesno'; +import { validate as uuidValidate } from 'uuid'; import { setupEnigmaConnection, addTrafficLogging } from '../../util/qseow/enigma_util.js'; import { logger, setLoggingLevel, isSea, execPath } from '../../../globals.js'; import { catchLog } from '../../util/log.js'; -import { deleteAppById, publishApp } from '../../util/qseow/app.js'; +import { deleteAppById, publishApp, replaceApp, getAppByName, getAppById } from '../../util/qseow/app.js'; +import { getStreamByName, getStreamById } from '../../util/qseow/stream.js'; /** * @@ -20,6 +22,108 @@ export async function scrambleField(options) { logger.debug(`Options: ${JSON.stringify(options, null, 2)}`); + // Keep track of the result of the scramble operation + const scrambleResult = { + newAppCmd: options.newAppCmd, + status: 'error', + }; + + // ------------------------------------------------ + // Verify parameters + + // --new-app-name is always required + if (!options.newAppName) { + logger.error('Option --new-app-name is required when --new-app-cmd is empty or set to "publish".'); + return scrambleResult; + } + + // No source app ID specified + if (!options.appId) { + logger.error('No source app ID specified.'); + return scrambleResult; + } + + // No fields specified + if (!options.fieldName || !Array.isArray(options.fieldName) || options?.fieldName?.length === 0) { + logger.error('No fields specified.'); + return scrambleResult; + } + + // Verify that --app-id is a valid GUID + if (!uuidValidate(options.appId)) { + logger.error(`Invalid GUID in --app-id: ${options.appId}`); + return scrambleResult; + } + + // Verify that source app exists, given --app-id + const appArray = await getAppById(options.appId, options); + if (appArray === false) { + logger.error(`App with ID ${options.appId} not found.`); + return scrambleResult; + } + + // Verify that --new-app-cmd is either '', 'publish' or 'replace' + if (options.newAppCmd !== '' && options.newAppCmd !== 'publish' && options.newAppCmd !== 'replace') { + logger.error(`Invalid value in --new-app-cmd: ${options.newAppCmd}`); + return scrambleResult; + } + + // Given --new-app-cmd-id and --new-app-cmd='publish' + if (options.newAppCmd === 'publish' && options.newAppCmdId) { + // Verify that stream ID is a valid GUID + if (!uuidValidate(options.newAppCmdId)) { + logger.error(`Invalid GUID in --new-app-cmd-id: ${options.newAppCmdId}`); + return scrambleResult; + } + + // Verify that stream exists, + const streamArray = await getStreamById(options.newAppCmdId, options); + if (streamArray === false || streamArray.length === 0) { + logger.error(`Stream with ID ${options.newAppCmdId} not found.`); + return scrambleResult; + } + } + + // Given --new-app-cmd-name and --new-app-cmd='publish' + if (options.newAppCmd === 'publish' && options.newAppCmdName) { + // Verify that stream exists, + const streamArray = await getStreamByName(options.newAppCmdName, options); + if (streamArray === false || streamArray.length === 0) { + logger.error(`Stream with name ${options.newAppCmdName} not found.`); + return scrambleResult; + } + } + + // Given --new-app-cmd-id and --new-app-cmd='replace' + if (options.newAppCmd === 'replace' && options.newAppCmdId) { + // Verify that app ID is a valid GUID + if (!uuidValidate(options.newAppCmdId)) { + logger.error(`Invalid GUID in --new-app-cmd-id: ${options.newAppCmdId}`); + return scrambleResult; + } + + // Verify that app exists + const appArray = await getAppById(options.newAppCmdId, options); + if (appArray === false) { + logger.error(`App with ID ${options.newAppCmdId} not found.`); + return scrambleResult; + } + } + + // Given --new-app-cmd-name and --new-app-cmd='replace' + if (options.newAppCmd === 'replace' && options.newAppCmdName) { + // Verify that app exists in singular + const appArray = await getAppByName(options.newAppCmdName, options); + if (appArray === false || appArray.length === 0) { + logger.error(`App with name ${options.newAppCmdName} not found.`); + return scrambleResult; + } + if (appArray.length > 1) { + logger.error(`More than one app with name ${options.newAppCmdName} found.`); + return scrambleResult; + } + } + // Session ID to use when connecting to the Qlik Sense server const sessionId = 'ctrlq'; @@ -58,11 +162,20 @@ export async function scrambleField(options) { const app = await global.openDoc(options.appId, '', '', '', false); logger.verbose(`Opened app ${options.appId}.`); - // Fields to be scrambled are availbel in array options.fieldName; - - if (options.fieldName.length === 0) { + // Fields to be scrambled are availble in array options.fieldName; + // If no fields are specified, no scrambling will be done + // options.fieldNams is an array of field names to be scrambled + // Verify it's an array + if (!options.fieldName || !Array.isArray(options.fieldName) || options?.fieldName?.length === 0) { // No fields specified logger.warn('No fields specified, no scrambling of data will be done, no new app will be created.'); + + // Close session + if ((await session.close()) === true) { + logger.verbose(`Closed session after scrambling fields in app ${options.appId} on host ${options.host}`); + } else { + logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); + } } else { for (const field of options.fieldName) { // TODO make sure field exists before trying to scramble it @@ -86,103 +199,136 @@ export async function scrambleField(options) { logger.error(`Error closing session for app ${options.appId} on host ${options.host}`); } + // Add new app ID to result object + scrambleResult.newAppId = newAppId; + scrambleResult.status = 'success'; + + // ------------------------------------------------ // We now have a new app with scrambled data - // Proceed with other operations on the new app, e.g. publish, publish-replace, delete, etc. - if (options.newAppPublish) { - // Publish the new app to stream specified in options.newAppPublishStreamId or options.newAppPublishStreamName + // Proceed with other operations on the new app, e.g. publish, replace, delete, etc. - // Is stream ID or stream name specified? + if (options.newAppCmd === 'publish') { + // Publish the new app to stream specified in options.newAppCmdId or options.newAppCmdName + + // Is ID or name specified? let resultPublish; - if (options.newAppPublishStreamId) { + if (options.newAppCmdId) { // Publish to stream by stream ID - resultPublish = await publishApp(newAppId, options.newAppName, options.newAppPublishStreamId, options); - } else if (options.newAppPublishStreamName) { + resultPublish = await publishApp(newAppId, options.newAppName, options.newAppCmdId, options); + scrambleResult.status = 'success'; + } else if (options.newAppCmdName) { // Publish to stream by stream name // First look up stream ID by name // If there are multiple streams with the same name, report error and skip publishing // If no stream with the specified name is found, report error and skip publishing // If one stream is found, publish to that stream - const streamArray = await app.getStreamByName(options.newAppPublishStreamName, options); + const streamArray = await getStreamByName(options.newAppCmdName, options); if (streamArray.length === 1) { - logger.verbose(`Found stream with name "${options.newAppPublishStreamName}" with ID: ${streamArray[0].id}`); + logger.verbose(`Found stream with name "${options.newAppCmdName}" with ID: ${streamArray[0].id}`); resultPublish = await publishApp(newAppId, options.newAppName, streamArray[0].id, options); + scrambleResult.status = 'success'; } else if (streamArray.length > 1) { - logger.error(`More than one stream with name "${options.newAppPublishStreamName}" found. Skipping publish.`); + logger.error(`More than one stream with name "${options.newAppCmdName}" found. Skipping publish.`); + scrambleResult.status = 'error'; } else { - logger.error(`No stream with name "${options.newAppPublishStreamName}" found. Skipping publish.`); + logger.error(`No stream with name "${options.newAppCmdName}" found. Skipping publish.`); + scrambleResult.status = 'error'; } } if (resultPublish) { - logger.info( - `Published new app "${options.newAppName}" with app ID: ${newAppId} to stream "${options.newAppPublishStreamName}"` - ); + logger.info(`Published new app "${options.newAppName}" with app ID: ${newAppId} to stream "${options.newAppCmdName}"`); + scrambleResult.cmdDone = 'publish'; + scrambleResult.status = 'success'; } else { logger.error(`Error publishing new app "${options.newAppName}" with app ID: ${newAppId} to stream.`); + scrambleResult.status = 'error'; } - } - - if (options.newAppPublishReplace) { - // Publish-replace the new app with an existing published app + } else if (options.newAppCmd === 'replace') { + // Replace an existing app with the new, scrambled app // If app ID is specified, use that // If app name is specified, look up app ID by name - // If no app is found, report error and skip publish-replace - // If more than one app is found, report error and skip publish-replace - // If one app is found, publish-replace - let resultPublishReplace; - if (options.newAppPublishReplaceAppId) { - // Publish-replace by app ID - resultPublishReplace = await replaceApp(newAppId, options.newAppName, options.newAppPublishReplaceAppId, options); - } else if (options.newAppPublishReplaceAppName) { - // Publish-replace by app name + // If no app is found, report error and skip replace + // If more than one app is found, report error and skip replace + // If one app is found, replace + + let resultReplace; + if (options.newAppCmdId) { + // Replace by app ID + if (!options.force) { + const answer = await yesno({ + question: `Do you want to replace the existing app with app ID ${options.newAppCmdId} with the new, scrambled app? (y/n)`, + }); + + if (answer) { + resultReplace = await replaceApp(newAppId, options.newAppCmdId, options); + scrambleResult.status = 'success'; + } else { + logger.warn( + `Did not replace existing app with app ID ${options.newAppCmdId} with new, scrambled app "${options.newAppName}" with app ID ${newAppId}. The scrambled app is still available in My Work.` + ); + scrambleResult.status = 'aborted'; + } + } else { + resultReplace = await replaceApp(newAppId, options.newAppCmdId, options); + scrambleResult.status = 'success'; + } + } else if (options.newAppCmdName) { + // Replace by app name // First look up app ID by name - // If there are multiple apps with the same name, report error and skip publish-replace - // If no app with the specified name is found, report error and skip publish-replace - // If one app is found, publish-replace - const appArray = await app.getAppByName(options.newAppPublishReplaceAppName, options); + // If there are multiple apps with the same name, report error and skip replace + // If no app with the specified name is found, report error and skip replace + // If one app is found, replace + const appArray = await getAppByName(options.newAppCmdName, options); if (appArray.length === 1) { - logger.verbose(`Found app with name "${options.newAppPublishReplaceAppName}" with ID: ${appArray[0].id}`); - resultPublishReplace = await replaceApp(newAppId, options.newAppName, appArray[0].id, options); + logger.info(`Found app with name "${options.newAppCmdName}" with ID: ${appArray[0].id}`); + + if (!options.force) { + const answer = await yesno({ + question: `Do you want to replace the existing app with name "${options.newAppCmdName}" with the new, scrambled app? (y/n)`, + }); + + if (answer) { + resultReplace = await replaceApp(newAppId, appArray[0].id, options); + scrambleResult.status = 'success'; + } else { + logger.warn( + `Did not replace existing app with name "${options.newAppCmdName}" with new, scrambled app "${options.newAppName}" with app ID ${newAppId}. The scrambled app is still available in My Work.` + ); + scrambleResult.status = 'aborted'; + } + } else { + resultReplace = await replaceApp(newAppId, appArray[0].id, options); + scrambleResult.status = 'success'; + } } else if (appArray.length > 1) { - logger.error( - `More than one app with name "${options.newAppPublishReplaceAppName}" found. Skipping publish-replace.` - ); + logger.error(`More than one app with name "${options.newAppCmdName}" found. Skipping replace.`); + scrambleResult.status = 'error'; } else { - logger.error(`No app with name "${options.newAppPublishReplaceAppName}" found. Skipping publish-replace.`); + logger.error(`No app with name "${options.newAppCmdName}" found. Skipping replace.`); + scrambleResult.status = 'error'; } - } - } - if (options.newAppDeleteExistingUnpublished) { - // Delete any already existing apps with the same name as the new app - } - - if (options.newAppDelete) { - // Delete the new app after all other operations are done - // Ask user for confirmation unless --force option is set - if (options.force) { - await deleteAppById(newAppId, options); - logger.info(`Deleted new app "${options.newAppName}" with app ID: ${newAppId}`); - } else { - const answer = await yesno({ - question: `Do you want to delete the new app "${options.newAppName}" with app ID: ${newAppId}? (y/n)`, - }); - - if (answer) { - try { - await deleteAppById(newAppId, options); - logger.info(`Deleted new, scrambled app "${options.newAppName}" with app ID: ${newAppId}`); - } catch (err) { - catchLog(`Error deleting new app "${options.newAppName}" with app ID: ${newAppId}`, err); - } + if (resultReplace) { + logger.info( + `Replaced existing app "${options.newAppCmdName}" (app ID: ${appArray[0].id}) with new, scrambled app "${options.newAppName}" (app ID: ${newAppId})` + ); + scrambleResult.cmdDone = 'replace'; + scrambleResult.status = 'success'; } else { - logger.info(`Did not delete new app "${options.newAppName}" with app ID: ${newAppId}`); + logger.error( + `Error replacing existing app "${options.newAppCmdName}" with new, scrambled app "${options.newAppName}"` + ); + scrambleResult.status = 'error'; } } } } + + // Return the result of the scramble operation + return scrambleResult; } catch (err) { catchLog('Error in scrambleField', err); } diff --git a/src/lib/util/qseow/app.js b/src/lib/util/qseow/app.js index 8389b5f..e0972f4 100644 --- a/src/lib/util/qseow/app.js +++ b/src/lib/util/qseow/app.js @@ -81,6 +81,42 @@ export async function getApps(options, idArray, tagArray) { } } +// Function to get app(s) from QRS, given app name +// Returns array of zero or more app objects, or false if error +export async function getAppByName(appName, options) { + try { + logger.debug(`GET APP BY NAME: Starting get app from QSEoW for app name ${appName}`); + + // Did we get an app name? + if (!appName) { + logger.error(`GET APP BY NAME: No app name provided.`); + return false; + } + + // Set up connection to QRS + const axiosConfig = setupQrsConnection(options, { + method: 'get', + path: '/qrs/app/full', + queryParameters: [{ name: 'filter', value: encodeURI(`name eq '${appName}'`) }], + }); + + const result = await axios.request(axiosConfig); + logger.debug(`GET APP BY NAME: Result=${result.status}`); + + if (result.status === 200) { + const apps = JSON.parse(result.data); + logger.debug(`GET APP BY NAME: App details: ${apps}`); + + return apps; + } + + return false; + } catch (err) { + catchLog('GET APP BY NAME', err); + return false; + } +} + // Function to get app info from QRS, given app ID export async function getAppById(appId, optionsParam) { try { diff --git a/src/lib/util/qseow/assert-options.js b/src/lib/util/qseow/assert-options.js index 2940b4b..2344dcb 100644 --- a/src/lib/util/qseow/assert-options.js +++ b/src/lib/util/qseow/assert-options.js @@ -310,101 +310,56 @@ export const userActivityBucketsCustomPropertyAssertOptions = (options) => { export async function qseowScrambleFieldAssertOptions(options) { // Rules for options: - // - --new-app-publish: Publish the scrambled app to a stream. Optional. - // - If true, --new-app-publish-stream-id and --new-app-publish-stream-name options are used to determine which stream to publish to. Exactly one of those options must be present in this case. - // --new-app-publish-stream-id: Stream to which the scrambled app will be published. Default is ''. - // --new-app-publish-stream-name: Stream to which the scrambled app will be published. Default is ''. If more than one stream matches this name an error is returned. - // --new-app-publish-replace: Do a publish-replace using the scrambled app as source. Optional. - // - If true, The --new-app-publish-replace-app-id and --new-app-publish-replace-app-name options are used to determine which published app should be replaced. Exactly one of those two options must be present in this case. - // - --new-app-publish-replace-app-id: App ID for published app that will be replaced by newly created scrambled app. Default is ''. - // - --new-app-publish-replace-app-name: App name of published app that will be replaced by newly created scrambled app. Default is ''. If more than one published app matches this name an error is returned. - // - --new-app-delete-existing-unpublished: - // - If true, all unpublished apps (irrespective of owner) matching the app name passed in --new-app-name will be deleted before the source app is copied and scrambled. - // - --new-app-delete: Once all other activities are done, delete the newly created scrambled app. - // - --force: Do not ask for acknowledgment before deleting or replacing existing apps. - - // --new-app-publish: Publish the scrambled app to a stream. Optional. + // - --new-app-cmd: Either "publish" or "replace". Optional. If not specified, the new app will be placed in My Work. + // - If true, --new-app-cmd-id and --new-app-cmd-name options are used to determine which stream to publish to. Exactly one of those options must be present in this case. + // - --new-app-cmd-id: Stream/app to which the scrambled app will be published. Default is ''. + // - --new-app-cmd-name: Stream/app to which the scrambled app will be published. Default is ''. If more than one stream/app matches this name an error is returned. + // - --force: Do not ask for acknowledgment before replacing existing app. // Variable to keep track of whether options are valid let validOptions = true; - if (options.newAppPublish) { - // Neither of --new-app-publish-stream-id or --new-app-publish-stream-name are non-empty strings, exit - if (options.newAppPublishStreamId === '' && options.newAppPublishStreamName === '') { + if (options.newAppCmd === 'publish' || options.newAppCmd === 'replace') { + // Neither of --new-app-cmd-id or --new-app-cmd-name are empty strings, exit + if (options.newAppCmdId === '' && options.newAppCmdName === '') { logger.error( - 'When --new-app-publish is true, exactly one of --new-app-publish-stream-id or --new-app-publish-stream-name must be present.' + 'When --new-app-cmd is either "publish" or "replace", exactly one of --new-app-cmd-id and --new-app-cmd-name must be present.' ); validOptions = false; } - // If both --new-app-publish-stream-id and --new-app-publish-stream-name are non-empty strings, exit - if (options.newAppPublishStreamId !== '' && options.newAppPublishStreamName !== '') { - logger.error( - 'When --new-app-publish is true, exactly one of --new-app-publish-stream-id or --new-app-publish-stream-name must be present.' - ); - validOptions = false; - } - - // If --new-app-publish-stream-id is a non-empty string, it must be a valid uuid - if (options.newAppPublishStreamId !== '' && !uuidValidate(options.newAppPublishStreamId)) { - logger.error(`Invalid format of stream ID "${options.newAppPublishStreamId}".`); + // If both --new-app-cmd-id and --new-app-cmd-name are non-empty strings, exit + if (options.newAppCmdId !== '' && options.newAppCmdName !== '') { + logger.error('When --new-app-cmd is true, exactly one of --new-app-cmd-id or --new-app-cmd-name must be present.'); validOptions = false; } - // If --new-app-publish-stream-name is a non-empty string, it must not contain any special characters - if (options.newAppPublishStreamName !== '' && !/^[a-zA-Z0-9_]+$/.test(options.newAppPublishStreamName)) { - logger.error(`Invalid stream name "${options.newAppPublishStreamName}". Only letters, numbers and underscores are allowed.`); + // If --new-app-cmd-id is a non-empty string, it must be a valid uuid + if (options.newAppCmdId !== '' && !uuidValidate(options.newAppCmdId)) { + logger.error(`Invalid format of --new-app-cmd-id (not a valid ID): "${options.newAppCmdId}".`); validOptions = false; } - // If --new-app-publish-stream-name is a non-empty string, it must exist in the Qlik Sense environment - if (options.newAppPublishStreamName !== '') { + // If --new-app-cmd-name is a non-empty string, it must exist in the Qlik Sense environment + if (options.newAppCmdName !== '') { // TODO: Implement this check - // const stream = await global.getStream(options.newAppPublishStreamName); + // const stream = await global.getStream(options.newAppCmdStreamName); // if (stream === null) { - // logger.error(`Stream "${options.newAppPublishStreamName}" does not exist in the Qlik Sense environment.`); + // logger.error(`Stream "${options.newAppCmdStreamName}" does not exist in the Qlik Sense environment.`); // validOptions = false; // } } - } - - // --new-app-publish-replace: Do a publish-replace using the scrambled app as source. Optional. - if (options.newAppPublishReplace) { - // Neither of --new-app-publish-replace-app-id or --new-app-publish-replace-app-name are non-empty strings, exit - if (options.newAppPublishReplaceAppId === '' && options.newAppPublishReplaceAppName === '') { - logger.error( - 'When --new-app-publish-replace is true, exactly one of --new-app-publish-replace-app-id or --new-app-publish-replace-app-name must be present.' - ); - validOptions = false; - } - // If both --new-app-publish-replace-app-id and --new-app-publish-replace-app-name are non-empty strings, exit - if (options.newAppPublishReplaceAppId !== '' && options.newAppPublishReplaceAppName !== '') { - logger.error( - 'When --new-app-publish-replace is true, exactly one of --new-app-publish-replace-app-id or --new-app-publish-replace-app-name must be present.' - ); - validOptions = false; - } - - // If --new-app-publish-replace-app-id is a non-empty string, it must be a valid uuid - if (options.newAppPublishReplaceAppId !== '' && !uuidValidate(options.newAppPublishReplaceAppId)) { - logger.error(`Invalid format of app ID "${options.newAppPublishReplaceAppId}".`); - validOptions = false; - } - - // If --new-app-publish-replace-app-name is a non-empty string, that app must exist in the Qlik Sense environment and be published - if (options.newAppPublishReplaceAppName !== '') { + // If --new-app-cmd-id is a non-empty string, it must exist in the Qlik Sense environment + if (options.newAppCmdId !== '') { // TODO: Implement this check } } - // --new-app-delete-existing-unpublished: If true, all unpublished apps (irrespective of owner) matching the app name passed in --new-app-name will be deleted before the source app is copied and scrambled. - if (options.newAppDeleteExistingUnpublished) { - // --new-app-delete-existing-unpublished is true, but --new-app-name is not a non-empty string - if (options.newAppName === '') { - logger.error('When --new-app-delete-existing-unpublished is true, --new-app-name must be a non-empty string.'); - validOptions = false; - } + // If publishing to a stream, --new-app-cmd-name must be a non-empty string + if (options.newAppCmd === 'publish' && options.newAppCmdName === '') { + logger.error('When --new-app-cmd is "publish", --new-app-cmd-name must be a non-empty string.'); + validOptions = false; } if (validOptions === false) { diff --git a/src/lib/util/qseow/stream.js b/src/lib/util/qseow/stream.js index aec1160..aa802ad 100644 --- a/src/lib/util/qseow/stream.js +++ b/src/lib/util/qseow/stream.js @@ -4,7 +4,7 @@ import { logger } from '../../../globals.js'; import { setupQrsConnection } from './qrs.js'; import { catchLog } from '../log.js'; -// Function to get stream(s) from QRS, given a stram name +// Function to get stream(s) from QRS, given a stream name // Parameters: // - streamName: Name of stream to get // - options: Command line options @@ -46,3 +46,46 @@ export async function getStreamByName(streamName, options) { return false; } } + +// Function to get stream(s) from QRS, given a stream ID +// Parameters: +// - streamId: ID of stream to get +// - options: Command line options +// +// Returns: +// - Array of zero or more stream objects. +// - false if error +export async function getStreamById(streamId, options) { + try { + logger.debug(`GET STREAM BY ID: Starting get stream by ID from QSEoW for stream ${streamId}`); + + // Did we get a stream ID? + if (!streamId) { + logger.error(`GET STREAM BY ID: No stream ID provided.`); + return false; + } + + // Set up connection to QRS + const axiosConfig = setupQrsConnection(options, { + method: 'get', + path: `/qrs/stream/full`, + queryParameters: [{ name: 'filter', value: `id eq ${streamId}` }], + }); + + const result = await axios.request(axiosConfig); + logger.debug(`GET STREAM BY ID: Result=${result.status}`); + + if (result.status === 200) { + const streamArray = JSON.parse(result.data); + logger.debug(`GET STREAM BY ID: Stream details: ${streamArray}`); + logger.verbose(`Found ${streamArray.length} streams with ID ${streamId}`); + + return streamArray; + } + + return false; + } catch (err) { + catchLog('GET STREAM BY ID', err); + return false; + } +} From 175616641e03e18f69e9bb584370ea281001fd59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 26 Nov 2024 08:11:12 +0100 Subject: [PATCH 4/4] chore(deps): Update dependencies --- package-lock.json | 41 +++++++++++++++++++++-------------------- package.json | 10 +++++----- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index eef065c..edeba4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,10 @@ "license": "MIT", "dependencies": { "@qlik/api": "^1.25.0", - "axios": "^1.7.7", + "axios": "^1.7.8", "commander": "^12.1.0", - "csv-parse": "^5.5.6", - "csv-stringify": "^6.5.1", + "csv-parse": "^5.6.0", + "csv-stringify": "^6.5.2", "enigma.js": "^2.14.0", "esbuild": "^0.24.0", "form-data": "^4.0.1", @@ -42,8 +42,8 @@ "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", "jest": "^29.7.0", - "prettier": "^3.3.3", - "snyk": "^1.1294.0" + "prettier": "^3.4.0", + "snyk": "^1.1294.1" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -2178,9 +2178,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", - "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "version": "1.7.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.8.tgz", + "integrity": "sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", @@ -2619,14 +2619,15 @@ } }, "node_modules/csv-parse": { - "version": "5.5.6", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz", - "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==" + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", + "license": "MIT" }, "node_modules/csv-stringify": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.1.tgz", - "integrity": "sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ==", + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", + "integrity": "sha512-RFPahj0sXcmUyjrObAK+DOWtMvMIFV328n4qZJhgX3x2RqkQgOTU2mCUmiFR0CzM6AzChlRSUErjiJeEt8BaQA==", "license": "MIT" }, "node_modules/debug": { @@ -5201,9 +5202,9 @@ } }, "node_modules/prettier": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", - "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.4.0.tgz", + "integrity": "sha512-/OXNZcLyWkfo13ofOW5M7SLh+k5pnIs07owXK2teFpnfaOEcycnSy7HQxldaVX1ZP/7Q8oO1eDuQJNwbomQq5Q==", "dev": true, "license": "MIT", "bin": { @@ -5577,9 +5578,9 @@ } }, "node_modules/snyk": { - "version": "1.1294.0", - "resolved": "https://registry.npmjs.org/snyk/-/snyk-1.1294.0.tgz", - "integrity": "sha512-4RBj3Lfccz5+6L2Kw9bt7icF+ex3antwt9PkSl2oEulI7mgqvc8VUFLnezg8c6PY60IPM9DrSSmNjXBac10I3Q==", + "version": "1.1294.1", + "resolved": "https://registry.npmjs.org/snyk/-/snyk-1.1294.1.tgz", + "integrity": "sha512-A4yXpzoaa3Be5W1XvhQtipEK24TwlhU7zC0cusq+e0ejPD2j1FKcRGGIbhEke5l0OjUN1R7N1h/N83OpMpe7Hw==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", diff --git a/package.json b/package.json index 2f231a5..e96d585 100644 --- a/package.json +++ b/package.json @@ -31,10 +31,10 @@ "type": "module", "dependencies": { "@qlik/api": "^1.25.0", - "axios": "^1.7.7", + "axios": "^1.7.8", "commander": "^12.1.0", - "csv-parse": "^5.5.6", - "csv-stringify": "^6.5.1", + "csv-parse": "^5.6.0", + "csv-stringify": "^6.5.2", "enigma.js": "^2.14.0", "esbuild": "^0.24.0", "form-data": "^4.0.1", @@ -63,7 +63,7 @@ "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", "jest": "^29.7.0", - "prettier": "^3.3.3", - "snyk": "^1.1294.0" + "prettier": "^3.4.0", + "snyk": "^1.1294.1" } }