From ef26d4ba4e8e5a277ce913b45c6c27d148b1a5dc Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:09:20 +0100 Subject: [PATCH 01/11] feat(lib): add copy-param README --- src/builtins/copy-param/README.md | 104 ++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/builtins/copy-param/README.md diff --git a/src/builtins/copy-param/README.md b/src/builtins/copy-param/README.md new file mode 100644 index 000000000..1ec04f249 --- /dev/null +++ b/src/builtins/copy-param/README.md @@ -0,0 +1,104 @@ +# Copy-param + +`copy-param` is a generic plugin that duplicates an existing parameter in the `input` data and assigns it to a new key. You can either keep or delete the original copied parameter. A common use case for this is to rename parameters in the `inputs` array. + +You provide the name of the value you want to copy, and a name to assign the copy to. You also toggle a `keep-existing` parameter to either persist or del;et the original copied value. + +For example, you could copy `energy` into `energy-copy`, with `keep-existing=true`. In this case your inputs: + +```yaml +- timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + energy: 30 +``` + +would become + +```yaml +- timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + energy: 30 + energy-copy: 30 +``` + +but with `keep-existing=false`, the same inputs would yield: + +```yaml +- timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + energy-copy: 30 +``` + +## Parameters + +### Config + +Three parameters are required in config: `from` and `to` and `keep-existing`. + +`from`: an array of strings. Each string should match an existing key in the `inputs` array +`to`: a string defining the name to use to add the result of summing the input parameters to the output array. +`keep-existing`: toggles whether to keep or delete the copied parameter (defined in `to`) + +### Inputs + +As with all plugins, `timestamp` and `duration` are required. The key passed to `from` must exist in the `input` data. + +## Returns + +The plugin adds a new parameter with the name defined in `to` to the `input` data. + + +## Implementation + +To run the plugin, you must first create an instance of `Copy`. Then, you can call `execute()`. + +```typescript +import { Copy } from "."; + +const plugin = Copy({ 'keep-existing': true, from: 'from-param', to: 'to-param' }); + +const result = plugin.execute([{ + timestamp: "2023-12-12T00:00:13.000Z", + duration: 30, + 'from-param': 'hello', +}]) + +console.log(result) + +``` + +## Example manifest + +IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by and does not have to be done explicitly by the user. The following is an example manifest that calls `sum`: + +```yaml +name: copy-param +description: +tags: +initialize: + plugins: + copy-param: + path: builtin + method: Copy + global-config: + keep-existing: true + from: original + to: copy +tree: + children: + child-1: + pipeline: + - copy-param + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + original: 'hello' + +``` + +You can run this example by saving it as `./manifests/examples/copy.yml` and executing the following command from the project root: + +```sh +if-run --manifest ./manifests/examples/copy.yml -s +``` + +The results will be displayed in the console. From 4fefa36c75c076dab96a9a89c1a5f40cb6b9de6c Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:09:54 +0100 Subject: [PATCH 02/11] feat(lib): export copy-param --- src/builtins/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/builtins/index.ts b/src/builtins/index.ts index 1a98e3273..d22491aca 100644 --- a/src/builtins/index.ts +++ b/src/builtins/index.ts @@ -13,3 +13,4 @@ export {Exponent} from './exponent'; export {CSVLookup} from './csv-lookup'; export {Shell} from './shell'; export {Regex} from './regex'; +export {Copy} from './copy-param'; From b95c7e1baf210f2cb66177be4c33bc3b2e6f70b3 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:10:28 +0100 Subject: [PATCH 03/11] feat(lib): add copy-param source code --- src/builtins/copy-param/index.ts | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/builtins/copy-param/index.ts diff --git a/src/builtins/copy-param/index.ts b/src/builtins/copy-param/index.ts new file mode 100644 index 000000000..3f3e1f5ae --- /dev/null +++ b/src/builtins/copy-param/index.ts @@ -0,0 +1,78 @@ +import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; + +import {validate} from '../../util/validations'; + +import {STRINGS} from '../../config'; + +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {GlobalConfigError, MissingInputDataError} = ERRORS; +// global-config: +// keep-existing: true/false (whether to remove the parameter you are copying from) +// from-param: the parameter you are copying from (e.g. cpu/name) +// to-field: the parameter you are copying to (e.g. cpu/processor-name) + +export const Copy = (globalConfig: Record): ExecutePlugin => { + const metadata = { + kind: 'execute', + }; + + /** + * Checks global config value are valid. + */ + const validateGlobalConfig = () => { + if (!globalConfig) { + throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); + } + + const globalConfigSchema = z.object({ + 'keep-existing': z.boolean(), + from: z.string().min(1), + to: z.string().min(1), + }); + + return validate>( + globalConfigSchema, + globalConfig + ); + }; + + /** + * Checks for required fields in input. + */ + const validateSingleInput = (input: PluginParams, parameter: string) => { + if (!input[parameter]) { + throw new MissingInputDataError(MISSING_INPUT_DATA(parameter)); + } + + return input; + }; + + const execute = (inputs: PluginParams[]) => { + const safeGlobalConfig = validateGlobalConfig(); + const keepExisting = safeGlobalConfig['keep-existing'] === true; + const from = safeGlobalConfig['from']; + const to = safeGlobalConfig['to']; + + return inputs.map(input => { + const safeInput = validateSingleInput(input, from); + + const outputValue = safeInput[from]; + if (safeInput[from]) { + if (!keepExisting) { + delete safeInput[from]; + } + } + return { + ...safeInput, // need to return or what you provide won't be outputted, don't be evil! + [to]: outputValue, + }; + }); + }; + + return { + metadata, + execute, + }; +}; From 57e64df04ce4be5819c19291dae1a5bd88e225df Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:10:58 +0100 Subject: [PATCH 04/11] feat(lib): add unit tests for copy-param --- .../unit/builtins/copy-param.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/__tests__/unit/builtins/copy-param.test.ts diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/unit/builtins/copy-param.test.ts new file mode 100644 index 000000000..c3fcd9639 --- /dev/null +++ b/src/__tests__/unit/builtins/copy-param.test.ts @@ -0,0 +1,95 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {Copy} from '../../../builtins/copy-param'; + +import {STRINGS} from '../../../config'; + +const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; + +describe('builtins/copy: ', () => { + describe('Copy: ', () => { + const globalConfig = { + 'keep-existing': true, + from: 'original', + to: 'copy', + }; + const copy = Copy(globalConfig); + + describe('init: ', () => { + it('successfully initalized.', () => { + expect(copy).toHaveProperty('metadata'); + expect(copy).toHaveProperty('execute'); + }); + }); + + describe('execute(): ', () => { + it('successfully applies Copy strategy to given input.', () => { + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + original: 'hello', + copy: 'hello', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 'hello', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when global config is not provided.', () => { + const config = undefined; + const copy = Copy(config!); + + expect.assertions(1); + + try { + copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 1, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new GlobalConfigError(MISSING_GLOBAL_CONFIG) + ); + } + }); + + it('throws an error on missing params in input.', () => { + const globalConfig = { + 'keep-existing': true, + from: 'original', + to: 'copy', + }; + const copy = Copy(globalConfig); + expect.assertions(1); + + try { + copy.execute([ + { + duration: 3600, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new MissingInputDataError(MISSING_INPUT_DATA('original')) + ); + } + }); + }); + }); +}); From 22a34001b723d1f05e9b2a2946f5c1597f92391d Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:12:15 +0100 Subject: [PATCH 05/11] feat(manifests): add example manifest for copy-param --- manifests/examples/copy.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 manifests/examples/copy.yaml diff --git a/manifests/examples/copy.yaml b/manifests/examples/copy.yaml new file mode 100644 index 000000000..f2ee816e4 --- /dev/null +++ b/manifests/examples/copy.yaml @@ -0,0 +1,20 @@ +name: copy-param +description: +tags: +initialize: + plugins: + copy-param: + path: builtin + method: Copy + global-config: + keep-existing: true + from: original + to: copy +tree: + children: + child-1: + pipeline: + - copy-param + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + original: 'hello' From e361448f5f62be3295e2fa52ea524ca8ce0f1ae8 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:38:00 +0100 Subject: [PATCH 06/11] feat(lib): add unit test to cover keep-existing==false --- .../unit/builtins/copy-param.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/unit/builtins/copy-param.test.ts index c3fcd9639..cf5166107 100644 --- a/src/__tests__/unit/builtins/copy-param.test.ts +++ b/src/__tests__/unit/builtins/copy-param.test.ts @@ -90,6 +90,33 @@ describe('builtins/copy: ', () => { ); } }); + it('does not persist the original value when keep-existing==false.', () => { + expect.assertions(1); + const globalConfig = { + 'keep-existing': false, + from: 'original', + to: 'copy', + }; + const copy = Copy(globalConfig); + + const expectedResult = [ + { + duration: 3600, + copy: 'hello', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 'hello', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); }); }); }); From fbd34a62436a1c3fe8e41ee5759fd731826950b6 Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:01:23 +0100 Subject: [PATCH 07/11] Update src/builtins/copy-param/index.ts Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/copy-param/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/builtins/copy-param/index.ts b/src/builtins/copy-param/index.ts index 3f3e1f5ae..4501dccc8 100644 --- a/src/builtins/copy-param/index.ts +++ b/src/builtins/copy-param/index.ts @@ -8,7 +8,6 @@ import {STRINGS} from '../../config'; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; const {GlobalConfigError, MissingInputDataError} = ERRORS; -// global-config: // keep-existing: true/false (whether to remove the parameter you are copying from) // from-param: the parameter you are copying from (e.g. cpu/name) // to-field: the parameter you are copying to (e.g. cpu/processor-name) From 85c89f25e03809b59c5dde49d89d842107c57da8 Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:35:24 +0100 Subject: [PATCH 08/11] fix(lib): use zod to validate single inputs --- src/builtins/copy-param/index.ts | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/builtins/copy-param/index.ts b/src/builtins/copy-param/index.ts index 4501dccc8..6301bbf25 100644 --- a/src/builtins/copy-param/index.ts +++ b/src/builtins/copy-param/index.ts @@ -6,8 +6,8 @@ import {validate} from '../../util/validations'; import {STRINGS} from '../../config'; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; -const {GlobalConfigError, MissingInputDataError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {GlobalConfigError} = ERRORS; // keep-existing: true/false (whether to remove the parameter you are copying from) // from-param: the parameter you are copying from (e.g. cpu/name) // to-field: the parameter you are copying to (e.g. cpu/processor-name) @@ -40,10 +40,22 @@ export const Copy = (globalConfig: Record): ExecutePlugin => { /** * Checks for required fields in input. */ - const validateSingleInput = (input: PluginParams, parameter: string) => { - if (!input[parameter]) { - throw new MissingInputDataError(MISSING_INPUT_DATA(parameter)); - } + const validateSingleInput = ( + input: PluginParams, + inputParameters: string[] + ) => { + const inputData = inputParameters.reduce( + (acc, param) => { + acc[param] = input[param]; + + return acc; + }, + {} as Record + ); + + const validationSchema = z.record(z.string(), z.string()); + + validate(validationSchema, inputData); return input; }; @@ -55,7 +67,7 @@ export const Copy = (globalConfig: Record): ExecutePlugin => { const to = safeGlobalConfig['to']; return inputs.map(input => { - const safeInput = validateSingleInput(input, from); + const safeInput = validateSingleInput(input, [from]); const outputValue = safeInput[from]; if (safeInput[from]) { From 46440a8c78fd8162e5bcf31d3276e629a862589f Mon Sep 17 00:00:00 2001 From: jmc <33655003+jmcook1186@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:37:59 +0100 Subject: [PATCH 09/11] fix(lib): update unit tests to expect errors to surface from zod --- src/__tests__/unit/builtins/copy-param.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/unit/builtins/copy-param.test.ts index cf5166107..64809e44a 100644 --- a/src/__tests__/unit/builtins/copy-param.test.ts +++ b/src/__tests__/unit/builtins/copy-param.test.ts @@ -4,8 +4,8 @@ import {Copy} from '../../../builtins/copy-param'; import {STRINGS} from '../../../config'; -const {GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {GlobalConfigError, InputValidationError} = ERRORS; +const {MISSING_GLOBAL_CONFIG} = STRINGS; describe('builtins/copy: ', () => { describe('Copy: ', () => { @@ -86,7 +86,9 @@ describe('builtins/copy: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new MissingInputDataError(MISSING_INPUT_DATA('original')) + new InputValidationError( + '"original" parameter is required. Error code: invalid_type.' + ) ); } }); From 93769573c2741e9d8463e5edcface259f060a88a Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:48:21 +0100 Subject: [PATCH 10/11] Update src/builtins/copy-param/README.md Co-authored-by: Manushak Keramyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/copy-param/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/copy-param/README.md b/src/builtins/copy-param/README.md index 1ec04f249..634b45448 100644 --- a/src/builtins/copy-param/README.md +++ b/src/builtins/copy-param/README.md @@ -2,7 +2,7 @@ `copy-param` is a generic plugin that duplicates an existing parameter in the `input` data and assigns it to a new key. You can either keep or delete the original copied parameter. A common use case for this is to rename parameters in the `inputs` array. -You provide the name of the value you want to copy, and a name to assign the copy to. You also toggle a `keep-existing` parameter to either persist or del;et the original copied value. +You provide the name of the value you want to copy, and a name to assign the copy to. You also toggle a `keep-existing` parameter to either persist or delete the original copied value. For example, you could copy `energy` into `energy-copy`, with `keep-existing=true`. In this case your inputs: From 700138c2035ca3403d49dd30361561bdf6097a80 Mon Sep 17 00:00:00 2001 From: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:48:28 +0100 Subject: [PATCH 11/11] Update src/builtins/copy-param/README.md Co-authored-by: Manushak Keramyan Signed-off-by: Joseph Cook <33655003+jmcook1186@users.noreply.github.com> --- src/builtins/copy-param/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/copy-param/README.md b/src/builtins/copy-param/README.md index 634b45448..78ab5f656 100644 --- a/src/builtins/copy-param/README.md +++ b/src/builtins/copy-param/README.md @@ -69,7 +69,7 @@ console.log(result) ## Example manifest -IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by and does not have to be done explicitly by the user. The following is an example manifest that calls `sum`: +IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by and does not have to be done explicitly by the user. The following is an example manifest that calls `copy-param`: ```yaml name: copy-param