From 66adcc3834461acbd8827e86f322c45b6fcf5124 Mon Sep 17 00:00:00 2001 From: josh-swerdlow Date: Sun, 7 Apr 2024 22:27:01 -0700 Subject: [PATCH] Implicitly called groupby logic --- manifests/group-by-manifest.yml | 18 ++----- manifests/realistic-manifest.yml | 48 ++++++++---------- manifests/simple-manifest.yml | 22 ++++++++ src/__tests__/unit/lib/branch/index.test.ts | 33 +++++++----- src/lib/branch/index.ts | 56 ++++++++++++++++++++- 5 files changed, 122 insertions(+), 55 deletions(-) create mode 100644 manifests/simple-manifest.yml diff --git a/manifests/group-by-manifest.yml b/manifests/group-by-manifest.yml index bf12633..ac8a173 100644 --- a/manifests/group-by-manifest.yml +++ b/manifests/group-by-manifest.yml @@ -13,7 +13,7 @@ tree: children: child-1: pipeline: - - branch-on + # - branch-on - group-by config: group-by: @@ -28,15 +28,7 @@ tree: region: uk-west duration: 1 cpu/utilization: 10 - - timestamp: '2023-12-12T00:00:01.000Z' - duration: 5 - cpu/utilization: 20 - region: uk-west - - timestamp: '2023-12-12T00:00:06.000Z' - duration: 7 - cpu/utilization: 15 - region: uk-west - - timestamp: '2023-12-12T00:00:13.000Z' - duration: 30 - region: uk-west - cpu/utilization: 15 + - timestamp: '2023-12-12T00:00:00.000Z' + region: uk-north + duration: 1 + cpu/utilization: 10 diff --git a/manifests/realistic-manifest.yml b/manifests/realistic-manifest.yml index 7db04b1..bb0b0f3 100644 --- a/manifests/realistic-manifest.yml +++ b/manifests/realistic-manifest.yml @@ -1,10 +1,10 @@ name: pipeline-demo description: tags: -aggregation: - metrics: - - 'carbon' - type: 'both' +# aggregation: # TODO: I think this is broken, perhaps a regression. +# metrics: +# - 'carbon' +# type: 'both' initialize: plugins: 'branch-on': @@ -38,24 +38,18 @@ initialize: end-time: '2023-12-12T00:01:00.000Z' interval: 5 allow-padding: true - 'group-by': - path: builtin - method: GroupBy tree: children: child-1: pipeline: - - teads-curve - - sci-e - - sci-m - - sci-o - - time-sync - - sci + - branch-on + # - teads-curve + # - sci-e + # - sci-m + # - sci-o + # - time-sync + # - sci config: - group-by: - group: - - region - - instance-type branch-on: region: - uk-north @@ -91,17 +85,17 @@ tree: cpu/utilization: 15 child-2: pipeline: - - teads-curve - - sci-e - - sci-m - - sci-o - - time-sync - - sci + - branch-on + # - teads-curve + # - sci-e + # - sci-m + # - sci-o + # - time-sync + # - sci config: - group-by: - group: - - region - - instance-type + branch-on: + region: + - uk-north defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/simple-manifest.yml b/manifests/simple-manifest.yml new file mode 100644 index 0000000..309935d --- /dev/null +++ b/manifests/simple-manifest.yml @@ -0,0 +1,22 @@ +name: Simple +description: Simplest possible manifest to test branching. +tags: +initialize: + plugins: + branch-on: + path: 'if-branch-plugin' + method: Branch +tree: + children: + child-1: + pipeline: + - branch-on + config: + branch-on: + region: + - uk-north + defaults: + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + region: uk-west + duration: 1 \ No newline at end of file diff --git a/src/__tests__/unit/lib/branch/index.test.ts b/src/__tests__/unit/lib/branch/index.test.ts index 2c95574..770a199 100644 --- a/src/__tests__/unit/lib/branch/index.test.ts +++ b/src/__tests__/unit/lib/branch/index.test.ts @@ -18,21 +18,28 @@ describe('lib/branch: ', () => { describe('execute(): ', () => { /** Currently does not test for grouping */ it('successfully applies branch to given input.', async () => { - const expectedResult = [ - { - timestamp: '2021-01-01T00:00:00Z', - duration: '15s', - 'cpu-util': 34, - region: 'uk-south', + const expectedResult = { + 'uk-north': { + inputs: [ + { + 'cpu-util': 34, + duration: '15s', + region: 'uk-north', + timestamp: '2021-01-01T00:00:00Z', + }, + ], }, - { - timestamp: '2021-01-01T00:00:00Z', - duration: '15s', - 'cpu-util': 34, - region: 'uk-north', + 'uk-south': { + inputs: [ + { + 'cpu-util': 34, + duration: '15s', + region: 'uk-south', + timestamp: '2021-01-01T00:00:00Z', + }, + ], }, - ]; - + }; const result = await branch.execute( [ { diff --git a/src/lib/branch/index.ts b/src/lib/branch/index.ts index bbaefed..d4a93ea 100644 --- a/src/lib/branch/index.ts +++ b/src/lib/branch/index.ts @@ -11,6 +11,37 @@ export const Branch = (): PluginInterface => { kind: 'execute', }; + /** + * Creates structure to insert inputs by groups. + */ + const appendGroup = ( + input: PluginParams, + object: any, + branches: string[] + ): any => { + if (branches.length > 0) { + const branch = branches.shift() as string; + + object.children = object.children ?? {}; + object.children[branch] = object.children[branch] ?? {}; + + if (branches.length === 0) { + if ( + object.children[branch].inputs && + object.children[branch].inputs.length > 0 + ) { + object.children[branch].inputs.push(input); + } else { + object.children[branch].inputs = [input]; + } + } + + appendGroup(input, object.children[branch], branches); + } + + return object; + }; + /** * Clones `inputs` with specified `branch-on` fields, returning originals plus * altered copies with updated values substitued from `component-config` specification. @@ -20,10 +51,10 @@ export const Branch = (): PluginInterface => { config: Record ): Promise => { const validConfig = validateConfig(config); - console.log(validConfig); // New logic to duplicate inputs and replace values const newInputs = [...inputs]; // Start with a copy of the original inputs + const branches = Array(); inputs.forEach(input => { for (const key in validConfig) { if (key in input) { @@ -31,12 +62,33 @@ export const Branch = (): PluginInterface => { validConfig[key].forEach((value: string) => { const newInput = {...input, [key]: value}; newInputs.push(newInput); // Add the new input to the newInputs array + + /** + * Extract the branches that exist in the input since we don't + * branch if it does not already exist. + */ + // if (!branches.includes(value)) { branches.push(value); } + // if (!branches.includes(input[key])) { branches.push(input[key]) }; }); } } }); + const groupedInputs = newInputs.reduce((acc, input) => { + for (const key in validConfig) { + if (!input[key]) { + throw new InputValidationError(key); + } + + branches.push(input[key]); + } + acc = { + ...acc, + ...appendGroup(input, acc, branches), + }; + return acc; + }, {}).children; // children refers to the inputs manifest structure - return newInputs; // Return the new array with the duplicates included + return groupedInputs; // Return the de-duplicated grouped arrays }; /**