From 1cf6be01feca11570a5791a84bbe7c0cb28be765 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 08:19:00 +0200 Subject: [PATCH 01/25] exhaust: added exhaust function, with stub for exhaust-csv-export and a sample impl Signed-off-by: Paz Barda --- .../impls/functional/sci-e-exhaust-csv.yml | 22 +++++++++++++++++++ src/index.ts | 2 ++ src/lib/exhaust.ts | 16 ++++++++++++++ src/models/exhaust-csv-export.ts | 21 ++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 examples/impls/functional/sci-e-exhaust-csv.yml create mode 100644 src/lib/exhaust.ts create mode 100644 src/models/exhaust-csv-export.ts diff --git a/examples/impls/functional/sci-e-exhaust-csv.yml b/examples/impls/functional/sci-e-exhaust-csv.yml new file mode 100644 index 000000000..543f45646 --- /dev/null +++ b/examples/impls/functional/sci-e-exhaust-csv.yml @@ -0,0 +1,22 @@ +name: sci-e-demo +description: +tags: +initialize: + plugins: + 'sci-e': + model: SciE + path: '@grnsft/if-models' +exhaust: + pipeline: ['yaml', 'csv', 'grafana'] + basepath: '' +tree: + children: + child: + pipeline: + - sci-e + config: + sci-e: + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + energy-cpu: 0.001 diff --git a/src/index.ts b/src/index.ts index 3ae19216d..af2a23008 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ import {initalize} from './lib/initialize'; import {compute} from './lib/compute'; import {load} from './lib/load'; import {aggregate} from './lib/aggregate'; +import {exhaust} from './lib/exhaust'; const {CliInputError} = ERRORS; @@ -27,6 +28,7 @@ const impactEngine = async () => { const plugins = await initalize(context.initialize.plugins); const computedTree = await compute(tree, context, plugins); const aggregatedTree = aggregate(computedTree, context.aggregation); + exhaust(context, aggregatedTree); const outputFile = { ...context, diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts new file mode 100644 index 000000000..8e1c12612 --- /dev/null +++ b/src/lib/exhaust.ts @@ -0,0 +1,16 @@ +const initialize = (pipeline: string[]) => {}; + +export const exhaust = (context: any, tree: any) => { + const pipe = + (...fns: any[]) => + (x: any) => + fns.reduce((v, f) => f(v), x); + + // TODO PB - validate exhaust options + const {pipeline, basePath} = context.exhaust; // = exhaust options + + // import models from pipelint + const importedModels = initialize(pipeline); + + return pipe(importedModels)({context, tree, basePath}); +}; diff --git a/src/models/exhaust-csv-export.ts b/src/models/exhaust-csv-export.ts new file mode 100644 index 000000000..046fbff9f --- /dev/null +++ b/src/models/exhaust-csv-export.ts @@ -0,0 +1,21 @@ +import {PluginParams} from '../types/interface'; + +export interface ExhaustPluginInterface { + /** + * execute exhaust based on context and tree, produce output to a file in basePath + */ + execute(context: any, tree: any, basePath: string): Promise; +} + +export class ExhaustCsvExporter implements ExhaustPluginInterface { + /** + * Export to CSV + */ + async execute( + context: any, + tree: any, + basePath: string + ): Promise<[any, any, string]> { + return Promise.resolve([context, tree, basePath]); + } +} From c6b84842f94a61c34f2f03a3aee0000659749630 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 08:43:46 +0200 Subject: [PATCH 02/25] exhaust csv exporter: initial implementation, few inline TODOs and also need to imnplement configuration of: file name, custom headers (optional) Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 19 ++++++++++++- src/models/exhaust-csv-export.ts | 49 ++++++++++++++++++++++++++------ src/models/exhaust-plugin.ts | 10 +++++++ 3 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 src/models/exhaust-plugin.ts diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 8e1c12612..bbe02719d 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,4 +1,21 @@ -const initialize = (pipeline: string[]) => {}; +import {ExhaustCsvExporter} from '../models/exhaust-csv-export'; +import {ExhaustPluginInterface} from '../models/exhaust-plugin'; + +const createExhaustPlugin = (pluginTypeName: string) => { + switch (pluginTypeName) { + case 'csv': + return new ExhaustCsvExporter(); + default: + throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); + } +}; + +const initialize = (pipeline: string[]) => { + const exhaustPlugins: ExhaustPluginInterface[] = pipeline.map( + exhaustPluginName => createExhaustPlugin(exhaustPluginName) + ); + return exhaustPlugins; +}; export const exhaust = (context: any, tree: any) => { const pipe = diff --git a/src/models/exhaust-csv-export.ts b/src/models/exhaust-csv-export.ts index 046fbff9f..323f76a10 100644 --- a/src/models/exhaust-csv-export.ts +++ b/src/models/exhaust-csv-export.ts @@ -1,13 +1,19 @@ -import {PluginParams} from '../types/interface'; - -export interface ExhaustPluginInterface { - /** - * execute exhaust based on context and tree, produce output to a file in basePath - */ - execute(context: any, tree: any, basePath: string): Promise; -} +import * as path from 'path'; +import * as fs from 'fs/promises'; +import {ERRORS} from '../util/errors'; +import {ExhaustPluginInterface} from './exhaust-plugin'; +const {InputValidationError} = ERRORS; export class ExhaustCsvExporter implements ExhaustPluginInterface { + private createCsvContent(tree: any, headers: string[]): string { + return [ + headers.join(','), + ...tree.map((row: {[x: string]: any}) => + headers.map(fieldName => row[fieldName]).join(',') + ), + ].join('\r\n'); + } + /** * Export to CSV */ @@ -16,6 +22,33 @@ export class ExhaustCsvExporter implements ExhaustPluginInterface { tree: any, basePath: string ): Promise<[any, any, string]> { + // create directory in base path, if doesnt exist + try { + await fs.mkdir(basePath, {recursive: true}); + } catch (error) { + // TODO PB -- suitable error (originally MakeDirectoryError) + throw new InputValidationError( + `Failed to write CSV to ${basePath} ${error}` + ); + } + + // determine headers + const headers = Object.keys(tree[0]); + + // create csv content from tree with headers + const contents = this.createCsvContent(tree, headers); + + // write content to csv file + const outputPath = path.join(basePath, 'csv-export.csv'); + try { + await fs.writeFile(outputPath, contents); + } catch (error) { + // TODO PB -- suitable error (originally WriteFileError) + throw new InputValidationError( + `Failed to write CSV to ${basePath} ${error}` + ); + } + return Promise.resolve([context, tree, basePath]); } } diff --git a/src/models/exhaust-plugin.ts b/src/models/exhaust-plugin.ts new file mode 100644 index 000000000..f0368d922 --- /dev/null +++ b/src/models/exhaust-plugin.ts @@ -0,0 +1,10 @@ +export interface ExhaustPluginInterface { + /** + * execute exhaust based on context and tree, produce output to a file in basePath + */ + execute( + context: any, + tree: any, + basePath: string + ): Promise<[any, any, string]>; +} From 461f4f7b6ef69f500a394ea36cbf5cee321f4171 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 21:40:45 +0200 Subject: [PATCH 03/25] refactor exhaust export csv to comply with functional arch Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 9 +++--- ...st-csv-export.ts => exhaust-export-csv.ts} | 31 ++++++++----------- src/models/index.ts | 1 + 3 files changed, 18 insertions(+), 23 deletions(-) rename src/models/{exhaust-csv-export.ts => exhaust-export-csv.ts} (80%) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index bbe02719d..0d7902aec 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,18 +1,17 @@ -import {ExhaustCsvExporter} from '../models/exhaust-csv-export'; -import {ExhaustPluginInterface} from '../models/exhaust-plugin'; +import {ExhaustExportCsv} from '../models/exhaust-export-csv'; const createExhaustPlugin = (pluginTypeName: string) => { switch (pluginTypeName) { case 'csv': - return new ExhaustCsvExporter(); + return ExhaustExportCsv; default: throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); } }; const initialize = (pipeline: string[]) => { - const exhaustPlugins: ExhaustPluginInterface[] = pipeline.map( - exhaustPluginName => createExhaustPlugin(exhaustPluginName) + const exhaustPlugins: any[] = pipeline.map(exhaustPluginName => + createExhaustPlugin(exhaustPluginName) ); return exhaustPlugins; }; diff --git a/src/models/exhaust-csv-export.ts b/src/models/exhaust-export-csv.ts similarity index 80% rename from src/models/exhaust-csv-export.ts rename to src/models/exhaust-export-csv.ts index 323f76a10..89bcdc5d8 100644 --- a/src/models/exhaust-csv-export.ts +++ b/src/models/exhaust-export-csv.ts @@ -4,24 +4,12 @@ import {ERRORS} from '../util/errors'; import {ExhaustPluginInterface} from './exhaust-plugin'; const {InputValidationError} = ERRORS; -export class ExhaustCsvExporter implements ExhaustPluginInterface { - private createCsvContent(tree: any, headers: string[]): string { - return [ - headers.join(','), - ...tree.map((row: {[x: string]: any}) => - headers.map(fieldName => row[fieldName]).join(',') - ), - ].join('\r\n'); - } - - /** - * Export to CSV - */ - async execute( +export const ExhaustExportCsv = (): ExhaustPluginInterface => { + const execute: ( context: any, tree: any, basePath: string - ): Promise<[any, any, string]> { + ) => Promise<[any, any, string]> = async (context, tree, basePath) => { // create directory in base path, if doesnt exist try { await fs.mkdir(basePath, {recursive: true}); @@ -36,7 +24,12 @@ export class ExhaustCsvExporter implements ExhaustPluginInterface { const headers = Object.keys(tree[0]); // create csv content from tree with headers - const contents = this.createCsvContent(tree, headers); + const contents = [ + headers.join(','), + ...tree.map((row: {[x: string]: any}) => + headers.map(fieldName => row[fieldName]).join(',') + ), + ].join('\r\n'); // write content to csv file const outputPath = path.join(basePath, 'csv-export.csv'); @@ -50,5 +43,7 @@ export class ExhaustCsvExporter implements ExhaustPluginInterface { } return Promise.resolve([context, tree, basePath]); - } -} + }; + + return {execute}; +}; diff --git a/src/models/index.ts b/src/models/index.ts index 3519f0ff8..44fcb0aca 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1 +1,2 @@ export {TimeSyncModel} from './time-sync'; +export {ExhaustExportCsv} from './exhaust-export-csv'; From 22b3c24d6b5ebb6febe06fb682d63cce322f2161 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 22:03:59 +0200 Subject: [PATCH 04/25] Better error and exhaust context validation Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 13 ++++++------- src/models/exhaust-export-csv.ts | 8 +++----- src/util/errors.ts | 1 + 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 0d7902aec..5065e341a 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -22,11 +22,10 @@ export const exhaust = (context: any, tree: any) => { (x: any) => fns.reduce((v, f) => f(v), x); - // TODO PB - validate exhaust options - const {pipeline, basePath} = context.exhaust; // = exhaust options - - // import models from pipelint - const importedModels = initialize(pipeline); - - return pipe(importedModels)({context, tree, basePath}); + if (context && context.exhaust) { + const pipeline = context.exhaust.pipeline; + const basePath = context.exhaust.basePath; + const importedModels = initialize(pipeline); + return pipe(importedModels)({context, tree, basePath}); + } }; diff --git a/src/models/exhaust-export-csv.ts b/src/models/exhaust-export-csv.ts index 89bcdc5d8..52ecda0f7 100644 --- a/src/models/exhaust-export-csv.ts +++ b/src/models/exhaust-export-csv.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs/promises'; import {ERRORS} from '../util/errors'; import {ExhaustPluginInterface} from './exhaust-plugin'; -const {InputValidationError} = ERRORS; +const {MakeDirectoryError} = ERRORS; export const ExhaustExportCsv = (): ExhaustPluginInterface => { const execute: ( @@ -14,8 +14,7 @@ export const ExhaustExportCsv = (): ExhaustPluginInterface => { try { await fs.mkdir(basePath, {recursive: true}); } catch (error) { - // TODO PB -- suitable error (originally MakeDirectoryError) - throw new InputValidationError( + throw new MakeDirectoryError( `Failed to write CSV to ${basePath} ${error}` ); } @@ -36,8 +35,7 @@ export const ExhaustExportCsv = (): ExhaustPluginInterface => { try { await fs.writeFile(outputPath, contents); } catch (error) { - // TODO PB -- suitable error (originally WriteFileError) - throw new InputValidationError( + throw new MakeDirectoryError( `Failed to write CSV to ${basePath} ${error}` ); } diff --git a/src/util/errors.ts b/src/util/errors.ts index 03024ef21..4ae93aa70 100644 --- a/src/util/errors.ts +++ b/src/util/errors.ts @@ -6,6 +6,7 @@ const CUSTOM_ERRORS = [ 'InvalidAggregationParams', 'ModelInitializationError', 'ModelCredentialError', + 'MakeDirectoryError', ] as const; type CustomErrors = { From cc3028fa35c2584a4a50ea7d1b6ebf38553da87a Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 08:19:00 +0200 Subject: [PATCH 05/25] exhaust: added exhaust function, with stub for exhaust-csv-export and a sample impl Signed-off-by: Paz Barda --- .../impls/functional/sci-e-exhaust-csv.yml | 22 +++++++++++++++++++ src/index.ts | 2 ++ src/lib/exhaust.ts | 16 ++++++++++++++ src/models/exhaust-csv-export.ts | 21 ++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 examples/impls/functional/sci-e-exhaust-csv.yml create mode 100644 src/lib/exhaust.ts create mode 100644 src/models/exhaust-csv-export.ts diff --git a/examples/impls/functional/sci-e-exhaust-csv.yml b/examples/impls/functional/sci-e-exhaust-csv.yml new file mode 100644 index 000000000..543f45646 --- /dev/null +++ b/examples/impls/functional/sci-e-exhaust-csv.yml @@ -0,0 +1,22 @@ +name: sci-e-demo +description: +tags: +initialize: + plugins: + 'sci-e': + model: SciE + path: '@grnsft/if-models' +exhaust: + pipeline: ['yaml', 'csv', 'grafana'] + basepath: '' +tree: + children: + child: + pipeline: + - sci-e + config: + sci-e: + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + energy-cpu: 0.001 diff --git a/src/index.ts b/src/index.ts index c2c9e878f..fe435fa34 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ import {initalize} from './lib/initialize'; import {compute} from './lib/compute'; import {load} from './lib/load'; import {aggregate} from './lib/aggregate'; +import {exhaust} from './lib/exhaust'; const {CliInputError} = ERRORS; @@ -27,6 +28,7 @@ const impactEngine = async () => { const plugins = await initalize(context.initialize.plugins); const computedTree = await compute(tree, context, plugins); const aggregatedTree = aggregate(computedTree, context.aggregation); + exhaust(context, aggregatedTree); const outputFile = { ...context, diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts new file mode 100644 index 000000000..8e1c12612 --- /dev/null +++ b/src/lib/exhaust.ts @@ -0,0 +1,16 @@ +const initialize = (pipeline: string[]) => {}; + +export const exhaust = (context: any, tree: any) => { + const pipe = + (...fns: any[]) => + (x: any) => + fns.reduce((v, f) => f(v), x); + + // TODO PB - validate exhaust options + const {pipeline, basePath} = context.exhaust; // = exhaust options + + // import models from pipelint + const importedModels = initialize(pipeline); + + return pipe(importedModels)({context, tree, basePath}); +}; diff --git a/src/models/exhaust-csv-export.ts b/src/models/exhaust-csv-export.ts new file mode 100644 index 000000000..046fbff9f --- /dev/null +++ b/src/models/exhaust-csv-export.ts @@ -0,0 +1,21 @@ +import {PluginParams} from '../types/interface'; + +export interface ExhaustPluginInterface { + /** + * execute exhaust based on context and tree, produce output to a file in basePath + */ + execute(context: any, tree: any, basePath: string): Promise; +} + +export class ExhaustCsvExporter implements ExhaustPluginInterface { + /** + * Export to CSV + */ + async execute( + context: any, + tree: any, + basePath: string + ): Promise<[any, any, string]> { + return Promise.resolve([context, tree, basePath]); + } +} From 39b9852a12ad8b5b4b3077dd6077368384e8b71c Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 08:43:46 +0200 Subject: [PATCH 06/25] exhaust csv exporter: initial implementation, few inline TODOs and also need to imnplement configuration of: file name, custom headers (optional) Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 19 ++++++++++++- src/models/exhaust-csv-export.ts | 49 ++++++++++++++++++++++++++------ src/models/exhaust-plugin.ts | 10 +++++++ 3 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 src/models/exhaust-plugin.ts diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 8e1c12612..bbe02719d 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,4 +1,21 @@ -const initialize = (pipeline: string[]) => {}; +import {ExhaustCsvExporter} from '../models/exhaust-csv-export'; +import {ExhaustPluginInterface} from '../models/exhaust-plugin'; + +const createExhaustPlugin = (pluginTypeName: string) => { + switch (pluginTypeName) { + case 'csv': + return new ExhaustCsvExporter(); + default: + throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); + } +}; + +const initialize = (pipeline: string[]) => { + const exhaustPlugins: ExhaustPluginInterface[] = pipeline.map( + exhaustPluginName => createExhaustPlugin(exhaustPluginName) + ); + return exhaustPlugins; +}; export const exhaust = (context: any, tree: any) => { const pipe = diff --git a/src/models/exhaust-csv-export.ts b/src/models/exhaust-csv-export.ts index 046fbff9f..323f76a10 100644 --- a/src/models/exhaust-csv-export.ts +++ b/src/models/exhaust-csv-export.ts @@ -1,13 +1,19 @@ -import {PluginParams} from '../types/interface'; - -export interface ExhaustPluginInterface { - /** - * execute exhaust based on context and tree, produce output to a file in basePath - */ - execute(context: any, tree: any, basePath: string): Promise; -} +import * as path from 'path'; +import * as fs from 'fs/promises'; +import {ERRORS} from '../util/errors'; +import {ExhaustPluginInterface} from './exhaust-plugin'; +const {InputValidationError} = ERRORS; export class ExhaustCsvExporter implements ExhaustPluginInterface { + private createCsvContent(tree: any, headers: string[]): string { + return [ + headers.join(','), + ...tree.map((row: {[x: string]: any}) => + headers.map(fieldName => row[fieldName]).join(',') + ), + ].join('\r\n'); + } + /** * Export to CSV */ @@ -16,6 +22,33 @@ export class ExhaustCsvExporter implements ExhaustPluginInterface { tree: any, basePath: string ): Promise<[any, any, string]> { + // create directory in base path, if doesnt exist + try { + await fs.mkdir(basePath, {recursive: true}); + } catch (error) { + // TODO PB -- suitable error (originally MakeDirectoryError) + throw new InputValidationError( + `Failed to write CSV to ${basePath} ${error}` + ); + } + + // determine headers + const headers = Object.keys(tree[0]); + + // create csv content from tree with headers + const contents = this.createCsvContent(tree, headers); + + // write content to csv file + const outputPath = path.join(basePath, 'csv-export.csv'); + try { + await fs.writeFile(outputPath, contents); + } catch (error) { + // TODO PB -- suitable error (originally WriteFileError) + throw new InputValidationError( + `Failed to write CSV to ${basePath} ${error}` + ); + } + return Promise.resolve([context, tree, basePath]); } } diff --git a/src/models/exhaust-plugin.ts b/src/models/exhaust-plugin.ts new file mode 100644 index 000000000..f0368d922 --- /dev/null +++ b/src/models/exhaust-plugin.ts @@ -0,0 +1,10 @@ +export interface ExhaustPluginInterface { + /** + * execute exhaust based on context and tree, produce output to a file in basePath + */ + execute( + context: any, + tree: any, + basePath: string + ): Promise<[any, any, string]>; +} From 15e5b853493dd39f25964f3af716517bae7b166b Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 21:40:45 +0200 Subject: [PATCH 07/25] refactor exhaust export csv to comply with functional arch Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 9 +++--- ...st-csv-export.ts => exhaust-export-csv.ts} | 31 ++++++++----------- src/models/index.ts | 1 + 3 files changed, 18 insertions(+), 23 deletions(-) rename src/models/{exhaust-csv-export.ts => exhaust-export-csv.ts} (80%) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index bbe02719d..0d7902aec 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,18 +1,17 @@ -import {ExhaustCsvExporter} from '../models/exhaust-csv-export'; -import {ExhaustPluginInterface} from '../models/exhaust-plugin'; +import {ExhaustExportCsv} from '../models/exhaust-export-csv'; const createExhaustPlugin = (pluginTypeName: string) => { switch (pluginTypeName) { case 'csv': - return new ExhaustCsvExporter(); + return ExhaustExportCsv; default: throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); } }; const initialize = (pipeline: string[]) => { - const exhaustPlugins: ExhaustPluginInterface[] = pipeline.map( - exhaustPluginName => createExhaustPlugin(exhaustPluginName) + const exhaustPlugins: any[] = pipeline.map(exhaustPluginName => + createExhaustPlugin(exhaustPluginName) ); return exhaustPlugins; }; diff --git a/src/models/exhaust-csv-export.ts b/src/models/exhaust-export-csv.ts similarity index 80% rename from src/models/exhaust-csv-export.ts rename to src/models/exhaust-export-csv.ts index 323f76a10..89bcdc5d8 100644 --- a/src/models/exhaust-csv-export.ts +++ b/src/models/exhaust-export-csv.ts @@ -4,24 +4,12 @@ import {ERRORS} from '../util/errors'; import {ExhaustPluginInterface} from './exhaust-plugin'; const {InputValidationError} = ERRORS; -export class ExhaustCsvExporter implements ExhaustPluginInterface { - private createCsvContent(tree: any, headers: string[]): string { - return [ - headers.join(','), - ...tree.map((row: {[x: string]: any}) => - headers.map(fieldName => row[fieldName]).join(',') - ), - ].join('\r\n'); - } - - /** - * Export to CSV - */ - async execute( +export const ExhaustExportCsv = (): ExhaustPluginInterface => { + const execute: ( context: any, tree: any, basePath: string - ): Promise<[any, any, string]> { + ) => Promise<[any, any, string]> = async (context, tree, basePath) => { // create directory in base path, if doesnt exist try { await fs.mkdir(basePath, {recursive: true}); @@ -36,7 +24,12 @@ export class ExhaustCsvExporter implements ExhaustPluginInterface { const headers = Object.keys(tree[0]); // create csv content from tree with headers - const contents = this.createCsvContent(tree, headers); + const contents = [ + headers.join(','), + ...tree.map((row: {[x: string]: any}) => + headers.map(fieldName => row[fieldName]).join(',') + ), + ].join('\r\n'); // write content to csv file const outputPath = path.join(basePath, 'csv-export.csv'); @@ -50,5 +43,7 @@ export class ExhaustCsvExporter implements ExhaustPluginInterface { } return Promise.resolve([context, tree, basePath]); - } -} + }; + + return {execute}; +}; diff --git a/src/models/index.ts b/src/models/index.ts index d227e22e4..1943a616a 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,2 +1,3 @@ export {GroupBy} from './group-by'; export {TimeSync} from './time-sync'; +export {ExhaustExportCsv} from './exhaust-export-csv'; From fc477535fbd3459dfa1f65639aad67ffcd08dd84 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 18 Feb 2024 22:03:59 +0200 Subject: [PATCH 08/25] Better error and exhaust context validation Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 13 ++++++------- src/models/exhaust-export-csv.ts | 8 +++----- src/util/errors.ts | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 0d7902aec..5065e341a 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -22,11 +22,10 @@ export const exhaust = (context: any, tree: any) => { (x: any) => fns.reduce((v, f) => f(v), x); - // TODO PB - validate exhaust options - const {pipeline, basePath} = context.exhaust; // = exhaust options - - // import models from pipelint - const importedModels = initialize(pipeline); - - return pipe(importedModels)({context, tree, basePath}); + if (context && context.exhaust) { + const pipeline = context.exhaust.pipeline; + const basePath = context.exhaust.basePath; + const importedModels = initialize(pipeline); + return pipe(importedModels)({context, tree, basePath}); + } }; diff --git a/src/models/exhaust-export-csv.ts b/src/models/exhaust-export-csv.ts index 89bcdc5d8..52ecda0f7 100644 --- a/src/models/exhaust-export-csv.ts +++ b/src/models/exhaust-export-csv.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs/promises'; import {ERRORS} from '../util/errors'; import {ExhaustPluginInterface} from './exhaust-plugin'; -const {InputValidationError} = ERRORS; +const {MakeDirectoryError} = ERRORS; export const ExhaustExportCsv = (): ExhaustPluginInterface => { const execute: ( @@ -14,8 +14,7 @@ export const ExhaustExportCsv = (): ExhaustPluginInterface => { try { await fs.mkdir(basePath, {recursive: true}); } catch (error) { - // TODO PB -- suitable error (originally MakeDirectoryError) - throw new InputValidationError( + throw new MakeDirectoryError( `Failed to write CSV to ${basePath} ${error}` ); } @@ -36,8 +35,7 @@ export const ExhaustExportCsv = (): ExhaustPluginInterface => { try { await fs.writeFile(outputPath, contents); } catch (error) { - // TODO PB -- suitable error (originally WriteFileError) - throw new InputValidationError( + throw new MakeDirectoryError( `Failed to write CSV to ${basePath} ${error}` ); } diff --git a/src/util/errors.ts b/src/util/errors.ts index db70c6c5c..88c998e37 100644 --- a/src/util/errors.ts +++ b/src/util/errors.ts @@ -7,7 +7,7 @@ const CUSTOM_ERRORS = [ 'InvalidGrouping', 'ModuleInitializationError', 'PluginCredentialError', - 'PluginInterfaceError', + 'PluginInterfaceError' ] as const; type CustomErrors = { From 130460525565496eaafe694b8ec3ad243fd5d017 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Mon, 19 Feb 2024 22:05:10 +0200 Subject: [PATCH 09/25] added exhaust to validations and manifest load Signed-off-by: Paz Barda --- .../time-sync-exhaust-csv-export.yml | 35 +++++++++++++++++++ src/lib/load.ts | 1 + src/types/manifest.ts | 6 ++++ src/util/validations.ts | 4 +++ 4 files changed, 46 insertions(+) create mode 100644 examples/impls/functional/time-sync-exhaust-csv-export.yml diff --git a/examples/impls/functional/time-sync-exhaust-csv-export.yml b/examples/impls/functional/time-sync-exhaust-csv-export.yml new file mode 100644 index 000000000..a3b509912 --- /dev/null +++ b/examples/impls/functional/time-sync-exhaust-csv-export.yml @@ -0,0 +1,35 @@ +name: sci-e-demo +description: +tags: +initialize: + plugins: + 'time-sync': + model: TimeSync + path: "builtin" + global-config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true +exhaust: + pipeline: ['csv'] + basePath: 'C:\dev\' +tree: + children: + child: + pipeline: + - time-sync + config: + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + energy-cpu: 0.001 diff --git a/src/lib/load.ts b/src/lib/load.ts index 1b34b27cf..564e7f326 100644 --- a/src/lib/load.ts +++ b/src/lib/load.ts @@ -15,5 +15,6 @@ export const load = async (inputPath: string): Promise => { return { tree, context: {name, description, tags, params, aggregation, initialize}, + exhaust }; }; diff --git a/src/types/manifest.ts b/src/types/manifest.ts index 7499b8132..6c05a6924 100644 --- a/src/types/manifest.ts +++ b/src/types/manifest.ts @@ -10,6 +10,11 @@ type Tags = | null | undefined; +export type ExhaustOptions = { + pipeline: string[]; + basePath: string; +}; + export type PluginOptions = { 'global-config'?: Record; method: string; @@ -34,6 +39,7 @@ export type ManifestCommon = { plugins: GlobalPlugins; }; aggregation?: AggregationParams; + exhaust: ExhaustOptions; }; export type Manifest = ManifestCommon & { diff --git a/src/util/validations.ts b/src/util/validations.ts index 5100318fa..69ae9b176 100644 --- a/src/util/validations.ts +++ b/src/util/validations.ts @@ -51,6 +51,10 @@ const manifestValidation = z.object({ ), }), tree: z.record(z.string(), z.any()), + exhaust: z.object({ + pipeline: z.array(z.string()), + basePath: z.string(), + }), }); /** From 5f8a9b1053b261b151a7b2aadc1a264f20413dc2 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Tue, 20 Feb 2024 18:01:25 +0200 Subject: [PATCH 10/25] refactored exhaust-csv-export to fit aggregated tree structure and also added id column to csv Signed-off-by: Paz Barda --- .../time-sync-exhaust-csv-export.yml | 19 ++- src/lib/load.ts | 1 - src/models/exhaust-export-csv.ts | 119 +++++++++++++----- src/util/errors.ts | 3 +- 4 files changed, 105 insertions(+), 37 deletions(-) diff --git a/examples/impls/functional/time-sync-exhaust-csv-export.yml b/examples/impls/functional/time-sync-exhaust-csv-export.yml index a3b509912..30dc2d204 100644 --- a/examples/impls/functional/time-sync-exhaust-csv-export.yml +++ b/examples/impls/functional/time-sync-exhaust-csv-export.yml @@ -16,7 +16,24 @@ exhaust: basePath: 'C:\dev\' tree: children: - child: + child1: + pipeline: + - time-sync + config: + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + energy-cpu: 0.001 + child2: pipeline: - time-sync config: diff --git a/src/lib/load.ts b/src/lib/load.ts index 564e7f326..1b34b27cf 100644 --- a/src/lib/load.ts +++ b/src/lib/load.ts @@ -15,6 +15,5 @@ export const load = async (inputPath: string): Promise => { return { tree, context: {name, description, tags, params, aggregation, initialize}, - exhaust }; }; diff --git a/src/models/exhaust-export-csv.ts b/src/models/exhaust-export-csv.ts index 52ecda0f7..cd2c7cfc9 100644 --- a/src/models/exhaust-export-csv.ts +++ b/src/models/exhaust-export-csv.ts @@ -2,46 +2,97 @@ import * as path from 'path'; import * as fs from 'fs/promises'; import {ERRORS} from '../util/errors'; import {ExhaustPluginInterface} from './exhaust-plugin'; -const {MakeDirectoryError} = ERRORS; +const {WriteFileError} = ERRORS; +// TODO PB -- need a way for the user to configure the file name, maybe add a field to 'exhaust' options in the manifest? +const OUTPUT_FILE_NAME = 'if-csv-export.csv'; // default -export const ExhaustExportCsv = (): ExhaustPluginInterface => { - const execute: ( - context: any, - tree: any, - basePath: string - ) => Promise<[any, any, string]> = async (context, tree, basePath) => { - // create directory in base path, if doesnt exist - try { - await fs.mkdir(basePath, {recursive: true}); - } catch (error) { - throw new MakeDirectoryError( - `Failed to write CSV to ${basePath} ${error}` - ); - } +const extractIdHelper = (key: string): string => { + const parts = key.split('.'); + parts.pop(); + return parts.join('.'); +}; - // determine headers - const headers = Object.keys(tree[0]); +const getCsvString = ( + dict: {[key: string]: any}, + headers: Set, + ids: Set +): string => { + const csvRows: string[] = []; + csvRows.push(['id', ...headers].join(',')); + ids.forEach(id => { + const rowData = [id]; + headers.forEach(header => { + const value = dict[`${id}.${header}`] ?? ''; + rowData.push(value.toString()); + }); + csvRows.push(rowData.join(',')); + }); + return csvRows.join('\n'); +}; - // create csv content from tree with headers - const contents = [ - headers.join(','), - ...tree.map((row: {[x: string]: any}) => - headers.map(fieldName => row[fieldName]).join(',') - ), - ].join('\r\n'); +const writeOutputFile = async (csvString: string, basePath: string) => { + const csvPath = path.join(basePath, OUTPUT_FILE_NAME); + try { + await fs.writeFile(csvPath, csvString); + } catch (error) { + throw new WriteFileError(`Failed to write CSV to ${csvPath}: ${error}`); + } +}; - // write content to csv file - const outputPath = path.join(basePath, 'csv-export.csv'); - try { - await fs.writeFile(outputPath, contents); - } catch (error) { - throw new MakeDirectoryError( - `Failed to write CSV to ${basePath} ${error}` - ); +const extractFlatDictAndHeaders = ( + obj: any, + prefix = '' +): [{[key: string]: any}, Set] => { + const headers: Set = new Set(); + const flatDict: {[key: string]: any} = []; + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + const value = obj[key]; + const fullPath = prefix ? `${prefix}.${key}` : key; + if (typeof value === 'object' && value !== null) { + const [subFlatDict, subHeaders] = extractFlatDictAndHeaders( + value, + fullPath + ); + if (Object.keys(subFlatDict).length > 0) { + for (const subKey in subFlatDict) { + if (Object.prototype.hasOwnProperty.call(subFlatDict, subKey)) { + flatDict[subKey] = subFlatDict[subKey]; + } + } + subHeaders.forEach(subHeader => headers.add(subHeader)); + } + } else { + if (fullPath.includes('outputs')) { + headers.add(key); + flatDict[fullPath] = value; + } + } } + } + return [flatDict, headers]; +}; - return Promise.resolve([context, tree, basePath]); +export const ExhaustExportCsv = (): ExhaustPluginInterface => { + const execute: ( + context: any, + tree: any, + basePath: string + ) => Promise<[any, any, string]> = async ( + context, + aggregatedTree, + basePath + ) => { + // TODO PB -- need a way for the user to configure the headers (projection), maybe add a field to 'exhaust' options in the manifest? + const [extractredFlatDict, extractedHeaders] = + extractFlatDictAndHeaders(aggregatedTree); + const ids = new Set( + Object.keys(extractredFlatDict).map(key => extractIdHelper(key)) + ); + const csvString = getCsvString(extractredFlatDict, extractedHeaders, ids); + writeOutputFile(csvString, basePath); + // TODO PB -- is what we want to return? + return Promise.resolve([context, aggregatedTree, basePath]); }; - return {execute}; }; diff --git a/src/util/errors.ts b/src/util/errors.ts index 88c998e37..cfd60e90b 100644 --- a/src/util/errors.ts +++ b/src/util/errors.ts @@ -7,7 +7,8 @@ const CUSTOM_ERRORS = [ 'InvalidGrouping', 'ModuleInitializationError', 'PluginCredentialError', - 'PluginInterfaceError' + 'PluginInterfaceError', + 'WriteFileError', ] as const; type CustomErrors = { From 1a9726847668d1fc8f279d738638fea83cb68cdc Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Wed, 21 Feb 2024 23:50:45 +0200 Subject: [PATCH 11/25] cleaner code, refactor exhaust to iterate through exhaust plugins instead of piping them Signed-off-by: Paz Barda --- src/lib/exhaust.ts | 24 ++--- src/models/exhaust-export-csv.ts | 169 +++++++++++++++++-------------- src/models/exhaust-plugin.ts | 6 +- src/models/time-sync.ts | 4 +- 4 files changed, 109 insertions(+), 94 deletions(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index 5065e341a..deb53267b 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,31 +1,31 @@ import {ExhaustExportCsv} from '../models/exhaust-export-csv'; +import {ExhaustPluginInterface} from '../models/exhaust-plugin'; -const createExhaustPlugin = (pluginTypeName: string) => { +const createExhaustPlugin = ( + pluginTypeName: string +): ExhaustPluginInterface => { switch (pluginTypeName) { case 'csv': - return ExhaustExportCsv; + return ExhaustExportCsv(); default: throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); } }; -const initialize = (pipeline: string[]) => { - const exhaustPlugins: any[] = pipeline.map(exhaustPluginName => - createExhaustPlugin(exhaustPluginName) +const initialize = (pipeline: string[]): ExhaustPluginInterface[] => { + const exhaustPlugins: ExhaustPluginInterface[] = pipeline.map( + exhaustPluginName => createExhaustPlugin(exhaustPluginName) ); return exhaustPlugins; }; export const exhaust = (context: any, tree: any) => { - const pipe = - (...fns: any[]) => - (x: any) => - fns.reduce((v, f) => f(v), x); - if (context && context.exhaust) { const pipeline = context.exhaust.pipeline; const basePath = context.exhaust.basePath; - const importedModels = initialize(pipeline); - return pipe(importedModels)({context, tree, basePath}); + const importedModels: ExhaustPluginInterface[] = initialize(pipeline); + importedModels.forEach(plugin => { + plugin.execute(tree, basePath); + }); } }; diff --git a/src/models/exhaust-export-csv.ts b/src/models/exhaust-export-csv.ts index cd2c7cfc9..2abdab1c1 100644 --- a/src/models/exhaust-export-csv.ts +++ b/src/models/exhaust-export-csv.ts @@ -6,93 +6,114 @@ const {WriteFileError} = ERRORS; // TODO PB -- need a way for the user to configure the file name, maybe add a field to 'exhaust' options in the manifest? const OUTPUT_FILE_NAME = 'if-csv-export.csv'; // default -const extractIdHelper = (key: string): string => { - const parts = key.split('.'); - parts.pop(); - return parts.join('.'); -}; +export const ExhaustExportCsv = (): ExhaustPluginInterface => { + const handleLeafValue = ( + value: any, + fullPath: string, + key: any, + flatMap: {[key: string]: any}, + headers: Set + ) => { + if (fullPath.includes('outputs')) { + headers.add(key); + flatMap[fullPath] = value; + } + }; -const getCsvString = ( - dict: {[key: string]: any}, - headers: Set, - ids: Set -): string => { - const csvRows: string[] = []; - csvRows.push(['id', ...headers].join(',')); - ids.forEach(id => { - const rowData = [id]; - headers.forEach(header => { - const value = dict[`${id}.${header}`] ?? ''; - rowData.push(value.toString()); - }); - csvRows.push(rowData.join(',')); - }); - return csvRows.join('\n'); -}; + const handleNodeValue = ( + value: any, + fullPath: string, + flatMap: {[key: string]: any}, + headers: Set + ) => { + const [subFlatMap, subHeaders] = extractFlatMapAndHeaders( + value, + fullPath + ); + if (Object.keys(subFlatMap).length > 0) { + Object.entries(subFlatMap).forEach(([subKey, value]) => { + if (subKey in subFlatMap) { + flatMap[subKey] = value; + } + }); + subHeaders.forEach(subHeader => headers.add(subHeader)); + } + }; -const writeOutputFile = async (csvString: string, basePath: string) => { - const csvPath = path.join(basePath, OUTPUT_FILE_NAME); - try { - await fs.writeFile(csvPath, csvString); - } catch (error) { - throw new WriteFileError(`Failed to write CSV to ${csvPath}: ${error}`); - } -}; + const handleKey = ( + value: any, + key: any, + prefix: string, + flatMap: {[key: string]: any}, + headers: Set + ) => { + const fullPath = prefix ? `${prefix}.${key}` : key; + if (value !== null && typeof value === 'object') { + handleNodeValue(value, fullPath, flatMap, headers); + } else { + handleLeafValue(value, fullPath, key, flatMap, headers); + } + }; -const extractFlatDictAndHeaders = ( - obj: any, - prefix = '' -): [{[key: string]: any}, Set] => { - const headers: Set = new Set(); - const flatDict: {[key: string]: any} = []; - for (const key in obj) { - if (Object.prototype.hasOwnProperty.call(obj, key)) { - const value = obj[key]; - const fullPath = prefix ? `${prefix}.${key}` : key; - if (typeof value === 'object' && value !== null) { - const [subFlatDict, subHeaders] = extractFlatDictAndHeaders( - value, - fullPath - ); - if (Object.keys(subFlatDict).length > 0) { - for (const subKey in subFlatDict) { - if (Object.prototype.hasOwnProperty.call(subFlatDict, subKey)) { - flatDict[subKey] = subFlatDict[subKey]; - } - } - subHeaders.forEach(subHeader => headers.add(subHeader)); - } - } else { - if (fullPath.includes('outputs')) { - headers.add(key); - flatDict[fullPath] = value; - } + const extractFlatMapAndHeaders = ( + tree: any, + prefix = '' + ): [{[key: string]: any}, Set] => { + const headers: Set = new Set(); + const flatMap: {[key: string]: any} = []; + for (const key in tree) { + if (key in tree) { + handleKey(tree[key], key, prefix, flatMap, headers); } } - } - return [flatDict, headers]; -}; + return [flatMap, headers]; + }; -export const ExhaustExportCsv = (): ExhaustPluginInterface => { - const execute: ( - context: any, - tree: any, - basePath: string - ) => Promise<[any, any, string]> = async ( - context, + const extractIdHelper = (key: string): string => { + const parts = key.split('.'); + parts.pop(); + return parts.join('.'); + }; + + const getCsvString = ( + map: {[key: string]: any}, + headers: Set, + ids: Set + ): string => { + const csvRows: string[] = []; + csvRows.push(['id', ...headers].join(',')); + ids.forEach(id => { + const rowData = [id]; + headers.forEach(header => { + const value = map[`${id}.${header}`] ?? ''; + rowData.push(value.toString()); + }); + csvRows.push(rowData.join(',')); + }); + return csvRows.join('\n'); + }; + + const writeOutputFile = async (csvString: string, basePath: string) => { + const csvPath = path.join(basePath, OUTPUT_FILE_NAME); + try { + await fs.writeFile(csvPath, csvString); + } catch (error) { + throw new WriteFileError(`Failed to write CSV to ${csvPath}: ${error}`); + } + }; + + const execute: (tree: any, basePath: string) => void = async ( aggregatedTree, basePath ) => { // TODO PB -- need a way for the user to configure the headers (projection), maybe add a field to 'exhaust' options in the manifest? - const [extractredFlatDict, extractedHeaders] = - extractFlatDictAndHeaders(aggregatedTree); + const [extractredFlatMap, extractedHeaders] = + extractFlatMapAndHeaders(aggregatedTree); const ids = new Set( - Object.keys(extractredFlatDict).map(key => extractIdHelper(key)) + Object.keys(extractredFlatMap).map(key => extractIdHelper(key)) ); - const csvString = getCsvString(extractredFlatDict, extractedHeaders, ids); + const csvString = getCsvString(extractredFlatMap, extractedHeaders, ids); writeOutputFile(csvString, basePath); - // TODO PB -- is what we want to return? - return Promise.resolve([context, aggregatedTree, basePath]); }; return {execute}; }; diff --git a/src/models/exhaust-plugin.ts b/src/models/exhaust-plugin.ts index f0368d922..c7de40c5a 100644 --- a/src/models/exhaust-plugin.ts +++ b/src/models/exhaust-plugin.ts @@ -2,9 +2,5 @@ export interface ExhaustPluginInterface { /** * execute exhaust based on context and tree, produce output to a file in basePath */ - execute( - context: any, - tree: any, - basePath: string - ): Promise<[any, any, string]>; + execute(tree: any, basePath: string): void; } diff --git a/src/models/time-sync.ts b/src/models/time-sync.ts index 86e788207..fa4763521 100644 --- a/src/models/time-sync.ts +++ b/src/models/time-sync.ts @@ -24,9 +24,7 @@ const { UNEXPECTED_TIME_CONFIG, } = STRINGS; -export const TimeSync = ( - globalConfig: TimeNormalizerConfig -): PluginInterface => { +export const TimeSync = (globalConfig: TimeNormalizerConfig): PluginInterface => { const metadata = { kind: 'execute', }; From fcbf8859e71962df6517fb62ce255b161c27e309 Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Thu, 22 Feb 2024 20:36:41 +0200 Subject: [PATCH 12/25] refactored exhaust-export-csv to fit revised manifast format Signed-off-by: Paz Barda --- .../impls/functional/sci-e-exhaust-csv.yml | 22 -------- .../time-sync-exhaust-csv-export.yml | 15 +++--- src/lib/exhaust.ts | 32 ++++++------ src/models/exhaust-export-csv.ts | 50 ++++++++++--------- ...-plugin.ts => exhaust-plugin-interface.ts} | 2 +- src/models/time-sync.ts | 4 +- src/types/manifest.ts | 7 +-- src/util/validations.ts | 5 +- 8 files changed, 57 insertions(+), 80 deletions(-) delete mode 100644 examples/impls/functional/sci-e-exhaust-csv.yml rename src/models/{exhaust-plugin.ts => exhaust-plugin-interface.ts} (75%) diff --git a/examples/impls/functional/sci-e-exhaust-csv.yml b/examples/impls/functional/sci-e-exhaust-csv.yml deleted file mode 100644 index 543f45646..000000000 --- a/examples/impls/functional/sci-e-exhaust-csv.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: sci-e-demo -description: -tags: -initialize: - plugins: - 'sci-e': - model: SciE - path: '@grnsft/if-models' -exhaust: - pipeline: ['yaml', 'csv', 'grafana'] - basepath: '' -tree: - children: - child: - pipeline: - - sci-e - config: - sci-e: - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - energy-cpu: 0.001 diff --git a/examples/impls/functional/time-sync-exhaust-csv-export.yml b/examples/impls/functional/time-sync-exhaust-csv-export.yml index 30dc2d204..b7d53228c 100644 --- a/examples/impls/functional/time-sync-exhaust-csv-export.yml +++ b/examples/impls/functional/time-sync-exhaust-csv-export.yml @@ -1,19 +1,19 @@ -name: sci-e-demo +name: exhaust-csv-export demo description: tags: initialize: plugins: 'time-sync': model: TimeSync - path: "builtin" - global-config: + path: 'builtin' + global-config: start-time: '2023-12-12T00:00:00.000Z' end-time: '2023-12-12T00:01:00.000Z' interval: 5 allow-padding: true -exhaust: - pipeline: ['csv'] - basePath: 'C:\dev\' + outputs: + 'csv': + output-path: 'C:\dev\demo-exhaust-csv-export.csv' tree: children: child1: @@ -24,6 +24,7 @@ tree: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 energy-cpu: 0.001 + custom-metric: 0.001 - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 energy-cpu: 0.001 @@ -33,6 +34,7 @@ tree: - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 energy-cpu: 0.001 + custom-metric: 0.002 child2: pipeline: - time-sync @@ -44,6 +46,7 @@ tree: - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 energy-cpu: 0.001 + custom-metric: 0.003 - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 energy-cpu: 0.001 diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index deb53267b..e0ab5f9a8 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,31 +1,31 @@ import {ExhaustExportCsv} from '../models/exhaust-export-csv'; -import {ExhaustPluginInterface} from '../models/exhaust-plugin'; +import {ExhaustPluginInterface} from '../models/exhaust-plugin-interface'; + +const createExhaustPlugins = (exhaustPluginConfigs: any) => { + return Object.keys(exhaustPluginConfigs).map((key: string) => + createExhaustPlugin(key, exhaustPluginConfigs[key]) + ); +}; const createExhaustPlugin = ( - pluginTypeName: string + pluginTypeName: string, + pluginConfigItems: any ): ExhaustPluginInterface => { switch (pluginTypeName) { case 'csv': - return ExhaustExportCsv(); + return ExhaustExportCsv(pluginConfigItems); default: throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); } }; -const initialize = (pipeline: string[]): ExhaustPluginInterface[] => { - const exhaustPlugins: ExhaustPluginInterface[] = pipeline.map( - exhaustPluginName => createExhaustPlugin(exhaustPluginName) - ); - return exhaustPlugins; -}; - export const exhaust = (context: any, tree: any) => { - if (context && context.exhaust) { - const pipeline = context.exhaust.pipeline; - const basePath = context.exhaust.basePath; - const importedModels: ExhaustPluginInterface[] = initialize(pipeline); - importedModels.forEach(plugin => { - plugin.execute(tree, basePath); + if (context && context.initialize.outputs) { + const exhaustPlugins: ExhaustPluginInterface[] = createExhaustPlugins( + context.initialize.outputs + ); + exhaustPlugins.forEach(plugin => { + plugin.execute(tree); }); } }; diff --git a/src/models/exhaust-export-csv.ts b/src/models/exhaust-export-csv.ts index 2abdab1c1..a280aecf7 100644 --- a/src/models/exhaust-export-csv.ts +++ b/src/models/exhaust-export-csv.ts @@ -1,12 +1,19 @@ -import * as path from 'path'; import * as fs from 'fs/promises'; import {ERRORS} from '../util/errors'; -import {ExhaustPluginInterface} from './exhaust-plugin'; -const {WriteFileError} = ERRORS; -// TODO PB -- need a way for the user to configure the file name, maybe add a field to 'exhaust' options in the manifest? -const OUTPUT_FILE_NAME = 'if-csv-export.csv'; // default +import {ExhaustPluginInterface} from './exhaust-plugin-interface'; +const {InputValidationError, WriteFileError} = ERRORS; + +export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { + const extractConfigParams = () => { + const outputPath: string = + 'output-path' in config + ? config['output-path'] + : (() => { + throw new InputValidationError("Config does not have 'outputPath'"); + })(); + return [outputPath]; + }; -export const ExhaustExportCsv = (): ExhaustPluginInterface => { const handleLeafValue = ( value: any, fullPath: string, @@ -26,17 +33,14 @@ export const ExhaustExportCsv = (): ExhaustPluginInterface => { flatMap: {[key: string]: any}, headers: Set ) => { - const [subFlatMap, subHeaders] = extractFlatMapAndHeaders( - value, - fullPath - ); + const [subFlatMap, subHeaders] = extractFlatMapAndHeaders(value, fullPath); if (Object.keys(subFlatMap).length > 0) { Object.entries(subFlatMap).forEach(([subKey, value]) => { - if (subKey in subFlatMap) { - flatMap[subKey] = value; - } + flatMap[subKey] = value; + }); + subHeaders.forEach(subHeader => { + headers.add(subHeader); }); - subHeaders.forEach(subHeader => headers.add(subHeader)); } }; @@ -93,27 +97,25 @@ export const ExhaustExportCsv = (): ExhaustPluginInterface => { return csvRows.join('\n'); }; - const writeOutputFile = async (csvString: string, basePath: string) => { - const csvPath = path.join(basePath, OUTPUT_FILE_NAME); + const writeOutputFile = async (csvString: string, outputPath: string) => { try { - await fs.writeFile(csvPath, csvString); + await fs.writeFile(outputPath, csvString); } catch (error) { - throw new WriteFileError(`Failed to write CSV to ${csvPath}: ${error}`); + throw new WriteFileError( + `Failed to write CSV to ${outputPath}: ${error}` + ); } }; - const execute: (tree: any, basePath: string) => void = async ( - aggregatedTree, - basePath - ) => { - // TODO PB -- need a way for the user to configure the headers (projection), maybe add a field to 'exhaust' options in the manifest? + const execute: (tree: any) => void = async aggregatedTree => { + const [outputPath] = extractConfigParams(); const [extractredFlatMap, extractedHeaders] = extractFlatMapAndHeaders(aggregatedTree); const ids = new Set( Object.keys(extractredFlatMap).map(key => extractIdHelper(key)) ); const csvString = getCsvString(extractredFlatMap, extractedHeaders, ids); - writeOutputFile(csvString, basePath); + writeOutputFile(csvString, outputPath); }; return {execute}; }; diff --git a/src/models/exhaust-plugin.ts b/src/models/exhaust-plugin-interface.ts similarity index 75% rename from src/models/exhaust-plugin.ts rename to src/models/exhaust-plugin-interface.ts index c7de40c5a..684cb7ad0 100644 --- a/src/models/exhaust-plugin.ts +++ b/src/models/exhaust-plugin-interface.ts @@ -2,5 +2,5 @@ export interface ExhaustPluginInterface { /** * execute exhaust based on context and tree, produce output to a file in basePath */ - execute(tree: any, basePath: string): void; + execute(tree: any): void; } diff --git a/src/models/time-sync.ts b/src/models/time-sync.ts index fa4763521..86e788207 100644 --- a/src/models/time-sync.ts +++ b/src/models/time-sync.ts @@ -24,7 +24,9 @@ const { UNEXPECTED_TIME_CONFIG, } = STRINGS; -export const TimeSync = (globalConfig: TimeNormalizerConfig): PluginInterface => { +export const TimeSync = ( + globalConfig: TimeNormalizerConfig +): PluginInterface => { const metadata = { kind: 'execute', }; diff --git a/src/types/manifest.ts b/src/types/manifest.ts index 6c05a6924..6a642770c 100644 --- a/src/types/manifest.ts +++ b/src/types/manifest.ts @@ -10,11 +10,6 @@ type Tags = | null | undefined; -export type ExhaustOptions = { - pipeline: string[]; - basePath: string; -}; - export type PluginOptions = { 'global-config'?: Record; method: string; @@ -37,9 +32,9 @@ export type ManifestCommon = { params?: ManifestParameter[] | undefined | null; initialize: { plugins: GlobalPlugins; + outputs: {[key: string]: any}; }; aggregation?: AggregationParams; - exhaust: ExhaustOptions; }; export type Manifest = ManifestCommon & { diff --git a/src/util/validations.ts b/src/util/validations.ts index 69ae9b176..5685c1ba0 100644 --- a/src/util/validations.ts +++ b/src/util/validations.ts @@ -49,12 +49,9 @@ const manifestValidation = z.object({ 'global-config': z.record(z.string(), z.any()).optional(), }) ), + outputs: z.record(z.string(), z.record(z.string(), z.any()).optional()), }), tree: z.record(z.string(), z.any()), - exhaust: z.object({ - pipeline: z.array(z.string()), - basePath: z.string(), - }), }); /** From 9f7ae1bb179a51aca50130c731d5bc1ff6b495fd Mon Sep 17 00:00:00 2001 From: Paz Barda Date: Sun, 25 Feb 2024 22:10:50 +0200 Subject: [PATCH 13/25] code review fixes Signed-off-by: Paz Barda --- .../time-sync-exhaust-csv-export.yml | 2 +- src/index.ts | 2 +- src/lib/exhaust.ts | 26 ++++++++----- .../{exhaust-export-csv.ts => export-csv.ts} | 39 +++++++++++++++++-- src/models/index.ts | 2 +- .../exhaust-plugin-interface.ts | 0 6 files changed, 55 insertions(+), 16 deletions(-) rename src/models/{exhaust-export-csv.ts => export-csv.ts} (71%) rename src/{models => types}/exhaust-plugin-interface.ts (100%) diff --git a/examples/impls/functional/time-sync-exhaust-csv-export.yml b/examples/impls/functional/time-sync-exhaust-csv-export.yml index b7d53228c..4a8fed57f 100644 --- a/examples/impls/functional/time-sync-exhaust-csv-export.yml +++ b/examples/impls/functional/time-sync-exhaust-csv-export.yml @@ -4,7 +4,7 @@ tags: initialize: plugins: 'time-sync': - model: TimeSync + method: TimeSync path: 'builtin' global-config: start-time: '2023-12-12T00:00:00.000Z' diff --git a/src/index.ts b/src/index.ts index fe435fa34..d3ddbe85c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,7 +28,7 @@ const impactEngine = async () => { const plugins = await initalize(context.initialize.plugins); const computedTree = await compute(tree, context, plugins); const aggregatedTree = aggregate(computedTree, context.aggregation); - exhaust(context, aggregatedTree); + exhaust(aggregatedTree, context.initialize.outputs); const outputFile = { ...context, diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index e0ab5f9a8..a6dcd8b5d 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,29 +1,37 @@ -import {ExhaustExportCsv} from '../models/exhaust-export-csv'; -import {ExhaustPluginInterface} from '../models/exhaust-plugin-interface'; +import {ExportCsv} from '../models/export-csv'; +import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; +/** + * create exhaust plugins based on the provided config + */ const createExhaustPlugins = (exhaustPluginConfigs: any) => { return Object.keys(exhaustPluginConfigs).map((key: string) => createExhaustPlugin(key, exhaustPluginConfigs[key]) ); }; +/** + * factory method for exhaust plugins + */ const createExhaustPlugin = ( pluginTypeName: string, - pluginConfigItems: any + pluginConfigItems: {[key: string]: string} ): ExhaustPluginInterface => { switch (pluginTypeName) { case 'csv': - return ExhaustExportCsv(pluginConfigItems); + return ExportCsv(pluginConfigItems); default: throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); } }; -export const exhaust = (context: any, tree: any) => { - if (context && context.initialize.outputs) { - const exhaustPlugins: ExhaustPluginInterface[] = createExhaustPlugins( - context.initialize.outputs - ); +/** + * execute exhaust functionality + */ +export const exhaust = (tree: any, outputs: any) => { + if (outputs) { + const exhaustPlugins: ExhaustPluginInterface[] = + createExhaustPlugins(outputs); exhaustPlugins.forEach(plugin => { plugin.execute(tree); }); diff --git a/src/models/exhaust-export-csv.ts b/src/models/export-csv.ts similarity index 71% rename from src/models/exhaust-export-csv.ts rename to src/models/export-csv.ts index a280aecf7..27e58d164 100644 --- a/src/models/exhaust-export-csv.ts +++ b/src/models/export-csv.ts @@ -1,9 +1,14 @@ import * as fs from 'fs/promises'; import {ERRORS} from '../util/errors'; -import {ExhaustPluginInterface} from './exhaust-plugin-interface'; +import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; const {InputValidationError, WriteFileError} = ERRORS; -export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { +export const ExportCsv = (config: { + [key: string]: string; +}): ExhaustPluginInterface => { + /** + * extract config parameters from config + */ const extractConfigParams = () => { const outputPath: string = 'output-path' in config @@ -14,6 +19,10 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { return [outputPath]; }; + /** + * handle a tree leaf, where there are no child nodes, by adding it as key->value pair to the flat map + * and capturing key as a header + */ const handleLeafValue = ( value: any, fullPath: string, @@ -27,6 +36,9 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { } }; + /** + * handle a tree node, recursively traverse the children and append their results to the flat map and captured headers + */ const handleNodeValue = ( value: any, fullPath: string, @@ -44,6 +56,9 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { } }; + /** + * handle a key at the top level of the tree + */ const handleKey = ( value: any, key: any, @@ -59,6 +74,9 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { } }; + /** + * qrecursively extract a flat map and headers from the hierarcial tree + */ const extractFlatMapAndHeaders = ( tree: any, prefix = '' @@ -73,12 +91,19 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { return [flatMap, headers]; }; + /** + * extract the id of the key, that is removing the last token (which is the index). + * in this manner, multiple keys that identical besides their index share the same id. + */ const extractIdHelper = (key: string): string => { const parts = key.split('.'); parts.pop(); return parts.join('.'); }; + /** + * generate a CSV formatted string based on a flat key->value map, headers and ids + */ const getCsvString = ( map: {[key: string]: any}, headers: Set, @@ -97,9 +122,12 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { return csvRows.join('\n'); }; - const writeOutputFile = async (csvString: string, outputPath: string) => { + /** + * write the given string content to a file at the provided path + */ + const writeOutputFile = async (content: string, outputPath: string) => { try { - await fs.writeFile(outputPath, csvString); + await fs.writeFile(outputPath, content); } catch (error) { throw new WriteFileError( `Failed to write CSV to ${outputPath}: ${error}` @@ -107,6 +135,9 @@ export const ExhaustExportCsv = (config: any): ExhaustPluginInterface => { } }; + /** + * export the provided tree content to a CSV file, represented in a flat structure + */ const execute: (tree: any) => void = async aggregatedTree => { const [outputPath] = extractConfigParams(); const [extractredFlatMap, extractedHeaders] = diff --git a/src/models/index.ts b/src/models/index.ts index 1943a616a..40c3db0fd 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,3 +1,3 @@ export {GroupBy} from './group-by'; export {TimeSync} from './time-sync'; -export {ExhaustExportCsv} from './exhaust-export-csv'; +export {ExportCsv as ExhaustExportCsv} from './export-csv'; diff --git a/src/models/exhaust-plugin-interface.ts b/src/types/exhaust-plugin-interface.ts similarity index 100% rename from src/models/exhaust-plugin-interface.ts rename to src/types/exhaust-plugin-interface.ts From 48863b49a3bd53c9787bda812e2d925361e94ba8 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:46:28 +0400 Subject: [PATCH 14/25] feat(util): make outputs array in validations --- src/util/validations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/validations.ts b/src/util/validations.ts index 5685c1ba0..86d91f3db 100644 --- a/src/util/validations.ts +++ b/src/util/validations.ts @@ -49,7 +49,7 @@ const manifestValidation = z.object({ 'global-config': z.record(z.string(), z.any()).optional(), }) ), - outputs: z.record(z.string(), z.record(z.string(), z.any()).optional()), + outputs: z.array(z.string()), }), tree: z.record(z.string(), z.any()), }); From a63cd5e0f319353020631cbc119220d8513760ba Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:47:08 +0400 Subject: [PATCH 15/25] feat(util): add Make directory error --- src/util/errors.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util/errors.ts b/src/util/errors.ts index cfd60e90b..0d425981b 100644 --- a/src/util/errors.ts +++ b/src/util/errors.ts @@ -1,11 +1,12 @@ const CUSTOM_ERRORS = [ 'CliInputError', 'FileNotFoundError', + 'MakeDirectoryError', 'ManifestValidationError', + 'ModuleInitializationError', 'InputValidationError', 'InvalidAggregationParams', 'InvalidGrouping', - 'ModuleInitializationError', 'PluginCredentialError', 'PluginInterfaceError', 'WriteFileError', From a8371452c63883b362bc85dbba0afc6e42f07c6b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:47:38 +0400 Subject: [PATCH 16/25] chore(types): make outputs array in manifest --- src/types/manifest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/manifest.ts b/src/types/manifest.ts index 5d08edd68..4b9b1cf2a 100644 --- a/src/types/manifest.ts +++ b/src/types/manifest.ts @@ -32,7 +32,7 @@ export type Context = { params?: ManifestParameter[] | undefined | null; initialize: { plugins: GlobalPlugins; - outputs: {[key: string]: any}; + outputs?: string[]; }; aggregation?: AggregationParams; }; From 0971139fea3e265e71e754052f904205af32d7ff Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:48:52 +0400 Subject: [PATCH 17/25] feat(types): tune exhaust plugin interface --- src/types/exhaust-plugin-interface.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/types/exhaust-plugin-interface.ts b/src/types/exhaust-plugin-interface.ts index 684cb7ad0..3dc00182a 100644 --- a/src/types/exhaust-plugin-interface.ts +++ b/src/types/exhaust-plugin-interface.ts @@ -1,6 +1,8 @@ +import {Context} from './manifest'; + export interface ExhaustPluginInterface { /** - * execute exhaust based on context and tree, produce output to a file in basePath + * Execute exhaust based on `context` and `tree`, produce output to a file in `outputPath`. */ - execute(tree: any): void; + execute(tree: any, context: Context, outputPath?: string): void; } From 10b5e5f2afd2ddc2396d9e348337dc946910e917 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:50:37 +0400 Subject: [PATCH 18/25] feat(models): implement export yaml --- src/models/export-yaml.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/models/export-yaml.ts diff --git a/src/models/export-yaml.ts b/src/models/export-yaml.ts new file mode 100644 index 000000000..bd1bcd59a --- /dev/null +++ b/src/models/export-yaml.ts @@ -0,0 +1,27 @@ +import {saveYamlFileAs} from '../util/yaml'; + +import {ERRORS} from '../util/errors'; + +import {Context} from '../types/manifest'; + +const {CliInputError} = ERRORS; + +export const ExportYaml = () => { + /** + * Saves output file in YAML format. + */ + const execute = async (tree: any, context: Context, outputPath: string) => { + if (!outputPath) { + throw new CliInputError('Output path is required.'); + } + + const outputFile = { + ...context, + tree, + }; + + await saveYamlFileAs(outputFile, outputPath); + }; + + return {execute}; +}; From 3e5c4f90f4abdf49672b3c5ee42815e50d2d2f7c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:52:18 +0400 Subject: [PATCH 19/25] feat(models): implement export log --- src/models/export-log.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/models/export-log.ts diff --git a/src/models/export-log.ts b/src/models/export-log.ts new file mode 100644 index 000000000..e8c9d9d0d --- /dev/null +++ b/src/models/export-log.ts @@ -0,0 +1,17 @@ +import {Context} from '../types/manifest'; + +export const ExportLog = () => { + /** + * Logs output manifest in console. + */ + const execute = async (tree: any, context: Context) => { + const outputFile = { + ...context, + tree, + }; + + console.log(JSON.stringify(outputFile, null, 2)); + }; + + return {execute}; +}; From 2b3154b3c8e094f35c391af4c29f91476561bb3f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:53:32 +0400 Subject: [PATCH 20/25] chore(models): pretty export csv --- src/models/export-csv.ts | 57 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/models/export-csv.ts b/src/models/export-csv.ts index 27e58d164..70b5ad94b 100644 --- a/src/models/export-csv.ts +++ b/src/models/export-csv.ts @@ -1,24 +1,13 @@ import * as fs from 'fs/promises'; + import {ERRORS} from '../util/errors'; + import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; -const {InputValidationError, WriteFileError} = ERRORS; +import {Context} from '../types/manifest'; -export const ExportCsv = (config: { - [key: string]: string; -}): ExhaustPluginInterface => { - /** - * extract config parameters from config - */ - const extractConfigParams = () => { - const outputPath: string = - 'output-path' in config - ? config['output-path'] - : (() => { - throw new InputValidationError("Config does not have 'outputPath'"); - })(); - return [outputPath]; - }; +const {WriteFileError, CliInputError} = ERRORS; +export const ExportCsv = (): ExhaustPluginInterface => { /** * handle a tree leaf, where there are no child nodes, by adding it as key->value pair to the flat map * and capturing key as a header @@ -42,14 +31,16 @@ export const ExportCsv = (config: { const handleNodeValue = ( value: any, fullPath: string, - flatMap: {[key: string]: any}, + flatMap: Record, headers: Set ) => { const [subFlatMap, subHeaders] = extractFlatMapAndHeaders(value, fullPath); + if (Object.keys(subFlatMap).length > 0) { Object.entries(subFlatMap).forEach(([subKey, value]) => { flatMap[subKey] = value; }); + subHeaders.forEach(subHeader => { headers.add(subHeader); }); @@ -57,21 +48,22 @@ export const ExportCsv = (config: { }; /** - * handle a key at the top level of the tree + * Handles a key at the top level of the tree */ const handleKey = ( value: any, key: any, prefix: string, - flatMap: {[key: string]: any}, + flatMap: Record, headers: Set ) => { const fullPath = prefix ? `${prefix}.${key}` : key; + if (value !== null && typeof value === 'object') { - handleNodeValue(value, fullPath, flatMap, headers); - } else { - handleLeafValue(value, fullPath, key, flatMap, headers); + return handleNodeValue(value, fullPath, flatMap, headers); } + + return handleLeafValue(value, fullPath, key, flatMap, headers); }; /** @@ -80,14 +72,16 @@ export const ExportCsv = (config: { const extractFlatMapAndHeaders = ( tree: any, prefix = '' - ): [{[key: string]: any}, Set] => { + ): [Record, Set] => { const headers: Set = new Set(); - const flatMap: {[key: string]: any} = []; + const flatMap: Record = []; + for (const key in tree) { if (key in tree) { handleKey(tree[key], key, prefix, flatMap, headers); } } + return [flatMap, headers]; }; @@ -98,6 +92,7 @@ export const ExportCsv = (config: { const extractIdHelper = (key: string): string => { const parts = key.split('.'); parts.pop(); + return parts.join('.'); }; @@ -111,14 +106,17 @@ export const ExportCsv = (config: { ): string => { const csvRows: string[] = []; csvRows.push(['id', ...headers].join(',')); + ids.forEach(id => { const rowData = [id]; + headers.forEach(header => { const value = map[`${id}.${header}`] ?? ''; rowData.push(value.toString()); }); csvRows.push(rowData.join(',')); }); + return csvRows.join('\n'); }; @@ -138,15 +136,20 @@ export const ExportCsv = (config: { /** * export the provided tree content to a CSV file, represented in a flat structure */ - const execute: (tree: any) => void = async aggregatedTree => { - const [outputPath] = extractConfigParams(); + const execute = async (tree: any, _context: Context, outputPath: string) => { + if (!outputPath) { + throw new CliInputError('Output path is required.'); + } + const [extractredFlatMap, extractedHeaders] = - extractFlatMapAndHeaders(aggregatedTree); + extractFlatMapAndHeaders(tree); const ids = new Set( Object.keys(extractredFlatMap).map(key => extractIdHelper(key)) ); const csvString = getCsvString(extractredFlatMap, extractedHeaders, ids); + writeOutputFile(csvString, outputPath); }; + return {execute}; }; From 6799541d0dabca8805a91f81b7a8638c8c417cf9 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:54:05 +0400 Subject: [PATCH 21/25] revert(models): drop unused exhausts --- src/models/exhaust-export-csv.ts | 47 -------------------------------- src/models/exhaust-plugin.ts | 10 ------- 2 files changed, 57 deletions(-) delete mode 100644 src/models/exhaust-export-csv.ts delete mode 100644 src/models/exhaust-plugin.ts diff --git a/src/models/exhaust-export-csv.ts b/src/models/exhaust-export-csv.ts deleted file mode 100644 index 52ecda0f7..000000000 --- a/src/models/exhaust-export-csv.ts +++ /dev/null @@ -1,47 +0,0 @@ -import * as path from 'path'; -import * as fs from 'fs/promises'; -import {ERRORS} from '../util/errors'; -import {ExhaustPluginInterface} from './exhaust-plugin'; -const {MakeDirectoryError} = ERRORS; - -export const ExhaustExportCsv = (): ExhaustPluginInterface => { - const execute: ( - context: any, - tree: any, - basePath: string - ) => Promise<[any, any, string]> = async (context, tree, basePath) => { - // create directory in base path, if doesnt exist - try { - await fs.mkdir(basePath, {recursive: true}); - } catch (error) { - throw new MakeDirectoryError( - `Failed to write CSV to ${basePath} ${error}` - ); - } - - // determine headers - const headers = Object.keys(tree[0]); - - // create csv content from tree with headers - const contents = [ - headers.join(','), - ...tree.map((row: {[x: string]: any}) => - headers.map(fieldName => row[fieldName]).join(',') - ), - ].join('\r\n'); - - // write content to csv file - const outputPath = path.join(basePath, 'csv-export.csv'); - try { - await fs.writeFile(outputPath, contents); - } catch (error) { - throw new MakeDirectoryError( - `Failed to write CSV to ${basePath} ${error}` - ); - } - - return Promise.resolve([context, tree, basePath]); - }; - - return {execute}; -}; diff --git a/src/models/exhaust-plugin.ts b/src/models/exhaust-plugin.ts deleted file mode 100644 index f0368d922..000000000 --- a/src/models/exhaust-plugin.ts +++ /dev/null @@ -1,10 +0,0 @@ -export interface ExhaustPluginInterface { - /** - * execute exhaust based on context and tree, produce output to a file in basePath - */ - execute( - context: any, - tree: any, - basePath: string - ): Promise<[any, any, string]>; -} From 79d5bebd17ddca1fde446da21aec4736f1a41c6c Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:54:34 +0400 Subject: [PATCH 22/25] refactor(lib): rewrite exhaust --- src/lib/exhaust.ts | 58 +++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts index a6dcd8b5d..0ae6b0e68 100644 --- a/src/lib/exhaust.ts +++ b/src/lib/exhaust.ts @@ -1,39 +1,55 @@ +/** + * @todo This is temporary solution, will be refactored to support dynamic plugins. + */ import {ExportCsv} from '../models/export-csv'; +import {ExportLog} from '../models/export-log'; +import {ExportYaml} from '../models/export-yaml'; + +import {ERRORS} from '../util/errors'; + +import {STRINGS} from '../config'; + import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; +import {Context} from '../types/manifest'; + +const {ModuleInitializationError} = ERRORS; +const {INVALID_EXHAUST_PLUGIN} = STRINGS; /** - * create exhaust plugins based on the provided config + * Initialize exhaust plugins based on the provided config */ -const createExhaustPlugins = (exhaustPluginConfigs: any) => { - return Object.keys(exhaustPluginConfigs).map((key: string) => - createExhaustPlugin(key, exhaustPluginConfigs[key]) - ); -}; +const initializeExhaustPlugins = (plugins: string[]) => + plugins.map(initializeExhaustPlugin); /** * factory method for exhaust plugins */ -const createExhaustPlugin = ( - pluginTypeName: string, - pluginConfigItems: {[key: string]: string} -): ExhaustPluginInterface => { - switch (pluginTypeName) { +const initializeExhaustPlugin = (name: string): ExhaustPluginInterface => { + switch (name) { + case 'yaml': + return ExportYaml(); case 'csv': - return ExportCsv(pluginConfigItems); + return ExportCsv(); + case 'log': + return ExportLog(); default: - throw new Error(`unkonwn exhaust plugin type: ${pluginTypeName}`); + throw new ModuleInitializationError(INVALID_EXHAUST_PLUGIN(name)); } }; /** - * execute exhaust functionality + * Output manager - Exhaust. + * Grabs output plugins from context, executes every. */ -export const exhaust = (tree: any, outputs: any) => { - if (outputs) { - const exhaustPlugins: ExhaustPluginInterface[] = - createExhaustPlugins(outputs); - exhaustPlugins.forEach(plugin => { - plugin.execute(tree); - }); +export const exhaust = (tree: any, context: Context, outputPath?: string) => { + const outputPlugins = context.initialize.outputs; + + if (!outputPlugins) { + ExportLog().execute(tree, context); + + return; } + + const exhaustPlugins = initializeExhaustPlugins(outputPlugins); + exhaustPlugins.forEach(plugin => plugin.execute(tree, context, outputPath)); }; From 401539af2c9fde86b43ae02bb68ce42c6920e132 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:54:56 +0400 Subject: [PATCH 23/25] feat(config): add invalid exhaust plugin to strings --- src/config/strings.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config/strings.ts b/src/config/strings.ts index ba9b4c1d3..4b959b043 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -45,4 +45,6 @@ https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=fee INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`, REJECTING_OVERRIDE: (param: ManifestParameter) => `Rejecting overriding of canonical parameter: ${param.name}.`, + INVALID_EXHAUST_PLUGIN: (pluginName: string) => + `Invalid exhaust plugin: ${pluginName}.`, }; From 5c247c8738e5f095401aba309f7c3e0fe292d8fa Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:55:19 +0400 Subject: [PATCH 24/25] feat(src): use exhaust as an output transport --- src/index.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/index.ts b/src/index.ts index c7387a3b0..f5d7dd48d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,6 @@ import {parseArgs} from './util/args'; import {ERRORS} from './util/errors'; import {andHandle} from './util/helpers'; import {logger} from './util/logger'; -import {saveYamlFileAs} from './util/yaml'; import {STRINGS} from './config'; @@ -30,19 +29,7 @@ const impactEngine = async () => { const plugins = await initalize(context.initialize.plugins); const computedTree = await compute(tree, {context, plugins}); const aggregatedTree = aggregate(computedTree, context.aggregation); - exhaust(aggregatedTree, context.initialize.outputs); - - const outputFile = { - ...context, - tree: aggregatedTree, - }; - - if (!outputPath) { - logger.info(JSON.stringify(outputFile, null, 2)); - return; - } - - await saveYamlFileAs(outputFile, outputPath); + exhaust(aggregatedTree, context, outputPath); return; } From 7c99326b94bf43f84fdc406d3248efc28e012417 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sat, 2 Mar 2024 01:56:07 +0400 Subject: [PATCH 25/25] feat(config): in commitlintrc add models and plugins as scope --- .commitlintrc.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.commitlintrc.js b/.commitlintrc.js index e6a9cdf01..d0ebf6e05 100644 --- a/.commitlintrc.js +++ b/.commitlintrc.js @@ -30,7 +30,9 @@ module.exports = { 'examples', '.github', '.husky', - 'scripts' + 'scripts', + 'models', + 'plugins' ] ], 'scope-empty': [