diff --git a/.commitlintrc.js b/.commitlintrc.js index e6a9cdf01..d0ebf6e05 100644 --- a/.commitlintrc.js +++ b/.commitlintrc.js @@ -30,7 +30,9 @@ module.exports = { 'examples', '.github', '.husky', - 'scripts' + 'scripts', + 'models', + 'plugins' ] ], 'scope-empty': [ diff --git a/examples/impls/functional/sci-e-exhaust-csv.yml b/examples/impls/functional/sci-e-exhaust-csv.yml new file mode 100644 index 000000000..543f45646 --- /dev/null +++ b/examples/impls/functional/sci-e-exhaust-csv.yml @@ -0,0 +1,22 @@ +name: sci-e-demo +description: +tags: +initialize: + plugins: + 'sci-e': + model: SciE + path: '@grnsft/if-models' +exhaust: + pipeline: ['yaml', 'csv', 'grafana'] + basepath: '' +tree: + children: + child: + pipeline: + - sci-e + config: + sci-e: + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + energy-cpu: 0.001 diff --git a/examples/impls/functional/time-sync-exhaust-csv-export.yml b/examples/impls/functional/time-sync-exhaust-csv-export.yml new file mode 100644 index 000000000..4a8fed57f --- /dev/null +++ b/examples/impls/functional/time-sync-exhaust-csv-export.yml @@ -0,0 +1,55 @@ +name: exhaust-csv-export demo +description: +tags: +initialize: + plugins: + 'time-sync': + method: TimeSync + path: 'builtin' + global-config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true + outputs: + 'csv': + output-path: 'C:\dev\demo-exhaust-csv-export.csv' +tree: + children: + child1: + pipeline: + - time-sync + config: + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + energy-cpu: 0.001 + custom-metric: 0.001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + energy-cpu: 0.001 + custom-metric: 0.002 + child2: + pipeline: + - time-sync + config: + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + energy-cpu: 0.001 + custom-metric: 0.003 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + energy-cpu: 0.001 diff --git a/src/config/strings.ts b/src/config/strings.ts index ba9b4c1d3..4b959b043 100644 --- a/src/config/strings.ts +++ b/src/config/strings.ts @@ -45,4 +45,6 @@ https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=fee INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`, REJECTING_OVERRIDE: (param: ManifestParameter) => `Rejecting overriding of canonical parameter: ${param.name}.`, + INVALID_EXHAUST_PLUGIN: (pluginName: string) => + `Invalid exhaust plugin: ${pluginName}.`, }; diff --git a/src/index.ts b/src/index.ts index f3b726948..f5d7dd48d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node import {aggregate} from './lib/aggregate'; import {compute} from './lib/compute'; +import {exhaust} from './lib/exhaust'; import {initalize} from './lib/initialize'; import {load} from './lib/load'; import {parameterize} from './lib/parameterize'; @@ -9,7 +10,6 @@ import {parseArgs} from './util/args'; import {ERRORS} from './util/errors'; import {andHandle} from './util/helpers'; import {logger} from './util/logger'; -import {saveYamlFileAs} from './util/yaml'; import {STRINGS} from './config'; @@ -29,18 +29,7 @@ const impactEngine = async () => { const plugins = await initalize(context.initialize.plugins); const computedTree = await compute(tree, {context, plugins}); const aggregatedTree = aggregate(computedTree, context.aggregation); - - const outputFile = { - ...context, - tree: aggregatedTree, - }; - - if (!outputPath) { - logger.info(JSON.stringify(outputFile, null, 2)); - return; - } - - await saveYamlFileAs(outputFile, outputPath); + exhaust(aggregatedTree, context, outputPath); return; } diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts new file mode 100644 index 000000000..0ae6b0e68 --- /dev/null +++ b/src/lib/exhaust.ts @@ -0,0 +1,55 @@ +/** + * @todo This is temporary solution, will be refactored to support dynamic plugins. + */ +import {ExportCsv} from '../models/export-csv'; +import {ExportLog} from '../models/export-log'; +import {ExportYaml} from '../models/export-yaml'; + +import {ERRORS} from '../util/errors'; + +import {STRINGS} from '../config'; + +import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; +import {Context} from '../types/manifest'; + +const {ModuleInitializationError} = ERRORS; +const {INVALID_EXHAUST_PLUGIN} = STRINGS; + +/** + * Initialize exhaust plugins based on the provided config + */ +const initializeExhaustPlugins = (plugins: string[]) => + plugins.map(initializeExhaustPlugin); + +/** + * factory method for exhaust plugins + */ +const initializeExhaustPlugin = (name: string): ExhaustPluginInterface => { + switch (name) { + case 'yaml': + return ExportYaml(); + case 'csv': + return ExportCsv(); + case 'log': + return ExportLog(); + default: + throw new ModuleInitializationError(INVALID_EXHAUST_PLUGIN(name)); + } +}; + +/** + * Output manager - Exhaust. + * Grabs output plugins from context, executes every. + */ +export const exhaust = (tree: any, context: Context, outputPath?: string) => { + const outputPlugins = context.initialize.outputs; + + if (!outputPlugins) { + ExportLog().execute(tree, context); + + return; + } + + const exhaustPlugins = initializeExhaustPlugins(outputPlugins); + exhaustPlugins.forEach(plugin => plugin.execute(tree, context, outputPath)); +}; diff --git a/src/models/export-csv.ts b/src/models/export-csv.ts new file mode 100644 index 000000000..70b5ad94b --- /dev/null +++ b/src/models/export-csv.ts @@ -0,0 +1,155 @@ +import * as fs from 'fs/promises'; + +import {ERRORS} from '../util/errors'; + +import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; +import {Context} from '../types/manifest'; + +const {WriteFileError, CliInputError} = ERRORS; + +export const ExportCsv = (): ExhaustPluginInterface => { + /** + * handle a tree leaf, where there are no child nodes, by adding it as key->value pair to the flat map + * and capturing key as a header + */ + const handleLeafValue = ( + value: any, + fullPath: string, + key: any, + flatMap: {[key: string]: any}, + headers: Set + ) => { + if (fullPath.includes('outputs')) { + headers.add(key); + flatMap[fullPath] = value; + } + }; + + /** + * handle a tree node, recursively traverse the children and append their results to the flat map and captured headers + */ + const handleNodeValue = ( + value: any, + fullPath: string, + flatMap: Record, + headers: Set + ) => { + const [subFlatMap, subHeaders] = extractFlatMapAndHeaders(value, fullPath); + + if (Object.keys(subFlatMap).length > 0) { + Object.entries(subFlatMap).forEach(([subKey, value]) => { + flatMap[subKey] = value; + }); + + subHeaders.forEach(subHeader => { + headers.add(subHeader); + }); + } + }; + + /** + * Handles a key at the top level of the tree + */ + const handleKey = ( + value: any, + key: any, + prefix: string, + flatMap: Record, + headers: Set + ) => { + const fullPath = prefix ? `${prefix}.${key}` : key; + + if (value !== null && typeof value === 'object') { + return handleNodeValue(value, fullPath, flatMap, headers); + } + + return handleLeafValue(value, fullPath, key, flatMap, headers); + }; + + /** + * qrecursively extract a flat map and headers from the hierarcial tree + */ + const extractFlatMapAndHeaders = ( + tree: any, + prefix = '' + ): [Record, Set] => { + const headers: Set = new Set(); + const flatMap: Record = []; + + for (const key in tree) { + if (key in tree) { + handleKey(tree[key], key, prefix, flatMap, headers); + } + } + + return [flatMap, headers]; + }; + + /** + * extract the id of the key, that is removing the last token (which is the index). + * in this manner, multiple keys that identical besides their index share the same id. + */ + const extractIdHelper = (key: string): string => { + const parts = key.split('.'); + parts.pop(); + + return parts.join('.'); + }; + + /** + * generate a CSV formatted string based on a flat key->value map, headers and ids + */ + const getCsvString = ( + map: {[key: string]: any}, + headers: Set, + ids: Set + ): string => { + const csvRows: string[] = []; + csvRows.push(['id', ...headers].join(',')); + + ids.forEach(id => { + const rowData = [id]; + + headers.forEach(header => { + const value = map[`${id}.${header}`] ?? ''; + rowData.push(value.toString()); + }); + csvRows.push(rowData.join(',')); + }); + + return csvRows.join('\n'); + }; + + /** + * write the given string content to a file at the provided path + */ + const writeOutputFile = async (content: string, outputPath: string) => { + try { + await fs.writeFile(outputPath, content); + } catch (error) { + throw new WriteFileError( + `Failed to write CSV to ${outputPath}: ${error}` + ); + } + }; + + /** + * export the provided tree content to a CSV file, represented in a flat structure + */ + const execute = async (tree: any, _context: Context, outputPath: string) => { + if (!outputPath) { + throw new CliInputError('Output path is required.'); + } + + const [extractredFlatMap, extractedHeaders] = + extractFlatMapAndHeaders(tree); + const ids = new Set( + Object.keys(extractredFlatMap).map(key => extractIdHelper(key)) + ); + const csvString = getCsvString(extractredFlatMap, extractedHeaders, ids); + + writeOutputFile(csvString, outputPath); + }; + + return {execute}; +}; diff --git a/src/models/export-log.ts b/src/models/export-log.ts new file mode 100644 index 000000000..e8c9d9d0d --- /dev/null +++ b/src/models/export-log.ts @@ -0,0 +1,17 @@ +import {Context} from '../types/manifest'; + +export const ExportLog = () => { + /** + * Logs output manifest in console. + */ + const execute = async (tree: any, context: Context) => { + const outputFile = { + ...context, + tree, + }; + + console.log(JSON.stringify(outputFile, null, 2)); + }; + + return {execute}; +}; diff --git a/src/models/export-yaml.ts b/src/models/export-yaml.ts new file mode 100644 index 000000000..bd1bcd59a --- /dev/null +++ b/src/models/export-yaml.ts @@ -0,0 +1,27 @@ +import {saveYamlFileAs} from '../util/yaml'; + +import {ERRORS} from '../util/errors'; + +import {Context} from '../types/manifest'; + +const {CliInputError} = ERRORS; + +export const ExportYaml = () => { + /** + * Saves output file in YAML format. + */ + const execute = async (tree: any, context: Context, outputPath: string) => { + if (!outputPath) { + throw new CliInputError('Output path is required.'); + } + + const outputFile = { + ...context, + tree, + }; + + await saveYamlFileAs(outputFile, outputPath); + }; + + return {execute}; +}; diff --git a/src/models/index.ts b/src/models/index.ts index d227e22e4..40c3db0fd 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,2 +1,3 @@ export {GroupBy} from './group-by'; export {TimeSync} from './time-sync'; +export {ExportCsv as ExhaustExportCsv} from './export-csv'; diff --git a/src/types/exhaust-plugin-interface.ts b/src/types/exhaust-plugin-interface.ts new file mode 100644 index 000000000..3dc00182a --- /dev/null +++ b/src/types/exhaust-plugin-interface.ts @@ -0,0 +1,8 @@ +import {Context} from './manifest'; + +export interface ExhaustPluginInterface { + /** + * Execute exhaust based on `context` and `tree`, produce output to a file in `outputPath`. + */ + execute(tree: any, context: Context, outputPath?: string): void; +} diff --git a/src/types/manifest.ts b/src/types/manifest.ts index 0598fa0ae..4b9b1cf2a 100644 --- a/src/types/manifest.ts +++ b/src/types/manifest.ts @@ -32,6 +32,7 @@ export type Context = { params?: ManifestParameter[] | undefined | null; initialize: { plugins: GlobalPlugins; + outputs?: string[]; }; aggregation?: AggregationParams; }; diff --git a/src/util/errors.ts b/src/util/errors.ts index db70c6c5c..0d425981b 100644 --- a/src/util/errors.ts +++ b/src/util/errors.ts @@ -1,13 +1,15 @@ const CUSTOM_ERRORS = [ 'CliInputError', 'FileNotFoundError', + 'MakeDirectoryError', 'ManifestValidationError', + 'ModuleInitializationError', 'InputValidationError', 'InvalidAggregationParams', 'InvalidGrouping', - 'ModuleInitializationError', 'PluginCredentialError', 'PluginInterfaceError', + 'WriteFileError', ] as const; type CustomErrors = { diff --git a/src/util/validations.ts b/src/util/validations.ts index 33dc7a512..a9a0030dc 100644 --- a/src/util/validations.ts +++ b/src/util/validations.ts @@ -49,6 +49,7 @@ const manifestValidation = z.object({ 'global-config': z.record(z.string(), z.any()).optional(), }) ), + outputs: z.array(z.string()), }), tree: z.record(z.string(), z.any()), });