From 098a23ed0893d1023ab9bab0591e4e80770876ff Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 9 Sep 2024 23:55:10 +0400 Subject: [PATCH 01/19] feat(util): enable granular aggregation in helper --- src/if-run/util/aggregation-helper.ts | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/if-run/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts index e573720b..211364f1 100644 --- a/src/if-run/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -3,13 +3,13 @@ import {PluginParams} from '@grnsft/if-core/types'; import {CONFIG, STRINGS} from '../config'; -import {AggregationMetric, AggregationResult} from '../types/aggregation'; +import {AggregationResult} from '../types/aggregation'; -import {getAggregationMethod} from '../lib/aggregate'; +import {getAggregationInfoFor} from '../lib/aggregate'; const {MissingAggregationParamError} = ERRORS; const {METRIC_MISSING} = STRINGS; -const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; +const {AGGREGATION_TIME_METRICS} = CONFIG; /** * Aggregates child node level metrics. Appends aggregation additional params to metrics. @@ -17,31 +17,32 @@ const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; */ export const aggregateInputsIntoOne = ( inputs: PluginParams[], - metrics: AggregationMetric[], + metrics: string[], isTemporal?: boolean ) => { - const metricsKeys: string[] = metrics.map(metric => Object.keys(metric)[0]); - const extendedMetrics = [...metricsKeys, ...AGGREGATION_ADDITIONAL_PARAMS]; + const metricsWithTime = metrics.concat(AGGREGATION_TIME_METRICS); return inputs.reduce((acc, input, index) => { - for (const metric of extendedMetrics) { + for (const metric of metricsWithTime) { if (!(metric in input)) { throw new MissingAggregationParamError(METRIC_MISSING(metric, index)); } /** Checks if metric is timestamp or duration, then adds to aggregated value. */ - if (AGGREGATION_ADDITIONAL_PARAMS.includes(metric)) { + if (AGGREGATION_TIME_METRICS.includes(metric)) { if (isTemporal) { acc[metric] = input[metric]; } } else { - const method = getAggregationMethod(metric); + const aggregationParams = getAggregationInfoFor(metric); + /** Checks either its a temporal aggregation (vertical), then chooses `component`, otherwise `time`. */ + const aggregationType = isTemporal ? 'component' : 'time'; - if (method === 'none') { + if (aggregationParams[aggregationType] === 'none') { return acc; } - if (method === 'copy') { + if (aggregationParams[aggregationType] === 'copy') { acc[metric] = input[metric]; return acc; } @@ -51,7 +52,7 @@ export const aggregateInputsIntoOne = ( /** Checks for the last iteration. */ if (index === inputs.length - 1) { - if (method === 'avg') { + if (aggregationParams[aggregationType] === 'avg') { acc[metric] /= inputs.length; } } From 5f6b05528b4dfab365f1399c74c79e64035f9f8f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 9 Sep 2024 23:56:08 +0400 Subject: [PATCH 02/19] feat(types): use aggregation options as method --- src/if-run/types/explain.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/if-run/types/explain.ts b/src/if-run/types/explain.ts index c32f9132..caedd399 100644 --- a/src/if-run/types/explain.ts +++ b/src/if-run/types/explain.ts @@ -1,4 +1,4 @@ -import {ParameterMetadata} from '@grnsft/if-core/types'; +import {AggregationOptions, ParameterMetadata} from '@grnsft/if-core/types'; import {PluginOptions} from '../../common/types/manifest'; @@ -14,6 +14,6 @@ export type ExplainStorageType = Record< plugins: string[]; unit: string; description: string; - 'aggregation-method': string; + 'aggregation-method': AggregationOptions; } >; From ea5270f50cb0eaf2b6b5405515d45a3c0601c1ed Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 9 Sep 2024 23:57:00 +0400 Subject: [PATCH 03/19] feat(types): enable time and component aggregations --- src/if-run/types/aggregation.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts index 5315b298..811833c7 100644 --- a/src/if-run/types/aggregation.ts +++ b/src/if-run/types/aggregation.ts @@ -1,7 +1,9 @@ -import {AggregationMethodTypes} from '@grnsft/if-core/types'; - export type AggregationResult = Record; -export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; - -export type AggregationMetric = Record; +export const AGGREGATION_TYPES = [ + 'horizontal', + 'time', + 'vertical', + 'component', + 'both', +] as const; From e4e01e2b50882a0ebe62555c114433f3a2e641b1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 9 Sep 2024 23:57:48 +0400 Subject: [PATCH 04/19] fix(lib): update docs, improve readablity in initialize --- src/if-run/lib/initialize.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/if-run/lib/initialize.ts b/src/if-run/lib/initialize.ts index 6deb73ce..9239fe3d 100644 --- a/src/if-run/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -103,6 +103,10 @@ const initPlugin = async ( /** * Registers all plugins from `manifest`.`initialize` property. + * 1. Initalizes plugin storage. + * 2. Iterates over plugin names array. + * 3. While iteration, initalizes current plugin, gathers it's parameters (input/output). + * Then stores the aggregation metrics for each parameter to override stub values. */ export const initialize = async ( context: Context @@ -117,9 +121,9 @@ export const initialize = async ( const plugin = await initPlugin(plugins[pluginName]); const parameters = {...plugin.metadata.inputs, ...plugin.metadata.outputs}; - Object.keys(parameters).forEach(key => { + Object.keys(parameters).forEach(current => { storeAggregationMetrics({ - [key]: parameters[key]['aggregation-method'], + [current]: parameters[current]['aggregation-method'], }); }); From bca762906d27f03e4f30aae20b5979dd65d30e8a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 9 Sep 2024 23:58:36 +0400 Subject: [PATCH 05/19] feat(lib): use granular aggregation in explain --- src/if-run/lib/explain.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/if-run/lib/explain.ts b/src/if-run/lib/explain.ts index eb42cfd4..470eed2e 100644 --- a/src/if-run/lib/explain.ts +++ b/src/if-run/lib/explain.ts @@ -40,7 +40,7 @@ export const addExplainData = (params: ExplainParams) => { const allParameters = { ...parameterMetadata?.inputs, ...parameterMetadata?.outputs, - }; + } as ExplainStorageType; Object.entries(allParameters).forEach(([name, meta]) => { const existingParameter = parameters[name]; @@ -55,7 +55,10 @@ export const addExplainData = (params: ExplainParams) => { } if ( - meta['aggregation-method'] !== existingParameter['aggregation-method'] + meta['aggregation-method'].component !== + existingParameter['aggregation-method'].component || + meta['aggregation-method'].time !== + existingParameter['aggregation-method'].time ) { throw new ManifestValidationError(AGGREGATION_METHODS_NOT_MATCH(name)); } From d4567890921c0d4a736e60bfbcbad5f681ff28da Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Mon, 9 Sep 2024 23:59:30 +0400 Subject: [PATCH 06/19] feat(lib): introduce granular aggregation to aggregate --- src/if-run/lib/aggregate.ts | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index a810ecc4..bee7d303 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -12,8 +12,6 @@ import { import {aggregateInputsIntoOne} from '../util/aggregation-helper'; import {memoizedLog} from '../util/log-memoize'; -import {AggregationMetric} from '../types/aggregation'; - import {STRINGS} from '../config/strings'; const { @@ -40,7 +38,7 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { +const temporalAggregation = (node: any, metrics: string[]) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); @@ -64,9 +62,7 @@ const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { * 5. Now a grouping node has it's own outputs, it can horizotnally aggregate them. */ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { - const metrics: AggregationMetric[] = aggregationParams!.metrics.map( - metric => ({[metric]: 'none'}) - ); + const metrics = aggregationParams!.metrics; const type = aggregationParams!.type; if (node.children) { @@ -78,11 +74,13 @@ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { } if (!node.children) { - if (type === 'horizontal' || type === 'both') { + /** `time` aggregation is the new name of `horizontal`. */ + if (type === 'horizontal' || type === 'time' || type === 'both') { node.aggregated = aggregateInputsIntoOne(node.outputs, metrics); } } else { - if (type === 'vertical' || type === 'both') { + /** `component` aggregation is the new name of `vertical`. */ + if (type === 'vertical' || type === 'component' || type === 'both') { const outputs = temporalAggregation(node, metrics); node.outputs = outputs; node.aggregated = aggregateInputsIntoOne(outputs, metrics); @@ -127,13 +125,13 @@ export const storeAggregationMetrics = ( * Creates an encapsulated object to retrieve the metrics. */ const metricManager = (() => { - let metric: AggregationMetric; + let metric: AggregationMetricsWithMethod; const manager = { get metrics() { return metric; }, - set metrics(value: AggregationMetric) { + set metrics(value: AggregationMetricsWithMethod) { metric = value; }, }; @@ -142,22 +140,25 @@ const metricManager = (() => { })(); /** - * Returns aggregation method for given `unitName`. If doesn't exist then returns value `sum`. + * Returns aggregation method for given `metric`. */ -export const getAggregationMethod = (unitName: string) => { +export const getAggregationInfoFor = (metric: string) => { debugLogger.setExecutingPluginName(); memoizedLog(console.debug, '\n'); - memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); + memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(metric)); const aggregationMetricsStorage = storeAggregationMetrics(); if ( aggregationMetricsStorage && - Object.keys(aggregationMetricsStorage).includes(unitName) + Object.keys(aggregationMetricsStorage).includes(metric) ) { - return aggregationMetricsStorage[unitName]; + return aggregationMetricsStorage[metric]; } - memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); + memoizedLog(logger.warn, UNKNOWN_PARAM(metric)); - return AGGREGATION_METHODS[AGGREGATION_METHODS.length - 1]; + return { + time: AGGREGATION_METHODS[3], + component: AGGREGATION_METHODS[3], + }; }; From 668dfbab8008ad680b012e716df3605b894a7c88 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:00:06 +0400 Subject: [PATCH 07/19] feat(config): rename additional params to time metrics --- src/if-run/config/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/if-run/config/config.ts b/src/if-run/config/config.ts index 9bd39d89..d151a038 100644 --- a/src/if-run/config/config.ts +++ b/src/if-run/config/config.ts @@ -71,5 +71,5 @@ export const CONFIG = { } as ParseOptions, GITHUB_PATH: 'https://github.com', NATIVE_PLUGIN: 'if-plugins', - AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], + AGGREGATION_TIME_METRICS: ['timestamp', 'duration'], }; From dc840e678f91f413f44bfb0786e319940a62f429 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:00:55 +0400 Subject: [PATCH 08/19] feat(builtins): use granular aggregation in time sync --- src/if-run/builtins/time-sync/index.ts | 59 +++++++++++++++----------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/if-run/builtins/time-sync/index.ts b/src/if-run/builtins/time-sync/index.ts index 02352212..e34c4ca3 100644 --- a/src/if-run/builtins/time-sync/index.ts +++ b/src/if-run/builtins/time-sync/index.ts @@ -21,7 +21,7 @@ import { import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -import {getAggregationMethod} from '../../lib/aggregate'; +import {getAggregationInfoFor} from '../../lib/aggregate'; Settings.defaultZone = 'utc'; @@ -69,12 +69,18 @@ export const TimeSync = ( timestamp: { description: 'refers to the time of occurrence of the input', unit: 'RFC3339', - 'aggregation-method': 'none', + 'aggregation-method': { + time: 'none', + component: 'none', + }, }, duration: { description: 'refers to the duration of the input', unit: 'seconds', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'sum', + component: 'none', + }, }, } as ParameterMetadata), ...parametersMetadata?.inputs, @@ -246,35 +252,35 @@ export const TimeSync = ( * Breaks down input per minimal time unit. */ const breakDownInput = (input: PluginParams, i: number) => { - const inputKeys = Object.keys(input); + const metrics = Object.keys(input); - return inputKeys.reduce((acc, key) => { - const method = getAggregationMethod(key); + return metrics.reduce((acc, metric) => { + const aggregationParams = getAggregationInfoFor(metric); - if (key === 'timestamp') { + if (metric === 'timestamp') { const perSecond = normalizeTimePerSecond(input.timestamp, i); - acc[key] = perSecond.toUTC().toISO() ?? ''; + acc[metric] = perSecond.toUTC().toISO() ?? ''; return acc; } /** @todo use user defined resolution later */ - if (key === 'duration') { - acc[key] = 1; + if (metric === 'duration') { + acc[metric] = 1; return acc; } - if (method === 'none') { - acc[key] = null; + if (aggregationParams.time === 'none') { + acc[metric] = null; return acc; } - acc[key] = - method === 'sum' - ? convertPerInterval(input[key], input['duration']) - : input[key]; + acc[metric] = + aggregationParams.time === 'sum' + ? convertPerInterval(input[metric], input['duration']) + : input[metric]; return acc; }, {} as PluginParams); @@ -314,21 +320,24 @@ export const TimeSync = ( return acc; } - const method = getAggregationMethod(metric); + const aggregationParams = getAggregationInfoFor(metric); - if (method === 'none') { + if (aggregationParams.time === 'none') { acc[metric] = null; return acc; } - if (method === 'avg' || method === 'sum') { + if ( + aggregationParams.time === 'avg' || + aggregationParams.time === 'sum' + ) { acc[metric] = 0; return acc; } - if (method === 'copy') { + if (aggregationParams.time === 'copy') { acc[metric] = input[metric]; return acc; } @@ -382,7 +391,7 @@ export const TimeSync = ( const metrics = Object.keys(input); metrics.forEach(metric => { - let method = getAggregationMethod(metric); + const aggregationParams = getAggregationInfoFor(metric); if (metric === 'timestamp') { acc[metric] = inputs[0][metric]; @@ -391,23 +400,23 @@ export const TimeSync = ( } if (metric === 'duration') { - method = 'sum'; + aggregationParams.time = 'sum'; } - if (method === 'none') { + if (aggregationParams.time === 'none') { acc[metric] = null; return; } acc[metric] = acc[metric] ?? 0; - if (method === 'sum') { + if (aggregationParams.time === 'sum') { acc[metric] += input[metric]; return; } - if (method === 'copy') { + if (aggregationParams.time === 'copy') { acc[metric] = input[metric]; return; From 2930d4315aa0fdce36a7055d818c810786b3d397 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:01:26 +0400 Subject: [PATCH 09/19] feat(builtins): use granular aggregation methods in sci embodied --- src/if-run/builtins/sci-embodied/index.ts | 35 ++++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/if-run/builtins/sci-embodied/index.ts b/src/if-run/builtins/sci-embodied/index.ts index 2b3c4ccf..657f68e8 100644 --- a/src/if-run/builtins/sci-embodied/index.ts +++ b/src/if-run/builtins/sci-embodied/index.ts @@ -28,39 +28,60 @@ export const SciEmbodied = ( vCPUs: { description: 'number of CPUs allocated to an application', unit: 'CPUs', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, memory: { description: 'RAM available for a resource, in GB', unit: 'GB', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, ssd: { description: 'number of SSDs available for a resource', unit: 'SSDs', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, hdd: { description: 'number of HDDs available for a resource', unit: 'HDDs', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, gpu: { description: 'number of GPUs available for a resource', unit: 'GPUs', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, 'usage-ratio': { description: 'a scaling factor that can be used to describe the ratio of actual resource usage comapred to real device usage, e.g. 0.25 if you are using 2 out of 8 vCPUs, 0.1 if you are responsible for 1 out of 10 GB of storage, etc', unit: 'dimensionless', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, time: { description: 'a time unit to scale the embodied carbon by, in seconds. If not provided,time defaults to the value of the timestep duration.', unit: 'seconds', - 'aggregation-method': 'copy', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, }, } as ParameterMetadata), ...parametersMetadata?.inputs, From 3a9c12f9636546d2617b78dd3e65fc3ad0e0ef4a Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:04:58 +0400 Subject: [PATCH 10/19] feat(builtins): use granular aggregation methods in sci --- src/if-run/builtins/sci/index.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/if-run/builtins/sci/index.ts b/src/if-run/builtins/sci/index.ts index 407621b8..81fd62d0 100644 --- a/src/if-run/builtins/sci/index.ts +++ b/src/if-run/builtins/sci/index.ts @@ -40,13 +40,19 @@ export const Sci = ( carbon: { description: 'an amount of carbon emitted into the atmosphere', unit: 'gCO2e', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, }, 'functional-unit': { description: 'the name of the functional unit in which the final SCI value should be expressed, e.g. requests, users', unit: 'none', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, }, } as ParameterMetadata), ...parametersMetadata?.inputs, @@ -55,7 +61,10 @@ export const Sci = ( sci: { description: 'carbon expressed in terms of the given functional unit', unit: 'gCO2e', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'avg', + component: 'sum', + }, }, }, }; From 1fa4dec8ec045b8e491181a686114c4d8bcad1ec Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:07:37 +0400 Subject: [PATCH 11/19] feat(src): simplify if-run index --- src/if-run/index.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/if-run/index.ts b/src/if-run/index.ts index 5607df6b..6b26b150 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -1,13 +1,11 @@ #!/usr/bin/env node -import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; - import {STRINGS as COMMON_STRINGS} from '../common/config'; import {validateManifest} from '../common/util/validations'; import {debugLogger} from '../common/util/debug-logger'; import {logger} from '../common/util/logger'; import {load} from '../common/lib/load'; -import {aggregate, storeAggregationMetrics} from './lib/aggregate'; +import {aggregate} from './lib/aggregate'; import {injectEnvironment} from './lib/environment'; import {initialize} from './lib/initialize'; import {compute} from './lib/compute'; @@ -44,16 +42,6 @@ const impactEngine = async () => { try { const {tree, ...context} = validateManifest(envManifest); - if (context.aggregation) { - const convertMetrics = context.aggregation?.metrics.map( - (metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], - }) - ); - - storeAggregationMetrics(...convertMetrics); - } - const pluginStorage = await initialize(context); const computedTree = await compute(tree, { context, @@ -69,6 +57,7 @@ const impactEngine = async () => { await exhaust(aggregatedTree, context, outputOptions); } catch (error) { if (error instanceof Error) { + /** Execution block exists because manifest is already processed. Set's status to `fail`. */ envManifest.execution!.status = 'fail'; envManifest.execution!.error = error.toString(); logger.error(error); From 6938764e9e8392c444cf561b0b929eaae0fbfde3 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:11:22 +0400 Subject: [PATCH 12/19] feat(util): introduce granular aggregation to validation --- src/common/util/validations.ts | 51 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index 646aee3f..5d0ac758 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -1,4 +1,5 @@ import {ZodIssue, ZodIssueCode, ZodSchema, z} from 'zod'; +import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../../if-run/config'; @@ -22,32 +23,35 @@ export const allDefined = (obj: Record) => Object.values(obj).every(v => v !== undefined); /** - * Schema for parameter metadata. + * Reusabe aggregation method schema for parameter metadata. + */ +const aggregationMethodSchema = z.object({ + time: z.enum(AGGREGATION_METHODS), + component: z.enum(AGGREGATION_METHODS), +}); + +/** + * Reusable metadata schema. + */ +const metadataSchema = z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': aggregationMethodSchema, + }) + ) + .optional() + .nullable(); + +/** + * Reusable parameter metadata schema. */ const parameterMetadataSchema = z .object({ - inputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), - outputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), + inputs: metadataSchema, + outputs: metadataSchema, }) .optional(); @@ -71,6 +75,7 @@ export const manifestSchema = z.object({ .object({ metrics: z.array(z.string()), type: z.enum(AGGREGATION_TYPES), + 'skip-components': z.array(z.string()).optional(), }) .optional() .nullable(), From 2d53237f628b686943b6e7fe7ed42e1d519a1528 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:17:54 +0400 Subject: [PATCH 13/19] feat(types): enable time and component aggregations in manifest --- src/common/types/manifest.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/types/manifest.ts b/src/common/types/manifest.ts index 488d8eeb..483af708 100644 --- a/src/common/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -1,5 +1,5 @@ import {z} from 'zod'; -import {AggregationMethodTypes} from '@grnsft/if-core/types'; +import {AggregationOptions} from '@grnsft/if-core/types'; import {manifestSchema} from '../util/validations'; @@ -11,7 +11,7 @@ export type PluginOptions = GlobalPlugins[string]; export type AggregationParams = Manifest['aggregation']; export type AggregationMetricsWithMethod = { - [key: string]: AggregationMethodTypes; + [key: string]: AggregationOptions; }; export type AggregationParamsSure = Extract; From c9aeaf9ae537eaae534e7942af48147c12b0e6d3 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:19:06 +0400 Subject: [PATCH 14/19] test(util): tune aggregation helper units --- .../if-run/util/aggregation-helper.test.ts | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/__tests__/if-run/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts index d76fcbb9..32a749a4 100644 --- a/src/__tests__/if-run/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -5,7 +5,6 @@ import {PluginParams} from '@grnsft/if-core/types'; import {AggregationParams} from '../../../common/types/manifest'; import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; -import {AggregationMetric} from '../../../if-run/types/aggregation'; import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {STRINGS} from '../../../if-run/config'; @@ -20,16 +19,24 @@ describe('util/aggregation-helper: ', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); - storeAggregationMetrics({carbon: 'sum'}); + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); }); describe('aggregateInputsIntoOne(): ', () => { it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: AggregationMetric[] = [{'cpu/utilization': 'sum'}]; + const metrics: string[] = ['cpu/utilization']; const isTemporal = false; expect.assertions(2); @@ -46,12 +53,17 @@ describe('util/aggregation-helper: ', () => { }); it('passes `timestamp`, `duration` to aggregator if aggregation is temporal.', () => { - storeAggregationMetrics({carbon: 'sum'}); + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric[] = [{carbon: 'sum'}]; + const metrics: string[] = ['carbon']; const isTemporal = true; const expectedValue = { @@ -68,7 +80,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric[] = [{carbon: 'sum'}]; + const metrics: string[] = ['carbon']; const isTemporal = false; const expectedValue = { @@ -84,16 +96,24 @@ describe('util/aggregation-helper: ', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); - storeAggregationMetrics({'cpu/utilization': 'avg'}); + storeAggregationMetrics({ + 'cpu/utilization': { + time: 'avg', + component: 'avg', + }, + }); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: AggregationMetric[] = [{'cpu/utilization': 'avg'}]; + const metrics: string[] = ['cpu/utilization']; const isTemporal = false; const expectedValue = { From a650544f7e06e31a25ecd07b335c1f31747611aa Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:19:43 +0400 Subject: [PATCH 15/19] test(lib): tune explain units --- src/__tests__/if-run/lib/explain.test.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/__tests__/if-run/lib/explain.test.ts b/src/__tests__/if-run/lib/explain.test.ts index 8178bf07..df86448a 100644 --- a/src/__tests__/if-run/lib/explain.test.ts +++ b/src/__tests__/if-run/lib/explain.test.ts @@ -214,19 +214,28 @@ describe('lib/explain: ', () => { 'cpu/energy': { unit: 'kWh', description: 'energy consumed by the cpu', - 'aggregation-method': 'avg', + 'aggregation-method': { + time: 'avg', + component: 'avg', + }, }, 'memory/energy': { unit: 'kWh', description: 'energy consumed by data from memory', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, }, }, outputs: { 'total/energy': { unit: 'kWh', description: 'sum of energy components', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, }, }, }, From 0113ed5db8b7c7be4be370f360b3213c652f1cfa Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:20:22 +0400 Subject: [PATCH 16/19] test(lib): tune aggregate units --- src/__tests__/if-run/lib/aggregate.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/__tests__/if-run/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts index d053fea9..f39efdfc 100644 --- a/src/__tests__/if-run/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -15,7 +15,10 @@ describe('lib/aggregate: ', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); @@ -23,7 +26,12 @@ describe('lib/aggregate: ', () => { describe('aggregate(): ', () => { beforeAll(() => { - storeAggregationMetrics({carbon: 'sum'}); + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); }); it('returns tree if aggregation is missing.', () => { From 79ead4c26c7b3789ded1bdd2ab5ce5843ae5d6ee Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 00:20:45 +0400 Subject: [PATCH 17/19] test(builtins): tune time-sync units --- .../if-run/builtins/time-sync.test.ts | 47 ++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/src/__tests__/if-run/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts index f3b26803..41e47548 100644 --- a/src/__tests__/if-run/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -66,7 +66,10 @@ describe('builtins/time-sync:', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); }); @@ -483,7 +486,12 @@ describe('builtins/time-sync:', () => { 'allow-padding': true, }; - storeAggregationMetrics({carbon: 'sum'}); + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); const timeModel = TimeSync(basicConfig, parametersMetadata, {}); @@ -598,8 +606,18 @@ describe('builtins/time-sync:', () => { 'allow-padding': true, }; - storeAggregationMetrics({'time-reserved': 'avg'}); - storeAggregationMetrics({'resources-total': 'sum'}); + storeAggregationMetrics({ + 'time-reserved': { + time: 'avg', + component: 'avg', + }, + }); + storeAggregationMetrics({ + 'resources-total': { + time: 'sum', + component: 'sum', + }, + }); const timeModel = TimeSync(basicConfig, parametersMetadata, {}); @@ -647,8 +665,18 @@ describe('builtins/time-sync:', () => { 'time-reserved': 'time-allocated', }; - storeAggregationMetrics({'time-allocated': 'avg'}); - storeAggregationMetrics({'resources-total': 'sum'}); + storeAggregationMetrics({ + 'time-allocated': { + time: 'avg', + component: 'avg', + }, + }); + storeAggregationMetrics({ + 'resources-total': { + time: 'sum', + component: 'sum', + }, + }); const timeModel = TimeSync(basicConfig, parametersMetadata, mapping); @@ -722,7 +750,12 @@ describe('builtins/time-sync:', () => { 'allow-padding': true, }; - storeAggregationMetrics({'resources-total': 'none'}); + storeAggregationMetrics({ + 'resources-total': { + time: 'none', + component: 'none', + }, + }); const timeModel = TimeSync(basicConfig, parametersMetadata, {}); From bfb7b29f8ae7fbdd64c68c83702b579c02ae1692 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 10:18:50 +0400 Subject: [PATCH 18/19] feat(package): update if-core version --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca145548..9f54932d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.20", + "@grnsft/if-core": "^0.0.22", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -1186,9 +1186,9 @@ } }, "node_modules/@grnsft/if-core": { - "version": "0.0.20", - "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.20.tgz", - "integrity": "sha512-D5P97E0O9qIw9Ok0qCcy3zmNl8j1sr/vwfPsueFzkbmF6ZnLDBL1vapn8MzSRFsp3lJoH/mStDZ3uf3+S1L17g==", + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.22.tgz", + "integrity": "sha512-uzgYrQNh/aecouRdM5xcdCMC8Wu7xAWrGqJWqABopi/2CGs0xbvrQU0bqtGkSs1otAMnv5t7ynr6mpUyBxdQrw==", "dependencies": { "typescript": "^5.1.6", "zod": "^3.23.8" diff --git a/package.json b/package.json index 802c9f13..eed57c1a 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.20", + "@grnsft/if-core": "^0.0.22", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", From 922cbc496fab50d41a1152085b5efe83365f4fda Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Tue, 10 Sep 2024 16:26:31 +0400 Subject: [PATCH 19/19] fix(builtins): fix output param of sci embodied --- src/if-run/builtins/sci-embodied/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/if-run/builtins/sci-embodied/index.ts b/src/if-run/builtins/sci-embodied/index.ts index 657f68e8..835284ea 100644 --- a/src/if-run/builtins/sci-embodied/index.ts +++ b/src/if-run/builtins/sci-embodied/index.ts @@ -90,7 +90,10 @@ export const SciEmbodied = ( 'embodied-carbon': { description: 'embodied carbon for a resource, scaled by usage', unit: 'gCO2e', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, }, }, };