From 8c41f70d3db843ed91af8a92db6305699e304ac2 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 17:52:29 +0400 Subject: [PATCH 1/9] fix(manifests): remove `method` from aggregation section --- manifests/examples/pipelines/nesting.yml | 26 +- .../pipeline-with-aggregate.yaml | 18 +- .../outputs-if-diff/pipeline-with-mocks.yaml | 655 +++++++++--------- .../pipelines/pipeline-with-aggregate.yml | 9 +- .../pipelines/pipeline-with-mocks.yml | 3 +- .../bugs/aggregation-error-wrong-metric.yaml | 3 +- .../aggregate-failure-invalid-metrics.yaml | 3 +- ...gate-failure-missing-metric-in-inputs.yaml | 3 +- .../features/aggregate-horizontal.yaml | 9 +- .../outputs/features/aggregate-vertical.yaml | 9 +- manifests/outputs/features/aggregate.yaml | 9 +- manifests/outputs/pipelines/nesting.yaml | 234 ++++--- 12 files changed, 519 insertions(+), 462 deletions(-) diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index 185c715e0..baf12a688 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -6,8 +6,7 @@ tags: category: on-premise aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -139,6 +138,21 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: method: TimeSync path: "builtin" @@ -169,10 +183,10 @@ initialize: unit: seconds description: time reserved for a component aggregation-method: avg - network/energy: - description: 'Energy consumed by the Network of the component' - unit: 'kWh' - aggregation-method: 'sum' + network/energy: + description: "Energy consumed by the Network of the component" + unit: "kWh" + aggregation-method: "sum" tree: children: child-0: diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml index ee14e4e4a..c1f92da04 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml @@ -3,8 +3,7 @@ description: a full pipeline with the aggregate feature enabled tags: null aggregation: metrics: - carbon: - method: sum + - carbon type: both initialize: plugins: @@ -159,6 +158,21 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: path: builtin method: TimeSync diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml index 983e7a638..77496bf23 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -3,8 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature tags: null aggregation: metrics: - carbon: - method: sum + - carbon type: both initialize: plugins: @@ -52,7 +51,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -247,8 +246,8 @@ initialize: path: builtin method: TimeSync global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true parameter-metadata: @@ -285,20 +284,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-30T05:35:42.937Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.16' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -340,7 +339,7 @@ tree: - operational-carbon - sum-carbon - time-sync - - sci + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -350,36 +349,36 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 10 requests: 30 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west requests: 40 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west requests: 30 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west cpu/utilization: 15 requests: 50 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -388,21 +387,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:05.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -411,21 +410,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:10.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -434,21 +433,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:15.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -457,21 +456,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:20.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -480,21 +479,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:25.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -503,21 +502,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:30.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -526,21 +525,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:35.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -549,21 +548,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:40.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -572,21 +571,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:45.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -595,21 +594,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:50.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -618,21 +617,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:55.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" requests: 2.5 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 @@ -641,21 +640,21 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.000010128107559614409 - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:01:00.000Z' + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 - cpu/utilization: '*' + cpu/utilization: "*" requests: 0.5 cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -664,18 +663,18 @@ tree: device/expected-lifespan: 1576800 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' + cpu-energy-kwh: "*" carbon-embodied: 0.0000020256215119228817 - carbon-operational: '*' - carbon: '*' - sci: '*' + carbon-operational: "*" + carbon: "*" + sci: "*" aggregated: - carbon: '*' + carbon: "*" child-2: pipeline: observe: @@ -694,7 +693,7 @@ tree: - operational-carbon - sum-carbon - time-sync - - sci + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -704,34 +703,34 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 cpu/utilization: 30 cloud/instance-type: A1 cloud/region: uk-west requests: 30 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 28 cloud/instance-type: A1 cloud/region: uk-west requests: 40 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 40 cloud/instance-type: A1 cloud/region: uk-west requests: 50 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west requests: 60 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -742,19 +741,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:05.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -765,19 +764,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:10.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -788,19 +787,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:15.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -811,19 +810,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:20.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -834,19 +833,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:25.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -857,19 +856,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:30.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -880,19 +879,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:35.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -903,19 +902,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:40.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -926,19 +925,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:45.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -949,19 +948,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:50.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -972,19 +971,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:00:55.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 2.5 @@ -995,19 +994,19 @@ tree: device/expected-lifespan: 7884000 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' - - timestamp: '2023-12-12T00:01:00.000Z' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 - cpu/utilization: '*' + cpu/utilization: "*" cloud/instance-type: A1 cloud/region: uk-west requests: 0.5 @@ -1018,57 +1017,57 @@ tree: device/expected-lifespan: 1576800 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" vcpu-ratio: 8 - cpu-energy-kwh: '*' - carbon-embodied: '*' - carbon-operational: '*' - carbon: '*' - sci: '*' + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" aggregated: - carbon: '*' + carbon: "*" outputs: - - carbon: '*' - timestamp: '2023-12-12T00:00:00.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:05.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:10.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:15.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:20.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:25.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:30.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:35.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:40.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:45.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:50.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:00:55.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - - carbon: '*' - timestamp: '2023-12-12T00:01:00.000Z' + - carbon: "*" + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: - carbon: '*' + carbon: "*" diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index dd4dffa1c..184820228 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -3,8 +3,7 @@ description: a full pipeline with the aggregate feature enabled tags: aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: @@ -145,6 +144,12 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + outputs: + carbon: + unit: gCO2eq + description: product of carbon + aggregation-method: sum "time-sync": method: TimeSync path: "builtin" diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index 58933b171..fc6f81cb0 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -3,8 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature tags: aggregation: metrics: - "carbon": - method: sum + - carbon type: "both" initialize: plugins: diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index 313834f76..92f85c0d7 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -5,8 +5,7 @@ description: >- tags: null aggregation: metrics: - "dummy-param": - method: sum + - dummy-param type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index 11e7f5865..aef8ef6c5 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Fails with invalid metric. aggregation: metrics: - "test": - method: sum + - test type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 0e308d509..2dfc9fe80 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Fails with missing metric in inputs. aggregation: metrics: - "cpu/utilization": - method: sum + - cpu/utilization type: both initialize: plugins: diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index 0fef8d756..d595d5bdb 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply `horizontal` aggregation aggregation: metrics: - "cpu/utilization": - method: avg + - cpu/utilization type: horizontal initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index dfd604c73..9213522af 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply `vertical` aggregation aggregation: metrics: - "cpu/utilization": - method: avg + - cpu/utilization type: vertical initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index 79736b9c0..cb9ed29dc 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -2,8 +2,7 @@ name: Aggregation description: Apply both `horizontal` and `vertical` aggregations aggregation: metrics: - "cpu/utilization": - method: avg + - cpu/utilization type: both initialize: plugins: @@ -18,6 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml index 5e26ae900..792f41243 100644 --- a/manifests/outputs/pipelines/nesting.yaml +++ b/manifests/outputs/pipelines/nesting.yaml @@ -6,8 +6,7 @@ tags: category: on-premise aggregation: metrics: - carbon: - method: sum + - carbon type: both initialize: plugins: @@ -21,7 +20,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -153,12 +152,27 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: path: builtin method: TimeSync global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true parameter-metadata: @@ -196,20 +210,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-31T13:17:29.944Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.16' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -256,28 +270,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 60 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 70 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -285,7 +299,7 @@ tree: network/energy: 0.000001 requests: 55 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -309,7 +323,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.00006983123919278518 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -333,7 +347,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.0001028924208718729 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -357,7 +371,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.0001544881286073813 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -381,7 +395,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -405,7 +419,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -429,7 +443,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -453,7 +467,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -477,7 +491,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0005673927632489277 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -501,7 +515,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0005673927632489276 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -525,7 +539,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -549,7 +563,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -573,7 +587,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -622,28 +636,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 10 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 90 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 30 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -651,7 +665,7 @@ tree: network/energy: 0.000001 requests: 22 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -675,7 +689,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.0000834568468401579 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -699,7 +713,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.00015224732194049487 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -723,7 +737,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.000363108733129716 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -747,7 +761,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -771,7 +785,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -795,7 +809,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -819,7 +833,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -843,7 +857,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -867,7 +881,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0014184819081223194 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -891,7 +905,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -915,7 +929,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -939,7 +953,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -990,28 +1004,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 65 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 80 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -1019,7 +1033,7 @@ tree: network/energy: 0.000001 requests: 40 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -1043,7 +1057,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.00006709275922444067 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -1067,7 +1081,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.0000911261343001502 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -1091,7 +1105,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.00014075142645028166 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1115,7 +1129,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1139,7 +1153,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1163,7 +1177,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1187,7 +1201,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1211,7 +1225,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1235,7 +1249,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0007801650494672755 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1259,7 +1273,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1283,7 +1297,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1307,7 +1321,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -1356,28 +1370,28 @@ tree: - time-sync - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 50 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 60 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -1385,7 +1399,7 @@ tree: network/energy: 0.000001 requests: 40 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -1409,7 +1423,7 @@ tree: carbon-operational: 0.006833333333333334 carbon: 0.0068434614408929475 sci: 0.00007603846045436609 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -1433,7 +1447,7 @@ tree: carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.00012081561676568304 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -1457,7 +1471,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.0001832602465191587 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1481,7 +1495,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1505,7 +1519,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1529,7 +1543,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1553,7 +1567,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1577,7 +1591,7 @@ tree: carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0007801650494672756 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1601,7 +1615,7 @@ tree: carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0007801650494672755 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1625,7 +1639,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1649,7 +1663,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1673,7 +1687,7 @@ tree: carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -1701,85 +1715,85 @@ tree: carbon: 0.04652112950279046 outputs: - carbon: 0.013686922881785895 - timestamp: '2023-12-12T00:00:00.000Z' + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - carbon: 0.010700811770674782 - timestamp: '2023-12-12T00:00:05.000Z' + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - carbon: 0.010402200659563673 - timestamp: '2023-12-12T00:00:10.000Z' + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:15.000Z' + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:20.000Z' + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:25.000Z' + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:30.000Z' + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - carbon: 0.010402200659563675 - timestamp: '2023-12-12T00:00:35.000Z' + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - carbon: 0.006241320395738204 - timestamp: '2023-12-12T00:00:40.000Z' + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:45.000Z' + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:50.000Z' + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:55.000Z' + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:01:00.000Z' + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: carbon: 0.09304225900558093 outputs: - carbon: 0.02737384576357179 - timestamp: '2023-12-12T00:00:00.000Z' + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - carbon: 0.021401623541349564 - timestamp: '2023-12-12T00:00:05.000Z' + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - carbon: 0.020804401319127346 - timestamp: '2023-12-12T00:00:10.000Z' + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:15.000Z' + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:20.000Z' + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:25.000Z' + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:30.000Z' + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - carbon: 0.02080440131912735 - timestamp: '2023-12-12T00:00:35.000Z' + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - carbon: 0.012482640791476408 - timestamp: '2023-12-12T00:00:40.000Z' + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:45.000Z' + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:50.000Z' + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:55.000Z' + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:01:00.000Z' + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: carbon: 0.18608451801116185 From 45821add239ffafc626b66290a85395bc6d30088 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:18:53 +0400 Subject: [PATCH 2/9] fix(util): update manifest schema --- src/common/util/validations.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index 95d9efeff..5cdbec5db 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -3,10 +3,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../../if-run/config'; -import { - AGGREGATION_METHODS, - AGGREGATION_TYPES, -} from '../../if-run/types/aggregation'; +import {AGGREGATION_TYPES} from '../../if-run/types/aggregation'; const {ManifestValidationError, InputValidationError} = ERRORS; const {VALIDATING_MANIFEST} = STRINGS; @@ -72,11 +69,7 @@ export const manifestSchema = z.object({ explain: z.record(z.string(), z.any()).optional(), aggregation: z .object({ - metrics: z.record( - z.object({ - method: z.enum(AGGREGATION_METHODS), - }) - ), + metrics: z.array(z.string()), type: z.enum(AGGREGATION_TYPES), }) .optional() From 719a239e1cea887935420046e03616cb84ced525 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:23:52 +0400 Subject: [PATCH 3/9] fix(lib): update aggregation logic without `method` --- src/if-run/lib/aggregate.ts | 21 +++++++++++++-------- src/if-run/lib/initialize.ts | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index 508117bb3..b80d0048a 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -5,7 +5,7 @@ import {logger} from '../../common/util/logger'; import { AggregationParams, AggregationParamsSure, - AggregationParamsWithoutType, + AggregationMetricsWithMethod, } from '../../common/types/manifest'; import {aggregateInputsIntoOne} from '../util/aggregation-helper'; @@ -39,7 +39,7 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: AggregationMetric) => { +const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); @@ -63,7 +63,9 @@ const temporalAggregation = (node: any, metrics: AggregationMetric) => { * 5. Now a grouping node has it's own outputs, it can horizotnally aggregate them. */ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { - const metrics = aggregationParams!.metrics; + const metrics: AggregationMetric[] = aggregationParams!.metrics.map( + metric => ({[metric]: 'none'}) + ); const type = aggregationParams!.type; if (node.children) { @@ -108,12 +110,12 @@ export const aggregate = (tree: any, aggregationParams: AggregationParams) => { * Gets or stores aggregation metrics. */ export const storeAggregationMetrics = ( - aggregationParams?: AggregationParamsWithoutType + aggregationMetrics?: AggregationMetricsWithMethod ) => { - if (aggregationParams?.metrics) { + if (aggregationMetrics) { metricManager.metrics = { ...metricManager.metrics, - ...aggregationParams?.metrics, + ...aggregationMetrics, }; } @@ -146,8 +148,11 @@ export const getAggregationMethod = (unitName: string) => { memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); const aggregationMetricsStorage = storeAggregationMetrics(); - if (aggregationMetricsStorage && `${unitName}` in aggregationMetricsStorage) { - return aggregationMetricsStorage[unitName].method; + if ( + aggregationMetricsStorage && + Object.keys(aggregationMetricsStorage).includes(unitName) + ) { + return aggregationMetricsStorage[unitName]; } memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); diff --git a/src/if-run/lib/initialize.ts b/src/if-run/lib/initialize.ts index f047be117..181e88d15 100644 --- a/src/if-run/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -118,7 +118,7 @@ export const initialize = async ( Object.keys(parameters).forEach(key => { storeAggregationMetrics({ - metrics: {[key]: {method: parameters[key]['aggregation-method']}}, + [key]: parameters[key]['aggregation-method'], }); }); From 6eeba6583e891e08c35ea9bb44bf5c860ceef024 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:24:55 +0400 Subject: [PATCH 4/9] fix(util): update aggregation logic --- src/if-run/util/aggregation-helper.ts | 29 +++++---------------------- src/if-run/util/helpers.ts | 7 +++---- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/if-run/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts index 87d3e15fc..10b1a0bdd 100644 --- a/src/if-run/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -7,40 +7,21 @@ import {AggregationMetric, AggregationResult} from '../types/aggregation'; import {getAggregationMethod} from '../lib/aggregate'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; -/** - * Validates metrics array before applying aggregator. - * If aggregation method is `none`, then throws error. - */ -const checkIfMetricsAreValid = (metrics: AggregationMetric) => { - Object.keys(metrics).forEach(metric => { - const method = metrics[metric].method; - - if (method === 'none') { - throw new InvalidAggregationMethodError( - INVALID_AGGREGATION_METHOD(metric) - ); - } - }); -}; - /** * Aggregates child node level metrics. Validates if metric aggregation type is `none`, then rejects with error. * Appends aggregation additional params to metrics. Otherwise iterates over inputs by aggregating per given `metrics`. */ export const aggregateInputsIntoOne = ( inputs: PluginParams[], - metrics: AggregationMetric, + metrics: AggregationMetric[], isTemporal?: boolean ) => { - checkIfMetricsAreValid(metrics); - const extendedMetrics = [ - ...Object.keys(metrics), - ...AGGREGATION_ADDITIONAL_PARAMS, - ]; + const metricsKeys: string[] = metrics.map(metric => Object.keys(metric)[0]); + const extendedMetrics = [...metricsKeys, ...AGGREGATION_ADDITIONAL_PARAMS]; return inputs.reduce((acc, input, index) => { for (const metric of extendedMetrics) { diff --git a/src/if-run/util/helpers.ts b/src/if-run/util/helpers.ts index da50f22f4..2939022e2 100644 --- a/src/if-run/util/helpers.ts +++ b/src/if-run/util/helpers.ts @@ -62,10 +62,9 @@ export const storeAggregationMethods = ( const {'aggregation-method': aggregationMethod} = parameterMetadata; if (aggregationMethod) { - const metrics = { - [parameterName]: {method: aggregationMethod}, - }; - storeAggregationMetrics({metrics}); + const metrics = {[parameterName]: aggregationMethod}; + + storeAggregationMetrics(metrics); } } ); From 914b8eb8cd1b91f0797d355c9df0aa3f7d92068a Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:26:13 +0400 Subject: [PATCH 5/9] fix(types): update aggregation types --- src/common/types/manifest.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/common/types/manifest.ts b/src/common/types/manifest.ts index c28fe7194..1efab523d 100644 --- a/src/common/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -1,5 +1,7 @@ import {z} from 'zod'; +import {AggregationMethodTypes} from '../../if-run/types/aggregation'; + import {manifestSchema} from '../util/validations'; export type Manifest = z.infer; @@ -9,10 +11,9 @@ export type GlobalPlugins = Manifest['initialize']['plugins']; export type PluginOptions = GlobalPlugins[string]; export type AggregationParams = Manifest['aggregation']; -export type AggregationParamsWithoutType = Omit< - Exclude, - 'type' ->; +export type AggregationMetricsWithMethod = { + [key: string]: AggregationMethodTypes; +}; export type AggregationParamsSure = Extract; From c69949718f867c005371de1d52132be46d32a550 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:27:05 +0400 Subject: [PATCH 6/9] fix(types): update aggregation types --- src/if-run/types/aggregation.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts index a7e895da4..c3b143a1f 100644 --- a/src/if-run/types/aggregation.ts +++ b/src/if-run/types/aggregation.ts @@ -3,7 +3,5 @@ export type AggregationResult = Record; export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; export const AGGREGATION_METHODS = ['sum', 'avg', 'none'] as const; -export type AggregationMetric = Record< - string, - {method: 'sum' | 'avg' | 'none'} ->; +export type AggregationMethodTypes = 'sum' | 'avg' | 'none'; +export type AggregationMetric = Record; From be28ccdadd3e7304e9add7416f48a4e6da87be4e Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:28:49 +0400 Subject: [PATCH 7/9] fix(src): fix adding aggregation metric into storage --- src/if-run/index.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/if-run/index.ts b/src/if-run/index.ts index 9b8d86f6c..6d9f61a78 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -12,6 +12,8 @@ import {compute} from './lib/compute'; import {exhaust} from './lib/exhaust'; import {explain} from './lib/explain'; +import {AGGREGATION_METHODS} from './types/aggregation'; + import {parseIfRunProcessArgs} from './util/args'; import {andHandle} from './util/helpers'; @@ -43,7 +45,13 @@ const impactEngine = async () => { const {tree, ...context} = validateManifest(envManifest); if (context.aggregation) { - storeAggregationMetrics({metrics: context.aggregation?.metrics}); + const convertMetrics = context.aggregation?.metrics.map( + (metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + }) + ); + + storeAggregationMetrics(...convertMetrics); } const pluginStorage = await initialize(context); From 1612a6293e843b2fd7e8bae461f91c8a565c0e50 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:29:44 +0400 Subject: [PATCH 8/9] test(src): update tests accroding to changes --- .../if-run/builtins/time-sync.test.ts | 23 +++++--- src/__tests__/if-run/lib/aggregate.test.ts | 16 ++--- .../if-run/util/aggregation-helper.test.ts | 59 +++++++------------ src/__tests__/if-run/util/helpers.test.ts | 12 +--- 4 files changed, 48 insertions(+), 62 deletions(-) diff --git a/src/__tests__/if-run/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts index 314ed06d4..38621e12f 100644 --- a/src/__tests__/if-run/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -7,6 +7,7 @@ import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {TimeSync} from '../../../if-run/builtins/time-sync'; import {STRINGS} from '../../../if-run/config'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; Settings.defaultZone = 'utc'; const { @@ -56,16 +57,18 @@ jest.mock('luxon', () => { describe('builtins/time-sync:', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - 'cpu/utilization': {method: 'sum'}, - 'time-reserved': {method: 'avg'}, - 'resources-total': {method: 'none'}, - }, + metrics: [ + 'carbon', + 'cpu/utilization', + 'time-reserved', + 'resources-total', + ], type: 'horizontal', }; - - storeAggregationMetrics(metricStorage); + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); }); describe('time-sync: ', () => { @@ -472,6 +475,7 @@ describe('execute(): ', () => { interval: 1, 'allow-padding': true, }; + storeAggregationMetrics({carbon: 'sum'}); const timeModel = TimeSync(basicConfig, parametersMetadata); @@ -586,6 +590,8 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'time-reserved': 'avg'}); + storeAggregationMetrics({'resources-total': 'sum'}); const timeModel = TimeSync(basicConfig, parametersMetadata); @@ -659,6 +665,7 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'resources-total': 'none'}); const timeModel = TimeSync(basicConfig, parametersMetadata); diff --git a/src/__tests__/if-run/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts index 36bbc0d1f..00d9c6d0e 100644 --- a/src/__tests__/if-run/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -6,17 +6,19 @@ import { aggregate, storeAggregationMetrics, } from '../../../if-run/lib/aggregate'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; describe('lib/aggregate: ', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - }, + metrics: ['carbon'], type: 'horizontal', }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); - storeAggregationMetrics(metricStorage); + storeAggregationMetrics(...convertedMetrics); }); describe('aggregate(): ', () => { @@ -60,7 +62,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'horizontal', }); const expectedAggregated = { @@ -108,7 +110,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'vertical', }); const expectedOutputs = [ @@ -169,7 +171,7 @@ describe('lib/aggregate: ', () => { }; const aggregatedTree = aggregate(tree, { - metrics: {carbon: {method: 'sum'}}, + metrics: ['carbon'], type: 'both', }); diff --git a/src/__tests__/if-run/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts index df8a74d9b..f83536f39 100644 --- a/src/__tests__/if-run/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -4,52 +4,33 @@ import {PluginParams} from '@grnsft/if-core/types'; import {AggregationParams} from '../../../common/types/manifest'; import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; -import {AggregationMetric} from '../../../if-run/types/aggregation'; +import { + AGGREGATION_METHODS, + AggregationMetric, +} from '../../../if-run/types/aggregation'; import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {STRINGS} from '../../../if-run/config'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; describe('util/aggregation-helper: ', () => { beforeAll(() => { const metricStorage: AggregationParams = { - metrics: { - carbon: {method: 'sum'}, - 'cpu/number-cores': {method: 'none'}, - 'cpu/utilization': {method: 'sum'}, - }, + metrics: ['carbon', 'cpu/number-cores', 'cpu/utilization'], type: 'horizontal', }; - - storeAggregationMetrics(metricStorage); + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); }); describe('aggregateInputsIntoOne(): ', () => { - it('throws error if aggregation method is none.', () => { - const inputs: PluginParams[] = []; - const metrics: AggregationMetric = {'cpu/number-cores': {method: 'none'}}; - const isTemporal = false; - - expect.assertions(2); - - try { - aggregateInputsIntoOne(inputs, metrics, isTemporal); - } catch (error) { - expect(error).toBeInstanceOf(InvalidAggregationMethodError); - - if (error instanceof InvalidAggregationMethodError) { - expect(error.message).toEqual( - INVALID_AGGREGATION_METHOD('cpu/number-cores') - ); - } - } - }); - it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: AggregationMetric = {'cpu/utilization': {method: 'sum'}}; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'sum'}]; const isTemporal = false; expect.assertions(2); @@ -70,7 +51,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric = {carbon: {method: 'sum'}}; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = true; const expectedValue = { @@ -87,7 +68,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric = {carbon: {method: 'sum'}}; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = false; const expectedValue = { @@ -99,18 +80,20 @@ describe('util/aggregation-helper: ', () => { it('calculates average of metrics.', () => { const metricStorage: AggregationParams = { - metrics: { - 'cpu/utilization': {method: 'avg'}, - }, + metrics: ['cpu/utilization'], type: 'horizontal', }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); + storeAggregationMetrics({'cpu/utilization': 'avg'}); - storeAggregationMetrics(metricStorage); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: AggregationMetric = {'cpu/utilization': {method: 'avg'}}; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'avg'}]; const isTemporal = false; const expectedValue = { diff --git a/src/__tests__/if-run/util/helpers.test.ts b/src/__tests__/if-run/util/helpers.test.ts index c694c0a9a..796f6b201 100644 --- a/src/__tests__/if-run/util/helpers.test.ts +++ b/src/__tests__/if-run/util/helpers.test.ts @@ -242,19 +242,13 @@ describe('if-run/util/helpers: ', () => { expect(storeAggregationMetrics).toHaveBeenCalledTimes(3); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(1, { - metrics: { - carbon: {method: 'sum'}, - }, + carbon: 'sum', }); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(2, { - metrics: { - cpu: {method: 'avg'}, - }, + cpu: 'avg', }); expect(storeAggregationMetrics).toHaveBeenNthCalledWith(3, { - metrics: { - carbon: {method: 'none'}, - }, + carbon: 'none', }); }); From 27fce36762d13ecbaafab9233fa91f45ed89f593 Mon Sep 17 00:00:00 2001 From: manushak Date: Wed, 7 Aug 2024 19:32:39 +0400 Subject: [PATCH 9/9] fix(doc): update aggregation part in the doc --- Refactor-migration-guide.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index e29a80d0e..95283c841 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -199,9 +199,7 @@ The aggregate plugin aggregates data in two ways: first it condenses individual This is a builtin feature of IF, meaning it does not have to be initialized as a plugin. Instead, you just have to include a short config block in the top of the manifest file. There are two pieces of information required: -- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array. - - - `method`: the aggregation method for the specied metric +- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array and be described in the `parameter-metadata` of the plugin. - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components. @@ -210,8 +208,7 @@ Here's what the config block should look like: ```yaml aggregation: metrics: - 'carbon': - method: 'sum' + - carbon type: 'both' ```