diff --git a/.github/workflows/dispatch-integration-test.yml b/.github/workflows/dispatch-integration-test.yml new file mode 100644 index 000000000..6fe4904d7 --- /dev/null +++ b/.github/workflows/dispatch-integration-test.yml @@ -0,0 +1,23 @@ +name: Run if-check on requested branch + +on: + workflow_dispatch: + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: 'npm' + + - name: Install dependencies + run: npm install + + - name: Run integration tests + run: npm run if-check -- -d manifests/outputs diff --git a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml index 938c126aa..5d497b948 100644 --- a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml +++ b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml index c02f42cf9..1ef932df0 100644 --- a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/coefficient/success.yml b/manifests/examples/builtins/coefficient/success.yml index bd7d2cc94..69dd764eb 100644 --- a/manifests/examples/builtins/coefficient/success.yml +++ b/manifests/examples/builtins/coefficient/success.yml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index befa110a6..21f9769b4 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml index 1cd9b609a..c9fbb8296 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml index a01156154..62de6150d 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml index 52bda60b6..84e10112b 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml index 68d3bc917..5b4e9583b 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml index 64354be35..71417932d 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml index c30e11f9c..4c2767a3a 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml index 64354be35..71417932d 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml index 52d2b660c..991bdbb8b 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml @@ -15,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml index f86767455..3d433d6b2 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml @@ -15,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml index b1953f5ca..a04288e0c 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml @@ -15,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml index d26272a82..96d627aee 100644 --- a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml +++ b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml index 01347d943..fef8a1a7c 100644 --- a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml +++ b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/failure-missing-numerator.yml b/manifests/examples/builtins/divide/failure-missing-numerator.yml index eda63d49d..5645f0ecf 100644 --- a/manifests/examples/builtins/divide/failure-missing-numerator.yml +++ b/manifests/examples/builtins/divide/failure-missing-numerator.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/success.yml b/manifests/examples/builtins/divide/success.yml index 02222a6f4..32b247a87 100644 --- a/manifests/examples/builtins/divide/success.yml +++ b/manifests/examples/builtins/divide/success.yml @@ -23,10 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/examples/builtins/exponent/success.yml b/manifests/examples/builtins/exponent/success.yml index fe54f30a2..163dd1460 100644 --- a/manifests/examples/builtins/exponent/success.yml +++ b/manifests/examples/builtins/exponent/success.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/interpolation/interpolation.yml b/manifests/examples/builtins/interpolation/interpolation.yml index 35d750e48..394946467 100644 --- a/manifests/examples/builtins/interpolation/interpolation.yml +++ b/manifests/examples/builtins/interpolation/interpolation.yml @@ -12,12 +12,12 @@ initialize: y: [0.12, 0.32, 0.75, 1.02] input-parameter: "cpu/utilization" output-parameter: "result" - tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/interpolation/success.yml b/manifests/examples/builtins/interpolation/success.yml index cce024727..394946467 100644 --- a/manifests/examples/builtins/interpolation/success.yml +++ b/manifests/examples/builtins/interpolation/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml index fc74d71a6..472f797fd 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml index 0184d280a..85e3f566b 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml index 72cd6c347..58545dcec 100644 --- a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml +++ b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/success.yml b/manifests/examples/builtins/mock-observations/success.yml index 7767a68bc..5ca2d1942 100644 --- a/manifests/examples/builtins/mock-observations/success.yml +++ b/manifests/examples/builtins/mock-observations/success.yml @@ -29,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml index d71f9f096..988a33fc2 100644 --- a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml +++ b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml index 0a70ba5b3..e6e138723 100644 --- a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml +++ b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/multiply/success.yml b/manifests/examples/builtins/multiply/success.yml index 88b7bbf71..c5d53e046 100644 --- a/manifests/examples/builtins/multiply/success.yml +++ b/manifests/examples/builtins/multiply/success.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/failure-missing-input-param.yml b/manifests/examples/builtins/regex/failure-missing-input-param.yml index f6732c632..84adc0cb2 100644 --- a/manifests/examples/builtins/regex/failure-missing-input-param.yml +++ b/manifests/examples/builtins/regex/failure-missing-input-param.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml index d8818265a..8341d08e6 100644 --- a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml +++ b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/success.yml b/manifests/examples/builtins/regex/success.yml index 8e97d4c43..425dd9e67 100644 --- a/manifests/examples/builtins/regex/success.yml +++ b/manifests/examples/builtins/regex/success.yml @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml index 6ccd41a6e..503300f84 100644 --- a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml +++ b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml @@ -10,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: "fail" # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml index 26edf154c..8fd3e5784 100644 --- a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml +++ b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml @@ -10,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci-embodied/success.yml b/manifests/examples/builtins/sci-embodied/success.yml index 219e76845..991569404 100644 --- a/manifests/examples/builtins/sci-embodied/success.yml +++ b/manifests/examples/builtins/sci-embodied/success.yml @@ -10,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci/failure-invalid-config-value.yml b/manifests/examples/builtins/sci/failure-invalid-config-value.yml index 798e1debd..5882111dd 100644 --- a/manifests/examples/builtins/sci/failure-invalid-config-value.yml +++ b/manifests/examples/builtins/sci/failure-invalid-config-value.yml @@ -13,7 +13,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci config: sci: functional-unit: 999 # factor to convert per time to per f.unit diff --git a/manifests/examples/builtins/sci/failure-missing-input-param.yml b/manifests/examples/builtins/sci/failure-missing-input-param.yml index d30f21550..0a7677261 100644 --- a/manifests/examples/builtins/sci/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sci/failure-missing-input-param.yml @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci/success.yml b/manifests/examples/builtins/sci/success.yml index 95979fd28..85dd19db2 100644 --- a/manifests/examples/builtins/sci/success.yml +++ b/manifests/examples/builtins/sci/success.yml @@ -13,8 +13,8 @@ tree: children: child: pipeline: - - sci - config: + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/shell/failure-invalid-command.yml b/manifests/examples/builtins/shell/failure-invalid-command.yml index 2944bc3ea..e67aff103 100644 --- a/manifests/examples/builtins/shell/failure-invalid-command.yml +++ b/manifests/examples/builtins/shell/failure-invalid-command.yml @@ -12,7 +12,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/examples/builtins/shell/success.yml b/manifests/examples/builtins/shell/success.yml index b44ca2892..a26af274d 100644 --- a/manifests/examples/builtins/shell/success.yml +++ b/manifests/examples/builtins/shell/success.yml @@ -12,7 +12,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/examples/builtins/subtract/success.yml b/manifests/examples/builtins/subtract/success.yml index b29d89fd0..745ceb067 100644 --- a/manifests/examples/builtins/subtract/success.yml +++ b/manifests/examples/builtins/subtract/success.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/failure-missing-input-param.yml b/manifests/examples/builtins/sum/failure-missing-input-param.yml index 65004b679..1e963b694 100644 --- a/manifests/examples/builtins/sum/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-input-param.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/failure-missing-output-param.yml b/manifests/examples/builtins/sum/failure-missing-output-param.yml index b3c21ac3e..0248bf449 100644 --- a/manifests/examples/builtins/sum/failure-missing-output-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-output-param.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/success.yml b/manifests/examples/builtins/sum/success.yml index 4f5f7452b..366ae5580 100644 --- a/manifests/examples/builtins/sum/success.yml +++ b/manifests/examples/builtins/sum/success.yml @@ -13,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml index e6bc3a230..ffed32274 100644 --- a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml +++ b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 @@ -31,4 +31,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 + energy-cpu: 0.001 \ No newline at end of file diff --git a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml b/manifests/examples/builtins/time-sync/failure-missing-global-config.yml index 7b8e110fb..c5715a26a 100644 --- a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml +++ b/manifests/examples/builtins/time-sync/failure-missing-global-config.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 3 @@ -31,4 +31,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 + energy-cpu: 0.001 \ No newline at end of file diff --git a/manifests/examples/builtins/time-sync/success.yml b/manifests/examples/builtins/time-sync/success.yml index 6d1841e97..8aac13740 100644 --- a/manifests/examples/builtins/time-sync/success.yml +++ b/manifests/examples/builtins/time-sync/success.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 diff --git a/manifests/examples/builtins/groupby/failure-invalid-config-group.yml b/manifests/examples/features/regroup/failure-invalid-config-group.yml similarity index 87% rename from manifests/examples/builtins/groupby/failure-invalid-config-group.yml rename to manifests/examples/features/regroup/failure-invalid-config-group.yml index eca1dd8a9..5d331ba05 100644 --- a/manifests/examples/builtins/groupby/failure-invalid-config-group.yml +++ b/manifests/examples/features/regroup/failure-invalid-config-group.yml @@ -1,18 +1,13 @@ -name: groupby +name: regroup description: failure when `config->group-by->group` is not an array initialize: - plugins: - group-by: - path: "builtin" - method: GroupBy + plugins: {} tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: cloud/region + regroup: + cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml similarity index 89% rename from manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml rename to manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml index 34f31266d..0b02593fd 100644 --- a/manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml +++ b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml @@ -1,4 +1,4 @@ -name: groupby +name: regroup description: initialize: plugins: @@ -9,12 +9,9 @@ tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/groupby/success.yml b/manifests/examples/features/regroup/success.yml similarity index 89% rename from manifests/examples/builtins/groupby/success.yml rename to manifests/examples/features/regroup/success.yml index f3f4161f6..4677e8edb 100644 --- a/manifests/examples/builtins/groupby/success.yml +++ b/manifests/examples/features/regroup/success.yml @@ -1,4 +1,4 @@ -name: groupby +name: regroup description: successful path initialize: plugins: @@ -9,12 +9,9 @@ tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/pipelines/generics.yml b/manifests/examples/pipelines/generics.yml index 4fd3fa018..f9a19a6b3 100644 --- a/manifests/examples/pipelines/generics.yml +++ b/manifests/examples/pipelines/generics.yml @@ -62,15 +62,15 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - coefficient - - multiply - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - coefficient + - multiply defaults: cpu/thermal-design-power: 100 vcpus-allocated: 1 diff --git a/manifests/examples/pipelines/instance-metadata.yml b/manifests/examples/pipelines/instance-metadata.yml index d79dedb8c..0a64e9ac3 100644 --- a/manifests/examples/pipelines/instance-metadata.yml +++ b/manifests/examples/pipelines/instance-metadata.yml @@ -23,8 +23,9 @@ tree: children: child: pipeline: - - cloud-instance-metadata - - extract-processor-name + compute: + - cloud-instance-metadata + - extract-processor-name inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index 2a154e41e..185c715e0 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -185,17 +185,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -235,17 +236,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -287,17 +289,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -337,17 +340,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 diff --git a/manifests/outputs/pipelines/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml similarity index 100% rename from manifests/outputs/pipelines/pipeline-with-aggregate.yaml rename to manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml diff --git a/manifests/outputs/pipelines/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml similarity index 100% rename from manifests/outputs/pipelines/pipeline-with-mocks.yaml rename to manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml diff --git a/manifests/examples/pipelines/pipeline-teads-sci.yml b/manifests/examples/pipelines/pipeline-teads-sci.yml index c07b093bd..d80d0e89e 100644 --- a/manifests/examples/pipelines/pipeline-teads-sci.yml +++ b/manifests/examples/pipelines/pipeline-teads-sci.yml @@ -79,17 +79,17 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - sci - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index f83760e0b..dd4dffa1c 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -184,22 +184,21 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -235,22 +234,21 @@ tree: requests: 30 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index 69605dd23..58933b171 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -13,9 +13,9 @@ initialize: method: MockObservations path: "builtin" global-config: - timestamp-from: 2023-12-12T00:00 - timestamp-to: 2023-12-12T00:10 - duration: 60 + timestamp-from: "2023-12-12T00:00:00.000Z" + timestamp-to: "2023-12-12T00:00:13.000Z" + duration: 30 components: - cloud/instance-type: A1 generators: @@ -238,52 +238,27 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - parameter-metadata: - inputs: - time-reserved: - description: time reserved for a component - unit: seconds - aggregation-method: avg - outputs: - synced-time: - description: Synced time - unit: none - aggregation-method: none - group-by: - path: builtin - method: GroupBy - parameter-metadata: - inputs: - group: - description: Group by fields - unit: none - aggregation-method: none - outputs: - grouped-data: - description: Grouped data - unit: none - aggregation-method: none tree: children: child-1: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - instance-type + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -293,49 +268,25 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - cloud/instance-type: A1 - cloud/region: uk-west - duration: 1 - cpu/utilization: 10 - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 20 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 15 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 15 - requests: 50 child-2: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -345,27 +296,3 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - duration: 1 - cpu/utilization: 30 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 28 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 40 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 50 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cpu/utilization: 33 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 60 diff --git a/manifests/examples/pipelines/scenario-1.yml b/manifests/examples/pipelines/scenario-1.yml new file mode 100644 index 000000000..9ab2d20f4 --- /dev/null +++ b/manifests/examples/pipelines/scenario-1.yml @@ -0,0 +1,34 @@ +name: demo +description: demo for observe feat +tags: +initialize: + plugins: + mock-observations: + kind: plugin + method: MockObservations + path: "builtin" + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:01 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 +tree: + children: + child: + pipeline: + observe: + - mock-observations + inputs: null diff --git a/manifests/examples/pipelines/scenario-2.yml b/manifests/examples/pipelines/scenario-2.yml new file mode 100644 index 000000000..569cbcb05 --- /dev/null +++ b/manifests/examples/pipelines/scenario-2.yml @@ -0,0 +1,52 @@ +name: regroup demo +description: +initialize: + plugins: + interpolate: + method: Interpolation + path: "builtin" + global-config: + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" +tree: + children: + child: + pipeline: + observe: + regroup: + - cloud/region + - cloud/instance-type + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-3.yml b/manifests/examples/pipelines/scenario-3.yml new file mode 100644 index 000000000..f5710b553 --- /dev/null +++ b/manifests/examples/pipelines/scenario-3.yml @@ -0,0 +1,86 @@ +name: groupby +description: successful path +initialize: + plugins: + "sum": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy +tree: + children: + my-app: + pipeline: + observe: + regroup: + - cloud/instance-type + - cloud/region + compute: + children: + uk-west: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 + uk-east: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 9 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-4.yml b/manifests/examples/pipelines/scenario-4.yml new file mode 100644 index 000000000..151db09be --- /dev/null +++ b/manifests/examples/pipelines/scenario-4.yml @@ -0,0 +1,46 @@ +name: demo +description: +tags: +initialize: + plugins: + "sum": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy-sum + "coefficient": + path: "builtin" + method: Coefficient + global-config: + input-parameter: energy + coefficient: 2 + output-parameter: energy-doubled + "multiply": + path: "builtin" + method: Multiply + global-config: + input-parameters: ["cpu/utilization", "duration"] + output-parameter: "cpu-times-duration" +tree: + children: + child-1: + pipeline: + observe: + compute: + - sum + - coefficient + - multiply + defaults: + cpu/thermal-design-power: 100 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + cpu/energy: 20 + network/energy: 10 + energy: 5 diff --git a/manifests/examples/pipelines/scenario-5.yml b/manifests/examples/pipelines/scenario-5.yml new file mode 100644 index 000000000..6990c33e3 --- /dev/null +++ b/manifests/examples/pipelines/scenario-5.yml @@ -0,0 +1,46 @@ +name: demo +description: +tags: +initialize: + plugins: + mock-observations: + kind: plugin + method: MockObservations + path: "builtin" + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:01 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 + sum: + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/utilization + - memory/utilization + output-parameter: util-sum +tree: + children: + child: + pipeline: + observe: + - mock-observations + regroup: + - cloud/instance-type + compute: + - sum + inputs: null diff --git a/manifests/examples/pipelines/sci.yml b/manifests/examples/pipelines/sci.yml index 927f4eaf5..949d9e7df 100644 --- a/manifests/examples/pipelines/sci.yml +++ b/manifests/examples/pipelines/sci.yml @@ -79,18 +79,18 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sum-energy-components - - embodied-carbon - - operational-carbon - - sum-carbon - - sci - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sum-energy-components + - embodied-carbon + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/examples/pipelines/teads-curve.yml b/manifests/examples/pipelines/teads-curve.yml index 924384c20..1679a0976 100644 --- a/manifests/examples/pipelines/teads-curve.yml +++ b/manifests/examples/pipelines/teads-curve.yml @@ -49,12 +49,13 @@ tree: children: child: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio defaults: thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/examples/pipelines/zeros.yml b/manifests/examples/pipelines/zeros.yml index da5541789..f76a2c79f 100644 --- a/manifests/examples/pipelines/zeros.yml +++ b/manifests/examples/pipelines/zeros.yml @@ -106,22 +106,21 @@ tree: children: child-1: pipeline: - - sum-zero-and-one - - sum-zero-and-zero - - subtract-one-and-zero - - subtract-zero-and-zero - - subtract-zero-and-one - - coefficient-one-times-zero - - coefficient-zero-times-one - - coefficient-zero-times-zero - - multiply-one-times-zero - - multiply-zero-times-one - - exponent-one-to-zero - - exponent-zero-to-one - - exponent-zero-to-zero - - sci - config: - defaults: + compute: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index ebf55ad73..313834f76 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -88,9 +88,6 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - group-by: - path: builtin - method: GroupBy execution: status: fail command: >- @@ -142,21 +139,20 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -192,21 +188,20 @@ tree: requests: 100 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/outputs/bugs/input-error-missing-duration.yaml b/manifests/outputs/bugs/input-error-missing-duration.yaml index 310a7d0ae..1006c1c0c 100644 --- a/manifests/outputs/bugs/input-error-missing-duration.yaml +++ b/manifests/outputs/bugs/input-error-missing-duration.yaml @@ -41,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -76,7 +75,8 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - interpolate + compute: + - interpolate inputs: - timestamp: 2023-07-06T00:00 cpu/utilization: 20 diff --git a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml index 56ed12cee..d137975b5 100644 --- a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml +++ b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml @@ -44,7 +44,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,11 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: "RangeError: Maximum call stack size exceeded" + error: "InputValidationError: \"duration\" parameter is number must be greater than 0. Error code: too_small." tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml index aaf987300..fef0fa246 100644 --- a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml +++ b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml @@ -41,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -76,7 +75,8 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - wrong-name + compute: + - wrong-name inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml index 1fd747ce9..7a0060013 100644 --- a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml +++ b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml @@ -41,7 +41,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -76,8 +75,9 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - interpolate - - multiply + compute: + - interpolate + - multiply inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/outputs/bugs/pipeline-ordering-error.yaml b/manifests/outputs/bugs/pipeline-ordering-error.yaml index 5a47d8d3f..d1bd27d27 100644 --- a/manifests/outputs/bugs/pipeline-ordering-error.yaml +++ b/manifests/outputs/bugs/pipeline-ordering-error.yaml @@ -78,7 +78,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -111,13 +110,13 @@ tree: children: child-1: pipeline: - - interpolate - - correct-cpu-energy-for-vcpu-ratio - - calculate-vcpu-ratio - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - config: null + compute: + - interpolate + - correct-cpu-energy-for-vcpu-ratio + - calculate-vcpu-ratio + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml index 5c93301e2..486947ebd 100644 --- a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml +++ b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml @@ -60,7 +60,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml index 78af0bab9..7b6051dce 100644 --- a/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml +++ b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - coefficient - config: - sum: null + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml index 91e6d064e..847e984e7 100644 --- a/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - coefficient - config: - sum: null + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/coefficient/success.yaml b/manifests/outputs/builtins/coefficient/success.yaml index c560d0411..14a235709 100644 --- a/manifests/outputs/builtins/coefficient/success.yaml +++ b/manifests/outputs/builtins/coefficient/success.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,8 +59,6 @@ tree: child: pipeline: - coefficient - config: - sum: null inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index d1a5f2cfb..6945e6012 100644 --- a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -66,8 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 cloud/vendor: aws diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml index 79a4960a5..333d9e1ce 100644 --- a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml @@ -66,8 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 duration: 100 diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml index 098a0d1c6..37fab5816 100644 --- a/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml @@ -62,8 +62,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 cloud/vendor: aws diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml index 4db0b6b2f..842080a77 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml @@ -66,7 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml index bdaa3b6ad..32ca690d9 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml @@ -32,7 +32,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -65,7 +64,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml index e7e04b535..2f6f23dcf 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml @@ -33,7 +33,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -64,7 +63,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml index e8d31f661..f8bd9fdba 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml @@ -31,7 +31,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,7 +61,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml index f842a3756..ce268cd02 100644 --- a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml @@ -32,7 +32,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -65,8 +64,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml index 7ccf90ff6..28dc0b1a0 100644 --- a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml @@ -65,8 +65,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml index 249d2d3f4..f085664ff 100644 --- a/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml @@ -61,8 +61,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml index 8b98a5a27..b1149c505 100644 --- a/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml +++ b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - divide - config: - divide: null + compute: + - divide inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/divide/failure-missing-numerator.yaml b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml index 41ac4b1ea..6c714bf18 100644 --- a/manifests/outputs/builtins/divide/failure-missing-numerator.yaml +++ b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -61,7 +60,8 @@ tree: children: child: pipeline: - - divide + compute: + - divide config: divide: null defaults: diff --git a/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml index 2384df53b..25033c32c 100644 --- a/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml +++ b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml @@ -39,7 +39,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/builtins/divide/success.yaml b/manifests/outputs/builtins/divide/success.yaml index 3e43a85de..f7bab8b04 100644 --- a/manifests/outputs/builtins/divide/success.yaml +++ b/manifests/outputs/builtins/divide/success.yaml @@ -39,7 +39,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/builtins/exponent/success.yaml b/manifests/outputs/builtins/exponent/success.yaml index 7ad94e5bc..8586b57f6 100644 --- a/manifests/outputs/builtins/exponent/success.yaml +++ b/manifests/outputs/builtins/exponent/success.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -59,9 +58,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: null + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/interpolation/interpolation.yaml b/manifests/outputs/builtins/interpolation/interpolation.yaml index 8eff7b1f1..2569b5eb4 100644 --- a/manifests/outputs/builtins/interpolation/interpolation.yaml +++ b/manifests/outputs/builtins/interpolation/interpolation.yaml @@ -38,7 +38,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -69,7 +68,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/interpolation/success.yaml b/manifests/outputs/builtins/interpolation/success.yaml index cbfc92011..80def13d9 100644 --- a/manifests/outputs/builtins/interpolation/success.yaml +++ b/manifests/outputs/builtins/interpolation/success.yaml @@ -38,7 +38,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -69,7 +68,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml index fda4ddba7..272d9940e 100644 --- a/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml @@ -47,7 +47,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -80,5 +79,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml index cfb073fc6..e7c7e6180 100644 --- a/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml @@ -45,7 +45,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -78,5 +77,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml index 184e86230..86f4afd9e 100644 --- a/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml @@ -44,7 +44,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -77,5 +76,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/builtins/mock-observations/success.yaml b/manifests/outputs/builtins/mock-observations/success.yaml index e92b933bf..bcc5ee034 100644 --- a/manifests/outputs/builtins/mock-observations/success.yaml +++ b/manifests/outputs/builtins/mock-observations/success.yaml @@ -27,37 +27,26 @@ initialize: execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/mock-observations/success.yml -o - manifests/outputs/plugins/mock-observations/success + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/builtins/mock-observations/success.yml -o + manifests/outputs/builtins/mock-observations/success environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: "13.2" + os-version: '14.5' node-version: 18.14.2 - date-time: 2024-07-02T19:15:43.108Z (UTC) + date-time: 2024-08-02T15:04:18.262Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.16' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -77,210 +66,293 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: child: pipeline: - - mock-observations - inputs: null + observe: + - mock-observations + inputs: + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' outputs: - - timestamp: "2023-07-06T00:00:00.000Z" - - timestamp: "2023-07-06T00:00:00.000Z" + - timestamp: '2023-07-06T00:00:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 81 - memory/utilization: 63 - - timestamp: "2023-07-06T00:01:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:01:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 92 - memory/utilization: 63 - - timestamp: "2023-07-06T00:02:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:02:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 63 - memory/utilization: 95 - - timestamp: "2023-07-06T00:03:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:03:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 37 - memory/utilization: 13 - - timestamp: "2023-07-06T00:04:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:04:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 28 - memory/utilization: 50 - - timestamp: "2023-07-06T00:05:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:05:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 98 - memory/utilization: 29 - - timestamp: "2023-07-06T00:06:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:06:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 10 - memory/utilization: 93 - - timestamp: "2023-07-06T00:07:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:07:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 84 - memory/utilization: 37 - - timestamp: "2023-07-06T00:08:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:08:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 10 - memory/utilization: 30 - - timestamp: "2023-07-06T00:09:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:09:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: 50 - memory/utilization: 10 - - timestamp: "2023-07-06T00:00:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:00:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:00:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 97 - memory/utilization: 89 - - timestamp: "2023-07-06T00:01:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:01:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 61 - memory/utilization: 83 - - timestamp: "2023-07-06T00:02:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:02:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 86 - memory/utilization: 52 - - timestamp: "2023-07-06T00:03:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:03:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 17 - memory/utilization: 60 - - timestamp: "2023-07-06T00:04:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:04:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 53 - memory/utilization: 84 - - timestamp: "2023-07-06T00:05:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:05:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 38 - memory/utilization: 58 - - timestamp: "2023-07-06T00:06:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:06:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 4 - memory/utilization: 39 - - timestamp: "2023-07-06T00:07:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:07:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 89 - memory/utilization: 52 - - timestamp: "2023-07-06T00:08:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:08:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: 38 - memory/utilization: 16 - - timestamp: "2023-07-06T00:09:00.000Z" - cpu/utilization: "*" - memory/utilization: "*" - - timestamp: "2023-07-06T00:09:00.000Z" + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: "*" - memory/utilization: "*" + cpu/utilization: '*' + memory/utilization: '*' diff --git a/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml index 8c12556fb..80738e80c 100644 --- a/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml +++ b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml @@ -30,7 +30,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -63,9 +62,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml index b9b2470c3..359e25a06 100644 --- a/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml +++ b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/multiply/success.yaml b/manifests/outputs/builtins/multiply/success.yaml index c2925f183..1f93a9140 100644 --- a/manifests/outputs/builtins/multiply/success.yaml +++ b/manifests/outputs/builtins/multiply/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/regex/failure-missing-input-param.yaml b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml index 5ea08772c..cd578218f 100644 --- a/manifests/outputs/builtins/regex/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - regex - config: - regex: null + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml b/manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml deleted file mode 100644 index d07bc0484..000000000 --- a/manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: regex -description: physical processor doesn't match the regex expression -tags: null -initialize: - plugins: - regex: - method: Regex - path: builtin - global-config: - parameter: physical-processor - match: ^ - output: cpu/name -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/regex/failure-not-matching-with-regex.yml -o - manifests/outputs/plugins/regex/failure-not-matching-with-regex - environment: - if-version: 0.4.0 - os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T20:24:39.241Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - RegexMismatchError: `Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 - GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz` does - not match the /^/ regex expression -tree: - children: - child: - pipeline: - - regex - config: - regex: null - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - physical-processor: >- - Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® - E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz diff --git a/manifests/outputs/builtins/regex/success.yaml b/manifests/outputs/builtins/regex/success.yaml index 745e3cf71..22988eead 100644 --- a/manifests/outputs/builtins/regex/success.yaml +++ b/manifests/outputs/builtins/regex/success.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -59,9 +58,8 @@ tree: children: child: pipeline: - - regex - config: - regex: null + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -75,5 +73,4 @@ tree: Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz cpu/name: >- - Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® - E5-2673 v4 2.3 GHz, + Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz, Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz diff --git a/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml index b1784d516..f6af79041 100644 --- a/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -61,7 +60,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: fail time-reserved: 3600 diff --git a/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml index 5959d2b21..43f22163e 100644 --- a/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml @@ -25,7 +25,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -58,7 +57,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/builtins/sci-embodied/success.yaml b/manifests/outputs/builtins/sci-embodied/success.yaml index f7c4e8dc0..e91ab9212 100644 --- a/manifests/outputs/builtins/sci-embodied/success.yaml +++ b/manifests/outputs/builtins/sci-embodied/success.yaml @@ -24,7 +24,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -55,7 +54,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml index 463a855da..384fe14b1 100644 --- a/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml +++ b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml @@ -26,7 +26,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -57,7 +56,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci config: sci: functional-unit: 999 diff --git a/manifests/outputs/builtins/sci/failure-missing-input-param.yaml b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml index 0aed8267c..b0fbce2f8 100644 --- a/manifests/outputs/builtins/sci/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml @@ -28,7 +28,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -61,7 +60,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/sci/success.yml.yaml b/manifests/outputs/builtins/sci/success.yaml similarity index 98% rename from manifests/outputs/builtins/sci/success.yml.yaml rename to manifests/outputs/builtins/sci/success.yaml index e08d7e27e..9f36fc9bd 100644 --- a/manifests/outputs/builtins/sci/success.yml.yaml +++ b/manifests/outputs/builtins/sci/success.yaml @@ -26,7 +26,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -57,8 +56,8 @@ tree: children: child: pipeline: - - sci - config: null + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/shell/failure-invalid-command.yaml b/manifests/outputs/builtins/shell/failure-invalid-command.yaml index ad11fecad..97d8bb6af 100644 --- a/manifests/outputs/builtins/shell/failure-invalid-command.yaml +++ b/manifests/outputs/builtins/shell/failure-invalid-command.yaml @@ -27,7 +27,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,7 +59,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/outputs/builtins/shell/success.yaml b/manifests/outputs/builtins/shell/success.yaml index 0c53a27aa..b85cbfb95 100644 --- a/manifests/outputs/builtins/shell/success.yaml +++ b/manifests/outputs/builtins/shell/success.yaml @@ -26,7 +26,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -57,7 +56,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/outputs/builtins/subtract/success.yaml b/manifests/outputs/builtins/subtract/success.yaml index ebf2197c5..5eb0a7bd3 100644 --- a/manifests/outputs/builtins/subtract/success.yaml +++ b/manifests/outputs/builtins/subtract/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: null + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/sum/failure-missing-input-param.yaml b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml index 85838df45..e2aaf8158 100644 --- a/manifests/outputs/builtins/sum/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml @@ -30,7 +30,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -63,9 +62,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/sum/failure-missing-output-param.yaml b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml index 624bf3fdf..d7a4d9302 100644 --- a/manifests/outputs/builtins/sum/failure-missing-output-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -62,9 +61,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/sum/success.yaml b/manifests/outputs/builtins/sum/success.yaml index 77b0a3f2d..6b5b4d973 100644 --- a/manifests/outputs/builtins/sum/success.yaml +++ b/manifests/outputs/builtins/sum/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,9 +59,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml index 81a430414..bb5ae79fb 100644 --- a/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml +++ b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml @@ -62,8 +62,8 @@ tree: children: child: pipeline: - - time-sync - config: null + compute: + - time-sync inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/builtins/time-sync/success.yaml b/manifests/outputs/builtins/time-sync/success.yaml index 17e1a71d1..3ed64a0e3 100644 --- a/manifests/outputs/builtins/time-sync/success.yaml +++ b/manifests/outputs/builtins/time-sync/success.yaml @@ -29,7 +29,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -60,8 +59,8 @@ tree: children: child: pipeline: - - time-sync - config: null + compute: + - time-sync inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index 870be308a..11e7f5865 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -70,7 +70,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 4a3da85ae..0e308d509 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -70,7 +70,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index f870bb4d2..0fef8d756 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -3,7 +3,7 @@ description: Apply `horizontal` aggregation aggregation: metrics: "cpu/utilization": - method: sum + method: avg type: horizontal initialize: plugins: @@ -36,7 +36,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -67,7 +66,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index 0fd5b170a..dfd604c73 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -3,7 +3,7 @@ description: Apply `vertical` aggregation aggregation: metrics: "cpu/utilization": - method: sum + method: avg type: vertical initialize: plugins: @@ -36,7 +36,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -67,7 +66,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index 35ab21423..79736b9c0 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -3,7 +3,7 @@ description: Apply both `horizontal` and `vertical` aggregations aggregation: metrics: "cpu/utilization": - method: sum + method: avg type: both initialize: plugins: @@ -36,7 +36,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -67,7 +66,8 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: diff --git a/manifests/outputs/builtins/groupby/failure-invalid-config-group.yaml b/manifests/outputs/features/failure-invalid-config-group.yaml similarity index 91% rename from manifests/outputs/builtins/groupby/failure-invalid-config-group.yaml rename to manifests/outputs/features/failure-invalid-config-group.yaml index 22506aa2e..1ed31b06d 100644 --- a/manifests/outputs/builtins/groupby/failure-invalid-config-group.yaml +++ b/manifests/outputs/features/failure-invalid-config-group.yaml @@ -1,10 +1,7 @@ name: groupby description: failure when `config->group-by->group` is not an array initialize: - plugins: - group-by: - path: builtin - method: GroupBy + plugins: {} execution: status: fail command: >- @@ -24,7 +21,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -51,16 +47,14 @@ execution: - winston@3.11.0 - zod@3.22.4 error: >- - InputValidationError: "group" parameter is expected array, received string. + InputValidationError: "groups" parameter is expected array, received string. Error code: invalid_type. tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: cloud/region + regroup: + cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/builtins/groupby/failure-missing-cloud-instance-type.yaml b/manifests/outputs/features/failure-missing-cloud-instance-type.yaml similarity index 92% rename from manifests/outputs/builtins/groupby/failure-missing-cloud-instance-type.yaml rename to manifests/outputs/features/failure-missing-cloud-instance-type.yaml index e6b2ea8ea..741544461 100644 --- a/manifests/outputs/builtins/groupby/failure-missing-cloud-instance-type.yaml +++ b/manifests/outputs/features/failure-missing-cloud-instance-type.yaml @@ -1,10 +1,7 @@ name: groupby description: null initialize: - plugins: - group-by: - path: builtin - method: GroupBy + plugins: {} execution: status: fail command: >- @@ -24,7 +21,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -55,12 +51,9 @@ tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/pipelines/cloud-metadata-divide.yaml b/manifests/outputs/pipelines/cloud-metadata-divide.yaml index f85725c5c..02ff19b86 100644 --- a/manifests/outputs/pipelines/cloud-metadata-divide.yaml +++ b/manifests/outputs/pipelines/cloud-metadata-divide.yaml @@ -39,7 +39,6 @@ execution: - "@commitlint/cli@18.6.0" - "@commitlint/config-conventional@18.6.0" - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - "@types/jest@29.5.8" - "@types/js-yaml@4.0.9" @@ -70,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/pipelines/generics.yaml b/manifests/outputs/pipelines/generics.yaml index 4f724754e..613a65f83 100644 --- a/manifests/outputs/pipelines/generics.yaml +++ b/manifests/outputs/pipelines/generics.yaml @@ -122,15 +122,15 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - coefficient - - multiply - config: null + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - coefficient + - multiply defaults: cpu/thermal-design-power: 100 vcpus-allocated: 1 diff --git a/manifests/outputs/pipelines/instance-metadata.yaml b/manifests/outputs/pipelines/instance-metadata.yaml index fbff57e45..369d54c27 100644 --- a/manifests/outputs/pipelines/instance-metadata.yaml +++ b/manifests/outputs/pipelines/instance-metadata.yaml @@ -67,8 +67,9 @@ tree: children: child: pipeline: - - cloud-instance-metadata - - extract-processor-name + compute: + - cloud-instance-metadata + - extract-processor-name inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/pipelines/mock-obs-time-sync.yaml b/manifests/outputs/pipelines/mock-obs-time-sync.yaml index d1340e974..04eb881ec 100644 --- a/manifests/outputs/pipelines/mock-obs-time-sync.yaml +++ b/manifests/outputs/pipelines/mock-obs-time-sync.yaml @@ -169,39 +169,101 @@ tree: children: child-1: pipeline: - - mock-observations - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - time-sync + observe: + - mock-observations + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - time-sync defaults: cpu/thermal-design-power: 100 vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - duration: 1 - cpu/utilization: 10 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 20 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:01:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:02:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:03:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 15 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:04:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:05:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 15 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:06:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:07:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:08:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:09:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 outputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml index 13980cecd..83ed612d5 100644 --- a/manifests/outputs/pipelines/nesting.yaml +++ b/manifests/outputs/pipelines/nesting.yaml @@ -243,17 +243,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 diff --git a/manifests/outputs/pipelines/pipeline-teads-sci.yaml b/manifests/outputs/pipelines/pipeline-teads-sci.yaml index fc8e6d0f8..1c2a9d005 100644 --- a/manifests/outputs/pipelines/pipeline-teads-sci.yaml +++ b/manifests/outputs/pipelines/pipeline-teads-sci.yaml @@ -83,14 +83,6 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon - time-sync: - path: builtin - method: TimeSync - global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' - interval: 5 - allow-padding: true execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -139,16 +131,17 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - sci config: null defaults: cpu/thermal-design-power: 100 diff --git a/manifests/outputs/pipelines/sci.yaml b/manifests/outputs/pipelines/sci.yaml index 5c5f0d822..c4fa641c2 100644 --- a/manifests/outputs/pipelines/sci.yaml +++ b/manifests/outputs/pipelines/sci.yaml @@ -138,18 +138,18 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sum-energy-components - - embodied-carbon - - operational-carbon - - sum-carbon - - sci - config: null + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sum-energy-components + - embodied-carbon + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/outputs/pipelines/teads-curve.yaml b/manifests/outputs/pipelines/teads-curve.yaml index 3f5d2d323..26885e1c7 100644 --- a/manifests/outputs/pipelines/teads-curve.yaml +++ b/manifests/outputs/pipelines/teads-curve.yaml @@ -105,12 +105,13 @@ tree: children: child: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio defaults: thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/outputs/pipelines/zeros.yaml b/manifests/outputs/pipelines/zeros.yaml index 523728097..3efd3c0fb 100644 --- a/manifests/outputs/pipelines/zeros.yaml +++ b/manifests/outputs/pipelines/zeros.yaml @@ -155,22 +155,21 @@ tree: children: child-1: pipeline: - - sum-zero-and-one - - sum-zero-and-zero - - subtract-one-and-zero - - subtract-zero-and-zero - - subtract-zero-and-one - - coefficient-one-times-zero - - coefficient-zero-times-one - - coefficient-zero-times-zero - - multiply-one-times-zero - - multiply-zero-times-one - - exponent-one-to-zero - - exponent-zero-to-one - - exponent-zero-to-zero - - sci - config: null - defaults: null + compute: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 diff --git a/src/__tests__/if-run/builtins/group-by.test.ts b/src/__tests__/if-run/builtins/group-by.test.ts deleted file mode 100644 index b59da6836..000000000 --- a/src/__tests__/if-run/builtins/group-by.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import {ERRORS} from '@grnsft/if-core/utils'; - -import {GroupBy} from '../../../if-run/builtins/group-by'; - -import {STRINGS} from '../../../if-run/config'; - -const {InvalidGroupingError, InputValidationError, GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, INVALID_GROUP_BY} = STRINGS; - -describe('builtins/group-by: ', () => { - describe('GroupBy: ', () => { - const plugin = GroupBy(); - - describe('init GroupBy: ', () => { - it('initalizes object with properties.', async () => { - expect(plugin).toHaveProperty('metadata'); - expect(plugin).toHaveProperty('execute'); - }); - }); - - describe('execute(): ', () => { - it('groups inputs correctly.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - const config = { - group: ['region', 'cloud/instance-type'], - }; - - const expectedOutput = { - 'uk-west': { - children: { - A1: { - inputs: [ - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T00:00', - }, - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T05:00', - }, - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T10:00', - }, - ], - }, - }, - }, - }; - - const result = plugin.execute(inputs, config); - expect(result).toEqual(expectedOutput); - }); - - it('throws an error when config is not provided.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - - const config = undefined; - - expect.assertions(2); - try { - plugin.execute(inputs, config!); - } catch (error) { - expect(error).toBeInstanceOf(GlobalConfigError); - expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); - } - }); - - it('throws an error if `group` is an empty array.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - const config = { - group: ['region', 'cloud/instance-type'], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InvalidGroupingError); - expect(error).toEqual( - new InvalidGroupingError('Invalid group region.') - ); - } - }); - - it('throws an error if group type is missing from the input.', () => { - const inputs = [ - {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, - ]; - const config = { - group: [], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError( - '"group" parameter is array must contain at least 1 element(s). Error code: too_small.' - ) - ); - } - }); - - it('throws an error if input does not have required group type.', () => { - const inputs = [ - {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, - ]; - const config = { - group: ['region', 'cloud/instance-type', 'unknown'], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InvalidGroupingError); - expect(error).toEqual( - new InvalidGroupingError(INVALID_GROUP_BY(config.group[2])) - ); - } - }); - }); - }); -}); diff --git a/src/__tests__/if-run/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts index 4ed48b5a5..912793381 100644 --- a/src/__tests__/if-run/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -19,10 +19,57 @@ describe('lib/compute: ', () => { kind: 'execute', }, }); - const mockGroupByPlugin = () => ({ - execute: (inputs: any) => ({children: inputs}), + const mockObservePlugin = () => ({ + execute: () => [ + {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, + {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, + ], metadata: { - kind: 'groupby', + kind: 'execute', + }, + }); + const mockObservePluginTimeSync = () => ({ + execute: () => [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 60, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:01:00.000Z', + duration: 60, + 'cpu/utilization': 40, + }, + ], + metadata: { + kind: 'execute', + }, + }); + const mockTimeSync = () => ({ + execute: () => [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 30, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:00:30.000Z', + duration: 30, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:01:00.000Z', + duration: 30, + 'cpu/utilization': 40, + }, + { + timestamp: '2023-12-12T00:01:30.000Z', + duration: 30, + 'cpu/utilization': 40, + }, + ], + metadata: { + kind: 'execute', }, }); /** @@ -41,22 +88,11 @@ describe('lib/compute: ', () => { }, }, }, - pluginStorage: pluginStorage().set('mock', mockExecutePlugin()), - }; - const params: ComputeParams = { - // @ts-ignore - context: { - name: 'mock-name', - initialize: { - plugins: { - mock: { - path: 'mockavizta', - method: 'Mockavizta', - }, - }, - }, - }, - pluginStorage: pluginStorage().set('mock', mockGroupByPlugin()), + pluginStorage: pluginStorage() + .set('mock', mockExecutePlugin()) + .set('mock-observe', mockObservePlugin()) + .set('mock-observe-time-sync', mockObservePluginTimeSync()) + .set('time-sync', mockTimeSync()), }; describe('compute(): ', () => { @@ -64,7 +100,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, inputs: [ {timestamp: 'mock-timestamp-1', duration: 10}, {timestamp: 'mock-timestamp-2', duration: 10}, @@ -85,7 +121,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {regroup: ['duration']}, inputs: [ {timestamp: 'mock-timestamp-1', duration: 10}, {timestamp: 'mock-timestamp-2', duration: 10}, @@ -93,19 +129,24 @@ describe('lib/compute: ', () => { }, }, }; - const response = await compute(tree, params); - const expectedResult = mockGroupByPlugin().execute( - tree.children.mockChild.inputs - ); + const response = await compute(tree, paramsExecute); + const expectedResponse = { + '10': { + inputs: [ + {duration: 10, timestamp: 'mock-timestamp-1'}, + {duration: 10, timestamp: 'mock-timestamp-2'}, + ], + }, + }; - expect(response.children.mockChild.children).toEqual(expectedResult); + expect(response.children.mockChild.children).toEqual(expectedResponse); }); it('computes simple tree with defaults and execute plugin.', async () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -132,7 +173,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild1: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -144,7 +185,7 @@ describe('lib/compute: ', () => { mockChild2: { children: { mockChild21: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -188,16 +229,17 @@ describe('lib/compute: ', () => { }, }; const response = await compute(tree, paramsExecute); - const expectedResult: any[] = []; - expect(response.children.mockChild.outputs).toEqual(expectedResult); + expect(response.children.mockChild.outputs).toBeUndefined(); }); it('computes simple tree with defaults and no inputs with execue plugin.', async () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: { + compute: ['mock'], + }, defaults: { carbon: 10, }, @@ -215,7 +257,9 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: { + compute: ['mock'], + }, config: { 'cpu/name': 'Intel CPU', }, @@ -234,4 +278,22 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.outputs).toEqual(expectedResult); }); }); + + it('computes simple tree with observe plugin.', async () => { + const tree = { + children: { + mockChild: { + pipeline: {observe: ['mock-observe']}, + }, + }, + }; + + const response = await compute(tree, paramsExecute); + const expectedResult = [ + {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, + {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, + ]; + + expect(response.children.mockChild.inputs).toEqual(expectedResult); + }); }); diff --git a/src/__tests__/if-run/lib/initialize.test.ts b/src/__tests__/if-run/lib/initialize.test.ts index f00142260..a1eda8924 100644 --- a/src/__tests__/if-run/lib/initialize.test.ts +++ b/src/__tests__/if-run/lib/initialize.test.ts @@ -18,7 +18,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {initialize} from '../../../if-run/lib/initialize'; import {STRINGS} from '../../../if-run/config'; -import {GlobalPlugins} from '../../../common/types/manifest'; const { MissingPluginPathError, @@ -30,8 +29,9 @@ const {MISSING_METHOD, MISSING_PATH, INVALID_MODULE_PATH} = STRINGS; describe('lib/initalize: ', () => { describe('initalize(): ', () => { it('creates instance with get and set methods.', async () => { - const plugins = {}; - const response = await initialize(plugins); + const context = {initialize: {plugins: {}}}; + // @ts-ignore + const response = await initialize(context); expect(response).toHaveProperty('get'); expect(response).toHaveProperty('set'); @@ -40,15 +40,20 @@ describe('lib/initalize: ', () => { }); it('checks if plugin is initalized, warning is logged and plugin has execute and metadata props.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'mockavizta', - method: 'Mockavizta', + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + method: 'Mockavizta', + }, + }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); @@ -56,36 +61,45 @@ describe('lib/initalize: ', () => { }); it('checks if plugin is initalized with global config and has execute and metadata.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'mockavizta', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('throws error if plugin does not have path property.', async () => { - const plugins: GlobalPlugins = { - // @ts-ignore - mockavizta: { - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error) { expect(error).toBeInstanceOf(MissingPluginPathError); @@ -96,18 +110,22 @@ describe('lib/initalize: ', () => { }); it('throws error if plugin does not have path property.', async () => { - const plugins: GlobalPlugins = { - // @ts-ignore - mockavizta: { - path: 'mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error) { expect(error).toBeInstanceOf(MissingPluginMethodError); @@ -118,59 +136,74 @@ describe('lib/initalize: ', () => { }); it('checks if builtin plugin is initalized.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'builtin', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'builtin', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('checks if github plugin is initalized.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'https://github.com/mockavizta', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'https://github.com/mockavizta', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('throws error if plugin path is invalid.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'failing-mock', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'failing-mock', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error: any) { expect(error).toBeInstanceOf(PluginInitializationError); expect(error.message).toEqual( INVALID_MODULE_PATH( - plugins.mockavizta.path, + context.initialize.plugins.mockavizta.path, new Error( "Cannot find module 'failing-mock' from 'src/if-run/lib/initialize.ts'" ) diff --git a/src/__tests__/if-run/lib/regroup.test.ts b/src/__tests__/if-run/lib/regroup.test.ts new file mode 100644 index 000000000..9f0b373ff --- /dev/null +++ b/src/__tests__/if-run/lib/regroup.test.ts @@ -0,0 +1,161 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {STRINGS} from '../../../if-run/config'; +import {Regroup} from '../../../if-run/lib/regroup'; + +const {InvalidGroupingError, InputValidationError} = ERRORS; +const {INVALID_GROUP_KEY} = STRINGS; + +describe('lib/regroup: ', () => { + describe('Regroup: ', () => { + it('groups inputs correctly.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + const groups = ['region', 'cloud/instance-type']; + + const expectedOutput = { + 'uk-west': { + children: { + A1: { + inputs: [ + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T00:00', + }, + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T05:00', + }, + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T10:00', + }, + ], + }, + }, + }, + }; + + const result = Regroup(inputs, groups); + expect(result).toEqual(expectedOutput); + }); + + it('throws an error when groups is not provided.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + + const groups = undefined; + + expect.assertions(2); + try { + Regroup(inputs, groups!); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"groups" parameter is required. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error if `group` is an empty array.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + const groups = ['region', 'cloud/instance-type']; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InvalidGroupingError); + expect(error).toEqual( + new InvalidGroupingError('Invalid group region.') + ); + } + }); + + it('throws an error if group type is missing from the input.', () => { + const inputs = [ + {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, + ]; + const groups: any[] = []; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"groups" parameter is regroup phase is not an array or should contain at least one key.. Error code: too_small.' + ) + ); + } + }); + + it('throws an error if input does not have required group type.', () => { + const inputs = [ + {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, + ]; + const groups = ['region', 'cloud/instance-type', 'unknown']; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InvalidGroupingError); + expect(error).toEqual( + new InvalidGroupingError(INVALID_GROUP_KEY(groups[2])) + ); + } + }); + }); +}); diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index 9f2d4c495..95d9efeff 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -24,6 +24,36 @@ export const atLeastOneDefined = ( export const allDefined = (obj: Record) => Object.values(obj).every(v => v !== undefined); +/** + * Schema for parameter metadata. + */ +const parameterMetadataSchema = z + .object({ + inputs: z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': z.string(), + }) + ) + .optional() + .nullable(), + outputs: z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': z.string(), + }) + ) + .optional() + .nullable(), + }) + .optional(); + /** * Validation schema for manifests. */ @@ -54,37 +84,14 @@ export const manifestSchema = z.object({ initialize: z.object({ plugins: z.record( z.string(), - z.object({ - path: z.string(), - method: z.string(), - 'global-config': z.record(z.string(), z.any()).optional(), - 'parameter-metadata': z - .object({ - inputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), - outputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), - }) - .optional(), - }) + z + .object({ + path: z.string(), + method: z.string(), + 'global-config': z.record(z.string(), z.any()).optional(), + 'parameter-metadata': parameterMetadataSchema, + }) + .optional() ), }), execution: z diff --git a/src/if-env/config/config.ts b/src/if-env/config/config.ts index cc603d6de..f3f21b0c8 100644 --- a/src/if-env/config/config.ts +++ b/src/if-env/config/config.ts @@ -7,36 +7,34 @@ import {STRINGS} from '../../common/config'; const {DISCLAIMER_MESSAGE} = STRINGS; export const CONFIG = { - IF_ENV: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the manifest file]', - }, - install: { - type: Boolean, - optional: true, - alias: 'i', - description: '[command to install package.json]', - }, - cwd: { - type: Boolean, - optional: true, - alias: 'c', - description: - '[command to generate the package.json in the command working directory]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - }, + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + install: { + type: Boolean, + optional: true, + alias: 'i', + description: '[command to install package.json]', + }, + cwd: { + type: Boolean, + optional: true, + alias: 'c', + description: + '[command to generate the package.json in the command working directory]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, }; diff --git a/src/if-env/util/args.ts b/src/if-env/util/args.ts index 940fa7b7a..525ae14f6 100644 --- a/src/if-env/util/args.ts +++ b/src/if-env/util/args.ts @@ -12,7 +12,7 @@ import {STRINGS as COMMON_STRINGS} from '../../common/config'; import {IFEnvArgs} from '../types/process-args'; const {ParseCliParamsError, CliSourceFileError} = ERRORS; -const {IF_ENV} = CONFIG; +const {ARGS, HELP} = CONFIG; const {MANIFEST_NOT_FOUND, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; /** @@ -20,7 +20,7 @@ const {MANIFEST_NOT_FOUND, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; */ const validateAndParseIfEnvArgs = () => { try { - return parse(IF_ENV.ARGS, IF_ENV.HELP); + return parse(ARGS, HELP); } catch (error) { if (error instanceof Error) { throw new ParseCliParamsError(error.message); diff --git a/src/if-run/builtins/group-by.ts b/src/if-run/builtins/group-by.ts deleted file mode 100644 index c8694d4cc..000000000 --- a/src/if-run/builtins/group-by.ts +++ /dev/null @@ -1,95 +0,0 @@ -import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; -import { - GroupByPlugin, - PluginParams, - GroupByConfig, -} from '@grnsft/if-core/types'; - -import {STRINGS} from '../config'; - -import {validate} from '../../common/util/validations'; - -const {InvalidGroupingError, GlobalConfigError} = ERRORS; - -const {INVALID_GROUP_BY, MISSING_GLOBAL_CONFIG} = STRINGS; - -/** - * Plugin for inputs grouping. - */ -export const GroupBy = (): GroupByPlugin => { - const metadata = { - kind: 'groupby', - }; - - /** - * Creates structure to insert inputs by groups. - */ - const appendGroup = ( - value: PluginParams, - object: any, - groups: string[] - ): any => { - if (groups.length > 0) { - const group = groups.shift() as string; - - object.children = object.children ?? {}; - object.children[group] = object.children[group] ?? {}; - - if (groups.length === 0) { - if ( - object.children[group].inputs && - object.children[group].inputs.length > 0 - ) { - object.children[group].inputs.push(value); - } else { - object.children[group].inputs = [value]; - } - } - - appendGroup(value, object.children[group], groups); - } - - return object; - }; - - /** - * Interates over inputs, grabs config-group types values for each one. - * Based on grouping types, initializes the structure grouped structure. - */ - const execute = (inputs: PluginParams[], config: GroupByConfig) => - inputs.reduce((acc, input) => { - const validatedConfig = validateConfig(config); - const groups = validatedConfig.group.map(groupType => { - if (!input[groupType]) { - throw new InvalidGroupingError(INVALID_GROUP_BY(groupType)); - } - - return input[groupType]; - }); - - acc = { - ...acc, - ...appendGroup(input, acc, groups), - }; - - return acc; - }, {} as any).children; - - /** - * Validates config parameter. - */ - const validateConfig = (config: GroupByConfig) => { - if (!config) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); - } - - const schema = z.object({ - group: z.array(z.string()).min(1), - }); - - return validate>(schema, config); - }; - - return {metadata, execute}; -}; diff --git a/src/if-run/builtins/index.ts b/src/if-run/builtins/index.ts index d22491aca..36a414c95 100644 --- a/src/if-run/builtins/index.ts +++ b/src/if-run/builtins/index.ts @@ -1,5 +1,3 @@ -export {GroupBy} from './group-by'; -export {TimeSync} from './time-sync'; export {Interpolation} from './interpolation'; export {MockObservations} from './mock-observations'; export {Divide} from './divide'; @@ -14,3 +12,4 @@ export {CSVLookup} from './csv-lookup'; export {Shell} from './shell'; export {Regex} from './regex'; export {Copy} from './copy-param'; +export {TimeSync} from './time-sync'; diff --git a/src/if-run/builtins/sci-embodied/index.ts b/src/if-run/builtins/sci-embodied/index.ts index 35e012b72..e8b69cd2b 100644 --- a/src/if-run/builtins/sci-embodied/index.ts +++ b/src/if-run/builtins/sci-embodied/index.ts @@ -35,6 +35,7 @@ export const SciEmbodied = ( 'resources-total': { description: 'total resources available', unit: 'count', + 'aggregation-method': 'none', }, 'vcpus-allocated': { description: 'number of vcpus allocated to particular resource', diff --git a/src/if-run/builtins/time-sync.ts b/src/if-run/builtins/time-sync.ts index 9d589c491..e9f4ee1bc 100644 --- a/src/if-run/builtins/time-sync.ts +++ b/src/if-run/builtins/time-sync.ts @@ -36,6 +36,21 @@ const { INVALID_DATETIME, } = STRINGS; +/** + * Time synchronization plugin converted into framework integrated tool. + * It can't be requested in `initialize.plugins` section anymore. Instead describe configuration in context. + * @example + * ```yaml + * name: time-sync + * description: sample in time sync lib + * tags: sample, time, sync + * time-sync: + * start-time: '2023-12-12T00:00:00.000Z' + * end-time: '2023-12-12T00:01:00.000Z' + * interval: 5 + * allow-padding: true + * ``` + */ export const TimeSync = ( globalConfig: TimeNormalizerConfig, parametersMetadata: PluginParametersMetadata @@ -131,14 +146,16 @@ export const TimeSync = ( return resampleInputs(sortedInputs, timeParams) as PluginParams[]; }; + /** + * Dates are passed to `time-sync` both in ISO 8601 format + * and as a Date object (from the deserialization of a YAML file). + * If the YAML parser fails to identify as a date, it passes as a string. + */ const parseDate = (date: Date | string) => { if (!date) { return DateTime.invalid('Invalid date'); } - // dates are passed to time-sync.ts both in ISO 8601 format - // and as a Date object (from the deserialization of a YAML file) - // if the YAML parser fails to identify as a date, it passes as a string if (isDate(date)) { return DateTime.fromJSDate(date); } diff --git a/src/if-run/config/config.ts b/src/if-run/config/config.ts index 91f366543..9bd39d89f 100644 --- a/src/if-run/config/config.ts +++ b/src/if-run/config/config.ts @@ -7,49 +7,68 @@ import {IfRunArgs} from '../types/process-args'; const {DISCLAIMER_MESSAGE} = STRINGS; export const CONFIG = { - IF_RUN: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the input file]', - }, - output: { - type: String, - optional: true, - alias: 'o', - description: '[path to the output file]', - }, - 'no-output': { - type: Boolean, - optional: true, - alias: 'n', - description: '[prevent output to the console]', - }, - help: { - type: Boolean, - optional: true, - alias: 'h', - description: '[prints out the above help instruction]', - }, - debug: { - type: Boolean, - optional: true, - alias: 'd', - description: '[prints out debug logs to the console]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - }, + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the input file]', + }, + output: { + type: String, + optional: true, + alias: 'o', + description: '[path to the output file]', + }, + 'override-params': { + type: String, + optional: true, + alias: 'p', + description: '[path to a parameter file that overrides our defaults]', + }, + 'no-output': { + type: Boolean, + optional: true, + alias: 'n', + description: '[prevent output to the console]', + }, + help: { + type: Boolean, + optional: true, + alias: 'h', + description: '[prints out the above help instruction]', + }, + debug: { + type: Boolean, + optional: true, + alias: 'd', + description: '[prints out debug logs to the console]', + }, + observe: { + type: Boolean, + optional: true, + description: '[executes only observe pipeline of the manifest]', + }, + regroup: { + type: Boolean, + optional: true, + description: '[executes only regroup pipeline of the manifest]', + }, + compute: { + type: Boolean, + optional: true, + description: '[executes only compute pipeline of the manifest]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, GITHUB_PATH: 'https://github.com', NATIVE_PLUGIN: 'if-plugins', AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index cdbc7c8c2..24835f8ed 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -23,7 +23,9 @@ export const STRINGS = { `Aggregation is not possible for given ${metric} since method is 'none'.`, METRIC_MISSING: (metric: string, index: number) => `Aggregation metric ${metric} is not found in inputs[${index}].`, - INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`, + INVALID_GROUP_KEY: (key: string) => `Invalid group ${key}.`, + REGROUP_ERROR: + 'Regroup phase is not an array or should contain at least one key.', INVALID_EXHAUST_PLUGIN: (pluginName: string) => `Invalid exhaust plugin: ${pluginName}.`, UNKNOWN_PARAM: (name: string) => diff --git a/src/if-run/index.ts b/src/if-run/index.ts index 5c1084cf9..9b8d86f6c 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -13,7 +13,7 @@ import {exhaust} from './lib/exhaust'; import {explain} from './lib/explain'; import {parseIfRunProcessArgs} from './util/args'; -import {andHandle, storeAggregationMethods} from './util/helpers'; +import {andHandle} from './util/helpers'; import {STRINGS} from './config'; @@ -22,7 +22,14 @@ const {DISCLAIMER_MESSAGE} = COMMON_STRINGS; const impactEngine = async () => { const options = parseIfRunProcessArgs(); - const {inputPath, outputOptions, debug} = options; + const { + inputPath, + outputOptions, + debug, + observe, + regroup, + compute: computeFlag, + } = options; debugLogger.overrideConsoleMethods(!!debug); @@ -34,17 +41,21 @@ const impactEngine = async () => { try { const {tree, ...context} = validateManifest(envManifest); - const pluginStorage = await initialize(context.initialize.plugins); if (context.aggregation) { storeAggregationMetrics({metrics: context.aggregation?.metrics}); } - storeAggregationMethods(context.initialize.plugins, pluginStorage); + const pluginStorage = await initialize(context); + const computedTree = await compute(tree, { + context, + pluginStorage, + observe, + regroup, + compute: computeFlag, + }); - const computedTree = await compute(tree, {context, pluginStorage}); const aggregatedTree = aggregate(computedTree, context.aggregation); - envManifest.explainer && (context.explain = explain()); await exhaust(aggregatedTree, context, outputOptions); diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts index 02cc8e0e4..41a153448 100644 --- a/src/if-run/lib/compute.ts +++ b/src/if-run/lib/compute.ts @@ -1,20 +1,22 @@ -import {PluginParams, GroupByConfig} from '@grnsft/if-core/types'; +import {PluginParams} from '@grnsft/if-core/types'; + +import {Regroup} from './regroup'; +import {addExplainData} from './explain'; import {mergeObjects} from '../util/helpers'; import {debugLogger} from '../../common/util/debug-logger'; import {STRINGS} from '../config/strings'; -import {isExecute, isGroupBy} from '../types/interface'; -import {ComputeParams, Node, Params} from '../types/compute'; -import {addExplainData} from './explain'; +import {ComputeParams, Node, PhasedPipeline} from '../types/compute'; +import {isExecute} from '../types/interface'; -const {MERGING_DEFAULTS_WITH_INPUT_DATA, COMPUTING_PIPELINE_FOR_NODE} = STRINGS; +const {MERGING_DEFAULTS_WITH_INPUT_DATA} = STRINGS; /** * Traverses all child nodes based on children grouping. */ -const traverse = async (children: any, params: Params) => { +const traverse = async (children: any, params: ComputeParams) => { for (const child in children) { await computeNode(children[child], params); } @@ -46,15 +48,21 @@ const mergeDefaults = ( * 2. If it's a grouping node, then first of all computes all it's children. * This is doing a depth first traversal. * 3. Otherwise merges the defaults into the inputs. - * 4. Goes through the pipeline plugins, by checking if it's `execute` plugin. If so sets outputs. - * If is a `groupby` plugin, it will return child components rather than outputs. - * 5. Since after `groupby`, there are new child components, then computes them. - * Note: `pipeline` now equals the remaining plu.gins to apply to each child + * 4. Iterates over pipeline phases (observe, regroup, compute). + * 5. Observe plugins are used to insert input values + * (isolated execution can be achived by passing `--observe` flag to CLI command). + * 6. Regroup plugin is used to group existing inputs by criteria + * (isolated execution can be achived by passing `--regroup` flag to CLI command). + * Since it creates new children for node, existing inputs and outputs are dropped and recursive traversal is called + * for newbord child component. + * 7. Compute plugins are used to do desired computations and appending the result to outputs + * (isolated execution can be achived by passing `--compute` flag to CLI command). */ -const computeNode = async (node: Node, params: Params): Promise => { - const pipeline = (node.pipeline || params.pipeline) as string[]; +const computeNode = async (node: Node, params: ComputeParams): Promise => { + const pipeline = node.pipeline || (params.pipeline as PhasedPipeline); const config = node.config || params.config; const defaults = node.defaults || params.defaults; + const noFlags = !params.observe && !params.regroup && !params.compute; if (node.children) { return traverse(node.children, { @@ -67,50 +75,67 @@ const computeNode = async (node: Node, params: Params): Promise => { let inputStorage = structuredClone(node.inputs) as PluginParams[]; inputStorage = mergeDefaults(inputStorage, defaults); - const pipelineCopy = structuredClone(pipeline); - - while (pipelineCopy.length !== 0) { - const pluginName = pipelineCopy.shift() as string; - const plugin = params.pluginStorage.get(pluginName); - const nodeConfig = config && config[pluginName]; - - console.debug(COMPUTING_PIPELINE_FOR_NODE(pluginName)); - debugLogger.setExecutingPluginName(pluginName); - - if (isExecute(plugin)) { - inputStorage = await plugin.execute(inputStorage, nodeConfig); - - if (params.context.explainer) { - addExplainData({ - pluginName, - metadata: plugin.metadata, - pluginData: params.context.initialize.plugins[pluginName], - }); + const pipelineCopy = structuredClone(pipeline) || {}; + + /** + * If iteration is on observe pipeline, then executes observe plugins and sets the inputs value. + */ + if ((noFlags || params.observe) && pipelineCopy.observe) { + while (pipelineCopy.observe.length !== 0) { + const pluginName = pipelineCopy.observe.shift() as string; + const plugin = params.pluginStorage.get(pluginName); + const nodeConfig = config && config[pluginName]; + + if (isExecute(plugin)) { + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.inputs = inputStorage; + + if (params.context.explainer) { + addExplainData({ + pluginName, + metadata: plugin.metadata, + pluginData: params.context.initialize!.plugins[pluginName], + }); + } + + node.outputs = inputStorage; } - - debugLogger.setExecutingPluginName(); - - node.outputs = inputStorage; } + } - if (isGroupBy(plugin)) { - node.children = await plugin.execute( - inputStorage, - nodeConfig as GroupByConfig - ); - delete node.inputs; - delete node.outputs; - - await traverse(node.children, { - ...params, - pipeline: pipelineCopy, - defaults, - config, - }); + /** + * If regroup is requested, execute regroup strategy, delete child's inputs, outputs and empty regroup array. + */ + if ((noFlags || params.regroup) && pipelineCopy.regroup) { + node.children = Regroup(inputStorage, pipelineCopy.regroup); + delete node.inputs; + delete node.outputs; - debugLogger.setExecutingPluginName(); + return traverse(node.children, { + ...params, + pipeline: { + ...pipelineCopy, + regroup: undefined, + }, + defaults, + config, + }); + } - break; + /** + * If iteration is on compute plugin, then executes compute plugins and sets the outputs value. + */ + if ((noFlags || params.compute) && pipelineCopy.compute) { + while (pipelineCopy.compute.length !== 0) { + const pluginName = pipelineCopy.compute.shift() as string; + const plugin = params.pluginStorage.get(pluginName); + const nodeConfig = config && config[pluginName]; + + if (isExecute(plugin)) { + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.outputs = inputStorage; + debugLogger.setExecutingPluginName(); + } } } }; diff --git a/src/if-run/lib/explain.ts b/src/if-run/lib/explain.ts index add1a02b6..dce49b75a 100644 --- a/src/if-run/lib/explain.ts +++ b/src/if-run/lib/explain.ts @@ -30,8 +30,8 @@ export const addExplainData = (params: ExplainParams) => { const {pluginName, pluginData, metadata} = params; const plugin = { [pluginName]: { - method: pluginData.method, - path: pluginData.path, + method: pluginData!.method, + path: pluginData!.path, inputs: metadata?.inputs || 'undefined', outputs: metadata?.outputs || 'undefined', }, diff --git a/src/if-run/lib/initialize.ts b/src/if-run/lib/initialize.ts index 5adcc2b18..f047be117 100644 --- a/src/if-run/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -9,8 +9,9 @@ import {pluginStorage} from '../util/plugin-storage'; import {CONFIG, STRINGS} from '../config'; import {PluginInterface} from '../types/interface'; -import {GlobalPlugins, PluginOptions} from '../../common/types/manifest'; +import {Context, PluginOptions} from '../../common/types/manifest'; import {PluginStorageInterface} from '../types/plugin-storage'; +import {storeAggregationMetrics} from './aggregate'; const { PluginInitializationError, @@ -84,7 +85,7 @@ const initPlugin = async ( path, 'global-config': globalConfig, 'parameter-metadata': parameterMetadata, - } = initPluginParams; + } = initPluginParams!; console.debug(INITIALIZING_PLUGIN(method)); @@ -105,14 +106,22 @@ const initPlugin = async ( * Registers all plugins from `manifest`.`initialize` property. */ export const initialize = async ( - plugins: GlobalPlugins + context: Context ): Promise => { console.debug(INITIALIZING_PLUGINS); - + const {plugins} = context.initialize; const storage = pluginStorage(); for await (const pluginName of Object.keys(plugins)) { const plugin = await initPlugin(plugins[pluginName]); + const parameters = {...plugin.metadata.inputs, ...plugin.metadata.outputs}; + + Object.keys(parameters).forEach(key => { + storeAggregationMetrics({ + metrics: {[key]: {method: parameters[key]['aggregation-method']}}, + }); + }); + storage.set(pluginName, plugin); } diff --git a/src/if-run/lib/regroup.ts b/src/if-run/lib/regroup.ts new file mode 100644 index 000000000..eba87a4d0 --- /dev/null +++ b/src/if-run/lib/regroup.ts @@ -0,0 +1,79 @@ +import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginParams} from '@grnsft/if-core/types'; + +import {validate} from '../../common/util/validations'; + +import {STRINGS} from '../config'; + +const {InvalidGroupingError} = ERRORS; + +const {INVALID_GROUP_KEY, REGROUP_ERROR} = STRINGS; + +/** + * Grouping strategy. + */ +export const Regroup = (inputs: PluginParams[], groups: string[]) => { + /** + * Creates structure to insert inputs by groups. + */ + const appendGroup = (value: PluginParams, object: any, groups: string[]) => { + if (groups.length > 0) { + const group = groups.shift() as string; + + object.children = object.children ?? {}; + object.children[group] = object.children[group] ?? {}; + + if (groups.length === 0) { + if ( + object.children[group].inputs && + object.children[group].inputs.length > 0 + ) { + object.children[group].inputs.push(value); + } else { + object.children[group].inputs = [value]; + } + } + + appendGroup(value, object.children[group], groups); + } + + return object; + }; + + /** + * Validates groups array. + */ + const validateGroups = (groups: string[]) => { + const inputData = {groups}; + const validationSchema = z.record( + z.string(), + z.array(z.string()).min(1, REGROUP_ERROR) + ); + validate(validationSchema, inputData); + + return groups; + }; + + /** + * Interates over inputs, grabs group values for each one. + * Based on grouping, initializes the structure. + */ + return inputs.reduce((acc, input) => { + const validtedGroups = validateGroups(groups); + const groupsWithData = validtedGroups.map(groupType => { + if (!input[groupType]) { + throw new InvalidGroupingError(INVALID_GROUP_KEY(groupType)); + } + + return input[groupType]; + }); + + acc = { + ...acc, + ...appendGroup(input, acc, groupsWithData), + }; + + return acc; + }, {} as any).children; +}; diff --git a/src/if-run/types/compute.ts b/src/if-run/types/compute.ts index bfe294241..6777bf553 100644 --- a/src/if-run/types/compute.ts +++ b/src/if-run/types/compute.ts @@ -7,24 +7,28 @@ export type NodeConfig = { [key: string]: Record; }; -export type Params = { +export type PhasedPipeline = { + observe?: string[]; + regroup?: string[]; + compute?: string[]; +}; + +export type ComputeParams = { pluginStorage: PluginStorageInterface; context: Context; - pipeline?: string[]; + pipeline?: PhasedPipeline; config?: NodeConfig; defaults?: PluginParams; + observe?: Boolean; + regroup?: Boolean; + compute?: Boolean; }; export type Node = { children?: any; - pipeline?: string[]; + pipeline?: PhasedPipeline; config?: NodeConfig; defaults?: PluginParams; inputs?: PluginParams[]; outputs?: PluginParams[]; }; - -export type ComputeParams = { - context: Context; - pluginStorage: PluginStorageInterface; -}; diff --git a/src/if-run/types/explain.ts b/src/if-run/types/explain.ts index b1b684882..c7f1a6d38 100644 --- a/src/if-run/types/explain.ts +++ b/src/if-run/types/explain.ts @@ -1,7 +1,9 @@ import {ParameterMetadata} from '@grnsft/if-core/types'; +import {PluginOptions} from '../../common/types/manifest'; + export type ExplainParams = { pluginName: string; - pluginData: {method: string; path: string}; + pluginData: PluginOptions; metadata: {inputs?: ParameterMetadata; outputs?: ParameterMetadata}; }; diff --git a/src/if-run/types/interface.ts b/src/if-run/types/interface.ts index 6834a3ebb..3884305a2 100644 --- a/src/if-run/types/interface.ts +++ b/src/if-run/types/interface.ts @@ -1,9 +1,6 @@ -import {ExecutePlugin, GroupByPlugin} from '@grnsft/if-core/types'; +import {ExecutePlugin} from '@grnsft/if-core/types'; -export type PluginInterface = ExecutePlugin | GroupByPlugin; +export type PluginInterface = ExecutePlugin; -export const isExecute = (plugin: PluginInterface): plugin is ExecutePlugin => +export const isExecute = (plugin: ExecutePlugin): plugin is ExecutePlugin => (plugin as ExecutePlugin).metadata.kind === 'execute'; - -export const isGroupBy = (plugin: PluginInterface): plugin is GroupByPlugin => - (plugin as GroupByPlugin).metadata.kind === 'groupby'; diff --git a/src/if-run/types/process-args.ts b/src/if-run/types/process-args.ts index 184877bd0..298cadda6 100644 --- a/src/if-run/types/process-args.ts +++ b/src/if-run/types/process-args.ts @@ -3,6 +3,9 @@ export interface IfRunArgs { output?: string; 'no-output'?: boolean; debug?: boolean; + observe?: boolean; + regroup?: boolean; + compute?: boolean; } export interface ProcessArgsOutputs { @@ -13,6 +16,9 @@ export interface ProcessArgsOutputs { }; paramPath?: string; debug?: boolean; + observe?: boolean; + regroup?: boolean; + compute?: boolean; } export interface Options { diff --git a/src/if-run/util/args.ts b/src/if-run/util/args.ts index 140c5d140..538d37a36 100644 --- a/src/if-run/util/args.ts +++ b/src/if-run/util/args.ts @@ -13,16 +13,16 @@ import {IfRunArgs, ProcessArgsOutputs} from '../types/process-args'; const {ParseCliParamsError, CliSourceFileError} = ERRORS; -const {IF_RUN} = CONFIG; +const {ARGS, HELP} = CONFIG; const {NO_OUTPUT} = STRINGS; const {SOURCE_IS_NOT_YAML, MANIFEST_IS_MISSING} = COMMON_STRINGS; /** - * Validates `ie` process arguments. + * Validates `if-run` process arguments. */ const validateAndParseProcessArgs = () => { try { - return parse(IF_RUN.ARGS, IF_RUN.HELP); + return parse(ARGS, HELP); } catch (error) { if (error instanceof Error) { throw new ParseCliParamsError(error.message); @@ -33,9 +33,8 @@ const validateAndParseProcessArgs = () => { }; /** - * 1. Parses process arguments like `manifest`, `output`, `help` and `debug`. - * 2. Checks if `help` param is provided, then logs help message and exits. - * 3. If output params are missing, warns user about it. + * 1. Parses process arguments for `if-run`. + * 2. If output params are missing, warns user about it. * 3. Otherwise checks if `manifest` param is there, then processes with checking if it's a yaml file. * If it is, then returns object containing full path. * 4. If params are missing or invalid, then rejects with `ParseCliParamsError`. @@ -46,6 +45,9 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { output, 'no-output': noOutput, debug, + observe, + regroup, + compute, } = validateAndParseProcessArgs(); if (!output && noOutput) { @@ -61,6 +63,9 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { ...(noOutput && {noOutput}), }, debug, + observe, + regroup, + compute, }; }