diff --git a/.github/workflows/dispatch-integration-test.yml b/.github/workflows/dispatch-integration-test.yml new file mode 100644 index 000000000..6fe4904d7 --- /dev/null +++ b/.github/workflows/dispatch-integration-test.yml @@ -0,0 +1,23 @@ +name: Run if-check on requested branch + +on: + workflow_dispatch: + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: 'npm' + + - name: Install dependencies + run: npm install + + - name: Run integration tests + run: npm run if-check -- -d manifests/outputs diff --git a/.github/workflows/nodejs-ci.yml b/.github/workflows/nodejs-ci.yml index fac415dc3..5b45181e7 100644 --- a/.github/workflows/nodejs-ci.yml +++ b/.github/workflows/nodejs-ci.yml @@ -3,21 +3,83 @@ on: push jobs: build: runs-on: ubuntu-latest + permissions: + pull-requests: write # this allows to show carbon and energy data table in PRs steps: + - name: Eco CI Energy Estimation - Initialize + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: start-measurement + continue-on-error: true + - name: Checkout code uses: actions/checkout@v4 + - name: Eco CI Energy Estimation - Get Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: get-measurement + label: "checkout" + continue-on-error: true + - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: 18 cache: 'npm' + - name: Eco CI Energy Estimation - Get Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: get-measurement + label: "setup node" + continue-on-error: true + + - name: Install dependencies run: npm install + - name: Eco CI Energy Estimation - Get Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: get-measurement + label: "npm install" + continue-on-error: true + + - name: Run lint run: npm run lint + + - name: Eco CI Energy Estimation - Get Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: get-measurement + label: "npm run lint" + continue-on-error: true + - name: Run unit tests run: npm run test --silent + + - name: Eco CI Energy Estimation - Get Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: get-measurement + label: "npm run test" + continue-on-error: true + + - name: Eco CI Energy Estimation - End Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3 + + with: + task: display-results + send-data: true + pr-comment: true + continue-on-error: true + diff --git a/.github/workflows/release-commit.yml b/.github/workflows/release-commit.yml index efcf8726e..8ffa665f7 100644 --- a/.github/workflows/release-commit.yml +++ b/.github/workflows/release-commit.yml @@ -8,7 +8,7 @@ env: PRE_RELEASE: ${{ github.event.release.prerelease}} jobs: - release-commit-pr: + unit-tests: runs-on: ubuntu-latest steps: - name: Checkout code @@ -28,9 +28,45 @@ jobs: - name: Run unit tests run: npm test + integration-tests: + runs-on: ubuntu-latest + needs: unit-tests + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + registry-url: https://registry.npmjs.org/ + + - name: Install dependencies + run: npm ci + - name: Run integration tests run: npm run if-check -- -d manifests/outputs + publish: + runs-on: ubuntu-latest + needs: integration-tests + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18 + registry-url: https://registry.npmjs.org/ + + - name: Install dependencies + run: npm ci + - name: Initialize git user email run: git config --global user.email "${{ vars.RELEASE_USER_EMAIL }}" diff --git a/README.md b/README.md index 6fe0f67a4..fba9fd18d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Read the [specification and design docs](https://if.greensoftware.foundation) to ## Get started -IF is a framework for running pipelines of plugins that operate on a set of observations. This is all configured using a manifest file. We provide a standard library of plugins that come bundled with IF - we refer to these as `builtins`. We also have an [Explorer](https://explorer.if.greensoftware.foundation) where anyone can list third party plugins you can install. +IF is a framework for running pipelines of plugins that operate on a set of observations. This is all configured using a manifest file. We provide a standard library of plugins that come bundled with IF - we refer to these as `builtins`. We also have an [Explorer](https://explorer.if.greensoftware.foundation) where anyone can list third party plugins you can install. Start by installing the latest version of IF: @@ -32,26 +32,18 @@ Then create a `manifest` file that describes your application (see our docs for Then, run `if` using the following command: ```sh -if-run --manifest --stdout -## or you can use aliases -if-run -m -s +if-run --manifest +## or you can use alias +if-run -m ``` -Note that above command will print your outputs to the console. You can also provide the `--output` command to save your outputs to a yaml file: +Note that above command will print your outputs to the console. If you do not want to print the outputs to the console, you need to provide `--no-output` command. You can also provide the `--output` command to save your outputs to a yaml file: ```sh if-run -m -o ``` -Note that you also have to add configuration to your manifest to enable this, as follows: - -```yaml -initialize: - output: - - yaml -``` - The `if-run` CLI tool will configure and run the plugins defined in your input `yaml` (`manifest`) and return the results as an output `yaml` (`output`). Use the `debug` command if you want to diagnose and fix errors in your plugin: @@ -72,19 +64,16 @@ if-run -h Please read our documentation at [if.greensoftware.foundation](https://if.greensoftware.foundation/) - ## Video walk-through Watch this video to learn how to create and run a `manifest`. [![Watch the walk-through video](https://i3.ytimg.com/vi/R-6eDM8AsvY/maxresdefault.jpg)](https://youtu.be/GW37Qd4AQbU) - ## Join our mailing list We have a public mailing list at [if-community@greensoftware.foundation](https://groups.google.com/u/1/a/greensoftware.foundation/g/if-community). We send out weekly updates that explain what we've shipped, what we're working on and how you can get involved each week! - ## Contributing To contribute to IF, please fork this repository and raise a pull request from your fork. @@ -93,7 +82,6 @@ You can check our issue board for issues. We mark some issues `core-only` if the Please read the full contribution guidelines at [if.greensoftware.foundation](https://if.greensoftware.foundation/Contributing) - ## Bug reports To report bugs please use our bug report template. You can do this by opening a new issue and selecting `Bug Report` when you are prompted to pick a template. The more information you provide,.the quicker we will be able to reproduce, diagnose and triage your issue. diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index d315d5c41..95283c841 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -8,43 +8,29 @@ If you are a IF user or developer, there are some changes you must be aware of r There have been some name changes to the CLI, specifically: -- `impact-engine` --> `ie` - The command line tool has been renamed from `impact-engine` to simply `ie`. This means that to invoke the Impact Framework on the command line you simply use +- `impact-engine` --> `if-run` + The command line tool has been renamed from `impact-engine` to simply `if-run`. This means that to invoke the Impact Framework on the command line you simply use - ``` - if-run ... - ``` -- `impl` --> `manifest` - We have deprecated the original `impl` and `ompl` terminology across all our repositories and on the command line. Now, to pass a manifest file to IF, you use the `--manifest` command, as follows: + ``` + if-run ... + ``` - ```sh - if-run --manifest - ``` +- `impl` --> `manifest` + We have deprecated the original `impl` and `ompl` terminology across all our repositories and on the command line. Now, to pass a manifest file to IF, you use the `--manifest` command, as follows: + ```sh + if-run --manifest + ``` - `ompl` --> `output` - - We have deprecated the original `impl` and `ompl` terminology across all our repositories and on the command line. Now, to define a savepath for your output file, you use the `--output` command, as follows: - ```sh - if-run --manifest --output - ``` + We have deprecated the original `impl` and `ompl` terminology across all our repositories and on the command line. Now, to define a savepath for your output file, you use the `--output` command, as follows: -## Outputs + ```sh + if-run --manifest --output + ``` -We currently default to exporting output data to the console only. If you want to export to a file, you have to add a small piece of config inside your manifest, as follows: - -```yaml -initalize: - plugins: ... - outputs: ['yaml'] -``` - -You then provide your savepatrh (without file extension) to the `--output` command on the CLI. - -The available options are currently `csv`, `yaml` or `log`. - -### Summary of how to run the refactored IF +### Summary of how to run the refactored IF As before, you can install IF from our npm package using @@ -64,7 +50,6 @@ This will dump the output to the console. If you want to save the output to a ya if-run --manifest --output ``` - If you want to clone the source code and install and run a local copy of IF, you can do so using: ```sh @@ -76,138 +61,136 @@ npm i Then run IF using the following command: ```sh -npm run if -- --manifest +npm run if-run -- --manifest ``` - ## Manifest files There have also been some changes to the structure of manifest files. Some of these are simple renaming changes, others are more functional. - **Rename `graph` -> `tree`** - The `graph` section of the manifest file is now renamed to `tree`. This is just to help us stay consistent in our metaphors and provide a more familiar naming convention for the data beneath. + The `graph` section of the manifest file is now renamed to `tree`. This is just to help us stay consistent in our metaphors and provide a more familiar naming convention for the data beneath. - **Use plugin name as key in `Initialize` block** - In the previous version of IF, the plugins were organized into an array each having a `name` key, with the plugin name as the value. In the refactored IF, we use the name as the key identifying the plugin. For example, this is the OLD way: - - ```yaml - initialize: - plugins: - - name: ccf - model: CloudCarbonFootprint - path: if-plugins - ``` - - This is the new way: - - ```yaml - initialize: - plugins: - "sci-e": - path: "@grnsft/if-plugins" - method: SciE - ``` + In the previous version of IF, the plugins were organized into an array each having a `name` key, with the plugin name as the value. In the refactored IF, we use the name as the key identifying the plugin. For example, this is the OLD way: + + ```yaml + initialize: + plugins: + - name: ccf + model: CloudCarbonFootprint + path: if-plugins + ``` + + This is the new way: + + ```yaml + initialize: + plugins: + 'sci-e': + path: '@grnsft/if-plugins' + method: SciE + ``` - **Rename `model` to `method` in `Initialize` block** - Each plugin in the initialize block - - Each plugin in the `Initialize` block has a field where the name of the exported function representing that plugin is defined. Previously, these were class names and they were defined using the `model` key. Now, they are functions, and they are defined using the `method` key. We use `method` instead of `function` because `function` is a reserved keyword in Typescript. - - For example: - - ```yaml - "sci-embodied": - path: "builtin" - method: SciEmbodied - ``` + Each plugin in the initialize block + + Each plugin in the `Initialize` block has a field where the name of the exported function representing that plugin is defined. Previously, these were class names and they were defined using the `model` key. Now, they are functions, and they are defined using the `method` key. We use `method` instead of `function` because `function` is a reserved keyword in Typescript. + + For example: + + ```yaml + 'sci-embodied': + path: 'builtin' + method: SciEmbodied + ``` - **Global config** - We have introduced the concept of global config to the plugins. This is truly global configuration data that should be kept constant regardless of where the plugin is invoked across the manifest file. - - A good example is the `interpolation` method to use in the Teads curve plugin - this is not expected to vary from component to component and can therefore be defined in global config. The plugin code itself must expect the global config. Then, the config can be passed in the `Initialize` block, for example: - - ```yaml - initialize: - plugins: - "time-sync": - method: TimeSync - path: "builtin" - global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" - interval: 5 - allow-padding: true - ``` + We have introduced the concept of global config to the plugins. This is truly global configuration data that should be kept constant regardless of where the plugin is invoked across the manifest file. + + A good example is the `interpolation` method to use in the Teads curve plugin - this is not expected to vary from component to component and can therefore be defined in global config. The plugin code itself must expect the global config. Then, the config can be passed in the `Initialize` block, for example: + + ```yaml + initialize: + plugins: + 'time-sync': + method: TimeSync + path: 'builtin' + global-config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true + ``` - **Node level config** - We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `groupby` plugin to group the outputs according to a different set of keys. + We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `regroup` feature to group the outputs according to a different set of keys. - ```yaml - tree: - children: - child-1: - pipeline: + ```yaml + tree: + children: + child-1: + pipeline: + compute: - teads-curve - sci-e - sci-embodied - sci-o - time-sync - sci - config: - group-by: - group: - - region - - cloud/instance-type - ``` + regroup: + - region + - cloud/instance-type + ``` - **Defaults** - We have also introduced the concept of `defaults`. This is a section in each component's definition that can be used to provide fallbacks for missing input data. For example, perhaps you have a value arriving from an external API that should be present in every observation in your inputs array, but for soem reason the API fails to deliver a value for some timestamps. In this case, IF would fallback to the value provided for that metric in the `defaults` section of the manifest for that component. + We have also introduced the concept of `defaults`. This is a section in each component's definition that can be used to provide fallbacks for missing input data. For example, perhaps you have a value arriving from an external API that should be present in every observation in your inputs array, but for soem reason the API fails to deliver a value for some timestamps. In this case, IF would fallback to the value provided for that metric in the `defaults` section of the manifest for that component. - You can also use `defaults` as a quick way to add values to everyobservation in your input array if those values are expected to be constant over time (e.g. some of the lifespan values for embodied carbon calculations could be appropriate to include in defaults). This saves you from having to enter the value in every observation in the input array, instead IF can automatically grab it from `defaults` for every timestamp. + You can also use `defaults` as a quick way to add values to everyobservation in your input array if those values are expected to be constant over time (e.g. some of the lifespan values for embodied carbon calculations could be appropriate to include in defaults). This saves you from having to enter the value in every observation in the input array, instead IF can automatically grab it from `defaults` for every timestamp. - ```yaml - tree: - children: - child-1: - pipeline: + ```yaml + tree: + children: + child-1: + pipeline: + compute: - teads-curve - sci-e - sci-embodied - sci-o - time-sync - sci - defaults: - cpu/thermal-design-power: 100 - grid/carbon-intensity: 800 - device/emissions-embodied: 1533.120 # gCO2eq - time-reserved: 3600 # 1hr in seconds - device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 - ``` - - + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.120 # gCO2eq + time-reserved: 3600 # 1hr in seconds + device/expected-lifespan: 94608000 # 3 years in seconds + resources-reserved: 1 + resources-total: 8 + ``` ## New IF features ### Time-sync -Technically time-sync is not a new feature as it was present in IF before the refactor, but there are some tweaks to how the plugin is configured that are worth explaining here. Time sync snaps all input arrays across an entire graph to a common time grid. + +Technically time-sync is not a new feature as it was present in IF before the refactor, but there are some tweaks to how the plugin is configured that are worth explaining here. Time sync snaps all input arrays across an entire graph to a common time grid. This means you have to define a global start time, end time and interval to use everywhere. There is also a boolean to toggle whether you should allow the time sync model to pad the start and end of your time series with zeroes. You should default to `true` unless you have a specific reason not to. In the refactored IF we expect this information to be provided in global config, as follows: ```yaml initialize: - plugins: - "time-sync": - method: TimeSync - path: "builtin" - global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" - interval: 5 - allow-padding: true + plugins: + 'time-sync': + method: TimeSync + path: 'builtin' + global-config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true ``` ### Aggregate @@ -216,7 +199,7 @@ The aggregate plugin aggregates data in two ways: first it condenses individual This is a builtin feature of IF, meaning it does not have to be initialized as a plugin. Instead, you just have to include a short config block in the top of the manifest file. There are two pieces of information required: -- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array. +- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array and be described in the `parameter-metadata` of the plugin. - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components. @@ -225,52 +208,16 @@ Here's what the config block should look like: ```yaml aggregation: metrics: - - 'carbon' + - carbon type: 'both' ``` - -### Groupby - -Groupby allows you to regroup your outputs according to keys you define. For example, maybe you want to group your outputs by region (show me all the outputs for applications run in `uk-south` etc). Groupby *is* a plugin that needs to be initialized in the manifest. - -You can initialize the plugin as follows: - -```yaml -initialize: - plugins: - 'group-by': - path: builtin - method: GroupBy -``` - -Then you configure groupby for each component in the node level config. In the following example we will regroup the outputs by the `region`: - -```yaml -tree: - children: - child-1: - pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - group-by - - sci - config: - group-by: - group: - - region -``` - ### Exhaust -We have introduced `exhaust` as an IF feature. This is a wrapper around export plugins and it allows community contributors to create plugins for exporting to different formats. +We have introduced `exhaust` as an IF feature. This is a wrapper around export plugins and it allows community contributors to create plugins for exporting to different formats. Details tbc... - ## Plugins The plugins themselves require some changes to keep them compatible with the refactored IF. @@ -295,18 +242,17 @@ The plugin still requires an execute function. This is where you implement the p Here's a minimal example for a plugin that sums some inputs defined in global config - see inline comments for some important notes: ```ts - // Here's the function definition - notice that global config is passed in here! export const Sum = (globalConfig: SumConfig): PluginInterface => { const inputParameters = globalConfig['input-parameters'] || []; const outputParameter = globalConfig['output-parameter']; - - // we also return metadata now too - you can add more or just use this default + + // we also return metadata now too - you can add more or just use this default const metadata = { kind: 'execute', }; - - /** + + /** * Calculate the sum of the input metrics for each timestamp. */ const execute = async (inputs: PluginParams[]): Promise => { @@ -332,7 +278,7 @@ export const Sum = (globalConfig: SumConfig): PluginInterface => { ); }; - // return the metadata and the execute function + // return the metadata and the execute function return { metadata, execute, diff --git a/jest.config.js b/jest.config.js index 016be5863..51f0cf3a6 100644 --- a/jest.config.js +++ b/jest.config.js @@ -10,10 +10,6 @@ module.exports = { }, ], }, - modulePathIgnorePatterns: [ - './build', - './src/__tests__/integration/helpers', - './src/__tests__/integration/test-data', - ], - coveragePathIgnorePatterns: ['src/config', 'src/types'], + modulePathIgnorePatterns: ['./build'], + coveragePathIgnorePatterns: ['src/.*/config/.*', 'src/.*/types/.*'], }; diff --git a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml index 9c0416c5e..5d497b948 100644 --- a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml +++ b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml @@ -2,7 +2,6 @@ name: coefficient-demo description: failure with ivalid `global-config.input-parameter` tags: initialize: - outputs: ['yaml'] plugins: coefficient: method: Coefficient @@ -15,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml index 795c04492..1ef932df0 100644 --- a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml @@ -2,7 +2,6 @@ name: coefficient-demo description: value of output-param is missing tags: initialize: - outputs: ['yaml'] plugins: coefficient: method: Coefficient @@ -10,12 +9,13 @@ initialize: global-config: input-parameter: "carbon" coefficient: 3 - output-parameter: + output-parameter: tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/coefficient/success.yml b/manifests/examples/builtins/coefficient/success.yml index 1ce61af1b..69dd764eb 100644 --- a/manifests/examples/builtins/coefficient/success.yml +++ b/manifests/examples/builtins/coefficient/success.yml @@ -2,7 +2,6 @@ name: coefficient-demo description: successful path tags: initialize: - outputs: ['yaml'] plugins: coefficient: method: Coefficient @@ -15,7 +14,8 @@ tree: children: child: pipeline: - - coefficient + compute: + - coefficient config: sum: inputs: diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index fa55623c2..21f9769b4 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -2,7 +2,6 @@ name: cloud-metadata description: cloud/instance-type instance type is not supported in the cloud vendor tags: initialize: - # outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,12 +11,13 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml index 6f247f6c4..c9fbb8296 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml @@ -2,7 +2,6 @@ name: cloud-metadata description: failure with invalid `inputs.cloud/vendor` tags: initialize: - # outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,12 +11,13 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml index 33c5cc4b1..62de6150d 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml @@ -2,7 +2,6 @@ name: cloud-metadata description: failing because cloud/vendor is not provided tags: initialize: - # outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,12 +11,13 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml index bd3cfdbb7..84e10112b 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml @@ -2,7 +2,6 @@ name: cloud-metadata description: successful path tags: initialize: - # outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,12 +11,13 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml index 85360f717..5b4e9583b 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml @@ -2,11 +2,10 @@ name: csv-lookup-demo description: tags: initialize: - # outputs: [yaml] plugins: cloud-metadata: method: CSVLookup - path: 'builtin' + path: "builtin" global-config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: @@ -17,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml index 005961ee0..71417932d 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml @@ -2,11 +2,10 @@ name: csv-lookup-demo description: tags: initialize: - # outputs: [yaml] plugins: cloud-metadata: method: CSVLookup - path: 'builtin' + path: "builtin" global-config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: @@ -17,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml index 66df39736..4c2767a3a 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml @@ -2,22 +2,22 @@ name: csv-lookup-demo description: tags: initialize: - # outputs: [yaml] plugins: cloud-metadata: method: CSVLookup - path: 'builtin' + path: "builtin" global-config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: "cloud/provider" cloud-region: "cloud/region" - output: ['em-zone-id','renamed-em-zone-data'] + output: ["em-zone-id", "renamed-em-zone-data"] tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml index 005961ee0..71417932d 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml @@ -2,11 +2,10 @@ name: csv-lookup-demo description: tags: initialize: - # outputs: [yaml] plugins: cloud-metadata: method: CSVLookup - path: 'builtin' + path: "builtin" global-config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: @@ -17,7 +16,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml index b14e57d1d..991bdbb8b 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml @@ -2,7 +2,6 @@ name: tdp-finder description: failure with `inputs` missing `physical-processor` param tags: initialize: - # outputs: ['yaml'] plugins: tdp-finder: method: CSVLookup @@ -16,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml index 47f392fcc..3d433d6b2 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml @@ -2,7 +2,6 @@ name: tdp-finder description: successful path tags: initialize: - # outputs: ['yaml'] plugins: tdp-finder: method: CSVLookup @@ -16,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml index 42545b0df..a04288e0c 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml @@ -2,7 +2,6 @@ name: tdp-finder description: successful path tags: initialize: - # outputs: ['yaml'] plugins: tdp-finder: method: CSVLookup @@ -16,7 +15,8 @@ tree: children: child: pipeline: - - tdp-finder + compute: + - tdp-finder config: inputs: - timestamp: 2023-07-06T00:00 diff --git a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml index 4022d4797..96d627aee 100644 --- a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml +++ b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml @@ -2,7 +2,6 @@ name: divide description: denominator is invalid, denominator is tags: initialize: -# outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,7 +11,7 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] divide: method: Divide path: "builtin" @@ -24,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml index 91129714a..fef8a1a7c 100644 --- a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml +++ b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml @@ -2,7 +2,6 @@ name: divide description: failure when `global-config.denominator` is string tags: initialize: - outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,20 +11,21 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] divide: method: Divide path: "builtin" global-config: numerator: vcpus-allocated - denominator: 'vcpus' + denominator: "vcpus" output: cpu/number-cores tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/failure-missing-numerator.yml b/manifests/examples/builtins/divide/failure-missing-numerator.yml index 7cf123976..5645f0ecf 100644 --- a/manifests/examples/builtins/divide/failure-missing-numerator.yml +++ b/manifests/examples/builtins/divide/failure-missing-numerator.yml @@ -2,7 +2,6 @@ name: divide description: success path tags: initialize: -# outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,7 +11,7 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] divide: method: Divide path: "builtin" @@ -24,8 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide + compute: + - cloud-metadata + - divide config: divide: defaults: diff --git a/manifests/examples/builtins/divide/success.yml b/manifests/examples/builtins/divide/success.yml index 6a32d7e2c..32b247a87 100644 --- a/manifests/examples/builtins/divide/success.yml +++ b/manifests/examples/builtins/divide/success.yml @@ -2,7 +2,6 @@ name: divide description: success path tags: initialize: - # outputs: ['yaml'] plugins: cloud-metadata: path: builtin @@ -12,7 +11,7 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] + output: ["cpu-cores-utilized", "vcpus-allocated"] divide: method: Divide path: "builtin" @@ -24,10 +23,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/examples/builtins/exponent/success.yml b/manifests/examples/builtins/exponent/success.yml index c8b59b78c..163dd1460 100644 --- a/manifests/examples/builtins/exponent/success.yml +++ b/manifests/examples/builtins/exponent/success.yml @@ -2,22 +2,20 @@ name: exponent demo description: tags: initialize: - # outputs: [yaml] plugins: exponent: method: Exponent - path: 'builtin' + path: "builtin" global-config: - input-parameter: 'cpu/energy' + input-parameter: "cpu/energy" exponent: 2 - output-parameter: 'energy' + output-parameter: "energy" tree: children: child: pipeline: - - exponent - config: - exponent: + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/interpolation/interpolation.yml b/manifests/examples/builtins/interpolation/interpolation.yml index 35d750e48..394946467 100644 --- a/manifests/examples/builtins/interpolation/interpolation.yml +++ b/manifests/examples/builtins/interpolation/interpolation.yml @@ -12,12 +12,12 @@ initialize: y: [0.12, 0.32, 0.75, 1.02] input-parameter: "cpu/utilization" output-parameter: "result" - tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/interpolation/success.yml b/manifests/examples/builtins/interpolation/success.yml index cce024727..394946467 100644 --- a/manifests/examples/builtins/interpolation/success.yml +++ b/manifests/examples/builtins/interpolation/success.yml @@ -16,7 +16,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml index 00243cc90..472f797fd 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml @@ -2,7 +2,6 @@ name: mock-observation-demo description: failure with `global-config->generators->randint->cpu/utilization->min` is greater than `max` tags: initialize: - outputs: ['yaml'] plugins: mock-observations: kind: plugin @@ -30,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml index ef7bd3dbd..85e3f566b 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml @@ -2,7 +2,6 @@ name: mock-observation-demo description: a manifest demonstrating how to use the mock observations feature tags: initialize: - # outputs: ['yaml'] plugins: mock-observations: kind: plugin @@ -30,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml index 2eaa8f02d..58545dcec 100644 --- a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml +++ b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml @@ -2,7 +2,6 @@ name: mock-observation-demo description: a manifest demonstrating how to use the mock observations feature tags: initialize: - outputs: ['yaml'] plugins: mock-observations: kind: plugin @@ -30,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/mock-observations/success.yml b/manifests/examples/builtins/mock-observations/success.yml index ac1464322..5ca2d1942 100644 --- a/manifests/examples/builtins/mock-observations/success.yml +++ b/manifests/examples/builtins/mock-observations/success.yml @@ -2,7 +2,6 @@ name: mock-observation-demo description: a manifest demonstrating how to use the mock observations feature tags: initialize: - # outputs: ['yaml'] plugins: mock-observations: kind: plugin @@ -30,5 +29,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: diff --git a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml index 90951d29f..988a33fc2 100644 --- a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml +++ b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml @@ -2,7 +2,6 @@ name: multiply description: failure `inputs` is missing `cpu/energy` parameter tags: initialize: - outputs: ['yaml'] plugins: multiply: method: Multiply @@ -14,9 +13,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml index cc2e7a90a..e6e138723 100644 --- a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml +++ b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml @@ -2,7 +2,6 @@ name: multiply description: a manifest demonstrating how to use the multiply feature tags: initialize: - outputs: ['yaml'] plugins: multiply: method: Multiply @@ -14,9 +13,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/multiply/success.yml b/manifests/examples/builtins/multiply/success.yml index f05adb071..c5d53e046 100644 --- a/manifests/examples/builtins/multiply/success.yml +++ b/manifests/examples/builtins/multiply/success.yml @@ -2,7 +2,7 @@ name: multiply description: a manifest demonstrating how to use the multiply feature tags: initialize: - # outputs: ['yaml'] + # plugins: multiply: method: Multiply @@ -14,9 +14,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/failure-missing-input-param.yml b/manifests/examples/builtins/regex/failure-missing-input-param.yml index 8ed278e41..84adc0cb2 100644 --- a/manifests/examples/builtins/regex/failure-missing-input-param.yml +++ b/manifests/examples/builtins/regex/failure-missing-input-param.yml @@ -2,7 +2,6 @@ name: regex description: failure with missing `physical-processor` field from `inputs` tags: initialize: - outputs: ['yaml'] plugins: regex: method: Regex @@ -15,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml index 84189dcde..8341d08e6 100644 --- a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml +++ b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml @@ -2,7 +2,6 @@ name: regex description: physical processor doesn't match the regex expression tags: initialize: - # outputs: ['yaml'] plugins: regex: method: Regex @@ -15,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/regex/success.yml b/manifests/examples/builtins/regex/success.yml index 0fa735fa6..425dd9e67 100644 --- a/manifests/examples/builtins/regex/success.yml +++ b/manifests/examples/builtins/regex/success.yml @@ -2,7 +2,6 @@ name: regex description: successful path tags: initialize: - # outputs: ['yaml'] plugins: regex: method: Regex @@ -15,9 +14,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml index 3f12dcdd7..503300f84 100644 --- a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml +++ b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml @@ -2,7 +2,6 @@ name: sci-embodied description: failure with `defaults.device/emissions-embodied` being string instead of number tags: initialize: - outputs: ['yaml'] plugins: "sci-embodied": # a model that calculates m from te, tir, el, rr and rtor method: SciEmbodied @@ -11,9 +10,10 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: - device/emissions-embodied: 'fail' # gCO2eq + device/emissions-embodied: "fail" # gCO2eq time-reserved: 3600 # 1hr in seconds device/expected-lifespan: 94608000 # 3 years in seconds resources-reserved: 1 diff --git a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml index 143bc7c95..8fd3e5784 100644 --- a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml +++ b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml @@ -2,7 +2,6 @@ name: sci-embodied description: missing device/expected-lifespan tags: initialize: - #outputs: ['yaml'] plugins: "sci-embodied": # a model that calculates m from te, tir, el, rr and rtor method: SciEmbodied @@ -11,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci-embodied/success.yml b/manifests/examples/builtins/sci-embodied/success.yml index 59ce00e52..991569404 100644 --- a/manifests/examples/builtins/sci-embodied/success.yml +++ b/manifests/examples/builtins/sci-embodied/success.yml @@ -2,7 +2,6 @@ name: sci-embodied description: successful path tags: initialize: - outputs: ['yaml'] plugins: "sci-embodied": # a model that calculates m from te, tir, el, rr and rtor method: SciEmbodied @@ -11,7 +10,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq time-reserved: 3600 # 1hr in seconds diff --git a/manifests/examples/builtins/sci/failure-invalid-config-value.yml b/manifests/examples/builtins/sci/failure-invalid-config-value.yml index f5a4def52..5882111dd 100644 --- a/manifests/examples/builtins/sci/failure-invalid-config-value.yml +++ b/manifests/examples/builtins/sci/failure-invalid-config-value.yml @@ -2,7 +2,6 @@ name: sci description: failure with `config.sci.functional-unit` value being number tags: initialize: - outputs: ['yaml'] plugins: sci: kind: plugin @@ -14,7 +13,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci config: sci: functional-unit: 999 # factor to convert per time to per f.unit diff --git a/manifests/examples/builtins/sci/failure-missing-input-param.yml b/manifests/examples/builtins/sci/failure-missing-input-param.yml index 54514a8e5..0a7677261 100644 --- a/manifests/examples/builtins/sci/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sci/failure-missing-input-param.yml @@ -2,7 +2,7 @@ name: sci description: missing input value (carbon-embodied) tags: initialize: - #outputs: ['yaml'] + # plugins: sci: kind: plugin @@ -14,7 +14,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci/success.yml b/manifests/examples/builtins/sci/success.yml index cddfe5755..85dd19db2 100644 --- a/manifests/examples/builtins/sci/success.yml +++ b/manifests/examples/builtins/sci/success.yml @@ -2,7 +2,6 @@ name: sci description: successful path tags: initialize: - # outputs: ['yaml'] plugins: sci: kind: plugin @@ -14,8 +13,8 @@ tree: children: child: pipeline: - - sci - config: + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/shell/failure-invalid-command.yml b/manifests/examples/builtins/shell/failure-invalid-command.yml index b395d6f4c..e67aff103 100644 --- a/manifests/examples/builtins/shell/failure-invalid-command.yml +++ b/manifests/examples/builtins/shell/failure-invalid-command.yml @@ -2,7 +2,6 @@ name: shell description: falure with `global-config.command` being number instead od string tags: initialize: - outputs: ['yaml'] plugins: shell: method: Shell @@ -13,7 +12,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/examples/builtins/shell/success.yml b/manifests/examples/builtins/shell/success.yml index b44ca2892..a26af274d 100644 --- a/manifests/examples/builtins/shell/success.yml +++ b/manifests/examples/builtins/shell/success.yml @@ -12,7 +12,8 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 diff --git a/manifests/examples/builtins/subtract/success.yml b/manifests/examples/builtins/subtract/success.yml index bcc27294c..745ceb067 100644 --- a/manifests/examples/builtins/subtract/success.yml +++ b/manifests/examples/builtins/subtract/success.yml @@ -2,23 +2,21 @@ name: subtract demo description: tags: initialize: - # outputs: ['yaml'] plugins: subtract: method: Subtract - path: 'builtin' + path: "builtin" global-config: - input-parameters: ['cpu/energy', 'network/energy'] - output-parameter: 'energy/diff' + input-parameters: ["cpu/energy", "network/energy"] + output-parameter: "energy/diff" tree: children: child: pipeline: - - subtract - config: - subtract: + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 cpu/energy: 0.003 - network/energy: 0.001 \ No newline at end of file + network/energy: 0.001 diff --git a/manifests/examples/builtins/sum/failure-missing-input-param.yml b/manifests/examples/builtins/sum/failure-missing-input-param.yml index 2d7c95ae1..1e963b694 100644 --- a/manifests/examples/builtins/sum/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-input-param.yml @@ -2,7 +2,6 @@ name: sum description: failure with `inputs[0]` misses one of `global-config.input-parameters` tags: initialize: - outputs: ['yaml'] plugins: sum: method: Sum @@ -14,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/failure-missing-output-param.yml b/manifests/examples/builtins/sum/failure-missing-output-param.yml index c99bc4ee2..0248bf449 100644 --- a/manifests/examples/builtins/sum/failure-missing-output-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-output-param.yml @@ -2,21 +2,19 @@ name: sum description: missing `output-parameter` in global-config tags: initialize: - outputs: ['yaml'] plugins: sum: method: Sum path: "builtin" global-config: input-parameters: ["cpu/energy", "network/energy"] - # output-parameter: "energy" + # output-parameter: "energy" tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sum/success.yml b/manifests/examples/builtins/sum/success.yml index 454efc9e0..366ae5580 100644 --- a/manifests/examples/builtins/sum/success.yml +++ b/manifests/examples/builtins/sum/success.yml @@ -2,7 +2,6 @@ name: sum description: successful path tags: initialize: - # outputs: ['yaml'] plugins: sum: method: Sum @@ -14,9 +13,8 @@ tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml index e6bc3a230..ffed32274 100644 --- a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml +++ b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 @@ -31,4 +31,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 + energy-cpu: 0.001 \ No newline at end of file diff --git a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml b/manifests/examples/builtins/time-sync/failure-missing-global-config.yml index 7b8e110fb..c5715a26a 100644 --- a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml +++ b/manifests/examples/builtins/time-sync/failure-missing-global-config.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 3 @@ -31,4 +31,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 + energy-cpu: 0.001 \ No newline at end of file diff --git a/manifests/examples/builtins/time-sync/success.yml b/manifests/examples/builtins/time-sync/success.yml index 6d1841e97..8aac13740 100644 --- a/manifests/examples/builtins/time-sync/success.yml +++ b/manifests/examples/builtins/time-sync/success.yml @@ -17,8 +17,8 @@ tree: children: child: pipeline: - - time-sync - config: + compute: + - time-sync inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 diff --git a/manifests/examples/builtins/groupby/failure-invalid-config-group.yml b/manifests/examples/features/regroup/failure-invalid-regroup.yml similarity index 80% rename from manifests/examples/builtins/groupby/failure-invalid-config-group.yml rename to manifests/examples/features/regroup/failure-invalid-regroup.yml index 7be0796b8..a121d82b5 100644 --- a/manifests/examples/builtins/groupby/failure-invalid-config-group.yml +++ b/manifests/examples/features/regroup/failure-invalid-regroup.yml @@ -1,19 +1,12 @@ -name: groupby -description: failure when `config->group-by->group` is not an array +name: regroup +description: failure when `regroup` is not an array initialize: - outputs: ['yaml'] - plugins: - group-by: - path: "builtin" - method: GroupBy + plugins: {} tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: cloud/region + regroup: cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml similarity index 82% rename from manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml rename to manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml index b87081318..8b8b44faf 100644 --- a/manifests/examples/builtins/groupby/failure-missing-cloud-instance-type.yml +++ b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml @@ -1,21 +1,14 @@ -name: groupby -description: +name: regroup +description: initialize: - # outputs: ['yaml'] plugins: - group-by: - path: "builtin" - method: GroupBy tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/groupby/success.yml b/manifests/examples/features/regroup/success.yml similarity index 83% rename from manifests/examples/builtins/groupby/success.yml rename to manifests/examples/features/regroup/success.yml index e8e164e91..6863dc0d6 100644 --- a/manifests/examples/builtins/groupby/success.yml +++ b/manifests/examples/features/regroup/success.yml @@ -1,21 +1,14 @@ -name: groupby +name: regroup description: successful path initialize: - # outputs: ['yaml'] plugins: - group-by: - path: "builtin" - method: GroupBy tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/pipelines/generics.yml b/manifests/examples/pipelines/generics.yml index 641bedcb1..f9a19a6b3 100644 --- a/manifests/examples/pipelines/generics.yml +++ b/manifests/examples/pipelines/generics.yml @@ -2,17 +2,16 @@ name: generics description: a pipeline that does arbitrary calculations using our generic arithmetic builtins tags: initialize: - # outputs: [yaml] plugins: "interpolate": method: Interpolation - path: 'builtin' + path: "builtin" global-config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" "cpu-factor-to-wattage": method: Multiply path: builtin @@ -63,15 +62,15 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - coefficient - - multiply - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - coefficient + - multiply defaults: cpu/thermal-design-power: 100 vcpus-allocated: 1 diff --git a/manifests/examples/pipelines/instance-metadata.yml b/manifests/examples/pipelines/instance-metadata.yml index 4ed443985..0a64e9ac3 100644 --- a/manifests/examples/pipelines/instance-metadata.yml +++ b/manifests/examples/pipelines/instance-metadata.yml @@ -11,7 +11,7 @@ initialize: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv query: instance-class: cloud/instance-type - output: '*' + output: "*" extract-processor-name: path: builtin method: Regex @@ -19,14 +19,13 @@ initialize: parameter: cpu-model-name match: /^([^,])+/g output: cpu/name - outputs: - - yaml tree: children: child: pipeline: - - cloud-instance-metadata - - extract-processor-name + compute: + - cloud-instance-metadata + - extract-processor-name inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index 978510f48..baf12a688 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -6,27 +6,51 @@ tags: category: on-premise aggregation: metrics: - - "carbon" + - carbon type: "both" -params: initialize: - # outputs: ['yaml'] plugins: "interpolate": method: Interpolation - path: 'builtin' + path: "builtin" global-config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg "cpu-factor-to-wattage": method: Multiply path: builtin global-config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum "wattage-times-duration": method: Multiply path: builtin @@ -40,6 +64,17 @@ initialize: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum "calculate-vcpu-ratio": method: Divide path: "builtin" @@ -47,6 +82,12 @@ initialize: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio + parameter-metadata: + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" @@ -63,11 +104,32 @@ initialize: global-config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum sci: path: "builtin" method: Sci global-config: functional-unit: "requests" + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum "sum-carbon": path: "builtin" method: Sum @@ -76,6 +138,21 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum time-sync: method: TimeSync path: "builtin" @@ -84,6 +161,32 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + network/energy: + description: "Energy consumed by the Network of the component" + unit: "kWh" + aggregation-method: "sum" tree: children: child-0: @@ -96,17 +199,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -146,17 +250,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -198,17 +303,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 @@ -248,17 +354,18 @@ tree: vcpus-allocated: 1 vcpus-total: 8 pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml new file mode 100644 index 000000000..c1f92da04 --- /dev/null +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml @@ -0,0 +1,997 @@ +name: pipeline-with-aggregate +description: a full pipeline with the aggregate feature enabled +tags: null +aggregation: + metrics: + - carbon + type: both +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + unit: count + description: total number of vcpus available on a particular resource + aggregation-method: none + vcpus-allocated: + unit: count + description: number of vcpus allocated to particular resource + aggregation-method: none + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + sci: + path: builtin + method: Sci + global-config: + functional-unit: requests + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum + time-sync: + path: builtin + method: TimeSync + global-config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/pipeline-with-aggregate.yml -o + manifests/outputs/pipelines/pipeline-with-aggregate-1.yaml + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-31T12:41:31.920Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.16' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-1: + pipeline: + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 10 + requests: 10 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 5 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 15 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + requests: 30 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 14 + requests: 14 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.3205 + cpu-wattage: 66.19999999999999 + cpu-wattage-times-duration: 203 + cpu-energy-raw: 0.0000563888888888889 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000007048611111111113 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.0056388888888888895 + carbon: 0.005649016996448503 + sci: 0.000403501214032036 + - timestamp: '2023-12-12T00:00:05.000Z' + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 9.571428571428571 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.0005589976298113692 + - timestamp: '2023-12-12T00:00:10.000Z' + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 8.428571428571429 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.0006170797001436077 + - timestamp: '2023-12-12T00:00:15.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:20.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:25.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:30.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:35.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + requests: 5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:40.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + requests: 3 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0010402200659563674 + - timestamp: '2023-12-12T00:00:45.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:50.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:55.000Z' + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:01:00.000Z' + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04532668505834602 + child-2: + pipeline: + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + cpu/utilization: 30 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 100 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 28 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 150 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 40 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 110 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cpu/utilization: 33 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 180 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 5 + cpu/utilization: 22.8 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 220 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.41509999999999997 + cpu-wattage: 94.57999999999998 + cpu-wattage-times-duration: 258.9 + cpu-energy-raw: 0.00007191666666666668 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008989583333333334 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007191666666666666 + carbon: 0.007201794774226282 + sci: 0.00003273543079193765 + - timestamp: '2023-12-12T00:00:05.000Z' + duration: 5 + cpu/utilization: 29.6 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 92.85714285714285 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.48819999999999997 + cpu-wattage: 46.98428571428572 + cpu-wattage-times-duration: 308.35 + cpu-energy-raw: 0.00008565277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000010706597222222223 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.008565277777777778 + carbon: 0.008575405885337391 + sci: 0.00009235052491901808 + - timestamp: '2023-12-12T00:00:10.000Z' + duration: 5 + cpu/utilization: 30.6 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 59.14285714285714 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.49894999999999995 + cpu-wattage: 31.31738095238095 + cpu-wattage-times-duration: 306.2 + cpu-energy-raw: 0.00008505555555555556 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000010631944444444445 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.008505555555555556 + carbon: 0.00851568366311517 + sci: 0.0001439849894729618 + - timestamp: '2023-12-12T00:00:15.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:20.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:25.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:30.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:35.000Z' + duration: 5 + cpu/utilization: 26.4 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.45380000000000004 + cpu-wattage: 9.454166666666667 + cpu-wattage-times-duration: 283.625 + cpu-energy-raw: 0.00007878472222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009848090277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.007878472222222222 + carbon: 0.007888600329781836 + sci: 0.0002629533443260612 + - timestamp: '2023-12-12T00:00:40.000Z' + duration: 5 + cpu/utilization: 19.8 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 18 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0.34035000000000004 + cpu-wattage: 5.6725 + cpu-wattage-times-duration: 170.175 + cpu-energy-raw: 0.00004727083333333333 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000005908854166666666 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.004727083333333333 + carbon: 0.0047331601978691015 + sci: 0.00026295334432606117 + - timestamp: '2023-12-12T00:00:45.000Z' + duration: 5 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:50.000Z' + duration: 5 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:00:55.000Z' + duration: 5 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: '2023-12-12T00:01:00.000Z' + duration: 1 + cpu/utilization: 0 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.06846904616945712 + outputs: + - carbon: 0.012850811770674785 + timestamp: '2023-12-12T00:00:00.000Z' + duration: 5 + - carbon: 0.013925811770674782 + timestamp: '2023-12-12T00:00:05.000Z' + duration: 5 + - carbon: 0.013716783992897007 + timestamp: '2023-12-12T00:00:10.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:15.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:20.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:25.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:30.000Z' + duration: 5 + - carbon: 0.013089700659563674 + timestamp: '2023-12-12T00:00:35.000Z' + duration: 5 + - carbon: 0.007853820395738204 + timestamp: '2023-12-12T00:00:40.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:00:45.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:00:50.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:00:55.000Z' + duration: 5 + - carbon: 0 + timestamp: '2023-12-12T00:01:00.000Z' + duration: 1 + aggregated: + carbon: 0.11379573122780316 diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml new file mode 100644 index 000000000..c86d6fec1 --- /dev/null +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -0,0 +1,1059 @@ +name: pipeline-with-mocks +description: a full pipeline seeded with data from mock-observations feature +tags: null +aggregation: + metrics: + - carbon + type: both +initialize: + plugins: + mock-observations: + path: builtin + method: MockObservations + global-config: + timestamp-from: 2023-12-12T00:00 + timestamp-to: 2023-12-12T00:10 + duration: 60 + components: + - cloud/instance-type: A1 + generators: + common: + cloud/region: uk-west + randint: + cpu/utilization: + min: 1 + max: 99 + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + "y": + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + parameter-metadata: + inputs: + cpu-wattage: + unit: kWh + description: Energy used by the CPU + aggregation-method: sum + duration: + unit: seconds + description: Duration of the observation + aggregation-method: sum + outputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + unit: count + description: total number of vcpus available on a particular resource + aggregation-method: none + vcpus-allocated: + unit: count + description: number of vcpus allocated to particular resource + aggregation-method: none + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + parameter-metadata: + inputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + outputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + carbon-embodied: + unit: gCO2eq + description: Embodied carbon footprint + aggregation-method: sum + outputs: + carbon: + unit: gCO2eq + description: Total carbon footprint + aggregation-method: sum + sci: + path: builtin + method: Sci + global-config: + functional-unit: requests + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum + outputs: + sci: + unit: none + description: Scientific Carbon Intensity + aggregation-method: none + time-sync: + path: builtin + method: TimeSync + global-config: + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" + interval: 5 + allow-padding: true + parameter-metadata: + inputs: + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + outputs: + synced-time: + unit: none + description: Synced time + aggregation-method: none +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/pipeline-with-mocks.yml -o + manifests/outputs/pipelines/pipeline-with-mocks-update + environment: + if-version: 0.5.0 + os: macOS + os-version: "14.5" + node-version: 18.14.2 + date-time: 2024-07-30T05:35:42.937Z (UTC) + dependencies: + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-1: + pipeline: + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 10 + requests: 30 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 40 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + requests: 50 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: "*" + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.000010128107559614409 + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: "*" + requests: 0.5 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 25.552 + time-reserved: 3600 + device/expected-lifespan: 1576800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: 0.0000020256215119228817 + carbon-operational: "*" + carbon: "*" + sci: "*" + aggregated: + carbon: "*" + child-2: + pipeline: + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 1 + cpu/utilization: 30 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 30 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 28 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 40 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 40 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 50 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cpu/utilization: 33 + cloud/instance-type: A1 + cloud/region: uk-west + requests: 60 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 2.5 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 127.75999999999999 + time-reserved: 2880 + device/expected-lifespan: 7884000 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cpu/utilization: "*" + cloud/instance-type: A1 + cloud/region: uk-west + requests: 0.5 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 25.552 + time-reserved: 3600 + device/expected-lifespan: 1576800 + vcpus-total: 8 + vcpus-allocated: 1 + cpu-factor: "*" + cpu-wattage: "*" + cpu-wattage-times-duration: "*" + cpu-energy-raw: "*" + vcpu-ratio: 8 + cpu-energy-kwh: "*" + carbon-embodied: "*" + carbon-operational: "*" + carbon: "*" + sci: "*" + aggregated: + carbon: "*" + outputs: + - carbon: "*" + timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + - carbon: "*" + timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + aggregated: + carbon: "*" diff --git a/manifests/examples/pipelines/pipeline-teads-sci.yml b/manifests/examples/pipelines/pipeline-teads-sci.yml index c07b093bd..d80d0e89e 100644 --- a/manifests/examples/pipelines/pipeline-teads-sci.yml +++ b/manifests/examples/pipelines/pipeline-teads-sci.yml @@ -79,17 +79,17 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - sci - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index 7b689f1d8..6f7822ad7 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -3,25 +3,51 @@ description: a full pipeline with the aggregate feature enabled tags: aggregation: metrics: - - "carbon" + - carbon type: "both" initialize: plugins: "interpolate": method: Interpolation - path: 'builtin' + path: "builtin" global-config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg "cpu-factor-to-wattage": method: Multiply path: builtin global-config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum "wattage-times-duration": method: Multiply path: builtin @@ -35,6 +61,17 @@ initialize: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum "calculate-vcpu-ratio": method: Divide path: "builtin" @@ -42,6 +79,21 @@ initialize: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + unit: count + description: total number of vcpus available on a particular resource + aggregation-method: none + vcpus-allocated: + unit: count + description: number of vcpus allocated to particular resource + aggregation-method: none + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" @@ -58,11 +110,32 @@ initialize: global-config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum "sci": path: "builtin" method: Sci global-config: functional-unit: requests # factor to convert per time to per f.unit + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum "sum-carbon": path: "builtin" method: Sum @@ -71,6 +144,12 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon + parameter-metadata: + outputs: + carbon: + unit: gCO2eq + description: product of carbon + aggregation-method: sum "time-sync": method: TimeSync path: "builtin" @@ -83,22 +162,21 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -134,22 +212,21 @@ tree: requests: 30 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -182,4 +259,4 @@ tree: cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west - requests: 180 \ No newline at end of file + requests: 180 diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index 168ebd7fe..fc6f81cb0 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -3,7 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature tags: aggregation: metrics: - - "carbon" + - carbon type: "both" initialize: plugins: @@ -12,71 +12,185 @@ initialize: method: MockObservations path: "builtin" global-config: - timestamp-from: 2023-07-06T00:00 - timestamp-to: 2023-07-06T00:10 - duration: 60 + timestamp-from: "2023-12-12T00:00:00.000Z" + timestamp-to: "2023-12-12T00:00:13.000Z" + duration: 30 components: - cloud/instance-type: A1 generators: common: cloud/region: uk-west - common-key: common-val randint: cpu/utilization: min: 1 max: 99 - "interpolate": + parameter-metadata: + inputs: + timestamp: + description: refers to the time of occurrence of the input + unit: RFC3339 + aggregation-method: none + duration: + description: refers to the duration of the input + unit: seconds + aggregation-method: sum + cloud/instance-type: + description: type of Cloud Instance name used in the cloud provider APIs + unit: none + aggregation-method: none + cloud/region: + description: region cloud instance + unit: none + aggregation-method: none + interpolate: method: Interpolation - path: 'builtin' + path: "builtin" global-config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' - "cpu-factor-to-wattage": + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" + parameter-metadata: + inputs: + cpu/utilization: + description: refers to CPU utilization. + unit: percentage + aggregation-method: avg + outputs: + cpu-factor: + description: result of interpolate + unit: kWh + aggregation-method: avg + cpu-factor-to-wattage: method: Multiply path: builtin global-config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" - "wattage-times-duration": + parameter-metadata: + inputs: + cpu-factor: + description: result of interpolate + unit: kWh + aggregation-method: avg + cpu/thermal-design-power: + description: thermal design power for a processor + unit: kWh + aggregation-method: avg + outputs: + cpu-wattage: + description: the energy used by the CPU + unit: kWh + aggregation-method: sum + wattage-times-duration: method: Multiply path: builtin global-config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" - "wattage-to-energy-kwh": + parameter-metadata: + inputs: + cpu-wattage: + description: Energy used by the CPU + unit: kWh + aggregation-method: sum + duration: + description: Duration of the observation + unit: seconds + aggregation-method: sum + outputs: + cpu-wattage-times-duration: + description: CPU wattage multiplied by duration + unit: kWh + aggregation-method: sum + wattage-to-energy-kwh: method: Divide path: "builtin" global-config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw - "calculate-vcpu-ratio": + parameter-metadata: + inputs: + cpu-wattage-times-duration: + description: CPU wattage multiplied by duration + unit: kWh + aggregation-method: sum + outputs: + cpu-energy-raw: + description: Raw energy used by CPU in kWh + unit: kWh + aggregation-method: sum + calculate-vcpu-ratio: method: Divide path: "builtin" global-config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio - "correct-cpu-energy-for-vcpu-ratio": + parameter-metadata: + inputs: + vcpus-total: + description: total number of vcpus available on a particular resource + unit: count + aggregation-method: none + vcpus-allocated: + description: number of vcpus allocated to particular resource + unit: count + aggregation-method: none + outputs: + vcpu-ratio: + description: Ratio of vCPUs + unit: none + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: method: Divide path: "builtin" global-config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh - "sci-embodied": + parameter-metadata: + inputs: + cpu-energy-raw: + description: Raw energy used by CPU in kWh + unit: kWh + aggregation-method: sum + vcpu-ratio: + description: Ratio of vCPUs + unit: none + aggregation-method: none + outputs: + cpu-energy-kwh: + description: Corrected CPU energy in kWh + unit: kWh + aggregation-method: sum + sci-embodied: path: "builtin" method: SciEmbodied - "operational-carbon": + operational-carbon: method: Multiply path: builtin global-config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" - "sum-carbon": + parameter-metadata: + inputs: + cpu-energy-kwh: + description: Corrected CPU energy in kWh + unit: kWh + aggregation-method: sum + grid/carbon-intensity: + description: Carbon intensity for the grid + unit: gCO2eq/kWh + aggregation-method: avg + outputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + sum-carbon: path: "builtin" method: Sum global-config: @@ -84,12 +198,38 @@ initialize: - carbon-operational - carbon-embodied output-parameter: carbon - "sci": + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum + sci: path: "builtin" method: Sci global-config: functional-unit: "requests" - "time-sync": + parameter-metadata: + inputs: + requests: + description: expressed the final SCI value + unit: none + aggregation-method: sum + outputs: + sci: + description: Scientific Carbon Intensity + unit: none + aggregation-method: none + time-sync: method: TimeSync path: "builtin" global-config: @@ -97,30 +237,27 @@ initialize: end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - "group-by": - path: builtin - method: GroupBy - # outputs: ['yaml'] tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - instance-type + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -130,48 +267,25 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - cloud/instance-type: A1 - cloud/region: uk-west - duration: 1 - cpu/utilization: 10 - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 20 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 15 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 15 - requests: 50 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - sum-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + observe: + - mock-observations + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -181,27 +295,3 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" - duration: 1 - cpu/utilization: 30 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 30 - - timestamp: "2023-12-12T00:00:01.000Z" - duration: 5 - cpu/utilization: 28 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 40 - - timestamp: "2023-12-12T00:00:06.000Z" - duration: 7 - cpu/utilization: 40 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 50 - - timestamp: "2023-12-12T00:00:13.000Z" - duration: 30 - cpu/utilization: 33 - cloud/instance-type: A1 - cloud/region: uk-west - requests: 60 diff --git a/manifests/examples/pipelines/scenario-1.yml b/manifests/examples/pipelines/scenario-1.yml new file mode 100644 index 000000000..9ab2d20f4 --- /dev/null +++ b/manifests/examples/pipelines/scenario-1.yml @@ -0,0 +1,34 @@ +name: demo +description: demo for observe feat +tags: +initialize: + plugins: + mock-observations: + kind: plugin + method: MockObservations + path: "builtin" + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:01 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 +tree: + children: + child: + pipeline: + observe: + - mock-observations + inputs: null diff --git a/manifests/examples/pipelines/scenario-2.yml b/manifests/examples/pipelines/scenario-2.yml new file mode 100644 index 000000000..569cbcb05 --- /dev/null +++ b/manifests/examples/pipelines/scenario-2.yml @@ -0,0 +1,52 @@ +name: regroup demo +description: +initialize: + plugins: + interpolate: + method: Interpolation + path: "builtin" + global-config: + method: linear + x: [0, 10, 50, 100] + y: [0.12, 0.32, 0.75, 1.02] + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" +tree: + children: + child: + pipeline: + observe: + regroup: + - cloud/region + - cloud/instance-type + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-3.yml b/manifests/examples/pipelines/scenario-3.yml new file mode 100644 index 000000000..f5710b553 --- /dev/null +++ b/manifests/examples/pipelines/scenario-3.yml @@ -0,0 +1,86 @@ +name: groupby +description: successful path +initialize: + plugins: + "sum": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy +tree: + children: + my-app: + pipeline: + observe: + regroup: + - cloud/instance-type + - cloud/region + compute: + children: + uk-west: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 + uk-east: + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 9 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-east + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-east + cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-4.yml b/manifests/examples/pipelines/scenario-4.yml new file mode 100644 index 000000000..151db09be --- /dev/null +++ b/manifests/examples/pipelines/scenario-4.yml @@ -0,0 +1,46 @@ +name: demo +description: +tags: +initialize: + plugins: + "sum": + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy-sum + "coefficient": + path: "builtin" + method: Coefficient + global-config: + input-parameter: energy + coefficient: 2 + output-parameter: energy-doubled + "multiply": + path: "builtin" + method: Multiply + global-config: + input-parameters: ["cpu/utilization", "duration"] + output-parameter: "cpu-times-duration" +tree: + children: + child-1: + pipeline: + observe: + compute: + - sum + - coefficient + - multiply + defaults: + cpu/thermal-design-power: 100 + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + cpu/energy: 20 + network/energy: 10 + energy: 5 diff --git a/manifests/examples/pipelines/scenario-5.yml b/manifests/examples/pipelines/scenario-5.yml new file mode 100644 index 000000000..6990c33e3 --- /dev/null +++ b/manifests/examples/pipelines/scenario-5.yml @@ -0,0 +1,46 @@ +name: demo +description: +tags: +initialize: + plugins: + mock-observations: + kind: plugin + method: MockObservations + path: "builtin" + global-config: + timestamp-from: 2023-07-06T00:00 + timestamp-to: 2023-07-06T00:01 + duration: 60 + components: + - cloud/instance-type: A1 + - cloud/instance-type: B1 + generators: + common: + region: uk-west + common-key: common-val + randint: + cpu/utilization: + min: 1 + max: 99 + memory/utilization: + min: 1 + max: 99 + sum: + path: "builtin" + method: Sum + global-config: + input-parameters: + - cpu/utilization + - memory/utilization + output-parameter: util-sum +tree: + children: + child: + pipeline: + observe: + - mock-observations + regroup: + - cloud/instance-type + compute: + - sum + inputs: null diff --git a/manifests/examples/pipelines/sci.yml b/manifests/examples/pipelines/sci.yml index 67c55cd64..949d9e7df 100644 --- a/manifests/examples/pipelines/sci.yml +++ b/manifests/examples/pipelines/sci.yml @@ -5,13 +5,13 @@ initialize: plugins: interpolate: method: Interpolation - path: 'builtin' + path: "builtin" global-config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" cpu-factor-to-wattage: method: Multiply path: builtin @@ -75,24 +75,22 @@ initialize: method: Sci global-config: functional-unit: "component" - outputs: - - yaml tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sum-energy-components - - embodied-carbon - - operational-carbon - - sum-carbon - - sci - config: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sum-energy-components + - embodied-carbon + - operational-carbon + - sum-carbon + - sci defaults: cpu/thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/examples/pipelines/teads-curve.yml b/manifests/examples/pipelines/teads-curve.yml index 924384c20..1679a0976 100644 --- a/manifests/examples/pipelines/teads-curve.yml +++ b/manifests/examples/pipelines/teads-curve.yml @@ -49,12 +49,13 @@ tree: children: child: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio defaults: thermal-design-power: 100 vcpus-total: 8 diff --git a/manifests/examples/pipelines/zeros.yml b/manifests/examples/pipelines/zeros.yml index da5541789..f76a2c79f 100644 --- a/manifests/examples/pipelines/zeros.yml +++ b/manifests/examples/pipelines/zeros.yml @@ -106,22 +106,21 @@ tree: children: child-1: pipeline: - - sum-zero-and-one - - sum-zero-and-zero - - subtract-one-and-zero - - subtract-zero-and-zero - - subtract-zero-and-one - - coefficient-one-times-zero - - coefficient-zero-times-one - - coefficient-zero-times-zero - - multiply-one-times-zero - - multiply-zero-times-one - - exponent-one-to-zero - - exponent-zero-to-one - - exponent-zero-to-zero - - sci - config: - defaults: + compute: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci inputs: - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index fc0ac48b0..ed4c00385 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -8,8 +8,6 @@ aggregation: - dummy-param type: both initialize: - outputs: - - yaml plugins: interpolate: method: Interpolation @@ -21,7 +19,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -85,13 +83,10 @@ initialize: method: TimeSync path: builtin global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - group-by: - path: builtin - method: GroupBy execution: status: fail command: >- @@ -102,20 +97,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T19:25:34.759Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -137,27 +132,25 @@ execution: - winston@3.11.0 - zod@3.22.4 error: >- - MissingAggregationParamError: Aggregation metric dummy-param is not found in - inputs[0]. + MissingInputDataError: `functional-unit` value is missing from input data or it is not a positive integer tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -167,25 +160,25 @@ tree: vcpus-allocated: 1 vcpus-total: 8 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 10 requests: 100 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -193,21 +186,20 @@ tree: requests: 100 child-2: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - sci-embodied - - operational-carbon - - time-sync - - sci - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - time-sync + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -217,25 +209,25 @@ tree: vcpus-allocated: 1 vcpus-total: 8 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 cpu/utilization: 30 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 28 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 40 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cpu/utilization: 33 cloud/instance-type: A1 diff --git a/manifests/outputs/bugs/initialize-error-no-config.yaml b/manifests/outputs/bugs/initialize-error-no-config.yaml deleted file mode 100644 index 56be9add7..000000000 --- a/manifests/outputs/bugs/initialize-error-no-config.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: initialize-error-no-config -description: >- - a negative test case that fails due to plugin initialization missing some - required config -tags: null -initialize: - outputs: - - yaml - plugins: - interpolate: - method: Interpolation - path: builtin - global-config: null -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/bugs/initialize-error-no-config.yml -o - manifests/outputs/bugs/initialize-error-no-config - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-01T19:45:41.936Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - ManifestValidationError: "initialize.plugins.interpolate.global-config" - parameter is expected object, received null. Error code: invalid_type. -tree: - children: - child-0: - defaults: - cpu/thermal-design-power: 100 - pipeline: - - interpolate - inputs: - - timestamp: 2023-07-06T00:00 - duration: 1 - cpu/utilization: 20 - - timestamp: 2023-07-06T00:01 - duration: 1 - cpu/utilization: 80 - - timestamp: 2023-07-06T00:02 - duration: 1 - cpu/utilization: 20 diff --git a/manifests/outputs/bugs/initialize-error-no-path.yaml b/manifests/outputs/bugs/initialize-error-no-path.yaml deleted file mode 100644 index e2ee4d616..000000000 --- a/manifests/outputs/bugs/initialize-error-no-path.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: initialize-error-no-path -description: >- - a negative test case that fails because the path is mising in a plugin - initialization -tags: null -initialize: - plugins: - interpolate: - method: Interpolation - path: null - global-config: - method: linear - x: - - 0 - - 10 - - 50 - - 100 - 'y': - - 0.12 - - 0.32 - - 0.75 - - 1.02 - input-parameter: cpu/utilization - output-parameter: cpu-factor - outputs: - - yaml -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/bugs/initialize-error-no-path.yml -o - manifests/outputs/bugs/initialize-error-no-path - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-01T19:57:11.499Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - ManifestValidationError: "initialize.plugins.interpolate.path" parameter is - expected string, received null. Error code: invalid_type. -tree: - children: - child-0: - defaults: - cpu/thermal-design-power: 100 - pipeline: - - interpolate - inputs: - - timestamp: 2023-07-06T00:00 - duration: 1 - cpu/utilization: 20 - - timestamp: 2023-07-06T00:01 - duration: 1 - cpu/utilization: 80 - - timestamp: 2023-07-06T00:02 - duration: 1 - cpu/utilization: 20 diff --git a/manifests/outputs/bugs/initialize-error-no-plugins.yaml b/manifests/outputs/bugs/initialize-error-no-plugins.yaml deleted file mode 100644 index 208010e00..000000000 --- a/manifests/outputs/bugs/initialize-error-no-plugins.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: initialize-error-no-path -description: >- - a negative test case that fails becuase no plugins are included in the - initialize block -tags: null -initialize: - plugins: null - outputs: - - yaml -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/bugs/initialize-error-no-plugins.yml -o - manifests/outputs/bugs/initialize-error-no-plugins - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-01T19:52:35.214Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - ManifestValidationError: "initialize.plugins" parameter is expected object, - received null. Error code: invalid_type. -tree: - children: - child-0: - defaults: - cpu/thermal-design-power: 100 - pipeline: - - teads-curve - inputs: - - timestamp: 2023-07-06T00:00 - duration: 1 - cpu/utilization: 20 - - timestamp: 2023-07-06T00:01 - duration: 1 - cpu/utilization: 80 - - timestamp: 2023-07-06T00:02 - duration: 1 - cpu/utilization: 20 diff --git a/manifests/outputs/bugs/input-error-missing-duration.yaml b/manifests/outputs/bugs/input-error-missing-duration.yaml index dd339eb4f..1006c1c0c 100644 --- a/manifests/outputs/bugs/input-error-missing-duration.yaml +++ b/manifests/outputs/bugs/input-error-missing-duration.yaml @@ -15,15 +15,13 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 - 1.02 input-parameter: cpu/utilization output-parameter: cpu-factor - outputs: - - yaml execution: status: fail command: >- @@ -34,21 +32,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T19:59:58.987Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -78,7 +75,8 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - interpolate + compute: + - interpolate inputs: - timestamp: 2023-07-06T00:00 cpu/utilization: 20 diff --git a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml index fedf856fb..d137975b5 100644 --- a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml +++ b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml @@ -2,8 +2,6 @@ name: mock-observation-demo description: a manifest demonstrating how to use the mock observations feature tags: null initialize: - outputs: - - yaml plugins: mock-observations: kind: plugin @@ -37,21 +35,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:02:22.596Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -72,10 +69,11 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: 'RangeError: Maximum call stack size exceeded' + error: "InputValidationError: \"duration\" parameter is number must be greater than 0. Error code: too_small." tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml index 21fc5e668..fef0fa246 100644 --- a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml +++ b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml @@ -15,15 +15,13 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 - 1.02 input-parameter: cpu/utilization output-parameter: cpu-factor - outputs: - - yaml execution: status: fail command: >- @@ -34,21 +32,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:06:51.254Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -78,7 +75,8 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - wrong-name + compute: + - wrong-name inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml index a37517432..7a0060013 100644 --- a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml +++ b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml @@ -4,8 +4,6 @@ description: >- in a pipeline tags: null initialize: - outputs: - - yaml plugins: interpolate: method: Interpolation @@ -17,7 +15,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -34,21 +32,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:08:51.260Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -78,8 +75,9 @@ tree: defaults: cpu/thermal-design-power: 100 pipeline: - - interpolate - - multiply + compute: + - interpolate + - multiply inputs: - timestamp: 2023-07-06T00:00 duration: 1 diff --git a/manifests/outputs/bugs/pipeline-ordering-error.yaml b/manifests/outputs/bugs/pipeline-ordering-error.yaml index fe59b03d8..d1bd27d27 100644 --- a/manifests/outputs/bugs/pipeline-ordering-error.yaml +++ b/manifests/outputs/bugs/pipeline-ordering-error.yaml @@ -4,8 +4,6 @@ description: >- pipeline (before its inputs are generated) tags: null initialize: - outputs: - - yaml plugins: interpolate: method: Interpolation @@ -17,7 +15,7 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -71,21 +69,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:14:29.109Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -113,13 +110,13 @@ tree: children: child-1: pipeline: - - interpolate - - correct-cpu-energy-for-vcpu-ratio - - calculate-vcpu-ratio - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - config: null + compute: + - interpolate + - correct-cpu-energy-for-vcpu-ratio + - calculate-vcpu-ratio + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -129,25 +126,25 @@ tree: vcpus-allocated: 1 vcpus-total: 8 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west diff --git a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml index e0a50cd90..486947ebd 100644 --- a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml +++ b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml @@ -4,8 +4,6 @@ description: >- `resources-total` issue tags: null initialize: - outputs: - - yaml plugins: sci-embodied: method: SciEmbodied @@ -20,21 +18,21 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:17:30.390Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,7 +60,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/plugins/coefficient/failure-invalid-config-input-param.yaml b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml similarity index 77% rename from manifests/outputs/plugins/coefficient/failure-invalid-config-input-param.yaml rename to manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml index ba10eaee3..7b6051dce 100644 --- a/manifests/outputs/plugins/coefficient/failure-invalid-config-input-param.yaml +++ b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml @@ -2,8 +2,6 @@ name: coefficient-demo description: failure with ivalid `global-config.input-parameter` tags: null initialize: - outputs: - - yaml plugins: coefficient: method: Coefficient @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:41:49.144Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,9 +61,8 @@ tree: children: child: pipeline: - - coefficient - config: - sum: null + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/coefficient/failure-output-param-is-null.yaml b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml similarity index 77% rename from manifests/outputs/plugins/coefficient/failure-output-param-is-null.yaml rename to manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml index dd2f6c952..847e984e7 100644 --- a/manifests/outputs/plugins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml @@ -2,8 +2,6 @@ name: coefficient-demo description: value of output-param is missing tags: null initialize: - outputs: - - yaml plugins: coefficient: method: Coefficient @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:38:48.831Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,9 +61,8 @@ tree: children: child: pipeline: - - coefficient - config: - sum: null + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/coefficient/success.yaml b/manifests/outputs/builtins/coefficient/success.yaml similarity index 78% rename from manifests/outputs/plugins/coefficient/success.yaml rename to manifests/outputs/builtins/coefficient/success.yaml index 86300012c..14a235709 100644 --- a/manifests/outputs/plugins/coefficient/success.yaml +++ b/manifests/outputs/builtins/coefficient/success.yaml @@ -10,8 +10,6 @@ initialize: input-parameter: carbon coefficient: 3 output-parameter: carbon-product - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -21,21 +19,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:37:08.297Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,8 +59,6 @@ tree: child: pipeline: - coefficient - config: - sum: null inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml similarity index 81% rename from manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml rename to manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index 6a8ed7fed..6945e6012 100644 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -2,8 +2,6 @@ name: cloud-metadata description: cloud/instance-type instance type is not supported in the cloud vendor tags: null initialize: - outputs: - - yaml plugins: cloud-metadata: path: builtin @@ -27,20 +25,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-04T13:44:58.641Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -68,8 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 cloud/vendor: aws diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml similarity index 80% rename from manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml rename to manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml index 074d1a4b0..333d9e1ce 100644 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml @@ -2,8 +2,6 @@ name: cloud-metadata description: failing because cloud/vendor is not provided tags: null initialize: - outputs: - - yaml plugins: cloud-metadata: path: builtin @@ -27,20 +25,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-05T09:07:35.386Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -68,8 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 duration: 100 diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml similarity index 81% rename from manifests/outputs/plugins/csv-lookup/cloud-metadata/success.yaml rename to manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml index 73c459201..37fab5816 100644 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml @@ -14,8 +14,6 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -25,20 +23,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-05T09:05:09.577Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,8 +62,8 @@ tree: children: child: pipeline: - - cloud-metadata - config: null + compute: + - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 cloud/vendor: aws diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-column.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml similarity index 79% rename from manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-column.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml index cad931fd1..842080a77 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-column.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml @@ -2,8 +2,6 @@ name: csv-lookup-demo description: null tags: null initialize: - outputs: - - yaml plugins: cloud-metadata: method: CSVLookup @@ -14,7 +12,7 @@ initialize: query: cloud-provider: nonexistant cloud-region: cloud/region - output: '*' + output: "*" execution: status: fail command: >- @@ -26,21 +24,21 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:26:59.352Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -68,7 +66,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml similarity index 80% rename from manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml index ad5ab9ec8..32ca690d9 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml @@ -2,8 +2,6 @@ name: csv-lookup-demo description: null tags: null initialize: - outputs: - - yaml plugins: cloud-metadata: method: CSVLookup @@ -25,21 +23,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:26:09.874Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -67,7 +64,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/success-renaming.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml similarity index 81% rename from manifests/outputs/plugins/csv-lookup/region-metadata/success-renaming.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml index 77aeecd66..2f6f23dcf 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/success-renaming.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml @@ -15,8 +15,6 @@ initialize: output: - em-zone-id - renamed-em-zone-data - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -26,21 +24,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:24:10.894Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -66,7 +63,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/region-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml similarity index 83% rename from manifests/outputs/plugins/csv-lookup/region-metadata/success.yaml rename to manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml index 10bb9a637..f8bd9fdba 100644 --- a/manifests/outputs/plugins/csv-lookup/region-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml @@ -12,9 +12,7 @@ initialize: query: cloud-provider: cloud/provider cloud-region: cloud/region - output: '*' - outputs: - - yaml + output: "*" execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -24,21 +22,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:23:15.965Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,7 +61,8 @@ tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml similarity index 79% rename from manifests/outputs/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yaml rename to manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml index 56b6c2630..ce268cd02 100644 --- a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml @@ -2,8 +2,6 @@ name: tdp-finder description: failure with `inputs` missing `physical-processor` param tags: null initialize: - outputs: - - yaml plugins: tdp-finder: method: CSVLookup @@ -25,21 +23,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:30:11.973Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -67,8 +64,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml similarity index 79% rename from manifests/outputs/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml rename to manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml index 888df1ce7..28dc0b1a0 100644 --- a/manifests/outputs/plugins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml @@ -2,8 +2,6 @@ name: tdp-finder description: successful path tags: null initialize: - outputs: - - yaml plugins: tdp-finder: method: CSVLookup @@ -25,21 +23,21 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:29:16.211Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -67,8 +65,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/csv-lookup/tdp-finder/success.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml similarity index 79% rename from manifests/outputs/plugins/csv-lookup/tdp-finder/success.yaml rename to manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml index bf12d4892..f085664ff 100644 --- a/manifests/outputs/plugins/csv-lookup/tdp-finder/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml @@ -12,8 +12,6 @@ initialize: query: name: physical-processor output: tdp - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -23,21 +21,21 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:28:19.790Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -63,8 +61,8 @@ tree: children: child: pipeline: - - tdp-finder - config: null + compute: + - tdp-finder inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/divide/failure-invalid-config-denominator.yaml b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml similarity index 77% rename from manifests/outputs/plugins/divide/failure-invalid-config-denominator.yaml rename to manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml index 299cc1ade..b1149c505 100644 --- a/manifests/outputs/plugins/divide/failure-invalid-config-denominator.yaml +++ b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml @@ -2,8 +2,6 @@ name: divide description: failure when `global-config.denominator` is string tags: null initialize: - outputs: - - yaml plugins: divide: method: Divide @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T06:02:25.409Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,9 +61,8 @@ tree: children: child: pipeline: - - divide - config: - divide: null + compute: + - divide inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/divide/failure-missing-numerator.yaml b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml similarity index 79% rename from manifests/outputs/plugins/divide/failure-missing-numerator.yaml rename to manifests/outputs/builtins/divide/failure-missing-numerator.yaml index 6aa908327..6c714bf18 100644 --- a/manifests/outputs/plugins/divide/failure-missing-numerator.yaml +++ b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml @@ -2,8 +2,6 @@ name: divide description: success path tags: null initialize: - outputs: - - yaml plugins: divide: method: Divide @@ -21,21 +19,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:49:51.802Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -63,7 +60,8 @@ tree: children: child: pipeline: - - divide + compute: + - divide config: divide: null defaults: diff --git a/manifests/outputs/plugins/divide/success-denominator-equal-zero.yaml b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml similarity index 82% rename from manifests/outputs/plugins/divide/success-denominator-equal-zero.yaml rename to manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml index 5617fdbf9..25033c32c 100644 --- a/manifests/outputs/plugins/divide/success-denominator-equal-zero.yaml +++ b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml @@ -21,8 +21,6 @@ initialize: numerator: vcpus-allocated denominator: 0 output: cpu/number-cores - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -32,21 +30,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T06:05:58.040Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -72,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/plugins/divide/success.yaml b/manifests/outputs/builtins/divide/success.yaml similarity index 81% rename from manifests/outputs/plugins/divide/success.yaml rename to manifests/outputs/builtins/divide/success.yaml index 3c741121f..f7bab8b04 100644 --- a/manifests/outputs/plugins/divide/success.yaml +++ b/manifests/outputs/builtins/divide/success.yaml @@ -21,8 +21,6 @@ initialize: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -32,21 +30,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:43:34.055Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -72,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/outputs/plugins/exponent/success.yaml b/manifests/outputs/builtins/exponent/success.yaml similarity index 77% rename from manifests/outputs/plugins/exponent/success.yaml rename to manifests/outputs/builtins/exponent/success.yaml index 1a8588693..8586b57f6 100644 --- a/manifests/outputs/plugins/exponent/success.yaml +++ b/manifests/outputs/builtins/exponent/success.yaml @@ -10,8 +10,6 @@ initialize: input-parameter: cpu/energy exponent: 2 output-parameter: energy - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -21,21 +19,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:07:19.737Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -61,9 +58,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: null + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/interpolation/interpolation.yaml b/manifests/outputs/builtins/interpolation/interpolation.yaml similarity index 79% rename from manifests/outputs/plugins/interpolation/interpolation.yaml rename to manifests/outputs/builtins/interpolation/interpolation.yaml index 9e3c45e4d..2569b5eb4 100644 --- a/manifests/outputs/plugins/interpolation/interpolation.yaml +++ b/manifests/outputs/builtins/interpolation/interpolation.yaml @@ -13,15 +13,13 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 - 1.02 input-parameter: cpu/utilization output-parameter: result - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -31,21 +29,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:14:18.033Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -71,7 +68,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/interpolation/success.yaml b/manifests/outputs/builtins/interpolation/success.yaml similarity index 79% rename from manifests/outputs/plugins/interpolation/success.yaml rename to manifests/outputs/builtins/interpolation/success.yaml index bad3425f9..80def13d9 100644 --- a/manifests/outputs/plugins/interpolation/success.yaml +++ b/manifests/outputs/builtins/interpolation/success.yaml @@ -13,15 +13,13 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 - 1.02 input-parameter: cpu/utilization output-parameter: result - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -31,21 +29,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:13:22.946Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -71,7 +68,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/mock-observations/failure-invalid-config-cpu-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml similarity index 82% rename from manifests/outputs/plugins/mock-observations/failure-invalid-config-cpu-range.yaml rename to manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml index d79f394f5..272d9940e 100644 --- a/manifests/outputs/plugins/mock-observations/failure-invalid-config-cpu-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml @@ -4,8 +4,6 @@ description: >- greater than `max` tags: null initialize: - outputs: - - yaml plugins: mock-observations: kind: plugin @@ -40,21 +38,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:19:22.233Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -82,5 +79,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/plugins/mock-observations/failure-invalid-memory-utilization-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml similarity index 82% rename from manifests/outputs/plugins/mock-observations/failure-invalid-memory-utilization-range.yaml rename to manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml index 3667dff6c..e7c7e6180 100644 --- a/manifests/outputs/plugins/mock-observations/failure-invalid-memory-utilization-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml @@ -2,8 +2,6 @@ name: mock-observation-demo description: a manifest demonstrating how to use the mock observations feature tags: null initialize: - outputs: - - yaml plugins: mock-observations: kind: plugin @@ -38,21 +36,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:18:11.486Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -80,5 +77,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/plugins/mock-observations/failure-missing-timestamp-from-param.yaml b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml similarity index 81% rename from manifests/outputs/plugins/mock-observations/failure-missing-timestamp-from-param.yaml rename to manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml index b8599cb08..86f4afd9e 100644 --- a/manifests/outputs/plugins/mock-observations/failure-missing-timestamp-from-param.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml @@ -2,8 +2,6 @@ name: mock-observation-demo description: a manifest demonstrating how to use the mock observations feature tags: null initialize: - outputs: - - yaml plugins: mock-observations: kind: plugin @@ -37,21 +35,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:17:01.277Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -79,5 +76,6 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: null diff --git a/manifests/outputs/plugins/mock-observations/success.yaml b/manifests/outputs/builtins/mock-observations/success.yaml similarity index 55% rename from manifests/outputs/plugins/mock-observations/success.yaml rename to manifests/outputs/builtins/mock-observations/success.yaml index e9629813c..bcc5ee034 100644 --- a/manifests/outputs/plugins/mock-observations/success.yaml +++ b/manifests/outputs/builtins/mock-observations/success.yaml @@ -24,27 +24,24 @@ initialize: memory/utilization: min: 1 max: 99 - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/mock-observations/success.yml -o - manifests/outputs/plugins/mock-observations/success + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/builtins/mock-observations/success.yml -o + manifests/outputs/builtins/mock-observations/success environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: '13.2' + os-version: '14.5' node-version: 18.14.2 - date-time: 2024-07-02T19:15:43.108Z (UTC) + date-time: 2024-08-02T15:04:18.262Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' - '@commitlint/cli@18.6.0' - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - + - '@grnsft/if-core@0.0.16' - '@jest/globals@29.7.0' - '@types/jest@29.5.8' - '@types/js-yaml@4.0.9' @@ -69,14 +66,155 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: child: pipeline: - - mock-observations - inputs: null + observe: + - mock-observations + inputs: + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: A1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:00:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:01:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:02:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:03:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:04:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:05:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:06:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:07:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:08:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' + - timestamp: '2023-07-06T00:09:00.000Z' + duration: 60 + cloud/instance-type: B1 + region: uk-west + common-key: common-val + cpu/utilization: '*' + memory/utilization: '*' outputs: - timestamp: '2023-07-06T00:00:00.000Z' duration: 60 diff --git a/manifests/outputs/plugins/multiply/failure-input-parameter-is-missing.yaml b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml similarity index 78% rename from manifests/outputs/plugins/multiply/failure-input-parameter-is-missing.yaml rename to manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml index 1ef59959d..80738e80c 100644 --- a/manifests/outputs/plugins/multiply/failure-input-parameter-is-missing.yaml +++ b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml @@ -2,8 +2,6 @@ name: multiply description: failure `inputs` is missing `cpu/energy` parameter tags: null initialize: - outputs: - - yaml plugins: multiply: method: Multiply @@ -23,21 +21,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:23:15.185Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -65,9 +62,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/multiply/success-with-multiple-inputs.yaml b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml similarity index 83% rename from manifests/outputs/plugins/multiply/success-with-multiple-inputs.yaml rename to manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml index 76c3d93d3..359e25a06 100644 --- a/manifests/outputs/plugins/multiply/success-with-multiple-inputs.yaml +++ b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml @@ -11,8 +11,6 @@ initialize: - cpu/energy - network/energy output-parameter: energy-product - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:22:04.324Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,9 +59,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/multiply/success.yaml b/manifests/outputs/builtins/multiply/success.yaml similarity index 78% rename from manifests/outputs/plugins/multiply/success.yaml rename to manifests/outputs/builtins/multiply/success.yaml index 60546f50e..1f93a9140 100644 --- a/manifests/outputs/plugins/multiply/success.yaml +++ b/manifests/outputs/builtins/multiply/success.yaml @@ -11,8 +11,6 @@ initialize: - cpu/energy - network/energy output-parameter: energy-product - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:20:34.016Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,9 +59,8 @@ tree: children: child: pipeline: - - multiply - config: - sum: null + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/regex/failure-missing-input-param.yaml b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml similarity index 79% rename from manifests/outputs/plugins/regex/failure-missing-input-param.yaml rename to manifests/outputs/builtins/regex/failure-missing-input-param.yaml index 8399ac20e..cd578218f 100644 --- a/manifests/outputs/plugins/regex/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml @@ -2,8 +2,6 @@ name: regex description: failure with missing `physical-processor` field from `inputs` tags: null initialize: - outputs: - - yaml plugins: regex: method: Regex @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:26:07.477Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,9 +61,8 @@ tree: children: child: pipeline: - - regex - config: - regex: null + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/regex/success.yaml b/manifests/outputs/builtins/regex/success.yaml similarity index 77% rename from manifests/outputs/plugins/regex/success.yaml rename to manifests/outputs/builtins/regex/success.yaml index 763cd4093..22988eead 100644 --- a/manifests/outputs/plugins/regex/success.yaml +++ b/manifests/outputs/builtins/regex/success.yaml @@ -10,8 +10,6 @@ initialize: parameter: physical-processor match: ^(.*), output: cpu/name - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -21,21 +19,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:23:25.744Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -61,9 +58,8 @@ tree: children: child: pipeline: - - regex - config: - regex: null + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -77,5 +73,4 @@ tree: IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz,IntelĀ® XeonĀ® E5-2673 v3 2.4 GHz cpu/name: >- - IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® - E5-2673 v4 2.3 GHz, + IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz, IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz diff --git a/manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yaml b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml similarity index 80% rename from manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yaml rename to manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml index a3df4f806..f6af79041 100644 --- a/manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml @@ -4,8 +4,6 @@ description: >- number tags: null initialize: - outputs: - - yaml plugins: sci-embodied: method: SciEmbodied @@ -21,21 +19,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:49:08.280Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -63,7 +60,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: fail time-reserved: 3600 diff --git a/manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yaml b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml similarity index 78% rename from manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yaml rename to manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml index 073b2cb45..43f22163e 100644 --- a/manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml @@ -2,8 +2,6 @@ name: sci-embodied description: missing device/expected-lifespan tags: null initialize: - outputs: - - yaml plugins: sci-embodied: method: SciEmbodied @@ -18,21 +16,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:42:51.951Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -60,7 +57,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/plugins/sci-embodied/success.yaml b/manifests/outputs/builtins/sci-embodied/success.yaml similarity index 80% rename from manifests/outputs/plugins/sci-embodied/success.yaml rename to manifests/outputs/builtins/sci-embodied/success.yaml index 5f74a121e..e91ab9212 100644 --- a/manifests/outputs/plugins/sci-embodied/success.yaml +++ b/manifests/outputs/builtins/sci-embodied/success.yaml @@ -6,8 +6,6 @@ initialize: sci-embodied: path: builtin method: SciEmbodied - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -17,21 +15,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:42:03.186Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -57,7 +54,8 @@ tree: children: child: pipeline: - - sci-embodied + compute: + - sci-embodied defaults: device/emissions-embodied: 1533.12 time-reserved: 3600 diff --git a/manifests/outputs/plugins/sci/failure-invalid-config-value.yaml b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml similarity index 76% rename from manifests/outputs/plugins/sci/failure-invalid-config-value.yaml rename to manifests/outputs/builtins/sci/failure-invalid-config-value.yaml index 61c424e91..384fe14b1 100644 --- a/manifests/outputs/plugins/sci/failure-invalid-config-value.yaml +++ b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml @@ -2,8 +2,6 @@ name: sci description: failure with `config.sci.functional-unit` value being number tags: null initialize: - outputs: - - yaml plugins: sci: kind: plugin @@ -19,21 +17,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:38:15.858Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -54,12 +51,13 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: 'InputValidationError: Required' + error: "InputValidationError: Required" tree: children: child: pipeline: - - sci + compute: + - sci config: sci: functional-unit: 999 diff --git a/manifests/outputs/plugins/sci/failure-missing-input-param.yaml b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml similarity index 78% rename from manifests/outputs/plugins/sci/failure-missing-input-param.yaml rename to manifests/outputs/builtins/sci/failure-missing-input-param.yaml index 656f21c1d..b0fbce2f8 100644 --- a/manifests/outputs/plugins/sci/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml @@ -2,8 +2,6 @@ name: sci description: missing input value (carbon-embodied) tags: null initialize: - outputs: - - yaml plugins: sci: kind: plugin @@ -21,21 +19,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:36:10.588Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -63,7 +60,8 @@ tree: children: child: pipeline: - - sci + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sci/success.yml.yaml b/manifests/outputs/builtins/sci/success.yaml similarity index 79% rename from manifests/outputs/plugins/sci/success.yml.yaml rename to manifests/outputs/builtins/sci/success.yaml index bd7b81f0c..9f36fc9bd 100644 --- a/manifests/outputs/plugins/sci/success.yml.yaml +++ b/manifests/outputs/builtins/sci/success.yaml @@ -8,8 +8,6 @@ initialize: method: Sci global-config: functional-unit: requests - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -19,21 +17,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:35:12.024Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -59,8 +56,8 @@ tree: children: child: pipeline: - - sci - config: null + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/shell/failure-invalid-command.yaml b/manifests/outputs/builtins/shell/failure-invalid-command.yaml similarity index 76% rename from manifests/outputs/plugins/shell/failure-invalid-command.yaml rename to manifests/outputs/builtins/shell/failure-invalid-command.yaml index f9ec8894b..97d8bb6af 100644 --- a/manifests/outputs/plugins/shell/failure-invalid-command.yaml +++ b/manifests/outputs/builtins/shell/failure-invalid-command.yaml @@ -2,8 +2,6 @@ name: shell description: falure with `global-config.command` being number instead od string tags: null initialize: - outputs: - - yaml plugins: shell: method: Shell @@ -20,21 +18,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:51:36.186Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,9 +59,10 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - - timestamp: '2023-11-02T10:35:31.820Z' + - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 cpu/energy: 0.002 memory/energy: 0.000005 diff --git a/manifests/outputs/plugins/shell/success.yaml b/manifests/outputs/builtins/shell/success.yaml similarity index 73% rename from manifests/outputs/plugins/shell/success.yaml rename to manifests/outputs/builtins/shell/success.yaml index 3a8825817..b85cbfb95 100644 --- a/manifests/outputs/plugins/shell/success.yaml +++ b/manifests/outputs/builtins/shell/success.yaml @@ -8,8 +8,6 @@ initialize: method: Shell global-config: command: python3 /usr/local/bin/sampler - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -19,21 +17,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:50:37.625Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -59,9 +56,10 @@ tree: children: child: pipeline: - - shell + compute: + - shell inputs: - - timestamp: '2023-11-02T10:35:31.820Z' + - timestamp: "2023-11-02T10:35:31.820Z" duration: 3600 cpu/energy: 0.002 memory/energy: 0.000005 diff --git a/manifests/outputs/plugins/subtract/success.yaml b/manifests/outputs/builtins/subtract/success.yaml similarity index 77% rename from manifests/outputs/plugins/subtract/success.yaml rename to manifests/outputs/builtins/subtract/success.yaml index 5d028f63f..5eb0a7bd3 100644 --- a/manifests/outputs/plugins/subtract/success.yaml +++ b/manifests/outputs/builtins/subtract/success.yaml @@ -11,8 +11,6 @@ initialize: - cpu/energy - network/energy output-parameter: energy/diff - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:52:42.932Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,9 +59,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: null + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sum/failure-missing-input-param.yaml b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml similarity index 78% rename from manifests/outputs/plugins/sum/failure-missing-input-param.yaml rename to manifests/outputs/builtins/sum/failure-missing-input-param.yaml index ce59a1f4e..e2aaf8158 100644 --- a/manifests/outputs/plugins/sum/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml @@ -2,8 +2,6 @@ name: sum description: failure with `inputs[0]` misses one of `global-config.input-parameters` tags: null initialize: - outputs: - - yaml plugins: sum: method: Sum @@ -23,21 +21,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:55:20.733Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -65,9 +62,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sum/failure-missing-output-param.yaml b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml similarity index 78% rename from manifests/outputs/plugins/sum/failure-missing-output-param.yaml rename to manifests/outputs/builtins/sum/failure-missing-output-param.yaml index 548a8b14e..d7a4d9302 100644 --- a/manifests/outputs/plugins/sum/failure-missing-output-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml @@ -2,8 +2,6 @@ name: sum description: missing `output-parameter` in global-config tags: null initialize: - outputs: - - yaml plugins: sum: method: Sum @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:54:23.422Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -64,9 +61,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/sum/success.yaml b/manifests/outputs/builtins/sum/success.yaml similarity index 77% rename from manifests/outputs/plugins/sum/success.yaml rename to manifests/outputs/builtins/sum/success.yaml index 1d9b47a3f..6b5b4d973 100644 --- a/manifests/outputs/plugins/sum/success.yaml +++ b/manifests/outputs/builtins/sum/success.yaml @@ -11,8 +11,6 @@ initialize: - cpu/energy - network/energy output-parameter: energy - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -22,21 +20,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T20:53:35.496Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,9 +59,8 @@ tree: children: child: pipeline: - - sum - config: - sum: null + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/plugins/time-sync/failure-config-start-later-end.yaml b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml similarity index 65% rename from manifests/outputs/plugins/time-sync/failure-config-start-later-end.yaml rename to manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml index e0957366a..bb5ae79fb 100644 --- a/manifests/outputs/plugins/time-sync/failure-config-start-later-end.yaml +++ b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml @@ -4,15 +4,13 @@ description: >- `global-config.end-time` tags: null initialize: - outputs: - - yaml plugins: time-sync: method: TimeSync path: builtin global-config: - start-time: '2023-12-12T00:01:00.000Z' - end-time: '2023-12-12T00:00:00.000Z' + start-time: "2023-12-12T00:01:00.000Z" + end-time: "2023-12-12T00:00:00.000Z" interval: 5 allow-padding: true execution: @@ -25,21 +23,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:13:59.114Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -60,23 +57,23 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: 'InputValidationError: `start-time` should be lower than `end-time`' + error: "InputValidationError: `start-time` should be lower than `end-time`" tree: children: child: pipeline: - - time-sync - config: null + compute: + - time-sync inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 energy-cpu: 0.001 diff --git a/manifests/outputs/plugins/time-sync/success.yaml b/manifests/outputs/builtins/time-sync/success.yaml similarity index 59% rename from manifests/outputs/plugins/time-sync/success.yaml rename to manifests/outputs/builtins/time-sync/success.yaml index 1f0a7f07e..7b1d6bbb8 100644 --- a/manifests/outputs/plugins/time-sync/success.yaml +++ b/manifests/outputs/builtins/time-sync/success.yaml @@ -7,12 +7,16 @@ initialize: path: builtin method: TimeSync global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - outputs: - - yaml + parameter-metadata: + outputs: + energy-cpu: + unit: KWH + description: energy + aggregation-method: sum execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -22,21 +26,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T21:12:32.629Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -62,58 +65,58 @@ tree: children: child: pipeline: - - time-sync - config: null + compute: + - time-sync inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 energy-cpu: 0.001 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 5 energy-cpu: 0.0018000000000000004 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 energy-cpu: 0.0007714285714285716 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 energy-cpu: 0.0004952380952380952 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 energy-cpu: 0.0001666666666666667 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 energy-cpu: 0.0001666666666666667 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 energy-cpu: 0.0001666666666666667 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 energy-cpu: 0.0001666666666666667 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 energy-cpu: 0.0001666666666666667 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 energy-cpu: 0.0001 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 energy-cpu: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 energy-cpu: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 energy-cpu: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 energy-cpu: 0 diff --git a/manifests/outputs/divide.yaml b/manifests/outputs/divide.yaml deleted file mode 100644 index e0875c420..000000000 --- a/manifests/outputs/divide.yaml +++ /dev/null @@ -1,93 +0,0 @@ -name: divide -description: success path -tags: null -initialize: - plugins: - cloud-metadata: - path: builtin - method: CSVLookup - global-config: - filepath: >- - https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv - query: - instance-class: cloud/instance-type - output: - - cpu-cores-utilized - - vcpus-allocated - divide: - path: builtin - method: Divide - global-config: - numerator: vcpus-allocated - denominator: 2 - output: cpu/number-cores - outputs: - - yaml -execution: - command: >- - /home/joe/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /home/joe/Code/if/src/index.ts -m ./manifests/plugins/divide/success.yml -o - ./manifests/outputs/divide - environment: - if-version: 0.4.0 - os: linux - os-version: 5.15.0-113-generic - node-version: 21.4.0 - date-time: 2024-07-03T14:02:26.789Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - children: - child: - pipeline: - - cloud-metadata - - divide - config: - divide: null - defaults: - cloud/vendor: aws - cloud/instance-type: m5n.large - cpu/name: IntelĀ® Coreā„¢ i7-1185G7 - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - cpu/utilization: 80 - outputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - cpu/utilization: 80 - cloud/vendor: aws - cloud/instance-type: m5n.large - cpu/name: IntelĀ® Coreā„¢ i7-1185G7 - vcpus-allocated: 2 - cpu/number-cores: 1 diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index 1a737e742..aef8ef6c5 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -5,8 +5,6 @@ aggregation: - test type: both initialize: - outputs: - - yaml plugins: cloud-metadata: path: builtin @@ -29,21 +27,21 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:25:17.348Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -71,30 +69,31 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: server-1: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 59 server-2: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 82d6759bc..2dfc9fe80 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -5,8 +5,6 @@ aggregation: - cpu/utilization type: both initialize: - outputs: - - yaml plugins: cloud-metadata: path: builtin @@ -29,21 +27,21 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:26:58.250Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -71,29 +69,30 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: server-1: inputs: - - timestamp: '2024-01-26 00:00:00' + - timestamp: "2024-01-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 59 server-2: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws - - timestamp: '2024-02-26 00:15:00' + - timestamp: "2024-02-26 00:15:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index 3110af664..d595d5bdb 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -17,8 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated - outputs: - - yaml + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -28,21 +32,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:28:27.168Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -68,30 +71,31 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: server-1: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 59 outputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 vcpus-allocated: 2 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws @@ -101,24 +105,24 @@ tree: cpu/utilization: 74 server-2: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 27 outputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 vcpus-allocated: 2 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index 1527a9280..9213522af 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -17,8 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated - outputs: - - yaml + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -28,21 +32,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:32:47.459Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -68,30 +71,31 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: server-1: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 59 outputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 vcpus-allocated: 2 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws @@ -99,24 +103,24 @@ tree: vcpus-allocated: 2 server-2: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 27 outputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 vcpus-allocated: 2 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws @@ -124,28 +128,28 @@ tree: vcpus-allocated: 2 outputs: - cpu/utilization: 56.5 - timestamp: '2024-02-26 00:00:00' + timestamp: "2024-02-26 00:00:00" duration: 300 - cpu/utilization: 43 - timestamp: '2024-02-26 00:05:00' + timestamp: "2024-02-26 00:05:00" duration: 300 aggregated: cpu/utilization: 49.75 outputs: - cpu/utilization: 56.5 - timestamp: '2024-02-26 00:00:00' + timestamp: "2024-02-26 00:00:00" duration: 300 - cpu/utilization: 43 - timestamp: '2024-02-26 00:05:00' + timestamp: "2024-02-26 00:05:00" duration: 300 aggregated: cpu/utilization: 49.75 outputs: - cpu/utilization: 56.5 - timestamp: '2024-02-26 00:00:00' + timestamp: "2024-02-26 00:00:00" duration: 300 - cpu/utilization: 43 - timestamp: '2024-02-26 00:05:00' + timestamp: "2024-02-26 00:05:00" duration: 300 aggregated: cpu/utilization: 49.75 diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index 062cd9932..cb9ed29dc 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -17,8 +17,12 @@ initialize: output: - cpu-cores-utilized - vcpus-allocated - outputs: - - yaml + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -28,21 +32,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-01T20:34:35.981Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -68,30 +71,31 @@ tree: children: application: pipeline: - - cloud-metadata + compute: + - cloud-metadata children: uk-west: children: server-1: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 59 outputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 89 vcpus-allocated: 2 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws @@ -101,24 +105,24 @@ tree: cpu/utilization: 74 server-2: inputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 27 outputs: - - timestamp: '2024-02-26 00:00:00' + - timestamp: "2024-02-26 00:00:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws cpu/utilization: 24 vcpus-allocated: 2 - - timestamp: '2024-02-26 00:05:00' + - timestamp: "2024-02-26 00:05:00" duration: 300 cloud/instance-type: m5n.large cloud/vendor: aws @@ -128,28 +132,28 @@ tree: cpu/utilization: 25.5 outputs: - cpu/utilization: 56.5 - timestamp: '2024-02-26 00:00:00' + timestamp: "2024-02-26 00:00:00" duration: 300 - cpu/utilization: 43 - timestamp: '2024-02-26 00:05:00' + timestamp: "2024-02-26 00:05:00" duration: 300 aggregated: cpu/utilization: 49.75 outputs: - cpu/utilization: 56.5 - timestamp: '2024-02-26 00:00:00' + timestamp: "2024-02-26 00:00:00" duration: 300 - cpu/utilization: 43 - timestamp: '2024-02-26 00:05:00' + timestamp: "2024-02-26 00:05:00" duration: 300 aggregated: cpu/utilization: 49.75 outputs: - cpu/utilization: 56.5 - timestamp: '2024-02-26 00:00:00' + timestamp: "2024-02-26 00:00:00" duration: 300 - cpu/utilization: 43 - timestamp: '2024-02-26 00:05:00' + timestamp: "2024-02-26 00:05:00" duration: 300 aggregated: cpu/utilization: 49.75 diff --git a/manifests/outputs/plugins/groupby/failure-invalid-config-group.yaml b/manifests/outputs/features/failure-invalid-regroup.yaml similarity index 69% rename from manifests/outputs/plugins/groupby/failure-invalid-config-group.yaml rename to manifests/outputs/features/failure-invalid-regroup.yaml index 3b188a07d..2a39ac9c8 100644 --- a/manifests/outputs/plugins/groupby/failure-invalid-config-group.yaml +++ b/manifests/outputs/features/failure-invalid-regroup.yaml @@ -1,32 +1,26 @@ -name: groupby -description: failure when `config->group-by->group` is not an array +name: regroup +description: failure when `regroup` is not an array initialize: - outputs: - - yaml - plugins: - group-by: - path: builtin - method: GroupBy + plugins: {} execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/groupby/failure-invalid-config-group.yml -o - manifests/outputs/plugins/groupby/failure-invalid-config-group + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/features/regroup/failure-invalid-regroup.yml -o + manifests/outputs/features/failure-invalid-regroup.yaml environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-02T19:11:23.166Z (UTC) + os-version: 13.6.7 + node-version: 18.20.0 + date-time: 2024-08-05T14:53:00.560Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' - '@commitlint/cli@18.6.0' - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - + - '@grnsft/if-core@0.0.16' - '@jest/globals@29.7.0' - '@types/jest@29.5.8' - '@types/js-yaml@4.0.9' @@ -40,6 +34,7 @@ execution: - fixpack@4.0.0 - gts@5.2.0 - husky@8.0.3 + - if-eco-ci-plugin@v0.0.1 extraneous -> file:../../if-eco-ci-plugin - jest@29.7.0 - js-yaml@4.1.0 - lint-staged@15.2.2 @@ -51,18 +46,14 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- - InputValidationError: "group" parameter is expected array, received string. - Error code: invalid_type. + InputValidationError: "regroup" parameter is not an array or should contain at least one key. Error code: invalid_type. tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: cloud/region + regroup: cloud/region inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/plugins/groupby/failure-missing-cloud-instance-type.yaml b/manifests/outputs/features/failure-missing-cloud-instance-type.yaml similarity index 75% rename from manifests/outputs/plugins/groupby/failure-missing-cloud-instance-type.yaml rename to manifests/outputs/features/failure-missing-cloud-instance-type.yaml index ae91af1e1..741544461 100644 --- a/manifests/outputs/plugins/groupby/failure-missing-cloud-instance-type.yaml +++ b/manifests/outputs/features/failure-missing-cloud-instance-type.yaml @@ -1,12 +1,7 @@ name: groupby description: null initialize: - outputs: - - yaml - plugins: - group-by: - path: builtin - method: GroupBy + plugins: {} execution: status: fail command: >- @@ -17,21 +12,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T19:10:00.820Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -52,17 +46,14 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: 'InvalidGroupingError: Invalid group cloud/instance-type.' + error: "InvalidGroupingError: Invalid group cloud/instance-type." tree: children: my-app: pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type + regroup: + - cloud/region + - cloud/instance-type inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/outputs/pipelines/cloud-metadata-divide.yaml b/manifests/outputs/pipelines/cloud-metadata-divide.yaml index 130c4361e..02ff19b86 100644 --- a/manifests/outputs/pipelines/cloud-metadata-divide.yaml +++ b/manifests/outputs/pipelines/cloud-metadata-divide.yaml @@ -21,8 +21,6 @@ initialize: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -32,21 +30,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:13:05.868Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -72,10 +69,9 @@ tree: children: child: pipeline: - - cloud-metadata - - divide - config: - divide: null + compute: + - cloud-metadata + - divide defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/outputs/examples/generics.yaml b/manifests/outputs/pipelines/generics.yaml similarity index 91% rename from outputs/examples/generics.yaml rename to manifests/outputs/pipelines/generics.yaml index 2408d6b3d..613a65f83 100644 --- a/outputs/examples/generics.yaml +++ b/manifests/outputs/pipelines/generics.yaml @@ -74,19 +74,18 @@ initialize: - cpu/utilization - duration output-parameter: cpu-times-duration - outputs: - - yaml execution: command: >- - /Users/admin/.nvm/versions/node/v18.19.0/bin/ts-node - /Users/admin/Projects/uk/if/src/index.ts --manifest - manifests/examples/generics.yml --output outputs/examples/generics + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/generics.yml -o + manifests/outputs/pipelines/generics environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: 14.3.1 - node-version: 18.19.0 - date-time: 2024-06-30T07:53:53.523Z (UTC) + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-17T20:30:54.004Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' @@ -123,15 +122,15 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - coefficient - - multiply - config: null + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - coefficient + - multiply defaults: cpu/thermal-design-power: 100 vcpus-allocated: 1 diff --git a/outputs/examples/instance-metadata.yaml b/manifests/outputs/pipelines/instance-metadata.yaml similarity index 84% rename from outputs/examples/instance-metadata.yaml rename to manifests/outputs/pipelines/instance-metadata.yaml index 41baa0b58..369d54c27 100644 --- a/outputs/examples/instance-metadata.yaml +++ b/manifests/outputs/pipelines/instance-metadata.yaml @@ -19,20 +19,18 @@ initialize: parameter: cpu-model-name match: /^([^,])+/g output: cpu/name - outputs: - - yaml execution: command: >- - /Users/admin/.nvm/versions/node/v18.19.0/bin/ts-node - /Users/admin/Projects/uk/if/src/index.ts --manifest - manifests/examples/instance-metadata.yml --output - outputs/examples/instance-metadata + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/instance-metadata.yml -o + manifests/outputs/pipelines/instance-metadata environment: - if-version: 0.4.0 + if-version: 0.5.0 os: macOS - os-version: 14.3.1 - node-version: 18.19.0 - date-time: 2024-06-30T07:54:34.384Z (UTC) + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:31:27.411Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' @@ -69,8 +67,9 @@ tree: children: child: pipeline: - - cloud-instance-metadata - - extract-processor-name + compute: + - cloud-instance-metadata + - extract-processor-name inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/pipelines/mock-obs-group-by-cloud-meta.yaml b/manifests/outputs/pipelines/mock-obs-group-by-cloud-meta.yaml deleted file mode 100644 index 5585d4c42..000000000 --- a/manifests/outputs/pipelines/mock-obs-group-by-cloud-meta.yaml +++ /dev/null @@ -1,645 +0,0 @@ -name: My Manifest File -description: integration of mock observations, group by and cloud metadata -aggregation: - metrics: - - cpu/utilization - type: both -initialize: - plugins: - group-by: - path: builtin - method: GroupBy - global-config: - input-parameters: - - cpu/energy - - grid/carbon-intensity - output-parameter: carbon - cloud-metadata: - path: builtin - method: CSVLookup - global-config: - filepath: >- - https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv - query: - instance-class: cloud/instance-type - output: - - cpu-tdp - mock-observations: - path: builtin - method: MockObservations - global-config: - timestamp-from: 2024-03-05T00:00 - timestamp-to: 2024-03-05T01:00 - duration: 300 - components: - - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - generators: - common: - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - randint: - cpu/utilization: - min: 1 - max: 99 - outputs: - - yaml -execution: - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/pipelines/mock-obs-group-by-cloud-meta.yml -o - manifests/outputs/pipelines/mock-obs-group-by-cloud-meta - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-02T05:21:45.561Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - pipeline: - - mock-observations - - group-by - - cloud-metadata - config: - group-by: - group: - - name - - cloud/instance-type - children: - server-1: - children: - Standard_E64_v3: - inputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 93 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 6 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 6 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 3 - - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 83 - - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 85 - - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 8 - - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 59 - - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 55 - - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 85 - - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 52 - - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 54 - outputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 93 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 6 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 6 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 3 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 83 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 85 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 8 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 59 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 55 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 85 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 52 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: eastus - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 54 - cpu-tdp: 270 - aggregated: - cpu/utilization: 49.083333333333336 - outputs: - - cpu/utilization: 93 - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - - cpu/utilization: 6 - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - - cpu/utilization: 6 - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - - cpu/utilization: 3 - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - - cpu/utilization: 83 - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - - cpu/utilization: 85 - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - - cpu/utilization: 8 - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - - cpu/utilization: 59 - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - - cpu/utilization: 55 - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - - cpu/utilization: 85 - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - - cpu/utilization: 52 - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - - cpu/utilization: 54 - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - aggregated: - cpu/utilization: 49.083333333333336 - server-2: - children: - Standard_E64_v3: - inputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 4 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 63 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 38 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 10 - - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 88 - - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 81 - - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 17 - - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 11 - - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 76 - - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 63 - - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 81 - - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 74 - outputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 4 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 63 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 38 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 10 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 88 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 81 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 17 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 11 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 76 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 63 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 81 - cpu-tdp: 270 - - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - geolocation: 37.7749,-122.4194 - cloud/vendor: azure - cpu/utilization: 74 - cpu-tdp: 270 - aggregated: - cpu/utilization: 50.5 - outputs: - - cpu/utilization: 4 - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - - cpu/utilization: 63 - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - - cpu/utilization: 38 - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - - cpu/utilization: 10 - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - - cpu/utilization: 88 - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - - cpu/utilization: 81 - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - - cpu/utilization: 17 - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - - cpu/utilization: 11 - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - - cpu/utilization: 76 - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - - cpu/utilization: 63 - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - - cpu/utilization: 81 - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - - cpu/utilization: 74 - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - aggregated: - cpu/utilization: 50.5 - outputs: - - cpu/utilization: 48.5 - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - - cpu/utilization: 34.5 - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - - cpu/utilization: 22 - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - - cpu/utilization: 6.5 - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - - cpu/utilization: 85.5 - timestamp: '2024-03-05T00:20:00.000Z' - duration: 300 - - cpu/utilization: 83 - timestamp: '2024-03-05T00:25:00.000Z' - duration: 300 - - cpu/utilization: 12.5 - timestamp: '2024-03-05T00:30:00.000Z' - duration: 300 - - cpu/utilization: 35 - timestamp: '2024-03-05T00:35:00.000Z' - duration: 300 - - cpu/utilization: 65.5 - timestamp: '2024-03-05T00:40:00.000Z' - duration: 300 - - cpu/utilization: 74 - timestamp: '2024-03-05T00:45:00.000Z' - duration: 300 - - cpu/utilization: 66.5 - timestamp: '2024-03-05T00:50:00.000Z' - duration: 300 - - cpu/utilization: 64 - timestamp: '2024-03-05T00:55:00.000Z' - duration: 300 - aggregated: - cpu/utilization: 49.791666666666664 diff --git a/manifests/outputs/pipelines/mock-obs-groupby.yaml b/manifests/outputs/pipelines/mock-obs-groupby.yaml deleted file mode 100644 index 4783dffde..000000000 --- a/manifests/outputs/pipelines/mock-obs-groupby.yaml +++ /dev/null @@ -1,178 +0,0 @@ -name: Mock observations + Group by -description: Integration of `mock observations` + `groupby plugins` -initialize: - plugins: - mock-observations: - path: builtin - method: MockObservations - global-config: - timestamp-from: '2024-03-05T00:00:00.000Z' - timestamp-to: '2024-03-05T00:01:00.000Z' - duration: 10 - components: - - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - generators: - common: - cloud/vendor: azure - randint: - cpu/utilization: - min: 1 - max: 99 - group-by: - path: builtin - method: GroupBy - outputs: - - yaml -execution: - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/pipelines/mock-obs-groupby.yml -o - manifests/outputs/pipelines/mock-obs-groupby - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-02T05:27:34.757Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - pipeline: - - mock-observations - - group-by - defaults: null - config: - group-by: - group: - - cloud/region - - name - children: - westus3: - children: - server-1: - inputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 10 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 12 - - timestamp: '2024-03-05T00:00:10.000Z' - duration: 10 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 71 - - timestamp: '2024-03-05T00:00:20.000Z' - duration: 10 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 75 - - timestamp: '2024-03-05T00:00:30.000Z' - duration: 10 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 39 - - timestamp: '2024-03-05T00:00:40.000Z' - duration: 10 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 42 - - timestamp: '2024-03-05T00:00:50.000Z' - duration: 10 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 98 - france: - children: - server-2: - inputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 10 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 14 - - timestamp: '2024-03-05T00:00:10.000Z' - duration: 10 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 80 - - timestamp: '2024-03-05T00:00:20.000Z' - duration: 10 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 53 - - timestamp: '2024-03-05T00:00:30.000Z' - duration: 10 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 10 - - timestamp: '2024-03-05T00:00:40.000Z' - duration: 10 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 40 - - timestamp: '2024-03-05T00:00:50.000Z' - duration: 10 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 92 diff --git a/manifests/outputs/pipelines/mock-obs-time-sync.yaml b/manifests/outputs/pipelines/mock-obs-time-sync.yaml index 271340c2b..04eb881ec 100644 --- a/manifests/outputs/pipelines/mock-obs-time-sync.yaml +++ b/manifests/outputs/pipelines/mock-obs-time-sync.yaml @@ -7,19 +7,36 @@ initialize: path: builtin method: MockObservations global-config: - timestamp-from: 2023-07-06T00:00 - timestamp-to: 2023-07-06T00:10 + timestamp-from: 2023-12-12T00:00 + timestamp-to: 2023-12-12T00:10 duration: 60 components: - cloud/instance-type: A1 generators: common: cloud/region: uk-west - common-key: common-val randint: cpu/utilization: min: 1 max: 99 + parameter-metadata: + inputs: + timestamp: + description: refers to the time of occurrence of the input + unit: RFC3339 + aggregation-method: none + duration: + description: refers to the duration of the input + unit: seconds + aggregation-method: sum + cloud/instance-type: + description: type of Cloud Instance name used in the cloud provider APIs + unit: none + aggregation-method: none + cloud/region: + description: region cloud instance + unit: none + aggregation-method: none interpolate: path: builtin method: Interpolation @@ -30,13 +47,19 @@ initialize: - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 - 1.02 input-parameter: cpu/utilization output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + description: refers to CPU utilization. + unit: percentage + aggregation-method: avg cpu-factor-to-wattage: path: builtin method: Multiply @@ -45,6 +68,12 @@ initialize: - cpu-factor - cpu/thermal-design-power output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu/thermal-design-power: + description: thermal design power for a processor + unit: kwh + aggregation-method: avg wattage-times-duration: path: builtin method: Multiply @@ -67,6 +96,16 @@ initialize: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio + parameter-metadata: + inputs: + vcpus-total: + description: total number of vcpus available on a particular resource + unit: count + aggregation-method: none + vcpus-allocated: + description: number of vcpus allocated to particular resource + unit: count + aggregation-method: none correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide @@ -78,12 +117,10 @@ initialize: path: builtin method: TimeSync global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true - outputs: - - yaml execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -93,21 +130,20 @@ execution: environment: if-version: 0.4.0 os: macOS - os-version: '13.2' + os-version: "13.2" node-version: 18.14.2 date-time: 2024-07-02T05:29:47.787Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -133,13 +169,16 @@ tree: children: child-1: pipeline: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - time-sync + observe: + - mock-observations + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - time-sync defaults: cpu/thermal-design-power: 100 vcpus-total: 8 @@ -148,203 +187,263 @@ tree: - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - duration: 1 - cpu/utilization: 10 - - timestamp: '2023-12-12T00:00:01.000Z' - duration: 5 - cpu/utilization: 20 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:01:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - - timestamp: '2023-12-12T00:00:06.000Z' - duration: 7 - cpu/utilization: 15 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:02:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:03:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - - timestamp: '2023-12-12T00:00:13.000Z' - duration: 30 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:04:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:05:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:06:00.000Z' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 15 + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:07:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:08:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 + - timestamp: '2023-12-12T00:09:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 60 + cpu/utilization: '*' + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 1 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 - cpu/utilization: 14 + cpu/utilization: '*' cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.662 - cpu-wattage: 66.19999999999999 - cpu-wattage-times-duration: 203 - cpu-energy-raw: 0.0000563888888888889 - vcpu-ratio: 14.399999999999999 - cpu-energy-kwh: 0.000007048611111111113 - - timestamp: '2023-12-12T00:00:05.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - cpu/utilization: 13 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.29907142857142854 - cpu-wattage: 29.907142857142862 - cpu-wattage-times-duration: 192.25 - cpu-energy-raw: 0.00005340277777777778 - vcpu-ratio: 6.171428571428571 - cpu-energy-kwh: 0.000006675347222222222 - - timestamp: '2023-12-12T00:00:10.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - cpu/utilization: 12 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.1850952380952381 - cpu-wattage: 18.50952380952381 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222222 - vcpu-ratio: 3.9619047619047616 - cpu-energy-kwh: 0.0000064887152777777775 - - timestamp: '2023-12-12T00:00:15.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 - - timestamp: '2023-12-12T00:00:20.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 - - timestamp: '2023-12-12T00:00:25.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 - - timestamp: '2023-12-12T00:00:30.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 - - timestamp: '2023-12-12T00:00:35.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 12 cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.06229166666666667 - cpu-wattage: 6.229166666666667 - cpu-wattage-times-duration: 186.875 - cpu-energy-raw: 0.00005190972222222223 - vcpu-ratio: 1.3333333333333333 - cpu-energy-kwh: 0.000006488715277777778 - - timestamp: '2023-12-12T00:00:40.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 9 - cpu/thermal-design-power: 60 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0.037375 - cpu-wattage: 3.7375 - cpu-wattage-times-duration: 112.125 - cpu-energy-raw: 0.000031145833333333336 - vcpu-ratio: 0.8 - cpu-energy-kwh: 0.000003893229166666667 - - timestamp: '2023-12-12T00:00:45.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 80 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 + cpu/utilization: '*' cloud/instance-type: A1 cloud/region: uk-west - cpu/utilization: 0 - cpu/thermal-design-power: 0 + cpu/thermal-design-power: 100 vcpus-total: 8 vcpus-allocated: 1 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 0 - cpu-energy-kwh: 0 + cpu-factor: '*' + cpu-wattage: '*' + cpu-wattage-times-duration: '*' + cpu-energy-raw: '*' + vcpu-ratio: '*' + cpu-energy-kwh: '*' diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml new file mode 100644 index 000000000..792f41243 --- /dev/null +++ b/manifests/outputs/pipelines/nesting.yaml @@ -0,0 +1,1799 @@ +name: nesting +description: a manifest that includes nested child components +tags: + kind: web + complexity: moderate + category: on-premise +aggregation: + metrics: + - carbon + type: both +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + "y": + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + parameter-metadata: + inputs: + cpu/utilization: + unit: percentage + description: refers to CPU utilization. + aggregation-method: avg + outputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + parameter-metadata: + inputs: + cpu-factor: + unit: kWh + description: result of interpolate + aggregation-method: avg + cpu/thermal-design-power: + unit: kWh + description: thermal design power for a processor + aggregation-method: avg + outputs: + cpu-wattage: + unit: kWh + description: the energy used by the CPU + aggregation-method: sum + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + parameter-metadata: + inputs: + cpu-wattage-times-duration: + unit: kWh + description: CPU wattage multiplied by duration + aggregation-method: sum + outputs: + cpu-energy-raw: + unit: kWh + description: Raw energy used by CPU in kWh + aggregation-method: sum + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + parameter-metadata: + outputs: + vcpu-ratio: + unit: none + description: Ratio of vCPUs + aggregation-method: none + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + parameter-metadata: + inputs: + cpu-energy-kwh: + unit: kWh + description: Corrected CPU energy in kWh + aggregation-method: sum + grid/carbon-intensity: + unit: gCO2eq/kWh + description: Carbon intensity for the grid + aggregation-method: avg + outputs: + carbon-operational: + unit: gCO2eq + description: Operational carbon footprint + aggregation-method: sum + sci: + path: builtin + method: Sci + global-config: + functional-unit: requests + parameter-metadata: + inputs: + requests: + unit: none + description: expressed the final SCI value + aggregation-method: sum + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + parameter-metadata: + inputs: + carbon-operational: + description: Operational carbon footprint + unit: gCO2eq + aggregation-method: sum + carbon-embodied: + description: Embodied carbon footprint + unit: gCO2eq + aggregation-method: sum + outputs: + carbon: + description: Total carbon footprint + unit: gCO2eq + aggregation-method: sum + time-sync: + path: builtin + method: TimeSync + global-config: + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" + interval: 5 + allow-padding: true + parameter-metadata: + inputs: + timestamp: + unit: RFC3339 + description: refers to the time of occurrence of the input + aggregation-method: none + duration: + unit: seconds + description: refers to the duration of the input + aggregation-method: sum + cloud/instance-type: + unit: none + description: type of Cloud Instance name used in the cloud provider APIs + aggregation-method: none + cloud/region: + unit: none + description: region cloud instance + aggregation-method: none + time-reserved: + unit: seconds + description: time reserved for a component + aggregation-method: avg + network/energy: + unit: kWh + description: Energy consumed by the Network of the component + aggregation-method: sum +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/nesting.yml -o + manifests/outputs/pipelines/nesting-1.yaml + environment: + if-version: 0.5.0 + os: macOS + os-version: "14.5" + node-version: 18.14.2 + date-time: 2024-07-31T13:17:29.944Z (UTC) + dependencies: + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-0: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 60 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 70 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 55 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 98 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.00006983123919278518 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 52 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.0001028924208718729 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 33.666666666666664 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.0001544881286073813 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 9.166666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0005673927632489277 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 5.5 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0005673927632489276 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + child-1: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 10 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 90 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 30 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 22 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 82 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.0000834568468401579 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 35.14285714285714 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.00015224732194049487 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 14.323809523809523 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.000363108733129716 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 3.6666666666666665 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 2.1999999999999997 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0014184819081223194 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + child-2: + children: + child-2-0: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 65 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 80 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 40 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 102 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.00006709275922444067 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 58.71428571428572 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.0000911261343001502 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 36.952380952380956 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.00014075142645028166 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 4 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0007801650494672755 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + child-2-1: + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-allocated: 1 + vcpus-total: 8 + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - time-sync + - sci + inputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:01.000Z" + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 50 + - timestamp: "2023-12-12T00:00:06.000Z" + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + requests: 60 + - timestamp: "2023-12-12T00:00:13.000Z" + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + requests: 40 + outputs: + - timestamp: "2023-12-12T00:00:00.000Z" + cloud/instance-type: A1 + cloud/region: uk-west + duration: 5 + cpu/utilization: 22 + network/energy: 0.0000018 + requests: 90 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 2759.6159999999995 + time-reserved: 2880 + device/expected-lifespan: 170294400 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.4065 + cpu-wattage: 109.19999999999999 + cpu-wattage-times-duration: 246 + cpu-energy-raw: 0.00006833333333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000008541666666666668 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.006833333333333334 + carbon: 0.0068434614408929475 + sci: 0.00007603846045436609 + - timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + cpu/utilization: 13 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 7.714285714285713e-7 + requests: 44.28571428571428 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 1182.6925714285712 + time-reserved: 2880 + device/expected-lifespan: 72983314.28571428 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.30975 + cpu-wattage: 29.907142857142862 + cpu-wattage-times-duration: 192.25 + cpu-energy-raw: 0.00005340277777777778 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006675347222222222 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005340277777777777 + carbon: 0.005350405885337391 + sci: 0.00012081561676568304 + - timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + cpu/utilization: 12 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 4.952380952380952e-7 + requests: 28.38095238095238 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 759.2594285714285 + time-reserved: 2880 + device/expected-lifespan: 46853485.71428572 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 18.50952380952381 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222222 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000064887152777777775 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.005190972222222222 + carbon: 0.0052011003297818366 + sci: 0.0001832602465191587 + - timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + network/energy: 1.6666666666666668e-7 + requests: 6.666666666666666 + cpu/thermal-design-power: 80 + grid/carbon-intensity: 640 + device/emissions-embodied: 255.51999999999998 + time-reserved: 2880 + device/expected-lifespan: 15768000 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.29900000000000004 + cpu-wattage: 6.229166666666667 + cpu-wattage-times-duration: 186.875 + cpu-energy-raw: 0.00005190972222222223 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000006488715277777778 + carbon-embodied: 0.000010128107559614409 + carbon-operational: 0.005190972222222222 + carbon: 0.005201100329781837 + sci: 0.0007801650494672756 + - timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 9 + network/energy: 1.e-7 + requests: 4 + cpu/thermal-design-power: 60 + grid/carbon-intensity: 480 + device/emissions-embodied: 153.312 + time-reserved: 2160.2 + device/expected-lifespan: 9460800 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0.22425 + cpu-wattage: 3.7375 + cpu-wattage-times-duration: 112.125 + cpu-energy-raw: 0.000031145833333333336 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000003893229166666667 + carbon-embodied: 0.000006076864535768645 + carbon-operational: 0.0031145833333333334 + carbon: 0.003120660197869102 + sci: 0.0007801650494672755 + - timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 0.8 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + - timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 0 + network/energy: 0 + requests: 0 + cpu/thermal-design-power: 0 + grid/carbon-intensity: 0 + device/emissions-embodied: 0 + time-reserved: 1 + device/expected-lifespan: 0 + vcpus-allocated: 1 + vcpus-total: 8 + cpu-factor: 0 + cpu-wattage: 0 + cpu-wattage-times-duration: 0 + cpu-energy-raw: 0 + vcpu-ratio: 8 + cpu-energy-kwh: 0 + carbon-embodied: 0 + carbon-operational: 0 + carbon: 0 + sci: 0 + aggregated: + carbon: 0.04652112950279046 + outputs: + - carbon: 0.013686922881785895 + timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + - carbon: 0.010700811770674782 + timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + - carbon: 0.010402200659563673 + timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + - carbon: 0.010402200659563675 + timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + - carbon: 0.006241320395738204 + timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + aggregated: + carbon: 0.09304225900558093 + outputs: + - carbon: 0.02737384576357179 + timestamp: "2023-12-12T00:00:00.000Z" + duration: 5 + - carbon: 0.021401623541349564 + timestamp: "2023-12-12T00:00:05.000Z" + duration: 5 + - carbon: 0.020804401319127346 + timestamp: "2023-12-12T00:00:10.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:15.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:20.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:25.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:30.000Z" + duration: 5 + - carbon: 0.02080440131912735 + timestamp: "2023-12-12T00:00:35.000Z" + duration: 5 + - carbon: 0.012482640791476408 + timestamp: "2023-12-12T00:00:40.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:45.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:50.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:00:55.000Z" + duration: 5 + - carbon: 0 + timestamp: "2023-12-12T00:01:00.000Z" + duration: 1 + aggregated: + carbon: 0.18608451801116185 diff --git a/manifests/outputs/pipelines/pipeline-teads-sci.yaml b/manifests/outputs/pipelines/pipeline-teads-sci.yaml new file mode 100644 index 000000000..1c2a9d005 --- /dev/null +++ b/manifests/outputs/pipelines/pipeline-teads-sci.yaml @@ -0,0 +1,276 @@ +name: pipeline-teads-sci +description: >- + a full pipeline seeded with some hardcoded input data and yielding an SCI + score +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh + sci-embodied: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-energy-kwh + - grid/carbon-intensity + output-parameter: carbon-operational + sci: + path: builtin + method: Sci + global-config: + functional-unit: component + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/pipeline-teads-sci.yml -o + manifests/outputs/pipelines/pipeline-teads-sci + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:32:50.994Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sci-embodied + - operational-carbon + - sum-carbon + - sci + config: null + defaults: + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 75 + cpu-energy-raw: 0.000020833333333333333 + vcpu-ratio: 8 + cpu-energy-kwh: 0.0000026041666666666666 + carbon-embodied: 0.0000020256215119228817 + carbon-operational: 0.0020833333333333333 + carbon: 0.002085358954845256 + sci: 0.002085358954845256 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.4275 + cpu-wattage: 42.75 + cpu-wattage-times-duration: 213.75 + cpu-energy-raw: 0.000059375 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000007421875 + carbon-embodied: 0.000010128107559614407 + carbon-operational: 0.0059375 + carbon: 0.005947628107559615 + sci: 0.005947628107559615 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 261.625 + cpu-energy-raw: 0.00007267361111111111 + vcpu-ratio: 8 + cpu-energy-kwh: 0.000009084201388888889 + carbon-embodied: 0.00001417935058346017 + carbon-operational: 0.007267361111111111 + carbon: 0.007281540461694571 + sci: 0.007281540461694571 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + vcpus-total: 8 + vcpus-allocated: 1 + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 1121.25 + cpu-energy-raw: 0.00031145833333333335 + vcpu-ratio: 8 + cpu-energy-kwh: 0.00003893229166666667 + carbon-embodied: 0.00006076864535768645 + carbon-operational: 0.031145833333333334 + carbon: 0.03120660197869102 + sci: 0.03120660197869102 diff --git a/manifests/outputs/pipelines/sci.yaml b/manifests/outputs/pipelines/sci.yaml new file mode 100644 index 000000000..c4fa641c2 --- /dev/null +++ b/manifests/outputs/pipelines/sci.yaml @@ -0,0 +1,297 @@ +name: pipeline-teads-sci +description: >- + a full pipeline seeded with some hardcoded input data and yielding an SCI + score +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - cpu/thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu/energy + sum-energy-components: + path: builtin + method: Sum + global-config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy + embodied-carbon: + path: builtin + method: SciEmbodied + operational-carbon: + path: builtin + method: Multiply + global-config: + input-parameters: + - energy + - grid/carbon-intensity + output-parameter: carbon-operational + sum-carbon: + path: builtin + method: Sum + global-config: + input-parameters: + - carbon-operational + - carbon-embodied + output-parameter: carbon + sci: + path: builtin + method: Sci + global-config: + functional-unit: component +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/sci.yml -o manifests/outputs/pipelines/sci + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:34:45.027Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + - sum-energy-components + - embodied-carbon + - operational-carbon + - sum-carbon + - sci + defaults: + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 75 + cpu-energy-raw: 0.000020833333333333333 + vcpu-ratio: 4 + cpu/energy: 0.000005208333333333333 + energy: 0.000006208333333333333 + carbon-embodied: 0.000004051243023845763 + carbon-operational: 0.004966666666666666 + carbon: 0.004970717909690512 + sci: 0.004970717909690512 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + cpu/utilization: 20 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.4275 + cpu-wattage: 42.75 + cpu-wattage-times-duration: 213.75 + cpu-energy-raw: 0.000059375 + vcpu-ratio: 4 + cpu/energy: 0.00001484375 + energy: 0.00001584375 + carbon-embodied: 0.000020256215119228814 + carbon-operational: 0.012674999999999999 + carbon: 0.012695256215119228 + sci: 0.012695256215119228 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + cpu/utilization: 15 + cloud/instance-type: A1 + cloud/region: uk-west + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 261.625 + cpu-energy-raw: 0.00007267361111111111 + vcpu-ratio: 4 + cpu/energy: 0.000018168402777777778 + energy: 0.000019168402777777778 + carbon-embodied: 0.00002835870116692034 + carbon-operational: 0.015334722222222222 + carbon: 0.015363080923389142 + sci: 0.015363080923389142 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 15 + network/energy: 0.000001 + cpu/thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + grid/carbon-intensity: 800 + device/emissions-embodied: 1533.12 + time-reserved: 3600 + device/expected-lifespan: 94608000 + resources-reserved: vcpus-allocated + resources-total: vcpus-total + component: 1 + cpu-factor: 0.37375 + cpu-wattage: 37.375 + cpu-wattage-times-duration: 1121.25 + cpu-energy-raw: 0.00031145833333333335 + vcpu-ratio: 4 + cpu/energy: 0.00007786458333333334 + energy: 0.00007886458333333333 + carbon-embodied: 0.0001215372907153729 + carbon-operational: 0.06309166666666667 + carbon: 0.06321320395738204 + sci: 0.06321320395738204 diff --git a/manifests/outputs/pipelines/teads-curve.yaml b/manifests/outputs/pipelines/teads-curve.yaml new file mode 100644 index 000000000..26885e1c7 --- /dev/null +++ b/manifests/outputs/pipelines/teads-curve.yaml @@ -0,0 +1,188 @@ +name: carbon-intensity plugin demo +description: null +tags: null +initialize: + plugins: + interpolate: + path: builtin + method: Interpolation + global-config: + method: linear + x: + - 0 + - 10 + - 50 + - 100 + 'y': + - 0.12 + - 0.32 + - 0.75 + - 1.02 + input-parameter: cpu/utilization + output-parameter: cpu-factor + cpu-factor-to-wattage: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-factor + - thermal-design-power + output-parameter: cpu-wattage + wattage-times-duration: + path: builtin + method: Multiply + global-config: + input-parameters: + - cpu-wattage + - duration + output-parameter: cpu-wattage-times-duration + wattage-to-energy-kwh: + path: builtin + method: Divide + global-config: + numerator: cpu-wattage-times-duration + denominator: 3600000 + output: cpu-energy-raw + calculate-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: vcpus-total + denominator: vcpus-allocated + output: vcpu-ratio + correct-cpu-energy-for-vcpu-ratio: + path: builtin + method: Divide + global-config: + numerator: cpu-energy-raw + denominator: vcpu-ratio + output: cpu-energy-kwh +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/teads-curve.yml -o + manifests/outputs/pipelines/teads-curve + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:35:33.728Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child: + pipeline: + compute: + - interpolate + - cpu-factor-to-wattage + - wattage-times-duration + - wattage-to-energy-kwh + - calculate-vcpu-ratio + - correct-cpu-energy-for-vcpu-ratio + defaults: + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + inputs: + - timestamp: 2023-08-06T00:00 + duration: 360 + cpu/utilization: 1 + carbon: 30 + - timestamp: 2023-09-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 10 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 50 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 100 + outputs: + - timestamp: 2023-08-06T00:00 + duration: 360 + cpu/utilization: 1 + carbon: 30 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 0.13999999999999999 + cpu-wattage: 13.999999999999998 + cpu-wattage-times-duration: 5039.999999999999 + cpu-energy-raw: 0.0013999999999999998 + vcpu-ratio: 4 + cpu-energy-kwh: 0.00034999999999999994 + - timestamp: 2023-09-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 10 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 0.32 + cpu-wattage: 32 + cpu-wattage-times-duration: 11520 + cpu-energy-raw: 0.0032 + vcpu-ratio: 4 + cpu-energy-kwh: 0.0008 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 50 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 0.75 + cpu-wattage: 75 + cpu-wattage-times-duration: 27000 + cpu-energy-raw: 0.0075 + vcpu-ratio: 4 + cpu-energy-kwh: 0.001875 + - timestamp: 2023-10-06T00:00 + duration: 360 + carbon: 30 + cpu/utilization: 100 + thermal-design-power: 100 + vcpus-total: 8 + vcpus-allocated: 2 + cpu-factor: 1.02 + cpu-wattage: 102 + cpu-wattage-times-duration: 36720 + cpu-energy-raw: 0.0102 + vcpu-ratio: 4 + cpu-energy-kwh: 0.00255 diff --git a/manifests/outputs/pipelines/zeros.yaml b/manifests/outputs/pipelines/zeros.yaml new file mode 100644 index 000000000..3efd3c0fb --- /dev/null +++ b/manifests/outputs/pipelines/zeros.yaml @@ -0,0 +1,198 @@ +name: generics +description: >- + a pipeline that does arbitrary calculations using our generic arithmetic + builtins +tags: null +initialize: + plugins: + sum-zero-and-one: + path: builtin + method: Sum + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-plus-zero + sum-zero-and-zero: + path: builtin + method: Sum + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-plus-zero + subtract-one-and-zero: + path: builtin + method: Subtract + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-minus-zero + subtract-zero-and-zero: + path: builtin + method: Sum + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-minus-zero + subtract-zero-and-one: + path: builtin + method: Subtract + global-config: + input-parameters: + - zero-value + - some-value + output-parameter: zero-minus-one + coefficient-one-times-zero: + path: builtin + method: Coefficient + global-config: + input-parameter: zero-value + coefficient: 1 + output-parameter: zero-times-one-coefficient + coefficient-zero-times-one: + path: builtin + method: Coefficient + global-config: + input-parameter: some-value + coefficient: 0 + output-parameter: one-times-zero-coefficient + coefficient-zero-times-zero: + path: builtin + method: Coefficient + global-config: + input-parameter: zero-value + coefficient: 0 + output-parameter: zero-times-zero-coefficient + multiply-one-times-zero: + path: builtin + method: Multiply + global-config: + input-parameters: + - some-value + - zero-value + output-parameter: one-times-zero + multiply-zero-times-one: + path: builtin + method: Multiply + global-config: + input-parameters: + - zero-value + - zero-value + output-parameter: zero-times-one + exponent-one-to-zero: + path: builtin + method: Exponent + global-config: + input-parameter: some-value + exponent: 0 + output-parameter: one-raised-to-zero-power + exponent-zero-to-zero: + path: builtin + method: Exponent + global-config: + input-parameter: zero-value + exponent: 0 + output-parameter: zero-raised-to-zero-power + exponent-zero-to-one: + path: builtin + method: Exponent + global-config: + input-parameter: zero-value + exponent: 1 + output-parameter: zero-raised-to-first-power + sci: + path: builtin + method: Sci + global-config: + functional-unit: zero-value +execution: + command: >- + /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + manifests/examples/pipelines/zeros.yml -o manifests/outputs/pipelines/zeros + environment: + if-version: 0.5.0 + os: macOS + os-version: '14.5' + node-version: 18.14.2 + date-time: 2024-07-19T06:36:00.790Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.10' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.22.4 + status: success +tree: + children: + child-1: + pipeline: + compute: + - sum-zero-and-one + - sum-zero-and-zero + - subtract-one-and-zero + - subtract-zero-and-zero + - subtract-zero-and-one + - coefficient-one-times-zero + - coefficient-zero-times-one + - coefficient-zero-times-zero + - multiply-one-times-zero + - multiply-zero-times-one + - exponent-one-to-zero + - exponent-zero-to-one + - exponent-zero-to-zero + - sci + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + some-value: 1 + zero-value: 0 + carbon: 10 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 1 + some-value: 1 + zero-value: 0 + carbon: 10 + one-plus-zero: 1 + zero-plus-zero: 0 + one-minus-zero: 1 + zero-minus-zero: 0 + zero-minus-one: -1 + zero-times-one-coefficient: 0 + one-times-zero-coefficient: 0 + zero-times-zero-coefficient: 0 + one-times-zero: 0 + zero-times-one: 0 + one-raised-to-zero-power: 1 + zero-raised-to-first-power: 0 + zero-raised-to-zero-power: 1 + sci: 10 diff --git a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml b/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml deleted file mode 100644 index c474c91fe..000000000 --- a/manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: cloud-metadata -description: failure with invalid `inputs.cloud/vendor` -tags: -initialize: - #outputs: ['yaml'] - plugins: - cloud-metadata: - path: builtin - method: CSVLookup - global-config: - filepath: >- - https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv - query: - instance-class: cloud/instance-type - output: ['cpu-cores-utilized', 'vcpus-allocated'] -tree: - children: - child: - pipeline: - - cloud-metadata - config: - inputs: - - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred - cloud/vendor: gcp - cloud/instance-type: m5n.large - duration: 100 - cpu/utilization: 10 diff --git a/manifests/outputs/plugins/groupby/success.yaml b/manifests/outputs/plugins/groupby/success.yaml deleted file mode 100644 index 04d54837b..000000000 --- a/manifests/outputs/plugins/groupby/success.yaml +++ /dev/null @@ -1,101 +0,0 @@ -name: groupby -description: successful path -initialize: - plugins: - group-by: - path: builtin - method: GroupBy - outputs: - - yaml -execution: - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/groupby/success.yml -o - manifests/outputs/plugins/groupby/success - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-02T19:08:31.858Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - cloud/instance-type - children: - uk-west: - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 12 - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - cloud/instance-type: B1 - cloud/region: uk-west - cpu/utilization: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - cloud/instance-type: B1 - cloud/region: uk-west - cpu/utilization: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - cloud/instance-type: B1 - cloud/region: uk-west - cpu/utilization: 1 diff --git a/manifests/outputs/plugins/regex/failure-not-matching-with-regex.yaml b/manifests/outputs/plugins/regex/failure-not-matching-with-regex.yaml deleted file mode 100644 index 4b1ae47d6..000000000 --- a/manifests/outputs/plugins/regex/failure-not-matching-with-regex.yaml +++ /dev/null @@ -1,76 +0,0 @@ -name: regex -description: physical processor doesn't match the regex expression -tags: null -initialize: - outputs: - - yaml - plugins: - regex: - method: Regex - path: builtin - global-config: - parameter: physical-processor - match: ^ - output: cpu/name -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/regex/failure-not-matching-with-regex.yml -o - manifests/outputs/plugins/regex/failure-not-matching-with-regex - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-02T20:24:39.241Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - RegexMismatchError: `IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 - GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz,IntelĀ® XeonĀ® E5-2673 v3 2.4 GHz` does - not match the /^/ regex expression -tree: - children: - child: - pipeline: - - regex - config: - regex: null - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® - E5-2673 v4 2.3 GHz,IntelĀ® XeonĀ® E5-2673 v3 2.4 GHz diff --git a/manifests/outputs/plugins/time-sync/failure-missing-global-config.yaml b/manifests/outputs/plugins/time-sync/failure-missing-global-config.yaml deleted file mode 100644 index 79f3bb69e..000000000 --- a/manifests/outputs/plugins/time-sync/failure-missing-global-config.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: time-sync -description: missing global config -tags: null -initialize: - outputs: - - yaml - plugins: - time-sync: - method: TimeSync - path: builtin - global-config: null -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/time-sync/failure-missing-global-config.yml -o - manifests/outputs/plugins/time-sync/failure-missing-global-config - environment: - if-version: 0.4.0 - os: macOS - os-version: '13.2' - node-version: 18.14.2 - date-time: 2024-07-02T21:13:12.360Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - ManifestValidationError: "initialize.plugins.time-sync.global-config" - parameter is expected object, received null. Error code: invalid_type. -tree: - children: - child: - pipeline: - - time-sync - config: null - inputs: - - timestamp: '2023-12-12T00:00:00.000Z' - duration: 3 - energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:01.000Z' - duration: 5 - energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:06.000Z' - duration: 7 - energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:13.000Z' - duration: 30 - energy-cpu: 0.001 diff --git a/outputs/examples/copy.yaml b/outputs/examples/copy.yaml deleted file mode 100644 index 6b088ab99..000000000 --- a/outputs/examples/copy.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: copy-param -description: null -tags: null -initialize: - plugins: - copy-param: - path: builtin - method: Copy - global-config: - keep-existing: true - from: original - to: copy - outputs: - - yaml -execution: - command: >- - /Users/admin/.nvm/versions/node/v18.19.0/bin/ts-node - /Users/admin/Projects/uk/if/src/index.ts --manifest - manifests/examples/copy.yaml --output outputs/examples/copy - environment: - if-version: 0.4.0 - os: macOS - os-version: 14.3.1 - node-version: 18.19.0 - date-time: 2024-06-30T07:53:07.361Z (UTC) - dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - children: - child-1: - pipeline: - - copy-param - inputs: - - timestamp: '2023-12-12T00:00:00.000Z' - original: hello - outputs: - - timestamp: '2023-12-12T00:00:00.000Z' - original: hello - copy: hello diff --git a/package-lock.json b/package-lock.json index 324104a9b..04b68854f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,17 @@ { "name": "@grnsft/if", - "version": "0.5.0", + "version": "0.6.0-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@grnsft/if", - "version": "0.5.0", + "version": "0.6.0-beta.0", "license": "MIT", "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.10", + "@grnsft/if-core": "^0.0.16", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -24,10 +24,11 @@ "zod": "^3.22.4" }, "bin": { - "if-check": "build/check.js", - "if-diff": "build/diff.js", - "if-env": "build/env.js", - "if-run": "build/index.js" + "if-check": "build/if-check/index.js", + "if-csv": "build/if-csv/index.js", + "if-diff": "build/if-diff/index.js", + "if-env": "build/if-env/index.js", + "if-run": "build/if-run/index.js" }, "devDependencies": { "@babel/core": "^7.22.10", @@ -1184,11 +1185,12 @@ } }, "node_modules/@grnsft/if-core": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.10.tgz", - "integrity": "sha512-WHCdr7H/dFO9gT5fbjrthjOU+4RoLZ5P1F84pbGwJiKLmcU7dvYRuNQKDVIQQ7YJfZl76KSaS7sYgqA+QG8Wpw==", + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.16.tgz", + "integrity": "sha512-Ep/YRk8rpFK7+kgD3iKon6PtY8jEj8H3ihYglw9Jli5lPszObwIMb4e6aHXmW2kcCndpBQKuSXaruGTgQ/d9ww==", "dependencies": { - "typescript": "^5.1.6" + "typescript": "^5.1.6", + "zod": "^3.23.8" }, "engines": { "node": ">=18", @@ -11869,8 +11871,9 @@ } }, "node_modules/zod": { - "version": "3.22.4", - "license": "MIT", + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 768ab1bd0..bd30ee973 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,17 @@ { "name": "@grnsft/if", "description": "Impact Framework", - "version": "0.5.0", + "version": "0.6.0-beta.0", "author": { "name": "Green Software Foundation", "email": "info@gsf.com" }, "bin": { - "if-diff": "./build/diff.js", - "if-run": "./build/index.js", - "if-env": "./build/env.js", - "if-check": "./build/check.js" + "if-diff": "./build/if-diff/index.js", + "if-run": "./build/if-run/index.js", + "if-env": "./build/if-env/index.js", + "if-check": "./build/if-check/index.js", + "if-csv": "./build/if-csv/index.js" }, "bugs": { "url": "https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=feedback&projects=&template=feedback.md&title=Feedback+-+" @@ -18,7 +19,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.10", + "@grnsft/if-core": "^0.0.16", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -63,7 +64,6 @@ "models" ], "license": "MIT", - "main": "build/index.js", "publishConfig": { "access": "public" }, @@ -73,21 +73,20 @@ "scripts": { "build": "npm run clean && tsc --project tsconfig.build.json", "clean": "rimraf build/", - "coverage": "jest --verbose --coverage --testPathPattern=src/__tests__/unit", + "coverage": "jest --verbose --coverage --testPathPattern=src/__tests__/", "fix": "gts fix", "fix:package": "fixpack", - "if-check": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/check.ts", - "if-diff": "npx ts-node src/diff.ts", - "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/env.ts", - "if-run": "npx ts-node src/index.ts", + "if-check": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/if-check/index.ts", + "if-csv": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/if-csv/index.ts", + "if-diff": "npx ts-node src/if-diff/index.ts", + "if-env": "cross-env CURRENT_DIR=$(node -p \"process.env.INIT_CWD\") npx ts-node src/if-env/index.ts", + "if-run": "npx ts-node src/if-run/index.ts", "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", "prepublish": "npm run build", "release": "release-it", - "test": "jest --verbose --testPathPattern=src/__tests__/unit", - "test:integration": "jest --verbose --testPathPattern=src/__tests__/integration" + "test": "jest --verbose --testPathPattern=src/__tests__/" }, - "stability": "stable", - "types": "src/index.d.ts" + "stability": "stable" } diff --git a/src/__mocks__/builtins/export-csv.ts b/src/__mocks__/builtins/export-yaml.ts similarity index 97% rename from src/__mocks__/builtins/export-csv.ts rename to src/__mocks__/builtins/export-yaml.ts index 1f0e00e73..85f54e966 100644 --- a/src/__mocks__/builtins/export-csv.ts +++ b/src/__mocks__/builtins/export-yaml.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -import {Context} from '../../types/manifest'; + +import {Context} from '../../common/types/manifest'; export const tree = { children: { @@ -161,8 +162,6 @@ export const aggregated = { }; export const aggregation = { - metrics: ['carbon'], + metrics: {carbon: {method: 'sum'}}, type: 'both', }; - -export const outputs = ['csv']; diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index 6d4f61b8b..491123143 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -10,27 +10,7 @@ export const readFile = async (filePath: string) => { return fs.readFileSync(updatedPath, 'utf8'); } - /** mock for util/json */ - if (filePath.includes('json-reject')) { - return Promise.reject(new Error('rejected')); - } - if (filePath.includes('json')) { - if (filePath.includes('param')) { - return JSON.stringify({ - 'mock-carbon': { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'mock-cpu': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - }); - } - return JSON.stringify(filePath); } @@ -72,8 +52,9 @@ cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,g tree: children: front-end: - pipeline: - - boavizta-cpu + pipeline: + compute: + - boavizta-cpu config: boavizta-cpu: core-units: 24 @@ -128,6 +109,9 @@ export const writeFile = async (pathToFile: string, content: string) => { } }; +export const appendFile = (file: string, appendContent: string) => + `${file}${appendContent}`; + export const stat = async (filePath: string) => { if (filePath === 'true') { return true; diff --git a/src/__mocks__/json.ts b/src/__mocks__/json.ts deleted file mode 100644 index d6bb24d2c..000000000 --- a/src/__mocks__/json.ts +++ /dev/null @@ -1,14 +0,0 @@ -export const readAndParseJson = async () => { - return { - 'mock-carbon': { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'mock-cpu': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - }; -}; diff --git a/src/__mocks__/mock-manifest.yaml b/src/__mocks__/mock-manifest.yaml index 1e38c1b13..a69009d1e 100644 --- a/src/__mocks__/mock-manifest.yaml +++ b/src/__mocks__/mock-manifest.yaml @@ -10,8 +10,6 @@ initialize: input-parameter: memory/utilization coefficient: 0.0001 output-parameter: memory/energy - outputs: - - yaml execution: command: >- /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -59,7 +57,8 @@ tree: children: child: pipeline: - - memory-energy-from-memory-util + compute: + - memory-energy-from-memory-util config: null inputs: - timestamp: 2023-12-12T00:00:00.000Z diff --git a/src/__tests__/common/lib/load.test.ts b/src/__tests__/common/lib/load.test.ts new file mode 100644 index 000000000..0c147271f --- /dev/null +++ b/src/__tests__/common/lib/load.test.ts @@ -0,0 +1,58 @@ +jest.mock( + 'mockavizta', + () => ({ + __esmodule: true, + Mockavizta: () => ({ + execute: (input: PluginParams) => input, + metadata: { + kind: 'execute', + }, + }), + }), + {virtual: true} +); +jest.mock('../../../common/util/helpers', () => ({ + parseManifestFromStdin: () => { + if (process.env.readline === 'valid-source') { + return ` +name: 'mock-name' +description: 'mock-description' +`; + } + return ''; + }, +})); +jest.mock('../../../common/util/yaml', () => ({ + openYamlFileAsObject: (path: string) => { + switch (path) { + case 'load-default.yml': + return 'raw-manifest'; + case 'source-path.yml': + return 'source-manifest'; + case 'target-path.yml': + return 'target-manifest'; + default: + return ''; + } + }, +})); + +import {PluginParams} from '@grnsft/if-core/types'; + +import {load} from '../../../common/lib/load'; + +describe('lib/load: ', () => { + describe('load(): ', () => { + it('successfully loads yaml.', async () => { + const inputPath = 'load-default.yml'; + + const result = await load(inputPath); + + const expectedValue = { + rawManifest: 'raw-manifest', + }; + + expect(result).toEqual(expectedValue); + }); + }); +}); diff --git a/src/__tests__/unit/util/debug-logger.test.ts b/src/__tests__/common/util/debug-logger.test.ts similarity index 97% rename from src/__tests__/unit/util/debug-logger.test.ts rename to src/__tests__/common/util/debug-logger.test.ts index 63bafa0af..affdca6c5 100644 --- a/src/__tests__/unit/util/debug-logger.test.ts +++ b/src/__tests__/common/util/debug-logger.test.ts @@ -4,7 +4,7 @@ const warnSpy = jest.spyOn(console, 'warn'); const errorSpy = jest.spyOn(console, 'error'); const debugSpy = jest.spyOn(console, 'debug'); -import {debugLogger} from '../../../util/debug-logger'; +import {debugLogger} from '../../../common/util/debug-logger'; describe('util/debug-logger: ', () => { beforeEach(() => { diff --git a/src/__tests__/unit/util/fs.test.ts b/src/__tests__/common/util/fs.test.ts similarity index 99% rename from src/__tests__/unit/util/fs.test.ts rename to src/__tests__/common/util/fs.test.ts index 77b3c0370..643d80df3 100644 --- a/src/__tests__/unit/util/fs.test.ts +++ b/src/__tests__/common/util/fs.test.ts @@ -6,7 +6,7 @@ import { isFileExists, getYamlFiles, removeFileIfExists, -} from '../../../util/fs'; +} from '../../../common/util/fs'; jest.mock('fs/promises', () => require('../../../__mocks__/fs')); diff --git a/src/__tests__/common/util/helpers.test.ts b/src/__tests__/common/util/helpers.test.ts new file mode 100644 index 000000000..92b4f9dab --- /dev/null +++ b/src/__tests__/common/util/helpers.test.ts @@ -0,0 +1,44 @@ +jest.mock('node:readline/promises', () => + require('../../../__mocks__/readline') +); + +import {parseManifestFromStdin} from '../../../common/util/helpers'; + +describe('common/util/helpers: ', () => { + describe('parseManifestFromStdin(): ', () => { + it('returns empty string if there is no data in stdin.', async () => { + const response = await parseManifestFromStdin(); + const expectedResult = ''; + + expect(response).toEqual(expectedResult); + }); + + it('returns empty string if nothing is piped.', async () => { + const originalIsTTY = process.stdin.isTTY; + process.stdin.isTTY = true; + const response = await parseManifestFromStdin(); + const expectedResult = ''; + + expect(response).toEqual(expectedResult); + process.stdin.isTTY = originalIsTTY; + }); + + it('throws error if there is no manifest in stdin.', async () => { + process.env.readline = 'no_manifest'; + expect.assertions(1); + + const response = await parseManifestFromStdin(); + + expect(response).toEqual(''); + }); + + it('returns empty string if there is no data in stdin.', async () => { + process.env.readline = 'manifest'; + const response = await parseManifestFromStdin(); + const expectedMessage = + '\nname: mock-name\ndescription: mock-description\n'; + + expect(response).toEqual(expectedMessage); + }); + }); +}); diff --git a/src/__tests__/unit/util/logger.test.ts b/src/__tests__/common/util/logger.test.ts similarity index 80% rename from src/__tests__/unit/util/logger.test.ts rename to src/__tests__/common/util/logger.test.ts index 9eebb4314..6d7e89ada 100644 --- a/src/__tests__/unit/util/logger.test.ts +++ b/src/__tests__/common/util/logger.test.ts @@ -1,6 +1,6 @@ import {Logger} from 'winston'; -import {logger} from '../../../util/logger'; +import {logger} from '../../../common/util/logger'; describe('util/logger: ', () => { describe('logger(): ', () => { diff --git a/src/__tests__/unit/util/yaml.test.ts b/src/__tests__/common/util/yaml.test.ts similarity index 98% rename from src/__tests__/unit/util/yaml.test.ts rename to src/__tests__/common/util/yaml.test.ts index 532ff3fd5..7ad06e13f 100644 --- a/src/__tests__/unit/util/yaml.test.ts +++ b/src/__tests__/common/util/yaml.test.ts @@ -4,7 +4,7 @@ import { checkIfFileIsYaml, openYamlFileAsObject, saveYamlFileAs, -} from '../../../util/yaml'; +} from '../../../common/util/yaml'; describe('util/yaml: ', () => { describe('checkIfFileIsYaml(): ', () => { diff --git a/src/__tests__/if-check/util/args.test.ts b/src/__tests__/if-check/util/args.test.ts new file mode 100644 index 000000000..de0f2a045 --- /dev/null +++ b/src/__tests__/if-check/util/args.test.ts @@ -0,0 +1,179 @@ +jest.mock('../../../common/util/fs', () => ({ + isFileExists: () => { + if (process.env.fileExists === 'true') { + return true; + } + return false; + }, + isDirectoryExists: () => { + if (process.env.directoryExists === 'true') { + return true; + } + return false; + }, +})); + +jest.mock('ts-command-line-args', () => ({ + __esModule: true, + parse: () => { + switch (process.env.result) { + case 'manifest-is-provided': + return {manifest: 'mock-manifest.yaml'}; + case 'directory-is-provided': + return {directory: '/mock-directory'}; + case 'flags-are-not-provided': + return {manifest: undefined, directory: undefined}; + default: + return { + manifest: 'mock-manifest.yaml', + output: 'mock-output', + }; + } + }, +})); + +jest.mock('ts-command-line-args', () => ({ + __esModule: true, + parse: () => { + switch (process.env.result) { + /** If-env mocks */ + case 'manifest-is-not-yaml': + return {manifest: 'manifest'}; + case 'env-throw-error': + throw new Error('mock-error'); + case 'env-throw': + throw 'mock-error'; + /** If-check */ + case 'manifest-is-provided': + return {manifest: 'mock-manifest.yaml'}; + case 'directory-is-provided': + return {directory: '/mock-directory'}; + case 'flags-are-not-provided': + return {manifest: undefined, directory: undefined}; + default: + return { + manifest: 'mock-manifest.yaml', + output: 'mock-output', + }; + } + }, +})); + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {parseIfCheckArgs} from '../../../if-check/util/args'; + +import {STRINGS} from '../../../if-check/config'; +import {STRINGS as COMMON_STRINGS} from '../../../common/config'; + +const { + InvalidDirectoryError, + MissingCliFlagsError, + CliSourceFileError, + ParseCliParamsError, +} = ERRORS; +const {DIRECTORY_NOT_FOUND, IF_CHECK_FLAGS_MISSING} = STRINGS; +const {SOURCE_IS_NOT_YAML, MANIFEST_NOT_FOUND} = COMMON_STRINGS; + +describe('if-check/util: ', () => { + const originalEnv = process.env; + + describe('parseIfCheckArgs(): ', () => { + it('executes when `manifest` is provided.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-provided'; + const response = await parseIfCheckArgs(); + + expect.assertions(1); + + expect(response).toEqual({manifest: 'mock-manifest.yaml'}); + }); + + it('executes when the `directory` is provided.', async () => { + process.env.directoryExists = 'true'; + process.env.result = 'directory-is-provided'; + + const response = await parseIfCheckArgs(); + + expect.assertions(1); + + expect(response).toEqual({directory: '/mock-directory'}); + }); + + it('throws an error when the `directory` does not exist.', async () => { + process.env.directoryExists = 'false'; + process.env.result = 'directory-is-provided'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + expect(error).toEqual(new InvalidDirectoryError(DIRECTORY_NOT_FOUND)); + } + }); + + it('throws an error when both `manifest` and `directory` flags are not provided.', async () => { + process.env.result = 'flags-are-not-provided'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + expect(error).toEqual(new MissingCliFlagsError(IF_CHECK_FLAGS_MISSING)); + } + }); + + it('throws an error if `manifest` is not a yaml.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-not-yaml'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); + } + } + }); + + it('throws an error if `manifest` path is invalid.', async () => { + process.env.fileExists = 'false'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError(MANIFEST_NOT_FOUND)); + } + } + }); + + it('throws an error if parsing failed.', async () => { + process.env.result = 'env-throw-error'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError('mock-error')); + } + } + }); + + it('throws error if parsing failed (not instance of error).', async () => { + process.env.result = 'env-throw'; + expect.assertions(1); + + try { + await parseIfCheckArgs(); + } catch (error) { + expect(error).toEqual('mock-error'); + } + }); + }); + + process.env = originalEnv; +}); diff --git a/src/__tests__/if-check/util/helpers.test.ts b/src/__tests__/if-check/util/helpers.test.ts new file mode 100644 index 000000000..381c7534c --- /dev/null +++ b/src/__tests__/if-check/util/helpers.test.ts @@ -0,0 +1,18 @@ +import {logStdoutFailMessage} from '../../../if-check/util/helpers'; + +describe('logStdoutFailMessage(): ', () => { + it('successfully logs the failed message.', () => { + const errorMessage = {stdout: '\n\nmock error message'}; + const mockFilename = 'mock-filename.yaml'; + const logSpy = jest.spyOn(global.console, 'log'); + logStdoutFailMessage(errorMessage, mockFilename); + + expect.assertions(2); + + expect(logSpy).toHaveBeenCalledWith( + `āœ– if-check could not verify ${mockFilename}. The re-executed file does not match the original.\n` + ); + + expect(logSpy).toHaveBeenCalledWith('mock error message'); + }); +}); diff --git a/src/__tests__/if-check/util/npm.test.ts b/src/__tests__/if-check/util/npm.test.ts new file mode 100644 index 000000000..4bdffc10f --- /dev/null +++ b/src/__tests__/if-check/util/npm.test.ts @@ -0,0 +1,54 @@ +jest.mock('fs/promises', () => require('../../../__mocks__/fs')); + +const mockInfo = jest.fn(); + +jest.mock('node:readline/promises', () => + require('../../../__mocks__/readline') +); +jest.mock('../../../common/util/logger', () => ({ + logger: { + info: mockInfo, + }, +})); + +jest.mock('../../../common/util/helpers', () => { + const originalModule = jest.requireActual('../../../common/util/helpers'); + return { + ...originalModule, + execPromise: async (param: any) => { + switch (process.env.NPM_INSTALL) { + case 'true': + expect(param).toEqual('npm install @grnsft/if@0.3.3-beta.0'); + break; + case 'npm init -y': + expect(param).toEqual('npm init -y'); + break; + case 'if-check': + expect(param).toEqual( + "npm run if-env -- -m ./src/__mocks__/mock-manifest.yaml && npm run if-run -- -m ./src/__mocks__/mock-manifest.yaml -o src/__mocks__/re-mock-manifest && node -p 'Boolean(process.stdout.isTTY)' | npm run if-diff -- -s src/__mocks__/re-mock-manifest.yaml -t ./src/__mocks__/mock-manifest.yaml" + ); + break; + } + return; + }, + }; +}); + +import {executeCommands} from '../../../if-check/util/npm'; + +describe('if-check/util/npm: ', () => { + describe('executeCommands(): ', () => { + it('successfully executes with correct commands.', async () => { + process.env.NPM_INSTALL = 'if-check'; + const manifest = './src/__mocks__/mock-manifest.yaml'; + const logSpy = jest.spyOn(global.console, 'log'); + + await executeCommands(manifest, false); + + expect.assertions(2); + expect(logSpy).toHaveBeenCalledWith( + 'āœ” if-check successfully verified mock-manifest.yaml\n' + ); + }); + }); +}); diff --git a/src/__tests__/if-csv/util/helpers.test.ts b/src/__tests__/if-csv/util/helpers.test.ts new file mode 100644 index 000000000..1781f5fbc --- /dev/null +++ b/src/__tests__/if-csv/util/helpers.test.ts @@ -0,0 +1,264 @@ +import {jest} from '@jest/globals'; +import * as fs from 'fs/promises'; +import {stringify} from 'csv-stringify/sync'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {generateCsv, getManifestData} from '../../../if-csv/util/helpers'; +import {CsvOptions} from '../../../if-csv/types/csv'; + +const {ManifestValidationError} = ERRORS; + +import { + tree, + context, + aggregated, + aggregation, +} from '../../../__mocks__/builtins/export-yaml'; + +jest.mock('fs/promises', () => { + const originalModule = + jest.requireActual('fs/promises'); + return { + __esModule: true, + ...originalModule, + writeFile: jest.fn(() => Promise.resolve()), + }; +}); + +describe('if-csv/util/helpers: ', () => { + describe('generateCsv(): ', () => { + it('generates CSV file with correct data.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, + }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context, + outputPath, + params: 'carbon', + }; + + await generateCsv(options); + + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + + it('generates CSV file when the `outputs` type is missing.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, + }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context, + outputPath, + params: 'carbon', + }; + + await generateCsv(options); + + expect.assertions(1); + + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + + it('generates CSV file when `aggregation` persists.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + + const reformedContext = Object.assign({}, context, {aggregation}); + const reformedTree = Object.assign({}, tree, { + children: { + ...tree.children, + 'child-1': { + ...tree.children['child-1'], + aggregated, + }, + 'child-2': { + ...tree.children['child-2'], + aggregated, + }, + }, + }); + + const options: CsvOptions = { + tree: reformedTree, + context: reformedContext, + outputPath, + params: 'carbon', + }; + + await generateCsv(options); + + expect.assertions(1); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + + it('returns string when `outputPath` is not provided', async () => { + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + + const options: CsvOptions = { + tree, + context, + outputPath: undefined, + params: 'carbon', + }; + + const result = await generateCsv(options); + + expect.assertions(1); + expect(result).toEqual(stringify(matrix, {columns})); + }); + + it('generates CSV file when `aggregation` is missing.', async () => { + const outputPath = 'output'; + const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; + const matrix = [ + columns, + ['tree.carbon', 8000.000004051243, 8000.000004051243], + [ + 'tree.children.child-1.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + [ + 'tree.children.child-2.carbon', + 4000.0000020256216, + 4000.0000020256216, + ], + ]; + + const options: CsvOptions = { + tree, + context, + outputPath, + params: 'carbon', + }; + + await generateCsv(options); + + expect.assertions(1); + expect(fs.writeFile).toHaveBeenCalledWith( + 'output.csv', + stringify(matrix, {columns}) + ); + }); + }); + + describe('getManifestData(): ', () => { + it('returns correct manifest data.', async () => { + process.env.MOCK_MANIFEST = 'true'; + const mockManifestPath = './src/__mocks__/mock-manifest.yaml'; + const result = await getManifestData(mockManifestPath); + + expect.assertions(2); + expect(result).toHaveProperty('description'); + expect(result).toHaveProperty('tree'); + }); + + it('throws an error when there is not `outputs` in the manifest.', async () => { + process.env.MOCK_MANIFEST = 'false'; + const mockManifestPath = ''; + const load = require('../../../common/lib/load'); + const spyLoad = jest.spyOn(load, 'load'); + + spyLoad.mockReturnValue({ + rawManifest: {tree: {children: {child: {}}}}, + }); + expect.assertions(1); + try { + await getManifestData(mockManifestPath); + } catch (error) { + expect(error).toEqual( + new ManifestValidationError('Manifest outputs are not available!') + ); + } + }); + }); +}); diff --git a/src/__tests__/unit/lib/compare.test.ts b/src/__tests__/if-diff/lib/compare.test.ts similarity index 97% rename from src/__tests__/unit/lib/compare.test.ts rename to src/__tests__/if-diff/lib/compare.test.ts index dbac5a490..f43e7c590 100644 --- a/src/__tests__/unit/lib/compare.test.ts +++ b/src/__tests__/if-diff/lib/compare.test.ts @@ -1,4 +1,4 @@ -import {compare} from '../../../lib/compare'; +import {compare} from '../../../if-diff/lib/compare'; describe('lib/compare: ', () => { describe('compare(): ', () => { diff --git a/src/__tests__/unit/lib/load.test.ts b/src/__tests__/if-diff/lib/load.test.ts similarity index 60% rename from src/__tests__/unit/lib/load.test.ts rename to src/__tests__/if-diff/lib/load.test.ts index ba3bca902..cd6fbb1ea 100644 --- a/src/__tests__/unit/lib/load.test.ts +++ b/src/__tests__/if-diff/lib/load.test.ts @@ -1,4 +1,3 @@ -jest.mock('../../../util/json', () => require('../../../__mocks__/json')); jest.mock( 'mockavizta', () => ({ @@ -12,7 +11,7 @@ jest.mock( }), {virtual: true} ); -jest.mock('../../../util/helpers', () => ({ +jest.mock('../../../common/util/helpers', () => ({ parseManifestFromStdin: () => { if (process.env.readline === 'valid-source') { return ` @@ -23,7 +22,7 @@ description: 'mock-description' return ''; }, })); -jest.mock('../../../util/yaml', () => ({ +jest.mock('../../../common/util/yaml', () => ({ openYamlFileAsObject: (path: string) => { switch (path) { case 'load-default.yml': @@ -40,58 +39,17 @@ jest.mock('../../../util/yaml', () => ({ import {PluginParams} from '@grnsft/if-core/types'; -import {load, loadIfDiffFiles} from '../../../lib/load'; +import {loadIfDiffFiles} from '../../../if-diff/lib/load'; -import {PARAMETERS, STRINGS} from '../../../config'; +import {parseManifestFromStdin} from '../../../common/util/helpers'; -import {parseManifestFromStdin} from '../../../util/helpers'; +import {STRINGS} from '../../../if-diff/config'; -import {LoadDiffParams} from '../../../types/util/args'; +import {LoadDiffParams} from '../../../if-diff/types/args'; const {INVALID_SOURCE} = STRINGS; -describe('lib/load: ', () => { - describe('load(): ', () => { - it('loads yaml with default parameters.', async () => { - const inputPath = 'load-default.yml'; - const paramPath = undefined; - - const result = await load(inputPath, paramPath); - - const expectedValue = { - rawManifest: 'raw-manifest', - parameters: PARAMETERS, - }; - - expect(result).toEqual(expectedValue); - }); - - it('loads yaml with custom parameters.', async () => { - const inputPath = 'load-default.yml'; - const paramPath = 'param-mock.json'; - - const result = await load(inputPath, paramPath); - - const expectedValue = { - rawManifest: 'raw-manifest', - parameters: { - 'mock-carbon': { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'mock-cpu': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - }, - }; - - expect(result).toEqual(expectedValue); - }); - }); - +describe('if-diff/lib/load: ', () => { describe('loadIfDiffFiles(): ', () => { it('rejects with invalid source error.', async () => { const params = { diff --git a/src/__tests__/if-diff/util/args.test.ts b/src/__tests__/if-diff/util/args.test.ts new file mode 100644 index 000000000..2f43ec6ab --- /dev/null +++ b/src/__tests__/if-diff/util/args.test.ts @@ -0,0 +1,133 @@ +jest.mock('ts-command-line-args', () => ({ + __esModule: true, + parse: () => { + switch (process.env.result) { + case 'only-target': + return { + target: 'target-mock.yml', + }; + case 'target-is-not-yaml': + return { + target: 'target-mock', + }; + case 'source-is-not-yaml': + return { + target: 'target-mock.yml', + source: 'source-mock', + }; + case 'target-source': + return { + target: 'target-mock.yml', + source: 'source-mock.yml', + }; + case 'diff-throw-error': + throw new Error('mock-error'); + case 'diff-throw': + throw 'mock-error'; + default: + return { + manifest: 'mock-manifest.yaml', + output: 'mock-output', + }; + } + }, +})); + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {parseIfDiffArgs} from '../../../if-diff/util/args'; + +import {STRINGS as COMMON_STRINGS} from '../../../common/config'; +import {STRINGS} from '../../../if-diff/config'; + +const {ParseCliParamsError} = ERRORS; + +const {TARGET_IS_NOT_YAML, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; +const {INVALID_TARGET} = STRINGS; + +describe('util/args: ', () => { + const originalEnv = process.env; + + describe('parseIfDiffArgs(): ', () => { + it('throws error if `target` is missing.', () => { + expect.assertions(1); + + try { + parseIfDiffArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError(INVALID_TARGET)); + } + } + }); + + it('throws error if `target` is not a yaml.', () => { + process.env.result = 'target-is-not-yaml'; + expect.assertions(1); + + try { + parseIfDiffArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError(TARGET_IS_NOT_YAML)); + } + } + }); + + it('returns `target`s full path.', () => { + process.env.result = 'only-target'; + expect.assertions(1); + + const response = parseIfDiffArgs(); + expect(response).toHaveProperty('targetPath'); + }); + + it('throws error if source is not a yaml.', () => { + process.env.result = 'source-is-not-yaml'; + expect.assertions(1); + + try { + parseIfDiffArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError(SOURCE_IS_NOT_YAML)); + } + } + }); + + it('returns target and source full paths.', () => { + process.env.result = 'target-source'; + expect.assertions(2); + + const response = parseIfDiffArgs(); + expect(response).toHaveProperty('targetPath'); + expect(response).toHaveProperty('sourcePath'); + }); + + it('throws error if parsing failed.', () => { + process.env.result = 'diff-throw-error'; + expect.assertions(1); + + try { + parseIfDiffArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError('mock-error')); + } + } + }); + + it('throws error if parsing failed (not instance of error).', () => { + process.env.result = 'diff-throw'; + expect.assertions(1); + + try { + parseIfDiffArgs(); + } catch (error) { + expect(error).toEqual('mock-error'); + } + }); + }); + + process.env = originalEnv; +}); diff --git a/src/__tests__/if-diff/util/helpers.test.ts b/src/__tests__/if-diff/util/helpers.test.ts new file mode 100644 index 000000000..eb9a0e553 --- /dev/null +++ b/src/__tests__/if-diff/util/helpers.test.ts @@ -0,0 +1,195 @@ +jest.mock('node:readline/promises', () => + require('../../../__mocks__/readline') +); + +import {Difference} from '../../../if-diff/types/compare'; +import { + checkIfEqual, + formatNotMatchingLog, + oneIsPrimitive, +} from '../../../if-diff/util/helpers'; + +describe('if-diff/util/helpers: ', () => { + describe('checkIfEqual(): ', () => { + it('checks if values are equal.', () => { + const a = 'mock'; + const b = 'mock'; + + const response = checkIfEqual(a, b); + expect(response).toBeTruthy(); + }); + + it('returns true if one of the values is wildcard.', () => { + const a = 'mock'; + const b = '*'; + + const response = checkIfEqual(a, b); + expect(response).toBeTruthy(); + }); + + it('returns false for number and string with the same value.', () => { + const a = 5; + const b = '5'; + + const response = checkIfEqual(a, b); + expect(response).toBeFalsy(); + }); + }); + + describe('oneIsPrimitive(): ', () => { + it('returns true if values are nullish.', () => { + const source = null; + const target = undefined; + + const result = oneIsPrimitive(source, target); + expect(result).toBeTruthy(); + }); + + it('returns true if values are string or number.', () => { + const source = 'string'; + const target = 10; + + const result = oneIsPrimitive(source, target); + expect(result).toBeTruthy(); + }); + + it('returns false if one of values is object.', () => { + const source = 'string'; + const target = {}; + + const result = oneIsPrimitive(source, target); + expect(result).toBeFalsy(); + }); + }); + + describe('formatNotMatchingLog(): ', () => { + const actualLogger = console.log; + const mockLogger = jest.fn(); + console.log = mockLogger; + + beforeEach(() => { + mockLogger.mockReset(); + }); + + it('logs the message.', () => { + const difference: Difference = { + message: 'mock-message', + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(1); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + }); + + it('logs message and path.', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(2); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + }); + + it('logs message, path and formatted source/target (one is missing).', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + source: 'mock-source', + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(4); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); + expect(mockLogger).toHaveBeenCalledWith('target: missing'); + }); + + it('logs message, path and formatted source/target.', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + source: 'mock-source', + target: 'mock-target', + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(4); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); + expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); + }); + + it('logs message, path and formatted source/target (numbers).', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + source: 10, + target: 0, + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(4); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); + expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); + }); + + it('logs message, path and formatted source/target (booleans).', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + source: true, + target: false, + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(4); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); + expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); + }); + + it('logs message, path and formatted source/target (objects).', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + source: {}, + target: false, + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(4); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); + expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); + }); + + it('logs message, path and formatted source/target (empty string).', () => { + const difference: Difference = { + message: 'mock-message', + path: 'mock.path', + source: '', + target: false, + }; + + formatNotMatchingLog(difference); + expect(mockLogger).toHaveBeenCalledTimes(4); + expect(mockLogger).toHaveBeenCalledWith(difference.message); + expect(mockLogger).toHaveBeenCalledWith(difference.path); + expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); + expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); + }); + + afterAll(() => { + console.log = actualLogger; + }); + }); +}); diff --git a/src/__tests__/if-env/util/args.test.ts b/src/__tests__/if-env/util/args.test.ts new file mode 100644 index 000000000..2b0dd13f9 --- /dev/null +++ b/src/__tests__/if-env/util/args.test.ts @@ -0,0 +1,124 @@ +jest.mock('../../../common/util/fs', () => ({ + isFileExists: () => { + if (process.env.fileExists === 'true') { + return true; + } + return false; + }, +})); + +jest.mock('ts-command-line-args', () => ({ + __esModule: true, + parse: () => { + switch (process.env.result) { + case 'manifest-is-missing': + return {}; + case 'manifest-install-provided': + return { + install: true, + manifest: 'mock-manifest.yaml', + }; + case 'manifest-is-not-yaml': + return {manifest: 'manifest'}; + case 'env-throw-error': + throw new Error('mock-error'); + case 'env-throw': + throw 'mock-error'; + default: + return { + manifest: 'mock-manifest.yaml', + output: 'mock-output', + }; + } + }, +})); + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {parseIfEnvArgs} from '../../../if-env/util/args'; + +import {STRINGS} from '../../../common/config'; + +const {CliSourceFileError, ParseCliParamsError} = ERRORS; + +const {SOURCE_IS_NOT_YAML, MANIFEST_NOT_FOUND} = STRINGS; + +describe('util/args: ', () => { + const originalEnv = process.env; + + describe('parseIfEnvArgs(): ', () => { + it('executes if `manifest` is missing.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-missing'; + const response = await parseIfEnvArgs(); + + expect.assertions(1); + + expect(response).toEqual({install: undefined}); + }); + + it('executes if `manifest` and `install` are provided.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-install-provided'; + + const response = await parseIfEnvArgs(); + + expect.assertions(2); + expect(response).toHaveProperty('install'); + expect(response).toHaveProperty('manifest'); + }); + + it('throws an error if `manifest` is not a yaml.', async () => { + process.env.fileExists = 'true'; + process.env.result = 'manifest-is-not-yaml'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); + } + } + }); + + it('throws an error if `manifest` path is invalid.', async () => { + process.env.fileExists = 'false'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError(MANIFEST_NOT_FOUND)); + } + } + }); + + it('throws an error if parsing failed.', async () => { + process.env.result = 'env-throw-error'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + if (error instanceof Error) { + expect(error).toEqual(new ParseCliParamsError('mock-error')); + } + } + }); + + it('throws error if parsing failed (not instance of error).', async () => { + process.env.result = 'env-throw'; + expect.assertions(1); + + try { + await parseIfEnvArgs(); + } catch (error) { + expect(error).toEqual('mock-error'); + } + }); + }); + + process.env = originalEnv; +}); diff --git a/src/__tests__/unit/util/npm.test.ts b/src/__tests__/if-env/util/npm.test.ts similarity index 89% rename from src/__tests__/unit/util/npm.test.ts rename to src/__tests__/if-env/util/npm.test.ts index 6d1476b42..be922f07c 100644 --- a/src/__tests__/unit/util/npm.test.ts +++ b/src/__tests__/if-env/util/npm.test.ts @@ -8,14 +8,15 @@ const mockInfo = jest.fn(); jest.mock('node:readline/promises', () => require('../../../__mocks__/readline') ); -jest.mock('../../../util/logger', () => ({ +jest.mock('../../../common/util/logger', () => ({ logger: { info: mockInfo, }, })); -jest.mock('../../../util/helpers', () => { - const originalModule = jest.requireActual('../../../util/helpers'); +jest.mock('../../../common/util/helpers', () => { + const originalModule = jest.requireActual('../../../if-run/util/helpers'); + return { ...originalModule, execPromise: async (param: any) => { @@ -43,12 +44,11 @@ import { updatePackageJsonDependencies, extractPathsWithVersion, updatePackageJsonProperties, - executeCommands, -} from '../../../util/npm'; -import {isFileExists} from '../../../util/fs'; +} from '../../../if-env/util/npm'; +import {isFileExists} from '../../../common/util/fs'; -import {STRINGS} from '../../../config/strings'; -import {ManifestPlugin} from '../../../types/npm'; +import {STRINGS} from '../../../if-env/config'; +import {ManifestPlugin} from '../../../if-env/types/npm'; const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; @@ -247,19 +247,4 @@ describe('util/npm: ', () => { expect(fsReadSpy).toHaveBeenCalledWith(packageJsonPath, 'utf8'); }); }); - - describe('executeCommands(): ', () => { - it('successfully executes with correct commands.', async () => { - process.env.NPM_INSTALL = 'if-check'; - const manifest = './src/__mocks__/mock-manifest.yaml'; - const logSpy = jest.spyOn(global.console, 'log'); - - await executeCommands(manifest, false); - - expect.assertions(2); - expect(logSpy).toHaveBeenCalledWith( - 'āœ” if-check successfully verified mock-manifest.yaml\n' - ); - }); - }); }); diff --git a/src/__tests__/unit/builtins/CommonGenerator.test.ts b/src/__tests__/if-run/builtins/CommonGenerator.test.ts similarity index 86% rename from src/__tests__/unit/builtins/CommonGenerator.test.ts rename to src/__tests__/if-run/builtins/CommonGenerator.test.ts index b0c165d1a..87b8457a8 100644 --- a/src/__tests__/unit/builtins/CommonGenerator.test.ts +++ b/src/__tests__/if-run/builtins/CommonGenerator.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {CommonGenerator} from '../../../builtins/mock-observations/helpers/common-generator'; +import {CommonGenerator} from '../../../if-run/builtins/mock-observations/helpers/common-generator'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {GlobalConfigError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; diff --git a/src/__tests__/unit/builtins/RandIntGenerator.test.ts b/src/__tests__/if-run/builtins/RandIntGenerator.test.ts similarity index 91% rename from src/__tests__/unit/builtins/RandIntGenerator.test.ts rename to src/__tests__/if-run/builtins/RandIntGenerator.test.ts index 328f7d6f4..05fdb37f2 100644 --- a/src/__tests__/unit/builtins/RandIntGenerator.test.ts +++ b/src/__tests__/if-run/builtins/RandIntGenerator.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {RandIntGenerator} from '../../../builtins/mock-observations/helpers/rand-int-generator'; +import {RandIntGenerator} from '../../../if-run/builtins/mock-observations/helpers/rand-int-generator'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {GlobalConfigError} = ERRORS; const {INVALID_NAME, MISSING_MIN_MAX, MISSING_GLOBAL_CONFIG} = STRINGS; diff --git a/src/__tests__/unit/builtins/coefficient.test.ts b/src/__tests__/if-run/builtins/coefficient.test.ts similarity index 86% rename from src/__tests__/unit/builtins/coefficient.test.ts rename to src/__tests__/if-run/builtins/coefficient.test.ts index 034d20f95..8d99c3e87 100644 --- a/src/__tests__/unit/builtins/coefficient.test.ts +++ b/src/__tests__/if-run/builtins/coefficient.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Coefficient} from '../../../builtins/coefficient'; +import {Coefficient} from '../../../if-run/builtins/coefficient'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {InputValidationError, GlobalConfigError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; @@ -14,7 +14,11 @@ describe('builtins/coefficient: ', () => { coefficient: 3, 'output-parameter': 'carbon-product', }; - const coefficient = Coefficient(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const coefficient = Coefficient(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -51,7 +55,7 @@ describe('builtins/coefficient: ', () => { it('throws an error when global config is not provided.', () => { const config = undefined; - const coefficient = Coefficient(config!); + const coefficient = Coefficient(config!, parametersMetadata); expect.assertions(1); @@ -76,7 +80,7 @@ describe('builtins/coefficient: ', () => { coefficient: 3, 'output-parameter': 'carbon-product', }; - const coefficient = Coefficient(invalidConfig); + const coefficient = Coefficient(invalidConfig, parametersMetadata); const expectedMessage = '"input-parameter" parameter is string must contain at least 1 character(s). Error code: too_small.'; @@ -103,7 +107,7 @@ describe('builtins/coefficient: ', () => { coefficient: 10, 'output-parameter': '', }; - const coefficient = Coefficient(invalidConfig); + const coefficient = Coefficient(invalidConfig, parametersMetadata); const expectedMessage = '"output-parameter" parameter is string must contain at least 1 character(s). Error code: too_small.'; diff --git a/src/__tests__/unit/builtins/copy-param.test.ts b/src/__tests__/if-run/builtins/copy-param.test.ts similarity index 87% rename from src/__tests__/unit/builtins/copy-param.test.ts rename to src/__tests__/if-run/builtins/copy-param.test.ts index 64809e44a..952546505 100644 --- a/src/__tests__/unit/builtins/copy-param.test.ts +++ b/src/__tests__/if-run/builtins/copy-param.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Copy} from '../../../builtins/copy-param'; +import {Copy} from '../../../if-run/builtins/copy-param'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {GlobalConfigError, InputValidationError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; @@ -14,7 +14,11 @@ describe('builtins/copy: ', () => { from: 'original', to: 'copy', }; - const copy = Copy(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const copy = Copy(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -49,7 +53,7 @@ describe('builtins/copy: ', () => { it('throws an error when global config is not provided.', () => { const config = undefined; - const copy = Copy(config!); + const copy = Copy(config!, parametersMetadata); expect.assertions(1); @@ -74,7 +78,7 @@ describe('builtins/copy: ', () => { from: 'original', to: 'copy', }; - const copy = Copy(globalConfig); + const copy = Copy(globalConfig, parametersMetadata); expect.assertions(1); try { @@ -99,7 +103,7 @@ describe('builtins/copy: ', () => { from: 'original', to: 'copy', }; - const copy = Copy(globalConfig); + const copy = Copy(globalConfig, parametersMetadata); const expectedResult = [ { diff --git a/src/__tests__/unit/builtins/csv-lookup.test.ts b/src/__tests__/if-run/builtins/csv-lookup.test.ts similarity index 92% rename from src/__tests__/unit/builtins/csv-lookup.test.ts rename to src/__tests__/if-run/builtins/csv-lookup.test.ts index 81e0498b4..49e4d45bb 100644 --- a/src/__tests__/unit/builtins/csv-lookup.test.ts +++ b/src/__tests__/if-run/builtins/csv-lookup.test.ts @@ -4,9 +4,9 @@ import axios from 'axios'; import AxiosMockAdapter from 'axios-mock-adapter'; import {ERRORS} from '@grnsft/if-core/utils'; -import {CSVLookup} from '../../../builtins'; +import {CSVLookup} from '../../../if-run/builtins'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const { GlobalConfigError, @@ -19,6 +19,10 @@ const { const {MISSING_GLOBAL_CONFIG, MISSING_CSV_COLUMN, NO_QUERY_DATA} = STRINGS; describe('builtins/CSVLookup: ', () => { + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; const mock = new AxiosMockAdapter(axios); describe('CSVLookup: ', () => { @@ -35,7 +39,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); expect(csvLookup).toHaveProperty('metadata'); expect(csvLookup).toHaveProperty('execute'); }); @@ -54,7 +58,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const responseData = `cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,gpu-count,gpu-model-name,Hardware Information on AWS Documentation & Comments,instance-class,instance-storage,memory-available,platform-memory,release-date,storage-drives 16,8,AWS,AWS Graviton,150.00,N/A,N/A,AWS Graviton (ARM),a1.2xlarge,EBS-Only,16,32,November 2018,0 @@ -93,7 +97,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const result = await csvLookup.execute([ { @@ -126,7 +130,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const input = [ { timestamp: '2024-03-01', @@ -155,7 +159,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const input = [ { timestamp: '2024-03-01', @@ -187,7 +191,7 @@ describe('builtins/CSVLookup: ', () => { }; mock.onGet(globalConfig.filepath).reply(404); - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const input = [ { timestamp: '2024-03-01', @@ -217,7 +221,7 @@ describe('builtins/CSVLookup: ', () => { }, output: '*', }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const result = await csvLookup.execute([ { @@ -265,7 +269,7 @@ describe('builtins/CSVLookup: ', () => { ['gpu-model-name', 'gpumodel'], ], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const result = await csvLookup.execute([ { @@ -300,7 +304,7 @@ describe('builtins/CSVLookup: ', () => { }, output: 'gpu-count', }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const result = await csvLookup.execute([ { @@ -335,7 +339,7 @@ describe('builtins/CSVLookup: ', () => { output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const input = [ { timestamp: '2024-03-01', @@ -392,7 +396,7 @@ describe('builtins/CSVLookup: ', () => { }, output: 'mock', }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const input = [ { timestamp: '2024-03-01', @@ -425,7 +429,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['gpu-count'], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const result = await csvLookup.execute([ { @@ -459,7 +463,7 @@ describe('builtins/CSVLookup: ', () => { }, output: [['gpu-count']], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); const result = await csvLookup.execute([ { @@ -495,7 +499,7 @@ describe('builtins/CSVLookup: ', () => { }, output: [['gpu-count']], }; - const csvLookup = CSVLookup(globalConfig); + const csvLookup = CSVLookup(globalConfig, parametersMetadata); try { await csvLookup.execute([ diff --git a/src/__tests__/unit/builtins/divide.test.ts b/src/__tests__/if-run/builtins/divide.test.ts similarity index 88% rename from src/__tests__/unit/builtins/divide.test.ts rename to src/__tests__/if-run/builtins/divide.test.ts index ba75e3879..e0a472998 100644 --- a/src/__tests__/unit/builtins/divide.test.ts +++ b/src/__tests__/if-run/builtins/divide.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Divide} from '../../../builtins'; +import {Divide} from '../../../if-run/builtins'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {InputValidationError, GlobalConfigError, MissingInputDataError} = ERRORS; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; @@ -14,7 +14,11 @@ describe('builtins/divide: ', () => { denominator: 2, output: 'cpu/number-cores', }; - const divide = Divide(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const divide = Divide(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -54,7 +58,7 @@ describe('builtins/divide: ', () => { denominator: 'duration', output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig); + const divide = Divide(globalConfig, parametersMetadata); const input = [ { @@ -86,7 +90,7 @@ describe('builtins/divide: ', () => { denominator: 3600, output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig); + const divide = Divide(globalConfig, parametersMetadata); expect.assertions(1); @@ -107,7 +111,7 @@ describe('builtins/divide: ', () => { it('throws an error on missing global config.', async () => { const config = undefined; - const divide = Divide(config!); + const divide = Divide(config!, parametersMetadata); expect.assertions(1); @@ -131,7 +135,7 @@ describe('builtins/divide: ', () => { denominator: 0, output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig); + const divide = Divide(globalConfig, parametersMetadata); expect.assertions(1); @@ -159,7 +163,7 @@ describe('builtins/divide: ', () => { denominator: '10', output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig); + const divide = Divide(globalConfig, parametersMetadata); expect.assertions(1); diff --git a/src/__tests__/unit/builtins/exponent.test.ts b/src/__tests__/if-run/builtins/exponent.test.ts similarity index 91% rename from src/__tests__/unit/builtins/exponent.test.ts rename to src/__tests__/if-run/builtins/exponent.test.ts index ad438179d..2e0419686 100644 --- a/src/__tests__/unit/builtins/exponent.test.ts +++ b/src/__tests__/if-run/builtins/exponent.test.ts @@ -1,6 +1,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Exponent} from '../../../builtins/exponent'; +import {Exponent} from '../../../if-run/builtins/exponent'; const {InputValidationError} = ERRORS; @@ -11,7 +11,11 @@ describe('builtins/exponent: ', () => { exponent: 3, 'output-parameter': 'energy', }; - const exponent = Exponent(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const exponent = Exponent(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -91,7 +95,7 @@ describe('builtins/exponent: ', () => { exponent: 4, 'output-parameter': 'carbon', }; - const exponent = Exponent(newConfig); + const exponent = Exponent(newConfig, parametersMetadata); const data = [ { diff --git a/src/__tests__/unit/builtins/export-log.test.ts b/src/__tests__/if-run/builtins/export-log.test.ts similarity index 94% rename from src/__tests__/unit/builtins/export-log.test.ts rename to src/__tests__/if-run/builtins/export-log.test.ts index f6990ab1e..8782142f6 100644 --- a/src/__tests__/unit/builtins/export-log.test.ts +++ b/src/__tests__/if-run/builtins/export-log.test.ts @@ -1,7 +1,7 @@ import * as YAML from 'js-yaml'; -import {ExportLog} from '../../../builtins/export-log'; -import {tree, context} from '../../../__mocks__/builtins/export-csv'; +import {ExportLog} from '../../../if-run/builtins/export-log'; +import {tree, context} from '../../../__mocks__/builtins/export-yaml'; describe('builtins/export-log:', () => { describe('ExportLog: ', () => { diff --git a/src/__tests__/unit/builtins/export-yaml.test.ts b/src/__tests__/if-run/builtins/export-yaml.test.ts similarity index 82% rename from src/__tests__/unit/builtins/export-yaml.test.ts rename to src/__tests__/if-run/builtins/export-yaml.test.ts index fb7954c1e..6a9a442e2 100644 --- a/src/__tests__/unit/builtins/export-yaml.test.ts +++ b/src/__tests__/if-run/builtins/export-yaml.test.ts @@ -1,13 +1,13 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {ExportYaml} from '../../../builtins/export-yaml'; -import {saveYamlFileAs} from '../../../util/yaml'; +import {ExportYaml} from '../../../if-run/builtins/export-yaml'; +import {saveYamlFileAs} from '../../../common/util/yaml'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; -import {tree, context} from '../../../__mocks__/builtins/export-csv'; +import {tree, context} from '../../../__mocks__/builtins/export-yaml'; -jest.mock('../../../util/yaml', () => ({ +jest.mock('../../../common/util/yaml', () => ({ saveYamlFileAs: jest.fn(), })); @@ -32,7 +32,7 @@ describe('builtins/export-yaml: ', () => { expect(saveYamlFileAs).toHaveBeenCalledWith( {...context, tree}, - `${outputPath.split('#')[0]}.yaml` + outputPath ); }); diff --git a/src/__tests__/unit/builtins/interpolation.test.ts b/src/__tests__/if-run/builtins/interpolation.test.ts similarity index 89% rename from src/__tests__/unit/builtins/interpolation.test.ts rename to src/__tests__/if-run/builtins/interpolation.test.ts index 6219e01ec..6634556dd 100644 --- a/src/__tests__/unit/builtins/interpolation.test.ts +++ b/src/__tests__/if-run/builtins/interpolation.test.ts @@ -1,9 +1,9 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Method} from '@grnsft/if-core/types'; -import {Interpolation} from '../../../builtins'; +import {Interpolation} from '../../../if-run/builtins'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {InputValidationError, GlobalConfigError} = ERRORS; const { @@ -22,6 +22,10 @@ describe('builtins/interpolation: ', () => { 'input-parameter': 'cpu/utilization', 'output-parameter': 'interpolation-result', }; + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; const inputs = [ { timestamp: '2023-07-06T00:00', @@ -29,7 +33,7 @@ describe('builtins/interpolation: ', () => { 'cpu/utilization': 45, }, ]; - const plugin = Interpolation(globalConfig); + const plugin = Interpolation(globalConfig, parametersMetadata); describe('init Interpolation: ', () => { it('initalizes object with properties.', async () => { @@ -59,7 +63,7 @@ describe('builtins/interpolation: ', () => { 'input-parameter': 'cpu/utilization', 'output-parameter': 'interpolation-result', }; - const plugin = Interpolation(globalConfig); + const plugin = Interpolation(globalConfig, parametersMetadata); const outputs = [ { @@ -75,7 +79,7 @@ describe('builtins/interpolation: ', () => { it('returns result when the `method` is `spline`.', () => { const config = Object.assign({}, globalConfig, {method: Method.SPLINE}); - const plugin = Interpolation(config); + const plugin = Interpolation(config, parametersMetadata); const outputs = [ { @@ -93,7 +97,7 @@ describe('builtins/interpolation: ', () => { const config = Object.assign({}, globalConfig, { method: Method.POLYNOMIAL, }); - const plugin = Interpolation(config); + const plugin = Interpolation(config, parametersMetadata); const outputs = [ { @@ -111,7 +115,7 @@ describe('builtins/interpolation: ', () => { const config = Object.assign({}, globalConfig, { x: [0, 10, 100, 50], }); - const plugin = Interpolation(config); + const plugin = Interpolation(config, parametersMetadata); const outputs = [ { @@ -147,7 +151,7 @@ describe('builtins/interpolation: ', () => { it('throws an when the global config is not provided.', () => { const config = undefined; - const plugin = Interpolation(config!); + const plugin = Interpolation(config!, parametersMetadata); expect.assertions(2); try { @@ -163,7 +167,7 @@ describe('builtins/interpolation: ', () => { x: [0, 10, 100], }); - const plugin = Interpolation(config); + const plugin = Interpolation(config, parametersMetadata); expect.assertions(2); try { @@ -198,7 +202,7 @@ describe('builtins/interpolation: ', () => { 'output-parameter': 'interpolation-result', }; const config = Object.assign({}, globalConfig, {method: Method.SPLINE}); - const plugin = Interpolation(config); + const plugin = Interpolation(config, parametersMetadata); const inputs = [ { timestamp: '2023-07-06T00:00', diff --git a/src/__tests__/unit/builtins/mock-observations.test.ts b/src/__tests__/if-run/builtins/mock-observations.test.ts similarity index 76% rename from src/__tests__/unit/builtins/mock-observations.test.ts rename to src/__tests__/if-run/builtins/mock-observations.test.ts index fd28d5595..0569b4c93 100644 --- a/src/__tests__/unit/builtins/mock-observations.test.ts +++ b/src/__tests__/if-run/builtins/mock-observations.test.ts @@ -1,31 +1,39 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {MockObservations} from '../../../builtins/mock-observations'; +import {MockObservations} from '../../../if-run/builtins/mock-observations'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {InputValidationError, GlobalConfigError} = ERRORS; const {INVALID_MIN_MAX} = STRINGS; describe('builtins/mock-observations: ', () => { + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + describe('init: ', () => { it('successfully initalized.', () => { - const mockObservations = MockObservations({ - 'timestamp-from': '2023-07-06T00:00', - 'timestamp-to': '2023-07-06T00:01', - duration: 5, - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, + const mockObservations = MockObservations( + { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + duration: 5, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, + }, }, }, - }); + parametersMetadata + ); expect(mockObservations).toHaveProperty('metadata'); expect(mockObservations).toHaveProperty('execute'); @@ -49,7 +57,7 @@ describe('builtins/mock-observations: ', () => { }, }, }; - const mockObservations = MockObservations(config); + const mockObservations = MockObservations(config, parametersMetadata); const result = await mockObservations.execute([]); expect.assertions(1); @@ -109,7 +117,7 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); - const mockObservations = MockObservations(config); + const mockObservations = MockObservations(config, parametersMetadata); try { await mockObservations.execute([]); } catch (error) { @@ -131,7 +139,7 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations(config); + const mockObservations = MockObservations(config, parametersMetadata); await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -165,7 +173,7 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations(config); + const mockObservations = MockObservations(config, parametersMetadata); await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -177,21 +185,24 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations({ - 'timestamp-from': '2023-07-06T00:00', - 'timestamp-to': '2023-07-06T00:01', - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, + const mockObservations = MockObservations( + { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, + }, }, }, - }); + parametersMetadata + ); await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -207,21 +218,25 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations({ - 'timestamp-from': '2023-07-06T00:00', - duration: 5, - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, + const mockObservations = MockObservations( + { + 'timestamp-from': '2023-07-06T00:00', + duration: 5, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, + }, }, }, - }); + parametersMetadata + ); + await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -237,21 +252,25 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations({ - 'timestamp-to': '2023-07-06T00:01', - duration: 5, - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, + const mockObservations = MockObservations( + { + 'timestamp-to': '2023-07-06T00:01', + duration: 5, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, + }, }, }, - }); + parametersMetadata + ); + await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -277,7 +296,7 @@ describe('builtins/mock-observations: ', () => { randint: null, }, }; - const mockObservations = MockObservations(config); + const mockObservations = MockObservations(config, parametersMetadata); expect.assertions(2); @@ -307,7 +326,7 @@ describe('builtins/mock-observations: ', () => { }, }, }; - const mockObservations = MockObservations(config); + const mockObservations = MockObservations(config, parametersMetadata); expect.assertions(2); diff --git a/src/__tests__/unit/builtins/multiply.test.ts b/src/__tests__/if-run/builtins/multiply.test.ts similarity index 90% rename from src/__tests__/unit/builtins/multiply.test.ts rename to src/__tests__/if-run/builtins/multiply.test.ts index d15a7e8f9..b3856dfd4 100644 --- a/src/__tests__/unit/builtins/multiply.test.ts +++ b/src/__tests__/if-run/builtins/multiply.test.ts @@ -1,6 +1,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Multiply} from '../../../builtins/multiply'; +import {Multiply} from '../../../if-run/builtins/multiply'; const {InputValidationError} = ERRORS; @@ -10,7 +10,11 @@ describe('builtins/multiply: ', () => { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], 'output-parameter': 'energy', }; - const multiply = Multiply(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const multiply = Multiply(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -72,7 +76,7 @@ describe('builtins/multiply: ', () => { 'input-parameters': ['carbon', 'other-carbon'], 'output-parameter': 'carbon-product', }; - const multiply = Multiply(newConfig); + const multiply = Multiply(newConfig, parametersMetadata); const data = [ { diff --git a/src/__tests__/unit/builtins/regex.test.ts b/src/__tests__/if-run/builtins/regex.test.ts similarity index 77% rename from src/__tests__/unit/builtins/regex.test.ts rename to src/__tests__/if-run/builtins/regex.test.ts index 54a70ed9e..ea45c49d1 100644 --- a/src/__tests__/unit/builtins/regex.test.ts +++ b/src/__tests__/if-run/builtins/regex.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Regex} from '../../../builtins/regex'; +import {Regex} from '../../../if-run/builtins/regex'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {GlobalConfigError, MissingInputDataError, RegexMismatchError} = ERRORS; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; @@ -14,7 +14,11 @@ describe('builtins/regex: ', () => { match: '^[^,]+', output: 'cpu/name', }; - const regex = Regex(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const regex = Regex(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -49,6 +53,33 @@ describe('builtins/regex: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully applies regex strategy with multiple matches', async () => { + const globalConfig = { + parameter: 'cloud/instance-type', + match: '/(?<=_)[^_]+?(?=_|$)/g', + output: 'cloud/instance-type', + }; + const regex = Regex(globalConfig, parametersMetadata); + + const expectedResult = [ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'cloud/instance-type': 'DS1 v2', + }, + ]; + + const result = await regex.execute([ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'cloud/instance-type': 'Standard_DS1_v2', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('returns a result when regex is not started and ended with ``.', async () => { const physicalProcessor = 'IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz,IntelĀ® XeonĀ® E5-2673 v3 2.4 GHz'; @@ -59,7 +90,7 @@ describe('builtins/regex: ', () => { match: '[^,]+/', output: 'cpu/name', }; - const regex = Regex(globalConfig); + const regex = Regex(globalConfig, parametersMetadata); const expectedResult = [ { @@ -90,7 +121,7 @@ describe('builtins/regex: ', () => { match: '^(^:)+', output: 'cpu/name', }; - const regex = Regex(globalConfig); + const regex = Regex(globalConfig, parametersMetadata); expect.assertions(1); @@ -113,7 +144,7 @@ describe('builtins/regex: ', () => { it('throws an error on missing global config.', async () => { const config = undefined; - const regex = Regex(config!); + const regex = Regex(config!, parametersMetadata); expect.assertions(1); diff --git a/src/__tests__/unit/builtins/sci-embodied.test.ts b/src/__tests__/if-run/builtins/sci-embodied.test.ts similarity index 97% rename from src/__tests__/unit/builtins/sci-embodied.test.ts rename to src/__tests__/if-run/builtins/sci-embodied.test.ts index a3f5bcbff..0e2c234f7 100644 --- a/src/__tests__/unit/builtins/sci-embodied.test.ts +++ b/src/__tests__/if-run/builtins/sci-embodied.test.ts @@ -1,15 +1,19 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {SciEmbodied} from '../../../builtins/sci-embodied'; +import {SciEmbodied} from '../../../if-run/builtins/sci-embodied'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {InputValidationError} = ERRORS; const {SCI_EMBODIED_ERROR} = STRINGS; describe('builtins/sci-embodied:', () => { describe('SciEmbodied: ', () => { - const sciEmbodied = SciEmbodied(); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const sciEmbodied = SciEmbodied(parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { diff --git a/src/__tests__/unit/builtins/sci.test.ts b/src/__tests__/if-run/builtins/sci.test.ts similarity index 82% rename from src/__tests__/unit/builtins/sci.test.ts rename to src/__tests__/if-run/builtins/sci.test.ts index 6a8ddc8db..0360149a9 100644 --- a/src/__tests__/unit/builtins/sci.test.ts +++ b/src/__tests__/if-run/builtins/sci.test.ts @@ -1,12 +1,16 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Sci} from '../../../builtins/sci'; +import {Sci} from '../../../if-run/builtins/sci'; const {MissingInputDataError} = ERRORS; describe('builtins/sci:', () => { describe('Sci: ', () => { - const sci = Sci({'functional-unit': 'users'}); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const sci = Sci({'functional-unit': 'users'}, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -17,9 +21,12 @@ describe('builtins/sci:', () => { describe('execute():', () => { it('returns a result with valid inputs.', async () => { - const sci = Sci({ - 'functional-unit': 'users', - }); + const sci = Sci( + { + 'functional-unit': 'users', + }, + parametersMetadata + ); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -48,9 +55,12 @@ describe('builtins/sci:', () => { }); it('returns the same result regardless of input duration.', async () => { - const sci = Sci({ - 'functional-unit': 'requests', - }); + const sci = Sci( + { + 'functional-unit': 'requests', + }, + parametersMetadata + ); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -96,9 +106,12 @@ describe('builtins/sci:', () => { }); it('throws exception on invalid functional unit data.', async () => { - const sci = Sci({ - 'functional-unit': 'requests', - }); + const sci = Sci( + { + 'functional-unit': 'requests', + }, + parametersMetadata + ); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -118,9 +131,12 @@ describe('builtins/sci:', () => { }); it('throws exception if functional unit value is not positive integer.', async () => { - const sci = Sci({ - 'functional-unit': 'requests', - }); + const sci = Sci( + { + 'functional-unit': 'requests', + }, + parametersMetadata + ); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -142,9 +158,12 @@ describe('builtins/sci:', () => { }); it('fallbacks to carbon value, if functional unit is 0.', async () => { - const sci = Sci({ - 'functional-unit': 'requests', - }); + const sci = Sci( + { + 'functional-unit': 'requests', + }, + parametersMetadata + ); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', diff --git a/src/__tests__/unit/builtins/shell.test.ts b/src/__tests__/if-run/builtins/shell.test.ts similarity index 86% rename from src/__tests__/unit/builtins/shell.test.ts rename to src/__tests__/if-run/builtins/shell.test.ts index 96de944f1..20b6a7e1c 100644 --- a/src/__tests__/unit/builtins/shell.test.ts +++ b/src/__tests__/if-run/builtins/shell.test.ts @@ -2,7 +2,7 @@ import {spawnSync} from 'child_process'; import {loadAll} from 'js-yaml'; import {ERRORS} from '@grnsft/if-core/utils'; -import {Shell} from '../../../builtins/shell'; +import {Shell} from '../../../if-run/builtins/shell'; const {InputValidationError, ProcessExecutionError} = ERRORS; @@ -10,8 +10,12 @@ jest.mock('child_process'); jest.mock('js-yaml'); describe('builtins/shell', () => { + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; describe('Shell', () => { - const shell = Shell({}); + const shell = Shell({}, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -22,7 +26,10 @@ describe('builtins/shell', () => { describe('execute(): ', () => { it('execute with valid inputs and command', async () => { - const shell = Shell({command: 'python3 /path/to/script.py'}); + const shell = Shell( + {command: 'python3 /path/to/script.py'}, + parametersMetadata + ); const mockSpawnSync = spawnSync as jest.MockedFunction< typeof spawnSync >; @@ -70,7 +77,10 @@ describe('builtins/shell', () => { }); it('throw an error when shell could not run command.', async () => { - const shell = Shell({command: 'python3 /path/to/script.py'}); + const shell = Shell( + {command: 'python3 /path/to/script.py'}, + parametersMetadata + ); (spawnSync as jest.Mock).mockImplementation(() => { throw new InputValidationError('Could not run the command'); }); diff --git a/src/__tests__/unit/builtins/subtract.test.ts b/src/__tests__/if-run/builtins/subtract.test.ts similarity index 90% rename from src/__tests__/unit/builtins/subtract.test.ts rename to src/__tests__/if-run/builtins/subtract.test.ts index fd4cf7c96..134cebfa7 100644 --- a/src/__tests__/unit/builtins/subtract.test.ts +++ b/src/__tests__/if-run/builtins/subtract.test.ts @@ -1,6 +1,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Subtract} from '../../../builtins/subtract'; +import {Subtract} from '../../../if-run/builtins/subtract'; const {InputValidationError} = ERRORS; @@ -10,7 +10,11 @@ describe('builtins/subtract: ', () => { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], 'output-parameter': 'energy/diff', }; - const subtract = Subtract(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const subtract = Subtract(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -72,7 +76,7 @@ describe('builtins/subtract: ', () => { 'input-parameters': ['carbon', 'other-carbon'], 'output-parameter': 'carbon-diff', }; - const subtract = Subtract(newConfig); + const subtract = Subtract(newConfig, parametersMetadata); const data = [ { diff --git a/src/__tests__/unit/builtins/sum.test.ts b/src/__tests__/if-run/builtins/sum.test.ts similarity index 90% rename from src/__tests__/unit/builtins/sum.test.ts rename to src/__tests__/if-run/builtins/sum.test.ts index 1810fc13a..9ccc64378 100644 --- a/src/__tests__/unit/builtins/sum.test.ts +++ b/src/__tests__/if-run/builtins/sum.test.ts @@ -1,8 +1,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {Sum} from '../../../builtins/sum'; +import {Sum} from '../../../if-run/builtins/sum'; -import {STRINGS} from '../../../config'; +import {STRINGS} from '../../../if-run/config'; const {GlobalConfigError, InputValidationError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; @@ -13,7 +13,11 @@ describe('builtins/sum: ', () => { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], 'output-parameter': 'energy', }; - const sum = Sum(globalConfig); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const sum = Sum(globalConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -52,7 +56,7 @@ describe('builtins/sum: ', () => { it('throws an error when global config is not provided.', () => { const config = undefined; - const sum = Sum(config!); + const sum = Sum(config!, parametersMetadata); expect.assertions(1); @@ -98,7 +102,7 @@ describe('builtins/sum: ', () => { 'input-parameters': ['carbon', 'other-carbon'], 'output-parameter': 'carbon-sum', }; - const sum = Sum(newConfig); + const sum = Sum(newConfig, parametersMetadata); const data = [ { diff --git a/src/__tests__/unit/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts similarity index 86% rename from src/__tests__/unit/builtins/time-sync.test.ts rename to src/__tests__/if-run/builtins/time-sync.test.ts index 6dd984659..a9fa13cd1 100644 --- a/src/__tests__/unit/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -1,9 +1,13 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Settings, DateTime} from 'luxon'; -import {TimeSync} from '../../../builtins/time-sync'; +import {AggregationParams} from '../../../common/types/manifest'; -import {STRINGS} from '../../../config'; +import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; +import {TimeSync} from '../../../if-run/builtins/time-sync'; + +import {STRINGS} from '../../../if-run/config'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; Settings.defaultZone = 'utc'; const { @@ -51,7 +55,27 @@ jest.mock('luxon', () => { }); describe('builtins/time-sync:', () => { + beforeAll(() => { + const metricStorage: AggregationParams = { + metrics: [ + 'carbon', + 'cpu/utilization', + 'time-reserved', + 'resources-total', + ], + type: 'horizontal', + }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); + }); + describe('time-sync: ', () => { + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; const basicConfig = { 'start-time': '2023-12-12T00:01:00.000Z', 'end-time': '2023-12-12T00:01:00.000Z', @@ -59,7 +83,7 @@ describe('builtins/time-sync:', () => { 'allow-padding': true, }; - const timeSync = TimeSync(basicConfig); + const timeSync = TimeSync(basicConfig, parametersMetadata); describe('init: ', () => { it('successfully initalized.', () => { @@ -71,6 +95,11 @@ describe('builtins/time-sync:', () => { }); describe('execute(): ', () => { + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + it('throws error if `start-time` is missing.', async () => { const invalidStartTimeConfig = { 'start-time': '', @@ -79,7 +108,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(invalidStartTimeConfig); + const timeModel = TimeSync(invalidStartTimeConfig, parametersMetadata); expect.assertions(1); @@ -114,7 +143,7 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; - const timeModel = TimeSync(invalidEndTimeConfig); + const timeModel = TimeSync(invalidEndTimeConfig, parametersMetadata); expect.assertions(1); @@ -143,7 +172,7 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; - const timeModel = TimeSync(invalidStartTimeConfig); + const timeModel = TimeSync(invalidStartTimeConfig, parametersMetadata); expect.assertions(1); try { await timeModel.execute([ @@ -169,7 +198,7 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; - const timeModel = TimeSync(invalidEndTimeConfig); + const timeModel = TimeSync(invalidEndTimeConfig, parametersMetadata); expect.assertions(1); try { @@ -191,7 +220,7 @@ describe('execute(): ', () => { it('throws error on missing global config.', async () => { const config = undefined; - const timeModel = TimeSync(config!); + const timeModel = TimeSync(config!, parametersMetadata); expect.assertions(1); @@ -223,7 +252,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(invalidIntervalConfig); + const timeModel = TimeSync(invalidIntervalConfig, parametersMetadata); expect.assertions(1); @@ -255,7 +284,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -285,7 +314,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -317,7 +346,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -336,7 +365,7 @@ describe('execute(): ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toStrictEqual( new InputValidationError( - '"timestamp" parameter is required in input[0]. Error code: invalid_union.' + '"timestamp" parameter is invalid datetime in input[0]. Error code: invalid_string.' ) ); } @@ -357,7 +386,7 @@ describe('execute(): ', () => { }, ]; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); expect.assertions(2); try { @@ -378,7 +407,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -408,7 +437,7 @@ describe('execute(): ', () => { 'allow-padding': false, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); const result = await timeModel.execute([ { @@ -427,12 +456,10 @@ describe('execute(): ', () => { { timestamp: '2023-12-12T00:00:00.000Z', duration: 1, - 'cpu/utilization': 10, }, { timestamp: '2023-12-12T00:00:01.000Z', duration: 1, - 'cpu/utilization': 10, }, ]; @@ -446,8 +473,9 @@ describe('execute(): ', () => { interval: 1, 'allow-padding': true, }; + storeAggregationMetrics({carbon: 'sum'}); - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); const result = await timeModel.execute([ { @@ -521,7 +549,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); const result = await timeModel.execute([ { @@ -536,17 +564,14 @@ describe('execute(): ', () => { }, ]); - /**In each 5 second interval, 60% of the time cpu/utilization = 10, 40% of the time it is 0, so cpu/utilization in the averaged result be 6 */ const expectedResult = [ { timestamp: '2023-12-12T00:00:00.000Z', duration: 5, - 'resources-total': 10, }, { timestamp: '2023-12-12T00:00:05.000Z', duration: 5, - 'resources-total': 10, }, ]; @@ -560,8 +585,10 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'time-reserved': 'avg'}); + storeAggregationMetrics({'resources-total': 'sum'}); - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); const result = await timeModel.execute([ { @@ -605,7 +632,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -619,7 +646,7 @@ describe('execute(): ', () => { expect(error).toBeInstanceOf(InputValidationError); expect(error).toStrictEqual( new InputValidationError( - '"timestamp" parameter is invalid datetime in input[1]. Error code: invalid_string.' + '"start-time" parameter is invalid datetime. Error code: invalid_string.' ) ); } @@ -633,8 +660,9 @@ describe('execute(): ', () => { interval: 5, 'allow-padding': true, }; + storeAggregationMetrics({'resources-total': 'none'}); - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); const result = await timeModel.execute([ { @@ -673,7 +701,7 @@ describe('execute(): ', () => { 'allow-padding': false, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -703,7 +731,7 @@ describe('execute(): ', () => { 'allow-padding': false, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -733,7 +761,7 @@ describe('execute(): ', () => { 'allow-padding': false, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); try { await timeModel.execute([ @@ -764,7 +792,7 @@ describe('execute(): ', () => { 'allow-padding': true, }; - const timeModel = TimeSync(basicConfig); + const timeModel = TimeSync(basicConfig, parametersMetadata); const result = await timeModel.execute([ { timestamp: '2023-12-12T00:00:00.000Z', diff --git a/src/__tests__/unit/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts similarity index 90% rename from src/__tests__/unit/lib/aggregate.test.ts rename to src/__tests__/if-run/lib/aggregate.test.ts index fb652e219..00d9c6d0e 100644 --- a/src/__tests__/unit/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -1,7 +1,26 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -import {aggregate} from '../../../lib/aggregate'; + +import {AggregationParams} from '../../../common/types/manifest'; + +import { + aggregate, + storeAggregationMetrics, +} from '../../../if-run/lib/aggregate'; +import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; describe('lib/aggregate: ', () => { + beforeAll(() => { + const metricStorage: AggregationParams = { + metrics: ['carbon'], + type: 'horizontal', + }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + + storeAggregationMetrics(...convertedMetrics); + }); + describe('aggregate(): ', () => { it('returns tree if aggregation is missing.', () => { const tree = {}; diff --git a/src/__tests__/unit/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts similarity index 66% rename from src/__tests__/unit/lib/compute.test.ts rename to src/__tests__/if-run/lib/compute.test.ts index 9ff951586..912793381 100644 --- a/src/__tests__/unit/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -import {compute} from '../../../lib/compute'; -import {ComputeParams} from '../../../types/compute'; -import {pluginStorage} from '../../../util/plugin-storage'; +import {compute} from '../../../if-run/lib/compute'; +import {ComputeParams} from '../../../if-run/types/compute'; +import {pluginStorage} from '../../../if-run/util/plugin-storage'; describe('lib/compute: ', () => { /** @@ -19,10 +19,57 @@ describe('lib/compute: ', () => { kind: 'execute', }, }); - const mockGroupByPlugin = () => ({ - execute: (inputs: any) => ({children: inputs}), + const mockObservePlugin = () => ({ + execute: () => [ + {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, + {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, + ], metadata: { - kind: 'groupby', + kind: 'execute', + }, + }); + const mockObservePluginTimeSync = () => ({ + execute: () => [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 60, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:01:00.000Z', + duration: 60, + 'cpu/utilization': 40, + }, + ], + metadata: { + kind: 'execute', + }, + }); + const mockTimeSync = () => ({ + execute: () => [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 30, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:00:30.000Z', + duration: 30, + 'cpu/utilization': 30, + }, + { + timestamp: '2023-12-12T00:01:00.000Z', + duration: 30, + 'cpu/utilization': 40, + }, + { + timestamp: '2023-12-12T00:01:30.000Z', + duration: 30, + 'cpu/utilization': 40, + }, + ], + metadata: { + kind: 'execute', }, }); /** @@ -41,22 +88,11 @@ describe('lib/compute: ', () => { }, }, }, - pluginStorage: pluginStorage().set('mock', mockExecutePlugin()), - }; - const params: ComputeParams = { - // @ts-ignore - context: { - name: 'mock-name', - initialize: { - plugins: { - mock: { - path: 'mockavizta', - method: 'Mockavizta', - }, - }, - }, - }, - pluginStorage: pluginStorage().set('mock', mockGroupByPlugin()), + pluginStorage: pluginStorage() + .set('mock', mockExecutePlugin()) + .set('mock-observe', mockObservePlugin()) + .set('mock-observe-time-sync', mockObservePluginTimeSync()) + .set('time-sync', mockTimeSync()), }; describe('compute(): ', () => { @@ -64,7 +100,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, inputs: [ {timestamp: 'mock-timestamp-1', duration: 10}, {timestamp: 'mock-timestamp-2', duration: 10}, @@ -85,7 +121,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {regroup: ['duration']}, inputs: [ {timestamp: 'mock-timestamp-1', duration: 10}, {timestamp: 'mock-timestamp-2', duration: 10}, @@ -93,19 +129,24 @@ describe('lib/compute: ', () => { }, }, }; - const response = await compute(tree, params); - const expectedResult = mockGroupByPlugin().execute( - tree.children.mockChild.inputs - ); + const response = await compute(tree, paramsExecute); + const expectedResponse = { + '10': { + inputs: [ + {duration: 10, timestamp: 'mock-timestamp-1'}, + {duration: 10, timestamp: 'mock-timestamp-2'}, + ], + }, + }; - expect(response.children.mockChild.children).toEqual(expectedResult); + expect(response.children.mockChild.children).toEqual(expectedResponse); }); it('computes simple tree with defaults and execute plugin.', async () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -132,7 +173,7 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild1: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -144,7 +185,7 @@ describe('lib/compute: ', () => { mockChild2: { children: { mockChild21: { - pipeline: ['mock'], + pipeline: {compute: ['mock']}, defaults: { 'cpu/name': 'Intel CPU', }, @@ -188,16 +229,17 @@ describe('lib/compute: ', () => { }, }; const response = await compute(tree, paramsExecute); - const expectedResult: any[] = []; - expect(response.children.mockChild.outputs).toEqual(expectedResult); + expect(response.children.mockChild.outputs).toBeUndefined(); }); it('computes simple tree with defaults and no inputs with execue plugin.', async () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: { + compute: ['mock'], + }, defaults: { carbon: 10, }, @@ -215,7 +257,9 @@ describe('lib/compute: ', () => { const tree = { children: { mockChild: { - pipeline: ['mock'], + pipeline: { + compute: ['mock'], + }, config: { 'cpu/name': 'Intel CPU', }, @@ -234,4 +278,22 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.outputs).toEqual(expectedResult); }); }); + + it('computes simple tree with observe plugin.', async () => { + const tree = { + children: { + mockChild: { + pipeline: {observe: ['mock-observe']}, + }, + }, + }; + + const response = await compute(tree, paramsExecute); + const expectedResult = [ + {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, + {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, + ]; + + expect(response.children.mockChild.inputs).toEqual(expectedResult); + }); }); diff --git a/src/__tests__/unit/lib/environment.test.ts b/src/__tests__/if-run/lib/environment.test.ts similarity index 93% rename from src/__tests__/unit/lib/environment.test.ts rename to src/__tests__/if-run/lib/environment.test.ts index c528eaee1..324bd1da6 100644 --- a/src/__tests__/unit/lib/environment.test.ts +++ b/src/__tests__/if-run/lib/environment.test.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ -import {injectEnvironment} from '../../../lib/environment'; + +import {injectEnvironment} from '../../../if-run/lib/environment'; describe('lib/envirnoment: ', () => { describe('injectEnvironment(): ', () => { @@ -9,7 +10,7 @@ describe('lib/envirnoment: ', () => { // @ts-ignore const response = await injectEnvironment(context); expect(response).toHaveProperty('execution'); - }); + }, 6000); it('checks `execution` to have `command` and `environment` props.', async () => { // @ts-ignore diff --git a/src/__tests__/if-run/lib/exhaust.test.ts b/src/__tests__/if-run/lib/exhaust.test.ts new file mode 100644 index 000000000..71d2367e6 --- /dev/null +++ b/src/__tests__/if-run/lib/exhaust.test.ts @@ -0,0 +1,59 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +jest.mock('fs', () => require('../../../__mocks__/fs')); + +import {exhaust} from '../../../if-run/lib/exhaust'; + +jest.mock('../../../if-run/builtins/export-yaml', () => ({ + ExportYaml: jest.fn().mockImplementation(() => ({ + // @ts-ignore + execute: (tree, context, outputOptions) => { + expect(outputOptions).toBe('mock-path'); + }, + })), +})); + +describe('lib/exhaust: ', () => { + describe('exhaust(): ', () => { + const spy = jest.spyOn(global.console, 'log'); + + beforeEach(() => { + spy.mockReset(); + }); + + it('successfully executes provided yaml file.', async () => { + const tree = {}; + const context = { + initialize: {}, + }; + const outputOptions = {outputPath: 'mock-path'}; + + // @ts-ignore + await exhaust(tree, context, outputOptions); + + expect.assertions(1); + }); + + it('returns void if no exhaust plugin selected.', async () => { + const tree = {}; + const context = { + initialize: {}, + }; + + // @ts-ignore + const result = await exhaust(tree, context, {}); + + expect(result).toBeUndefined(); + }); + + it('uses log exhaust plugin as export.', async () => { + const tree = {}; + const context = { + initialize: {}, + }; + + // @ts-ignore + await exhaust(tree, context, {'no-outout': false}); + expect(spy).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/src/__tests__/if-run/lib/explain.test.ts b/src/__tests__/if-run/lib/explain.test.ts new file mode 100644 index 000000000..506c62669 --- /dev/null +++ b/src/__tests__/if-run/lib/explain.test.ts @@ -0,0 +1,99 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +import {explain, addExplainData} from '../../../if-run/lib/explain'; + +describe('lib/explain: ', () => { + it('successfully adds explain data if `inputs` and `outputs` of `metadata` are `undefined`.', () => { + const mockData = { + pluginName: 'divide', + metadata: {kind: 'execute', inputs: undefined, outputs: undefined}, + pluginData: { + path: 'builtin', + method: 'Divide', + }, + }; + const expectedResult = { + divide: { + method: 'Divide', + path: 'builtin', + inputs: 'undefined', + outputs: 'undefined', + }, + }; + + addExplainData(mockData); + const result = explain(); + expect.assertions(1); + expect(result).toEqual(expectedResult); + }); + + it('successfully adds explain data if `inputs` and `outputs` of `metadata` are valid data.', () => { + const mockData = { + pluginName: 'sum', + metadata: { + kind: 'execute', + inputs: { + 'cpu/energy': { + unit: 'kWh', + description: 'energy consumed by the cpu', + 'aggregation-method': 'sum', + }, + 'network/energy': { + unit: 'kWh', + description: 'energy consumed by data ingress and egress', + 'aggregation-method': 'sum', + }, + }, + outputs: { + 'energy-sum': { + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': 'sum', + }, + }, + }, + pluginData: { + path: 'builtin', + method: 'Sum', + }, + }; + const expectedResult = { + divide: { + method: 'Divide', + path: 'builtin', + inputs: 'undefined', + outputs: 'undefined', + }, + sum: { + method: 'Sum', + path: 'builtin', + inputs: { + 'cpu/energy': { + unit: 'kWh', + description: 'energy consumed by the cpu', + 'aggregation-method': 'sum', + }, + 'network/energy': { + unit: 'kWh', + description: 'energy consumed by data ingress and egress', + 'aggregation-method': 'sum', + }, + }, + outputs: { + 'energy-sum': { + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': 'sum', + }, + }, + }, + }; + + // @ts-ignore + addExplainData(mockData); + + const result = explain(); + + expect.assertions(1); + expect(result).toEqual(expectedResult); + }); +}); diff --git a/src/__tests__/unit/lib/initialize.test.ts b/src/__tests__/if-run/lib/initialize.test.ts similarity index 51% rename from src/__tests__/unit/lib/initialize.test.ts rename to src/__tests__/if-run/lib/initialize.test.ts index ff3c10c45..a1eda8924 100644 --- a/src/__tests__/unit/lib/initialize.test.ts +++ b/src/__tests__/if-run/lib/initialize.test.ts @@ -2,21 +2,22 @@ jest.mock('mockavizta', () => require('../../../__mocks__/plugin'), { virtual: true, }); -jest.mock('../../../builtins', () => require('../../../__mocks__/plugin'), { - virtual: true, -}); +jest.mock( + '../../../if-run/builtins', + () => require('../../../__mocks__/plugin'), + { + virtual: true, + } +); const mockLog = jest.fn(); -jest.mock('../../../util/log-memoize', () => ({ +jest.mock('../../../if-run/util/log-memoize', () => ({ memoizedLog: mockLog, })); import {ERRORS} from '@grnsft/if-core/utils'; -import {initialize} from '../../../lib/initialize'; - -import {STRINGS} from '../../../config'; - -import {GlobalPlugins} from '../../../types/manifest'; +import {initialize} from '../../../if-run/lib/initialize'; +import {STRINGS} from '../../../if-run/config'; const { MissingPluginPathError, @@ -28,8 +29,9 @@ const {MISSING_METHOD, MISSING_PATH, INVALID_MODULE_PATH} = STRINGS; describe('lib/initalize: ', () => { describe('initalize(): ', () => { it('creates instance with get and set methods.', async () => { - const plugins = {}; - const response = await initialize(plugins); + const context = {initialize: {plugins: {}}}; + // @ts-ignore + const response = await initialize(context); expect(response).toHaveProperty('get'); expect(response).toHaveProperty('set'); @@ -38,15 +40,20 @@ describe('lib/initalize: ', () => { }); it('checks if plugin is initalized, warning is logged and plugin has execute and metadata props.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'mockavizta', - method: 'Mockavizta', + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + method: 'Mockavizta', + }, + }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); @@ -54,36 +61,45 @@ describe('lib/initalize: ', () => { }); it('checks if plugin is initalized with global config and has execute and metadata.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'mockavizta', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('throws error if plugin does not have path property.', async () => { - const plugins: GlobalPlugins = { - // @ts-ignore - mockavizta: { - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error) { expect(error).toBeInstanceOf(MissingPluginPathError); @@ -94,18 +110,22 @@ describe('lib/initalize: ', () => { }); it('throws error if plugin does not have path property.', async () => { - const plugins: GlobalPlugins = { - // @ts-ignore - mockavizta: { - path: 'mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error) { expect(error).toBeInstanceOf(MissingPluginMethodError); @@ -116,61 +136,76 @@ describe('lib/initalize: ', () => { }); it('checks if builtin plugin is initalized.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'builtin', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'builtin', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('checks if github plugin is initalized.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'https://github.com/mockavizta', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'https://github.com/mockavizta', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; - const storage = await initialize(plugins); + // @ts-ignore + const storage = await initialize(context); - const pluginName = Object.keys(plugins)[0]; + const pluginName = Object.keys(context.initialize.plugins)[0]; const module = storage.get(pluginName); expect(module).toHaveProperty('execute'); expect(module).toHaveProperty('metadata'); }); it('throws error if plugin path is invalid.', async () => { - const plugins: GlobalPlugins = { - mockavizta: { - path: 'failing-mock', - method: 'Mockavizta', - 'global-config': { - verbose: true, + const context = { + initialize: { + plugins: { + mockavizta: { + path: 'failing-mock', + method: 'Mockavizta', + 'global-config': { + verbose: true, + }, + }, }, }, }; try { - await initialize(plugins); + // @ts-ignore + await initialize(context); } catch (error: any) { expect(error).toBeInstanceOf(PluginInitializationError); expect(error.message).toEqual( INVALID_MODULE_PATH( - plugins.mockavizta.path, + context.initialize.plugins.mockavizta.path, new Error( - "Cannot find module 'failing-mock' from 'src/lib/initialize.ts'" + "Cannot find module 'failing-mock' from 'src/if-run/lib/initialize.ts'" ) ) ); diff --git a/src/__tests__/if-run/lib/regroup.test.ts b/src/__tests__/if-run/lib/regroup.test.ts new file mode 100644 index 000000000..67ff72e53 --- /dev/null +++ b/src/__tests__/if-run/lib/regroup.test.ts @@ -0,0 +1,161 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {STRINGS} from '../../../if-run/config'; +import {Regroup} from '../../../if-run/lib/regroup'; + +const {InvalidGroupingError, InputValidationError} = ERRORS; +const {INVALID_GROUP_KEY} = STRINGS; + +describe('lib/regroup: ', () => { + describe('Regroup: ', () => { + it('groups inputs correctly.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + const groups = ['region', 'cloud/instance-type']; + + const expectedOutput = { + 'uk-west': { + children: { + A1: { + inputs: [ + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T00:00', + }, + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T05:00', + }, + { + 'cloud/instance-type': 'A1', + region: 'uk-west', + timestamp: '2023-07-06T10:00', + }, + ], + }, + }, + }, + }; + + const result = Regroup(inputs, groups); + expect(result).toEqual(expectedOutput); + }); + + it('throws an error when groups is not provided.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + + const groups = undefined; + + expect.assertions(2); + try { + Regroup(inputs, groups!); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"regroup" parameter is not an array or should contain at least one key. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error if `group` is an empty array.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-west', + 'cloud/instance-type': 'A1', + }, + ]; + const groups = ['region', 'cloud/instance-type']; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InvalidGroupingError); + expect(error).toEqual( + new InvalidGroupingError('Invalid group region.') + ); + } + }); + + it('throws an error if group type is missing from the input.', () => { + const inputs = [ + {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, + ]; + const groups: any[] = []; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toEqual( + new InputValidationError( + '"regroup" parameter is array must contain at least 1 element(s). Error code: too_small.' + ) + ); + } + }); + + it('throws an error if input does not have required group type.', () => { + const inputs = [ + {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, + ]; + const groups = ['region', 'cloud/instance-type', 'unknown']; + + expect.assertions(2); + try { + Regroup(inputs, groups); + } catch (error) { + expect(error).toBeInstanceOf(InvalidGroupingError); + expect(error).toEqual( + new InvalidGroupingError(INVALID_GROUP_KEY(groups[2])) + ); + } + }); + }); +}); diff --git a/src/__tests__/unit/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts similarity index 61% rename from src/__tests__/unit/util/aggregation-helper.test.ts rename to src/__tests__/if-run/util/aggregation-helper.test.ts index 1a23bc3c1..f83536f39 100644 --- a/src/__tests__/unit/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -1,36 +1,36 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {PluginParams} from '@grnsft/if-core/types'; -import {aggregateInputsIntoOne} from '../../../util/aggregation-helper'; +import {AggregationParams} from '../../../common/types/manifest'; -import {STRINGS} from '../../../config'; +import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; +import { + AGGREGATION_METHODS, + AggregationMetric, +} from '../../../if-run/types/aggregation'; +import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; +import {STRINGS} from '../../../if-run/config'; -describe('util/aggregation-helper: ', () => { - describe('aggregateInputsIntoOne(): ', () => { - it('throws error if aggregation method is none.', () => { - const inputs: PluginParams[] = []; - const metrics: string[] = ['cpu/number-cores']; - const isTemporal = false; - - expect.assertions(2); - - try { - aggregateInputsIntoOne(inputs, metrics, isTemporal); - } catch (error) { - expect(error).toBeInstanceOf(InvalidAggregationMethodError); +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; - if (error instanceof InvalidAggregationMethodError) { - expect(error.message).toEqual(INVALID_AGGREGATION_METHOD(metrics[0])); - } - } - }); +describe('util/aggregation-helper: ', () => { + beforeAll(() => { + const metricStorage: AggregationParams = { + metrics: ['carbon', 'cpu/number-cores', 'cpu/utilization'], + type: 'horizontal', + }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); + }); + describe('aggregateInputsIntoOne(): ', () => { it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: string[] = ['cpu/utilization']; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'sum'}]; const isTemporal = false; expect.assertions(2); @@ -41,7 +41,7 @@ describe('util/aggregation-helper: ', () => { expect(error).toBeInstanceOf(MissingAggregationParamError); if (error instanceof MissingAggregationParamError) { - expect(error.message).toEqual(METRIC_MISSING(metrics[0], 0)); + expect(error.message).toEqual(METRIC_MISSING('cpu/utilization', 0)); } } }); @@ -51,7 +51,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: string[] = ['carbon']; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = true; const expectedValue = { @@ -68,7 +68,7 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: string[] = ['carbon']; + const metrics: AggregationMetric[] = [{carbon: 'sum'}]; const isTemporal = false; const expectedValue = { @@ -79,11 +79,21 @@ describe('util/aggregation-helper: ', () => { }); it('calculates average of metrics.', () => { + const metricStorage: AggregationParams = { + metrics: ['cpu/utilization'], + type: 'horizontal', + }; + const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + })); + storeAggregationMetrics(...convertedMetrics); + storeAggregationMetrics({'cpu/utilization': 'avg'}); + const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: string[] = ['cpu/utilization']; + const metrics: AggregationMetric[] = [{'cpu/utilization': 'avg'}]; const isTemporal = false; const expectedValue = { diff --git a/src/__tests__/if-run/util/args.test.ts b/src/__tests__/if-run/util/args.test.ts new file mode 100644 index 000000000..c43d49eb9 --- /dev/null +++ b/src/__tests__/if-run/util/args.test.ts @@ -0,0 +1,164 @@ +jest.mock('ts-command-line-args', () => ({ + __esModule: true, + parse: () => { + switch (process.env.result) { + case 'throw-error-object': + throw new Error(MANIFEST_IS_MISSING); + case 'error': + throw MANIFEST_IS_MISSING; + case 'manifest-is-missing': + return {}; + case 'manifest': + return { + manifest: 'manifest-mock.yml', + }; + case 'absolute-path': + return { + manifest: path.normalize(`${processRunningPath}/manifest-mock.yml`), + }; + case 'manifest-output': + return { + manifest: 'manifest-mock.yml', + output: 'output-mock.yml', + }; + case 'not-yaml': + return { + manifest: 'mock.notyaml', + }; + case 'no-output': + return { + manifest: 'manifest-mock.yaml', + 'no-output': false, + }; + default: + return { + manifest: 'mock-manifest.yaml', + output: 'mock-output', + }; + } + }, +})); + +const processRunningPath = process.cwd(); +import * as path from 'node:path'; + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {parseIfRunProcessArgs} from '../../../if-run/util/args'; + +import {STRINGS as COMMON_STRINGS} from '../../../common/config'; + +const {SOURCE_IS_NOT_YAML, MANIFEST_IS_MISSING} = COMMON_STRINGS; +const {CliSourceFileError, ParseCliParamsError} = ERRORS; + +describe('if-run/util/args: ', () => { + describe('parseIfRunProcessArgs(): ', () => { + it('throws error if there is no argument passed.', () => { + expect.assertions(4); + + process.env.result = 'error'; // used for mocking + + try { + parseIfRunProcessArgs(); + } catch (error) { + expect(error).toEqual(MANIFEST_IS_MISSING); + } + + process.env.result = 'throw-error-object'; + + try { + parseIfRunProcessArgs(); + } catch (error) { + expect(error).toEqual(new ParseCliParamsError(MANIFEST_IS_MISSING)); + } + + process.env.result = 'manifest-is-missing'; + + try { + parseIfRunProcessArgs(); + } catch (error) { + expect(error).toBeInstanceOf(CliSourceFileError); + expect(error).toEqual(new CliSourceFileError(MANIFEST_IS_MISSING)); + } + }); + + it('returns manifest path.', () => { + expect.assertions(1); + + process.env.result = 'manifest'; + + const result = parseIfRunProcessArgs(); + const manifestPath = 'manifest-mock.yml'; + const expectedResult = { + inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), + outputOptions: { + 'no-output': undefined, + }, + }; + + expect(result).toEqual(expectedResult); + }); + + it('returns manifest with absolute path.', () => { + expect.assertions(1); + + process.env.result = 'absolute-path'; + + const result = parseIfRunProcessArgs(); + const manifestPath = 'manifest-mock.yml'; + const expectedResult = { + inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), + outputOptions: {}, + }; + + expect(result).toEqual(expectedResult); + }); + + it('returns manifest and output path.', () => { + expect.assertions(1); + + process.env.result = 'manifest-output'; + + const result = parseIfRunProcessArgs(); + const manifestPath = 'manifest-mock.yml'; + const outputPath = 'output-mock.yml'; + const expectedResult = { + inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), + outputOptions: { + outputPath: path.normalize(`${processRunningPath}/${outputPath}`), + 'no-output': undefined, + }, + }; + + expect(result).toEqual(expectedResult); + }); + + it('throws error if file is not yaml.', () => { + expect.assertions(2); + + process.env.result = 'not-yaml'; + + try { + parseIfRunProcessArgs(); + } catch (error) { + expect(error).toBeInstanceOf(CliSourceFileError); + expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); + } + }); + + it('returns `no-output` and manifest.', () => { + expect.assertions(1); + + process.env.result = 'no-output'; + const manifestPath = 'manifest-mock.yaml'; + + const response = parseIfRunProcessArgs(); + const expectedResult = { + inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), + outputOptions: {}, + }; + + expect(response).toEqual(expectedResult); + }); + }); +}); diff --git a/src/__tests__/if-run/util/helpers.test.ts b/src/__tests__/if-run/util/helpers.test.ts new file mode 100644 index 000000000..796f6b201 --- /dev/null +++ b/src/__tests__/if-run/util/helpers.test.ts @@ -0,0 +1,274 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +const mockWarn = jest.fn(); +const mockError = jest.fn(); + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {GlobalPlugins} from '../../../common/types/manifest'; + +import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; + +import { + andHandle, + mergeObjects, + storeAggregationMethods, +} from '../../../if-run/util/helpers'; + +const {WriteFileError} = ERRORS; + +jest.mock('../../../common/util/logger', () => ({ + logger: { + warn: mockWarn, + error: mockError, + }, +})); + +jest.mock('../../../if-run/lib/aggregate', () => ({ + storeAggregationMetrics: jest.fn(), +})); + +describe('if-run/util/helpers: ', () => { + describe('andHandle(): ', () => { + afterEach(() => { + mockWarn.mockReset(); + mockError.mockReset(); + }); + + it('logs error in case of error is unknown.', () => { + const message = 'mock-message'; + + andHandle(new WriteFileError(message)); + expect(mockWarn).toHaveBeenCalledTimes(0); + expect(mockError).toHaveBeenCalledTimes(1); + }); + }); + + describe('mergeObjects(): ', () => { + it('does not override input.', () => { + expect.assertions(1); + + const input = { + a: 1, + b: false, + c: 'testInput', + }; + + const defaults = { + c: 'testDefault', + }; + const result = mergeObjects(defaults, input); + + expect(result).toEqual(input); + }); + + it('overrides null/undefined inputs.', () => { + expect.assertions(1); + + const input = { + a: 1, + b: false, + c: 'testInput', + d: null, + e: undefined, + }; + + const defaults = { + c: 'testDefault', + d: 'testDefault', + e: 'testDefault', + }; + const result = mergeObjects(defaults, input); + const expectedResult = { + a: 1, + b: false, + c: 'testInput', + d: 'testDefault', + e: 'testDefault', + }; + + expect(result).toEqual(expectedResult); + }); + + it('adds only properties missing in input.', () => { + expect.assertions(1); + + const input = { + a: 1, + b: false, + c: 'testInput', + }; + + const defaults = { + b: true, + c: 'testDefault', + d: 25, + }; + + const result = mergeObjects(defaults, input); + const expectedResult = { + a: 1, + b: false, + c: 'testInput', + d: 25, + }; + + expect(result).toEqual(expectedResult); + }); + + it('keeps values from input in case of nested objects.', () => { + expect.assertions(1); + + const input = { + a: 1, + b: false, + c: 'testInput', + d: { + e: 1, + }, + }; + + const defaults = { + b: true, + c: 'testDefault1', + d: { + e: 25, + f: 'testDefault2', + }, + }; + + const result = mergeObjects(defaults, input); + const expectedResult = { + a: 1, + b: false, + c: 'testInput', + d: { + e: 1, + }, + }; + + expect(result).toEqual(expectedResult); + }); + + it('keeps value from input in case of arrays.', () => { + expect.assertions(1); + + const input = { + a: 1, + b: false, + c: 'testInput1', + d: [1, 2, 3, 4], + e: 'testInput2', + }; + + const defaults = { + b: true, + c: 'testDefault1', + d: [5, 6, 7, 8, 9], + e: [1, 2, 3], + }; + + const result = mergeObjects(defaults, input); + const expectedResult = { + a: 1, + b: false, + c: 'testInput1', + d: [1, 2, 3, 4], + e: 'testInput2', + }; + + expect(result).toEqual(expectedResult); + }); + }); + + describe('storeAggregationMethods(): ', () => { + // @typescript-eslint/no-unused-vars + const mockPluginStorage = { + get: jest.fn(), + set: jest.fn(() => {}), + }; + + const mockPlugins: GlobalPlugins = { + multiply: { + path: 'builtin', + method: 'Multiply', + }, + sci: { + path: 'builtin', + method: 'Sci', + }, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('succefully executes with correct metrics.', () => { + const mockPlugin1 = { + execute: () => [{}], + metadata: { + kind: 'execute', + inputs: { + carbon: { + description: 'mock description', + unit: 'none', + 'aggregation-method': 'sum', + }, + }, + outputs: { + cpu: { + description: 'mock description', + unit: 'none', + 'aggregation-method': 'avg', + }, + }, + }, + }; + + const mockPlugin2 = { + metadata: { + inputs: {}, + outputs: { + carbon: {'aggregation-method': 'none'}, + }, + }, + }; + + mockPluginStorage.get + .mockReturnValueOnce(mockPlugin1) + .mockReturnValueOnce(mockPlugin2); + + // @ts-ignore + storeAggregationMethods(mockPlugins, mockPluginStorage); + + expect(storeAggregationMetrics).toHaveBeenCalledTimes(3); + expect(storeAggregationMetrics).toHaveBeenNthCalledWith(1, { + carbon: 'sum', + }); + expect(storeAggregationMetrics).toHaveBeenNthCalledWith(2, { + cpu: 'avg', + }); + expect(storeAggregationMetrics).toHaveBeenNthCalledWith(3, { + carbon: 'none', + }); + }); + + it('does not execute if there are no inputs or outputs.', () => { + mockPluginStorage.get.mockReturnValueOnce({ + execute: () => [{}], + metadata: {}, + }); + + const mockPlugin = { + execute: () => [{}], + metadata: { + kind: 'execute', + }, + }; + + mockPluginStorage.get.mockReturnValueOnce(mockPlugin); + // @ts-ignore + storeAggregationMethods(mockPlugins, mockPluginStorage); + expect(storeAggregationMetrics).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/__tests__/unit/util/log-memoize.test.ts b/src/__tests__/if-run/util/log-memoize.test.ts similarity index 92% rename from src/__tests__/unit/util/log-memoize.test.ts rename to src/__tests__/if-run/util/log-memoize.test.ts index 275ad9c22..a5d73b15b 100644 --- a/src/__tests__/unit/util/log-memoize.test.ts +++ b/src/__tests__/if-run/util/log-memoize.test.ts @@ -1,5 +1,6 @@ import {LeveledLogMethod} from 'winston'; -import {memoizedLog} from '../../../util/log-memoize'; + +import {memoizedLog} from '../../../if-run/util/log-memoize'; describe('util/log-memoize: ', () => { describe('momoizedLog(): ', () => { diff --git a/src/__tests__/unit/util/os-checker.test.ts b/src/__tests__/if-run/util/os-checker.test.ts similarity index 97% rename from src/__tests__/unit/util/os-checker.test.ts rename to src/__tests__/if-run/util/os-checker.test.ts index d01a7a8da..20d5ffea0 100644 --- a/src/__tests__/unit/util/os-checker.test.ts +++ b/src/__tests__/if-run/util/os-checker.test.ts @@ -9,7 +9,7 @@ jest.mock('os', () => ({ }, release: () => 'm.m.m', })); -jest.mock('../../../util/helpers', () => ({ +jest.mock('../../../common/util/helpers', () => ({ execPromise: async () => { if (process.env.KIND === 'darwin' && process.env.REJECT === 'true') return { @@ -58,7 +58,7 @@ OS Version: 10.0.22631 N/A Build 22631 }, })); -import {osInfo} from '../../../util/os-checker'; +import {osInfo} from '../../../if-run/util/os-checker'; describe('util/os-checker: ', () => { describe('osInfo(): ', () => { diff --git a/src/__tests__/unit/util/plugin-storage.test.ts b/src/__tests__/if-run/util/plugin-storage.test.ts similarity index 96% rename from src/__tests__/unit/util/plugin-storage.test.ts rename to src/__tests__/if-run/util/plugin-storage.test.ts index 8b76f36cd..885ccb107 100644 --- a/src/__tests__/unit/util/plugin-storage.test.ts +++ b/src/__tests__/if-run/util/plugin-storage.test.ts @@ -1,6 +1,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {pluginStorage} from '../../../util/plugin-storage'; +import {pluginStorage} from '../../../if-run/util/plugin-storage'; const {PluginInitializationError} = ERRORS; diff --git a/src/__tests__/integration/helpers/common.ts b/src/__tests__/integration/helpers/common.ts deleted file mode 100644 index 47b013e88..000000000 --- a/src/__tests__/integration/helpers/common.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {exec} from 'node:child_process'; -import {promisify} from 'node:util'; - -export const execPromise = promisify(exec); - -const stripAnsi = (text: string) => { - const pattern = [ - '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', - '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-nq-uy=><~]))', - ].join('|'); - - const ansiRegex = new RegExp(pattern, 'g'); - - return text.replaceAll(ansiRegex, ''); -}; - -export const getJSONFromText = (text: string) => { - const textWithoutAnsi = stripAnsi(text); - const jsonRegex = /\{[^]*\}/; - const jsonMatch = textWithoutAnsi.match(jsonRegex); - - if (jsonMatch) { - const jsonString = jsonMatch[0]; - - return JSON.parse(jsonString); - } else { - throw new Error('No JSON found in the log.'); - } -}; diff --git a/src/__tests__/integration/helpers/module-installer.ts b/src/__tests__/integration/helpers/module-installer.ts deleted file mode 100644 index a70ede21a..000000000 --- a/src/__tests__/integration/helpers/module-installer.ts +++ /dev/null @@ -1,11 +0,0 @@ -import {execPromise} from './common'; - -const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; - -export const npmInstallPackage = (module: string) => { - return execPromise(`${npmCommand} install ${module} --s`); -}; - -export const npmUninstallPackage = (module: string) => { - return execPromise(`${npmCommand} uninstall ${module}`); -}; diff --git a/src/__tests__/integration/scenarios/sci-e.ts b/src/__tests__/integration/scenarios/sci-e.ts deleted file mode 100644 index 0342240c2..000000000 --- a/src/__tests__/integration/scenarios/sci-e.ts +++ /dev/null @@ -1,75 +0,0 @@ -import * as YAML from 'js-yaml'; - -import {openYamlFileAsObject, saveYamlFileAs} from '../../../util/yaml'; -import {Manifest} from '../../../types/manifest'; -import { - npmInstallPackage, - npmUninstallPackage, -} from '../helpers/module-installer'; -import {execPromise} from '../helpers/common'; -import {sciEInputData} from '../test-data/sci-e'; - -describe('integration/sci-e', () => { - const modelName = 'sci-e'; - const absoluteManifestPath = `${__dirname}/../manifest/sci-e.yaml`; - const relativeManifestPath = 'src/__tests__/integration/manifest/sci-e.yaml'; - const implTemplatePath = `${__dirname}/../templates/integration.yaml`; - - beforeAll(() => { - return npmInstallPackage('@grnsft/if-plugins'); - }, 15000); - - it('output creation without ompl path.', async () => { - const file = await openYamlFileAsObject(implTemplatePath); - - file.initialize.plugins[modelName] = { - method: 'SciE', - path: '@grnsft/if-plugins', - }; - file.initialize.outputs = []; - - file.tree.children.child.pipeline = [modelName]; - file.tree.children.child.config = {}; - file.tree.children.child.config[modelName] = {}; - file.tree.children.child.inputs = sciEInputData['success-3-params']; - - await saveYamlFileAs(file, absoluteManifestPath); // save yaml uses absolute path - const response = ( - await execPromise( - `npm run if-run -- --manifest ${relativeManifestPath} --stdout` - ) - ).stdout; // exec promise uses relative path - - const yamlPart = response.substring( - response.indexOf('name:'), - response.length - ); - const finalOutputParsed: any = YAML.load(yamlPart); - - // assertions - const path = finalOutputParsed.tree.children.child.outputs[0]; - const manifestPath = file.tree.children.child.inputs[0]; - - // assert timestamp - expect(finalOutputParsed.tree.children.child.inputs[0].timestamp).toEqual( - file.tree.children.child.inputs[0].timestamp - ); - - // assert duration - expect(finalOutputParsed.tree.children.child.inputs[0].duration).toEqual( - file.tree.children.child.inputs[0].duration - ); - - // assert total energy - const sum = - manifestPath['cpu/energy'] + - manifestPath['memory/energy'] + - manifestPath['network/energy']; - - expect(path.energy).toEqual(sum); - }, 15000); - - afterAll(() => { - return npmUninstallPackage('@grnsft/if-plugins'); - }, 15000); -}); diff --git a/src/__tests__/integration/templates/integration.yaml b/src/__tests__/integration/templates/integration.yaml deleted file mode 100644 index 2d1523e91..000000000 --- a/src/__tests__/integration/templates/integration.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: integration-template -description: -tags: -initialize: - plugins: - 'sci-e': - model: ${model} - path: ${path} -tree: - children: - child: - pipeline: - - ${name} - config: - inputs: - - ${input} diff --git a/src/__tests__/integration/test-data/sci-e.ts b/src/__tests__/integration/test-data/sci-e.ts deleted file mode 100644 index e6a5acde7..000000000 --- a/src/__tests__/integration/test-data/sci-e.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const sciEInputData = { - 'success-3-params': [ - { - timestamp: '2023-08-06T00:00', - duration: 3600, - 'cpu/energy': 10, - 'memory/energy': 100, - 'network/energy': 100, - }, - ], -}; diff --git a/src/__tests__/unit/builtins/export-csv-raw.test.ts b/src/__tests__/unit/builtins/export-csv-raw.test.ts deleted file mode 100644 index e97242a4d..000000000 --- a/src/__tests__/unit/builtins/export-csv-raw.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import * as fs from 'fs/promises'; - -import {jest} from '@jest/globals'; -import {ERRORS} from '@grnsft/if-core/utils'; - -import {ExportCSVRaw} from '../../../builtins/export-csv-raw'; - -import {STRINGS} from '../../../config'; - -import {tree, context, outputs} from '../../../__mocks__/builtins/export-csv'; - -const {ExhaustOutputArgError} = ERRORS; -const {WRITE_CSV_ERROR, OUTPUT_REQUIRED} = STRINGS; - -jest.mock('fs/promises', () => ({ - __esModule: true, - writeFile: jest.fn<() => Promise>().mockResolvedValue(), -})); - -describe('builtins/export-csv-raw: ', () => { - describe('ExportCSVRaw: ', () => { - const exportCSVRaw = ExportCSVRaw(); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - describe('init GroupBy: ', () => { - it('initalizes object with properties.', async () => { - expect(exportCSVRaw).toHaveProperty('execute'); - }); - }); - - describe('execute(): ', () => { - it('generates CSV file with correct data.', async () => { - const outputPath = 'output#carbon'; - const content = - "id,timestamp,cloud/instance-type,region,duration,cpu/utilization,network/energy,energy,cpu/thermal-design-power,grid/carbon-intensity,device/emissions-embodied,time-reserved,device/expected-lifespan,resources-reserved,resources-total,cpu/energy,carbon-plus-energy',carbon-embodied,carbon-operational,carbon,sci\nchildren.child-1.outputs.0,2023-12-12T00:00:00.000Z,A1,uk-west,1,10,10,5,100,800,1533.12,3600,94608000,1,8,0.000008888888888888888,10.000008888888889,0.0000020256215119228817,4000,4000.0000020256216,240000.0001215373\nchildren.child-2.outputs.0,2023-12-12T00:00:00.000Z,A1,uk-west,1,30,10,5,100,800,1533.12,3600,94608000,1,8,0.00001650338753387534,10.000016503387533,0.0000020256215119228817,4000,4000.0000020256216,240000.0001215373\noutputs.0,2023-12-12T00:00:00.000Z,,,1,,,,,,,,,,,,,,,8000.000004051243,"; - - await exportCSVRaw.execute(tree, context, outputPath); - - expect(fs.writeFile).toHaveBeenCalledWith(`${outputPath}.csv`, content); - }); - - it('throws an error when the CSV file could not be created.', async () => { - const outputPath = 'output#carbon'; - const expectedMessage = 'Could not write CSV file.'; - - expect.assertions(1); - - jest.spyOn(fs, 'writeFile').mockRejectedValue(expectedMessage); - - await expect( - exportCSVRaw.execute(tree, context, outputPath) - ).rejects.toThrow( - new ExhaustOutputArgError( - WRITE_CSV_ERROR(outputPath, expectedMessage) - ) - ); - }); - - it('throws an error when output path is empty.', async () => { - const outputPath = ''; - - context.initialize = Object.assign({}, context.initialize, outputs); - - expect.assertions(2); - try { - await exportCSVRaw.execute(tree, context, outputPath); - } catch (error) { - expect(error).toBeInstanceOf(ExhaustOutputArgError); - expect(error).toEqual(new ExhaustOutputArgError(OUTPUT_REQUIRED)); - } - }); - }); - }); -}); diff --git a/src/__tests__/unit/builtins/export-csv.test.ts b/src/__tests__/unit/builtins/export-csv.test.ts deleted file mode 100644 index 78feaf534..000000000 --- a/src/__tests__/unit/builtins/export-csv.test.ts +++ /dev/null @@ -1,236 +0,0 @@ -import * as fs from 'fs/promises'; - -import {stringify} from 'csv-stringify/sync'; -import {jest} from '@jest/globals'; -import {ERRORS} from '@grnsft/if-core/utils'; - -import {ExportCSV} from '../../../builtins/export-csv'; - -import {STRINGS} from '../../../config'; - -import { - tree, - context, - outputs, - aggregated, - aggregation, -} from '../../../__mocks__/builtins/export-csv'; - -const {ExhaustOutputArgError} = ERRORS; -const {OUTPUT_REQUIRED, CSV_EXPORT} = STRINGS; - -jest.mock('fs/promises', () => ({ - writeFile: jest.fn<() => Promise>().mockResolvedValue(), -})); - -describe('builtins/export-csv: ', () => { - describe('ExportCSV: ', () => { - const exportCSV = ExportCSV(); - - describe('init GroupBy: ', () => { - it('initalizes object with properties.', async () => { - expect(exportCSV).toMatchObject({metadata: {kind: 'exhaust'}}); - expect(exportCSV).toHaveProperty('execute'); - }); - }); - - describe('execute(): ', () => { - it('generates CSV file with correct data.', async () => { - const outputPath = 'output#carbon'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - [ - 'tree.children.child-1.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - [ - 'tree.children.child-2.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - ]; - const reformedContext = Object.assign({}, context, {outputs}); - const reformedTree = Object.assign({}, tree, { - children: { - ...tree.children, - 'child-1': { - ...tree.children['child-1'], - aggregated, - }, - 'child-2': { - ...tree.children['child-2'], - aggregated, - }, - }, - }); - - await exportCSV.execute(reformedTree, reformedContext, outputPath); - - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); - - it('generates CSV file when the `outputs` type is missing.', async () => { - const outputPath = 'output#carbon'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - [ - 'tree.children.child-1.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - [ - 'tree.children.child-2.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - ]; - - const reformedTree = Object.assign({}, tree, { - children: { - ...tree.children, - 'child-1': { - ...tree.children['child-1'], - aggregated, - }, - 'child-2': { - ...tree.children['child-2'], - aggregated, - }, - }, - }); - - await exportCSV.execute(reformedTree, context, outputPath); - - expect.assertions(1); - - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); - - it('generates CSV file when `aggregation` persists.', async () => { - const outputPath = 'output#carbon'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - [ - 'tree.children.child-1.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - [ - 'tree.children.child-2.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - ]; - - const reformedContext = Object.assign( - {}, - context, - {outputs}, - {aggregation} - ); - const reformedTree = Object.assign({}, tree, { - children: { - ...tree.children, - 'child-1': { - ...tree.children['child-1'], - aggregated, - }, - 'child-2': { - ...tree.children['child-2'], - aggregated, - }, - }, - }); - - await exportCSV.execute(reformedTree, reformedContext, outputPath); - - expect.assertions(1); - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); - - it('generates CSV file when `aggregation` is missing.', async () => { - const outputPath = 'output#carbon'; - const columns = ['Path', 'Aggregated', '2023-12-12T00:00:00.000Z']; - const matrix = [ - columns, - ['tree.carbon', 8000.000004051243, 8000.000004051243], - [ - 'tree.children.child-1.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - [ - 'tree.children.child-2.carbon', - 4000.0000020256216, - 4000.0000020256216, - ], - ]; - - const reformedContext = Object.assign({}, context, {outputs}); - - await exportCSV.execute(tree, reformedContext, outputPath); - - expect.assertions(1); - expect(fs.writeFile).toHaveBeenCalledWith( - 'output.csv', - stringify(matrix, {columns}) - ); - }); - - it('throws an error when output path is empty.', async () => { - const outputPath = ''; - - context.initialize = Object.assign({}, context.initialize, outputs); - - try { - await exportCSV.execute(tree, context, outputPath); - } catch (error) { - expect(error).toBeInstanceOf(ExhaustOutputArgError); - expect(error).toEqual(new ExhaustOutputArgError(OUTPUT_REQUIRED)); - } - }); - - it('throws an error when output path does not contains `#`.', async () => { - const outputPath = 'output.csv'; - - context.initialize = Object.assign({}, context.initialize, outputs); - - try { - await exportCSV.execute(tree, context, outputPath); - } catch (error) { - expect(error).toBeInstanceOf(ExhaustOutputArgError); - expect(error).toEqual(new ExhaustOutputArgError(CSV_EXPORT)); - } - }); - - it('throws an error when output path does not contains a criteria.', async () => { - const outputPath = 'output.csv#'; - - context.initialize = Object.assign({}, context.initialize, outputs); - - try { - await exportCSV.execute(tree, context, outputPath); - } catch (error) { - expect(error).toBeInstanceOf(ExhaustOutputArgError); - expect(error).toEqual(new ExhaustOutputArgError(CSV_EXPORT)); - } - }); - }); - }); -}); diff --git a/src/__tests__/unit/builtins/group-by.test.ts b/src/__tests__/unit/builtins/group-by.test.ts deleted file mode 100644 index 3d676148b..000000000 --- a/src/__tests__/unit/builtins/group-by.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import {ERRORS} from '@grnsft/if-core/utils'; - -import {GroupBy} from '../../../builtins/group-by'; - -import {STRINGS} from '../../../config'; - -const {InvalidGroupingError, InputValidationError, GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, INVALID_GROUP_BY} = STRINGS; - -describe('builtins/group-by: ', () => { - describe('GroupBy: ', () => { - const plugin = GroupBy(); - - describe('init GroupBy: ', () => { - it('initalizes object with properties.', async () => { - expect(plugin).toHaveProperty('metadata'); - expect(plugin).toHaveProperty('execute'); - }); - }); - - describe('execute(): ', () => { - it('groups inputs correctly.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - const config = { - group: ['region', 'cloud/instance-type'], - }; - - const expectedOutput = { - 'uk-west': { - children: { - A1: { - inputs: [ - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T00:00', - }, - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T05:00', - }, - { - 'cloud/instance-type': 'A1', - region: 'uk-west', - timestamp: '2023-07-06T10:00', - }, - ], - }, - }, - }, - }; - - const result = plugin.execute(inputs, config); - expect(result).toEqual(expectedOutput); - }); - - it('throws an error when config is not provided.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - - const config = undefined; - - expect.assertions(2); - try { - plugin.execute(inputs, config!); - } catch (error) { - expect(error).toBeInstanceOf(GlobalConfigError); - expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); - } - }); - - it('throws an error if `group` is an empty array.', () => { - const inputs = [ - { - timestamp: '2023-07-06T00:00', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T05:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - { - timestamp: '2023-07-06T10:00', - region: 'uk-west', - 'cloud/instance-type': 'A1', - }, - ]; - const config = { - group: ['region', 'cloud/instance-type'], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InvalidGroupingError); - expect(error).toEqual( - new InvalidGroupingError('Invalid group region.') - ); - } - }); - - it('throws an error if group type is missing from the input.', () => { - const inputs = [ - {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, - ]; - const config = { - group: [], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toEqual( - new InputValidationError( - '"group" parameter is array must contain at least 1 element(s). Error code: too_small.' - ) - ); - } - }); - - it('throws an error if input does not have required group type.', () => { - const inputs = [ - {timestamp: 1, region: 'uk-west', 'cloud/instance-type': 'A1'}, - ]; - const config = { - group: ['region', 'cloud/instance-type', 'unknown'], - }; - - expect.assertions(2); - try { - plugin.execute(inputs, config); - } catch (error) { - expect(error).toBeInstanceOf(InvalidGroupingError); - expect(error).toEqual( - new InvalidGroupingError(INVALID_GROUP_BY(config.group[2])) - ); - } - }); - }); - }); -}); diff --git a/src/__tests__/unit/lib/exhaust.test.ts b/src/__tests__/unit/lib/exhaust.test.ts deleted file mode 100644 index 49db14370..000000000 --- a/src/__tests__/unit/lib/exhaust.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment */ -jest.mock('fs', () => require('../../../__mocks__/fs')); - -import {ERRORS} from '@grnsft/if-core/utils'; - -import {exhaust} from '../../../lib/exhaust'; - -import {STRINGS} from '../../../config'; - -const {ExhaustOutputArgError, InvalidExhaustPluginError} = ERRORS; -const {INVALID_EXHAUST_PLUGIN, OUTPUT_REQUIRED} = STRINGS; - -describe('lib/exhaust: ', () => { - describe('exhaust(): ', () => { - const spy = jest.spyOn(global.console, 'log'); - - beforeEach(() => { - spy.mockReset(); - }); - - it('returns void if no exhaust plugin selected.', async () => { - const tree = {}; - const context = { - initialize: { - outputs: null, - }, - }; - - // @ts-ignore - const result = await exhaust(tree, context, {}); - - expect(result).toBeUndefined(); - }); - - it('uses log exhaust plugin as export.', async () => { - const tree = {}; - const context = { - initialize: {}, - }; - - // @ts-ignore - await exhaust(tree, context, {stdout: true}); - expect(spy).toHaveBeenCalledTimes(1); - }); - - it('rejects with cli input error if output path is not provided with yaml.', async () => { - const tree = {}; - const context = { - initialize: { - outputs: ['yaml'], - }, - }; - - expect.assertions(2); - - try { - // @ts-ignore - await exhaust(tree, context, {}); - } catch (error) { - expect(error).toBeInstanceOf(ExhaustOutputArgError); - - if (error instanceof ExhaustOutputArgError) { - expect(error.message).toEqual(OUTPUT_REQUIRED); - } - } - }); - - it('rejects with module init error if output module is not supported.', async () => { - const tree = {}; - const context = { - initialize: { - outputs: ['mock'], - }, - }; - - expect.assertions(2); - - try { - // @ts-ignore - await exhaust(tree, context, {}); - } catch (error) { - expect(error).toBeInstanceOf(InvalidExhaustPluginError); - - if (error instanceof InvalidExhaustPluginError) { - expect(error.message).toEqual( - INVALID_EXHAUST_PLUGIN(context.initialize.outputs[0]) - ); - } - } - }); - }); -}); diff --git a/src/__tests__/unit/lib/parameterize.test.ts b/src/__tests__/unit/lib/parameterize.test.ts deleted file mode 100644 index 430d8e043..000000000 --- a/src/__tests__/unit/lib/parameterize.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import {LeveledLogMethod} from 'winston'; - -const mockLog = jest.fn((message: string) => message); - -jest.mock('../../../util/log-memoize', () => ({ - memoizedLog: mockLog, -})); -jest.mock('../../../util/logger', () => ({ - logger: { - warn: mockLog, - debug: mockLog, - }, -})); - -import {PARAMETERS} from '../../../config'; -import {parameterize} from '../../../lib/parameterize'; - -import {STRINGS} from '../../../config'; - -import {ManifestParameter} from '../../../types/manifest'; - -const {REJECTING_OVERRIDE, CHECKING_AGGREGATION_METHOD} = STRINGS; - -describe('lib/parameterize: ', () => { - afterEach(() => { - (mockLog as jest.Mock).mockReset(); - }); - - describe('getAggregationMethod(): ', () => { - it('returns method for average aggregation method metric.', () => { - const metric = 'cpu/utilization'; - const method = parameterize.getAggregationMethod(metric); - - const expectedMethod = 'avg'; - - expect(method).toEqual(expectedMethod); - }); - - it('returns method for unknown aggregation method metric.', () => { - const metric = 'mock/metric'; - const method = parameterize.getAggregationMethod(metric); - - const expectedMethod = 'sum'; - - expect(method).toEqual(expectedMethod); - expect(mockLog as unknown as LeveledLogMethod).toHaveBeenCalledTimes(2); - }); - - it('prints debug log for first input.', () => { - const unitName = 'timestamp'; - - parameterize.getAggregationMethod(unitName); - - expect(mockLog as typeof console.debug).toHaveBeenCalledWith( - console.debug, - CHECKING_AGGREGATION_METHOD(unitName) - ); - }); - }); - - describe('combine(): ', () => { - it('checks if return type is undefined.', () => { - const params = {}; - const response = parameterize.combine(null, params); - - expect(response).toBeUndefined(); - }); - - it('checks if uninitialized custom param is requested, then returns fallback `sum` method.', () => { - const name = 'mock-name'; - const method = parameterize.getAggregationMethod(name); - - const expectedMethodName = 'sum'; - expect(method).toEqual(expectedMethodName); - }); - - it('checks if custom params are inserted successfully.', () => { - const params = [ - { - name: 'mock-name', - description: 'mock-description', - unit: 'mock/sq', - aggregation: 'none', - }, - ] as ManifestParameter[]; - const object = {}; - - parameterize.combine(params, object); - const method = parameterize.getAggregationMethod(params[0].name); - - expect(method).toEqual(params[0].aggregation); - }); - - it('rejects on default param override.', () => { - const params = [ - { - name: 'carbon', - description: 'mock-description', - unit: 'mock/co', - aggregation: 'none', - }, - ] as ManifestParameter[]; - - parameterize.combine(params, PARAMETERS); - const method = parameterize.getAggregationMethod(params[0].name); - - const expectedMethodName = 'sum'; - const expectedMessage = REJECTING_OVERRIDE(params[0]); - - expect(method).toEqual(expectedMethodName); - expect(mockLog).toHaveBeenCalledWith(expectedMessage); - }); - }); -}); diff --git a/src/__tests__/unit/util/args.test.ts b/src/__tests__/unit/util/args.test.ts deleted file mode 100644 index f4238cbce..000000000 --- a/src/__tests__/unit/util/args.test.ts +++ /dev/null @@ -1,522 +0,0 @@ -const processRunningPath = process.cwd(); - -jest.mock('../../../util/fs', () => ({ - isFileExists: () => { - if (process.env.fileExists === 'true') { - return true; - } - return false; - }, - isDirectoryExists: () => { - if (process.env.directoryExists === 'true') { - return true; - } - return false; - }, -})); - -jest.mock('ts-command-line-args', () => ({ - __esModule: true, - parse: () => { - switch (process.env.result) { - case 'throw-error-object': - throw new Error(MANIFEST_IS_MISSING); - case 'error': - throw MANIFEST_IS_MISSING; - case 'manifest-is-missing': - return {}; - case 'manifest': - return { - manifest: 'manifest-mock.yml', - }; - case 'absolute-path': - return { - manifest: path.normalize(`${processRunningPath}/manifest-mock.yml`), - }; - case 'manifest-output': - return { - manifest: 'manifest-mock.yml', - output: 'output-mock.yml', - }; - case 'override-params': - return { - manifest: 'manifest-mock.yml', - 'override-params': 'override-params-mock.yml', - }; - case 'not-yaml': - return { - manifest: 'mock.notyaml', - }; - case 'stdout': - return { - manifest: 'manifest-mock.yaml', - stdout: true, - }; - /** If-diff mocks */ - case 'only-target': - return { - target: 'target-mock.yml', - }; - case 'target-is-not-yaml': - return { - target: 'target-mock', - }; - case 'source-is-not-yaml': - return { - target: 'target-mock.yml', - source: 'source-mock', - }; - case 'target-source': - return { - target: 'target-mock.yml', - source: 'source-mock.yml', - }; - case 'diff-throw-error': - throw new Error('mock-error'); - case 'diff-throw': - throw 'mock-error'; - /** If-env mocks */ - case 'manifest-install-provided': - return { - install: true, - manifest: 'mock-manifest.yaml', - }; - case 'manifest-is-not-yaml': - return {manifest: 'manifest'}; - case 'manifest-path-invalid': - throw new Error(MANIFEST_NOT_FOUND); - case 'env-throw-error': - throw new Error('mock-error'); - case 'env-throw': - throw 'mock-error'; - /** If-check */ - case 'manifest-is-provided': - return {manifest: 'mock-manifest.yaml'}; - case 'directory-is-provided': - return {directory: '/mock-directory'}; - case 'flags-are-not-provided': - return {manifest: undefined, directory: undefined}; - default: - return { - manifest: 'mock-manifest.yaml', - output: 'mock-output', - }; - } - }, -})); - -import * as path from 'node:path'; -import {ERRORS} from '@grnsft/if-core/utils'; - -import { - parseIEProcessArgs, - parseIfCheckArgs, - parseIfDiffArgs, - parseIfEnvArgs, -} from '../../../util/args'; - -import {STRINGS} from '../../../config'; - -const { - CliSourceFileError, - ParseCliParamsError, - InvalidDirectoryError, - MissingCliFlagsError, -} = ERRORS; - -const { - MANIFEST_IS_MISSING, - TARGET_IS_NOT_YAML, - INVALID_TARGET, - SOURCE_IS_NOT_YAML, - MANIFEST_NOT_FOUND, - DIRECTORY_NOT_FOUND, - IF_CHECK_FLAGS_MISSING, -} = STRINGS; - -describe('util/args: ', () => { - const originalEnv = process.env; - - describe('parseIEProcessArgs(): ', () => { - it('throws error if there is no argument passed.', () => { - expect.assertions(5); - - process.env.result = 'error'; // used for mocking - - try { - parseIEProcessArgs(); - } catch (error) { - expect(error).toEqual(MANIFEST_IS_MISSING); - } - - process.env.result = 'throw-error-object'; - - try { - parseIEProcessArgs(); - } catch (error) { - expect(error).toBeInstanceOf(ParseCliParamsError); - expect(error).toEqual(new ParseCliParamsError(MANIFEST_IS_MISSING)); - } - - process.env.result = 'manifest-is-missing'; - - try { - parseIEProcessArgs(); - } catch (error) { - expect(error).toBeInstanceOf(CliSourceFileError); - expect(error).toEqual(new CliSourceFileError(MANIFEST_IS_MISSING)); - } - }); - - it('returns manifest path.', () => { - expect.assertions(1); - - process.env.result = 'manifest'; - - const result = parseIEProcessArgs(); - const manifestPath = 'manifest-mock.yml'; - const expectedResult = { - inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), - outputOptions: { - stdout: undefined, - }, - }; - - expect(result).toEqual(expectedResult); - }); - - it('returns manifest with absolute path.', () => { - expect.assertions(1); - - process.env.result = 'absolute-path'; - - const result = parseIEProcessArgs(); - const manifestPath = 'manifest-mock.yml'; - const expectedResult = { - inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), - outputOptions: {}, - }; - - expect(result).toEqual(expectedResult); - }); - - it('returns manifest with `paramPath`.', () => { - expect.assertions(1); - - process.env.result = 'override-params'; - - const result = parseIEProcessArgs(); - const manifestPath = 'manifest-mock.yml'; - const expectedResult = { - inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), - paramPath: 'override-params-mock.yml', - outputOptions: {}, - }; - - expect(result).toEqual(expectedResult); - }); - - it('returns manifest and output path.', () => { - expect.assertions(1); - - process.env.result = 'manifest-output'; - - const result = parseIEProcessArgs(); - const manifestPath = 'manifest-mock.yml'; - const outputPath = 'output-mock.yml'; - const expectedResult = { - inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), - outputOptions: { - outputPath: path.normalize(`${processRunningPath}/${outputPath}`), - stdout: undefined, - }, - }; - - expect(result).toEqual(expectedResult); - }); - - it('throws error if file is not yaml.', () => { - expect.assertions(2); - - process.env.result = 'not-yaml'; - - try { - parseIEProcessArgs(); - } catch (error) { - expect(error).toBeInstanceOf(CliSourceFileError); - expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); - } - }); - - it('returns stdout and manifest.', () => { - expect.assertions(1); - - process.env.result = 'stdout'; - const manifestPath = 'manifest-mock.yaml'; - - const response = parseIEProcessArgs(); - const expectedResult = { - inputPath: path.normalize(`${processRunningPath}/${manifestPath}`), - outputOptions: { - stdout: true, - }, - }; - - expect(response).toEqual(expectedResult); - }); - }); - - describe('parseIfDiffArgs(): ', () => { - it('throws error if `target` is missing.', () => { - expect.assertions(1); - - try { - parseIfDiffArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError(INVALID_TARGET)); - } - } - }); - - it('throws error if `target` is not a yaml.', () => { - process.env.result = 'target-is-not-yaml'; - expect.assertions(1); - - try { - parseIfDiffArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError(TARGET_IS_NOT_YAML)); - } - } - }); - - it('returns `target`s full path.', () => { - process.env.result = 'only-target'; - expect.assertions(1); - - const response = parseIfDiffArgs(); - expect(response).toHaveProperty('targetPath'); - }); - - it('throws error if source is not a yaml.', () => { - process.env.result = 'source-is-not-yaml'; - expect.assertions(1); - - try { - parseIfDiffArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError(SOURCE_IS_NOT_YAML)); - } - } - }); - - it('returns target and source full paths.', () => { - process.env.result = 'target-source'; - expect.assertions(2); - - const response = parseIfDiffArgs(); - expect(response).toHaveProperty('targetPath'); - expect(response).toHaveProperty('sourcePath'); - }); - - it('throws error if parsing failed.', () => { - process.env.result = 'diff-throw-error'; - expect.assertions(1); - - try { - parseIfDiffArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError('mock-error')); - } - } - }); - - it('throws error if parsing failed (not instance of error).', () => { - process.env.result = 'diff-throw'; - expect.assertions(1); - - try { - parseIfDiffArgs(); - } catch (error) { - expect(error).toEqual('mock-error'); - } - }); - }); - - describe('parseIfEnvArgs(): ', () => { - it('executes if `manifest` is missing.', async () => { - process.env.fileExists = 'true'; - process.env.result = 'manifest-is-missing'; - const response = await parseIfEnvArgs(); - - expect.assertions(1); - - expect(response).toEqual({install: undefined}); - }); - - it('executes if `manifest` and `install` are provided.', async () => { - process.env.fileExists = 'true'; - process.env.result = 'manifest-install-provided'; - - const response = await parseIfEnvArgs(); - - expect.assertions(2); - expect(response).toHaveProperty('install'); - expect(response).toHaveProperty('manifest'); - }); - - it('throws an error if `manifest` is not a yaml.', async () => { - process.env.fileExists = 'true'; - process.env.result = 'manifest-is-not-yaml'; - expect.assertions(1); - - try { - await parseIfEnvArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); - } - } - }); - - it('throws an error if `manifest` path is invalid.', async () => { - process.env.fileExists = 'false'; - expect.assertions(1); - - try { - await parseIfEnvArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError(MANIFEST_NOT_FOUND)); - } - } - }); - - it('throws an error if parsing failed.', async () => { - process.env.result = 'env-throw-error'; - expect.assertions(1); - - try { - await parseIfEnvArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError('mock-error')); - } - } - }); - - it('throws error if parsing failed (not instance of error).', async () => { - process.env.result = 'env-throw'; - expect.assertions(1); - - try { - await parseIfEnvArgs(); - } catch (error) { - expect(error).toEqual('mock-error'); - } - }); - }); - - describe('parseIfCheckArgs(): ', () => { - it('executes when `manifest` is provided.', async () => { - process.env.fileExists = 'true'; - process.env.result = 'manifest-is-provided'; - const response = await parseIfCheckArgs(); - - expect.assertions(1); - - expect(response).toEqual({manifest: 'mock-manifest.yaml'}); - }); - - it('executes when the `directory` is provided.', async () => { - process.env.directoryExists = 'true'; - process.env.result = 'directory-is-provided'; - - const response = await parseIfCheckArgs(); - - expect.assertions(1); - - expect(response).toEqual({directory: '/mock-directory'}); - }); - - it('throws an error when the `directory` does not exist.', async () => { - process.env.directoryExists = 'false'; - process.env.result = 'directory-is-provided'; - expect.assertions(1); - - try { - await parseIfCheckArgs(); - } catch (error) { - expect(error).toEqual(new InvalidDirectoryError(DIRECTORY_NOT_FOUND)); - } - }); - - it('throws an error when both `manifest` and `directory` flags are not provided.', async () => { - process.env.result = 'flags-are-not-provided'; - expect.assertions(1); - - try { - await parseIfCheckArgs(); - } catch (error) { - expect(error).toEqual(new MissingCliFlagsError(IF_CHECK_FLAGS_MISSING)); - } - }); - - it('throws an error if `manifest` is not a yaml.', async () => { - process.env.fileExists = 'true'; - process.env.result = 'manifest-is-not-yaml'; - expect.assertions(1); - - try { - await parseIfCheckArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new CliSourceFileError(SOURCE_IS_NOT_YAML)); - } - } - }); - - it('throws an error if `manifest` path is invalid.', async () => { - process.env.fileExists = 'false'; - expect.assertions(1); - - try { - await parseIfCheckArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError(MANIFEST_NOT_FOUND)); - } - } - }); - - it('throws an error if parsing failed.', async () => { - process.env.result = 'env-throw-error'; - expect.assertions(1); - - try { - await parseIfCheckArgs(); - } catch (error) { - if (error instanceof Error) { - expect(error).toEqual(new ParseCliParamsError('mock-error')); - } - } - }); - - it('throws error if parsing failed (not instance of error).', async () => { - process.env.result = 'env-throw'; - expect.assertions(1); - - try { - await parseIfCheckArgs(); - } catch (error) { - expect(error).toEqual('mock-error'); - } - }); - }); - - process.env = originalEnv; -}); diff --git a/src/__tests__/unit/util/helpers.test.ts b/src/__tests__/unit/util/helpers.test.ts deleted file mode 100644 index ed46dc0d9..000000000 --- a/src/__tests__/unit/util/helpers.test.ts +++ /dev/null @@ -1,687 +0,0 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment */ -const mockWarn = jest.fn(); -const mockError = jest.fn(); - -jest.mock('node:readline/promises', () => - require('../../../__mocks__/readline') -); -jest.mock('../../../util/logger', () => ({ - logger: { - warn: mockWarn, - error: mockError, - }, -})); - -jest.mock('path', () => { - const actualPath = jest.requireActual('path') as Record; - return { - __esModule: true, - ...actualPath, - dirname: jest.fn(() => './mock-path'), - }; -}); - -jest.mock('fs/promises', () => require('../../../__mocks__/fs')); - -jest.mock('../../../lib/load', () => ({ - load: jest.fn(() => { - if (process.env.manifest === 'true') { - return { - rawManifest: { - name: 'divide', - initialize: { - plugins: { - 'cloud-metadata': { - path: '@grnsft/if-plugins', - method: 'CloudMetadata', - }, - divide: { - path: 'builtin', - method: 'Divide', - 'global-config': { - numerator: 'vcpus-allocated', - denominator: 2, - output: 'cpu/number-cores', - }, - }, - }, - }, - execution: { - environment: { - dependencies: [ - '@grnsft/if-core@0.0.7', - '@grnsft/if-plugins@v0.3.2 extraneous -> file:../../../if-models', - '@grnsft/if-unofficial-plugins@v0.3.0 extraneous -> file:../../../if-unofficial-models', - ], - }, - }, - }, - }; - } - return { - rawManifest: { - initialize: { - plugins: {'@grnsft/if-plugins': '1.0.0'}, - }, - execution: { - environment: { - dependencies: [], - }, - }, - }, - }; - }), -})); - -const initPackage = jest.fn(() => Promise.resolve('mock-path')); -const updatePackage = jest.fn(() => Promise.resolve(true)); -const installdeps = jest.fn(); -const updatedeps = jest.fn(); -jest.mock('../../../util/npm', () => { - const actualNPMUtil = jest.requireActual('../../../util/npm'); - - return { - ...actualNPMUtil, - initPackageJsonIfNotExists: (folderPath: string) => { - if (process.env.NPM_MOCK === 'true') { - return initPackage(); - } - - if (process.env.NPM_MOCK === 'error') { - throw new Error('mock-error'); - } - - return actualNPMUtil.initPackageJsonIfNotExists(folderPath); - }, - updatePackageJsonProperties: ( - newPackageJsonPath: string, - appendDependencies: boolean - ) => { - if (process.env.NPM_MOCK === 'true') { - return updatePackage(); - } - - return actualNPMUtil.updatePackageJsonProperties( - newPackageJsonPath, - appendDependencies - ); - }, - installDependencies: ( - folderPath: string, - dependencies: {[path: string]: string} - ) => { - if (process.env.NPM_MOCK === 'true') { - return installdeps(); - } - - return actualNPMUtil.installDependencies(folderPath, dependencies); - }, - updatePackageJsonDependencies: ( - packageJsonPath: string, - dependencies: any, - cwd: boolean - ) => { - if (process.env.NPM_MOCK === 'true') { - return updatedeps(); - } - - return actualNPMUtil.updatePackageJsonDependencies( - packageJsonPath, - dependencies, - cwd - ); - }, - }; -}); - -import {ERRORS} from '@grnsft/if-core/utils'; - -import { - andHandle, - checkIfEqual, - formatNotMatchingLog, - mergeObjects, - oneIsPrimitive, - parseManifestFromStdin, - getOptionsFromArgs, - addTemplateManifest, - initializeAndInstallLibs, - logStdoutFailMessage, -} from '../../../util/helpers'; -import {CONFIG} from '../../../config'; -import {Difference} from '../../../types/lib/compare'; - -const {IF_ENV} = CONFIG; -const {FAILURE_MESSAGE_DEPENDENCIES, FAILURE_MESSAGE} = IF_ENV; - -const {WriteFileError} = ERRORS; - -describe('util/helpers: ', () => { - describe('andHandle(): ', () => { - afterEach(() => { - mockWarn.mockReset(); - mockError.mockReset(); - }); - - it('logs error in case of error is unknown.', () => { - const message = 'mock-message'; - - andHandle(new WriteFileError(message)); - expect(mockWarn).toHaveBeenCalledTimes(0); - expect(mockError).toHaveBeenCalledTimes(1); - }); - }); - - describe('mergeObjects(): ', () => { - it('does not override input.', () => { - expect.assertions(1); - - const input = { - a: 1, - b: false, - c: 'testInput', - }; - - const defaults = { - c: 'testDefault', - }; - const result = mergeObjects(defaults, input); - - expect(result).toEqual(input); - }); - - it('overrides null/undefined inputs.', () => { - expect.assertions(1); - - const input = { - a: 1, - b: false, - c: 'testInput', - d: null, - e: undefined, - }; - - const defaults = { - c: 'testDefault', - d: 'testDefault', - e: 'testDefault', - }; - const result = mergeObjects(defaults, input); - const expectedResult = { - a: 1, - b: false, - c: 'testInput', - d: 'testDefault', - e: 'testDefault', - }; - - expect(result).toEqual(expectedResult); - }); - - it('adds only properties missing in input.', () => { - expect.assertions(1); - - const input = { - a: 1, - b: false, - c: 'testInput', - }; - - const defaults = { - b: true, - c: 'testDefault', - d: 25, - }; - - const result = mergeObjects(defaults, input); - const expectedResult = { - a: 1, - b: false, - c: 'testInput', - d: 25, - }; - - expect(result).toEqual(expectedResult); - }); - - it('keeps values from input in case of nested objects.', () => { - expect.assertions(1); - - const input = { - a: 1, - b: false, - c: 'testInput', - d: { - e: 1, - }, - }; - - const defaults = { - b: true, - c: 'testDefault1', - d: { - e: 25, - f: 'testDefault2', - }, - }; - - const result = mergeObjects(defaults, input); - const expectedResult = { - a: 1, - b: false, - c: 'testInput', - d: { - e: 1, - }, - }; - - expect(result).toEqual(expectedResult); - }); - - it('keeps value from input in case of arrays.', () => { - expect.assertions(1); - - const input = { - a: 1, - b: false, - c: 'testInput1', - d: [1, 2, 3, 4], - e: 'testInput2', - }; - - const defaults = { - b: true, - c: 'testDefault1', - d: [5, 6, 7, 8, 9], - e: [1, 2, 3], - }; - - const result = mergeObjects(defaults, input); - const expectedResult = { - a: 1, - b: false, - c: 'testInput1', - d: [1, 2, 3, 4], - e: 'testInput2', - }; - - expect(result).toEqual(expectedResult); - }); - }); - - describe('formatNotMatchingLog(): ', () => { - const actualLogger = console.log; - const mockLogger = jest.fn(); - console.log = mockLogger; - - beforeEach(() => { - mockLogger.mockReset(); - }); - - it('logs the message.', () => { - const difference: Difference = { - message: 'mock-message', - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(1); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - }); - - it('logs message and path.', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(2); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - }); - - it('logs message, path and formatted source/target (one is missing).', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - source: 'mock-source', - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(4); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); - expect(mockLogger).toHaveBeenCalledWith('target: missing'); - }); - - it('logs message, path and formatted source/target.', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - source: 'mock-source', - target: 'mock-target', - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(4); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); - expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); - }); - - it('logs message, path and formatted source/target (numbers).', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - source: 10, - target: 0, - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(4); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); - expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); - }); - - it('logs message, path and formatted source/target (booleans).', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - source: true, - target: false, - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(4); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); - expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); - }); - - it('logs message, path and formatted source/target (objects).', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - source: {}, - target: false, - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(4); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); - expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); - }); - - it('logs message, path and formatted source/target (empty string).', () => { - const difference: Difference = { - message: 'mock-message', - path: 'mock.path', - source: '', - target: false, - }; - - formatNotMatchingLog(difference); - expect(mockLogger).toHaveBeenCalledTimes(4); - expect(mockLogger).toHaveBeenCalledWith(difference.message); - expect(mockLogger).toHaveBeenCalledWith(difference.path); - expect(mockLogger).toHaveBeenCalledWith(`source: ${difference.source}`); - expect(mockLogger).toHaveBeenCalledWith(`target: ${difference.target}`); - }); - - afterAll(() => { - console.log = actualLogger; - }); - }); - - describe('oneIsPrimitive(): ', () => { - it('returns true if values are nullish.', () => { - const source = null; - const target = undefined; - - const result = oneIsPrimitive(source, target); - expect(result).toBeTruthy(); - }); - - it('returns true if values are string or number.', () => { - const source = 'string'; - const target = 10; - - const result = oneIsPrimitive(source, target); - expect(result).toBeTruthy(); - }); - - it('returns false if one of values is object.', () => { - const source = 'string'; - const target = {}; - - const result = oneIsPrimitive(source, target); - expect(result).toBeFalsy(); - }); - }); - - describe('parseManifestFromStdin(): ', () => { - it('returns empty string if there is no data in stdin.', async () => { - const response = await parseManifestFromStdin(); - const expectedResult = ''; - - expect(response).toEqual(expectedResult); - }); - - it('returns empty string if nothing is piped.', async () => { - const originalIsTTY = process.stdin.isTTY; - process.stdin.isTTY = true; - const response = await parseManifestFromStdin(); - const expectedResult = ''; - - expect(response).toEqual(expectedResult); - process.stdin.isTTY = originalIsTTY; - }); - - it('throws error if there is no manifest in stdin.', async () => { - process.env.readline = 'no_manifest'; - expect.assertions(1); - - const response = await parseManifestFromStdin(); - - expect(response).toEqual(''); - }); - - it('returns empty string if there is no data in stdin.', async () => { - process.env.readline = 'manifest'; - const response = await parseManifestFromStdin(); - const expectedMessage = ` -name: mock-name -description: mock-description -`; - - expect(response).toEqual(expectedMessage); - }); - }); - - describe('checkIfEqual(): ', () => { - it('checks if values are equal.', () => { - const a = 'mock'; - const b = 'mock'; - - const response = checkIfEqual(a, b); - expect(response).toBeTruthy(); - }); - - it('returns true if one of the values is wildcard.', () => { - const a = 'mock'; - const b = '*'; - - const response = checkIfEqual(a, b); - expect(response).toBeTruthy(); - }); - - it('returns false for number and string with the same value.', () => { - const a = 5; - const b = '5'; - - const response = checkIfEqual(a, b); - expect(response).toBeFalsy(); - }); - }); - - describe('getOptionsFromArgs(): ', () => { - it('returns the correct options when dependencies are present.', async () => { - const commandArgs = { - manifest: '/path/to/mock-manifest.json', - install: false, - }; - - process.env.manifest = 'true'; - - const result = await getOptionsFromArgs(commandArgs); - expect.assertions(1); - - expect(result).toEqual({ - folderPath: './mock-path', - dependencies: { - '@grnsft/if-plugins': '^v0.3.2', - }, - install: false, - }); - }); - - it('throws an error when there are no dependencies.', async () => { - const commandArgs = { - manifest: '/path/to/mock-manifest.json', - install: false, - }; - - process.env.manifest = 'false'; - - expect.assertions(1); - try { - await getOptionsFromArgs(commandArgs); - } catch (error) { - expect(error).toEqual(new Error(FAILURE_MESSAGE_DEPENDENCIES)); - } - }); - }); - - describe('addTemplateManifest(): ', () => { - it('successfully adds the template manifest to the directory.', async () => { - await addTemplateManifest('./'); - - expect.assertions(1); - }); - - it('throws an error when the manifest is not added into the directory.', async () => { - expect.assertions(1); - - try { - await addTemplateManifest(''); - } catch (error) { - const logSpy = jest.spyOn(global.console, 'log'); - expect(logSpy).toEqual(FAILURE_MESSAGE); - } - }); - }); - - describe('initializeAndInstallLibs(): ', () => { - beforeEach(() => { - initPackage.mockReset(); - updatePackage.mockReset(); - installdeps.mockReset(); - updatedeps.mockReset(); - }); - - it('installs dependencies if install flag is truthy.', async () => { - process.env.NPM_MOCK = 'true'; - // @ts-ignore - process.exit = (code: any) => code; - const options = { - folderPath: 'mock-folderPath', - install: true, - cwd: true, - dependencies: { - mock: 'mock-dependencies', - }, - }; - - expect.assertions(4); - await initializeAndInstallLibs(options); - - expect(initPackage).toHaveBeenCalledTimes(1); - expect(updatePackage).toHaveBeenCalledTimes(1); - expect(installdeps).toHaveBeenCalledTimes(1); - expect(updatedeps).toHaveBeenCalledTimes(0); - }); - - it('updates dependencies if install flag is falsy.', async () => { - process.env.NPM_MOCK = 'true'; - // @ts-ignore - process.exit = (code: any) => code; - const options = { - folderPath: 'mock-folderPath', - install: false, - cwd: true, - dependencies: { - mock: 'mock-dependencies', - }, - }; - - expect.assertions(4); - await initializeAndInstallLibs(options); - - expect(initPackage).toHaveBeenCalledTimes(1); - expect(updatePackage).toHaveBeenCalledTimes(1); - expect(installdeps).toHaveBeenCalledTimes(0); - expect(updatedeps).toHaveBeenCalledTimes(1); - }); - - it('exits process if error is thrown.', async () => { - process.env.NPM_MOCK = 'error'; - const originalProcessExit = process.exit; - const mockExit = jest.fn(); - // @ts-ignore - process.exit = mockExit; - const options = { - folderPath: 'mock-folderPath', - install: false, - cwd: true, - dependencies: { - mock: 'mock-dependencies', - }, - }; - - expect.assertions(5); - await initializeAndInstallLibs(options); - - expect(initPackage).toHaveBeenCalledTimes(0); - expect(updatePackage).toHaveBeenCalledTimes(0); - expect(installdeps).toHaveBeenCalledTimes(0); - expect(updatedeps).toHaveBeenCalledTimes(0); - expect(mockExit).toHaveBeenCalledTimes(1); - - process.exit = originalProcessExit; - }); - }); - - describe('logStdoutFailMessage(): ', () => { - it('successfully logs the failed message.', () => { - const errorMessage = {stdout: '\n\nmock error message'}; - const mockFilename = 'mock-filename.yaml'; - const logSpy = jest.spyOn(global.console, 'log'); - logStdoutFailMessage(errorMessage, mockFilename); - - expect.assertions(2); - - expect(logSpy).toHaveBeenCalledWith( - `āœ– if-check could not verify ${mockFilename}. The re-executed file does not match the original.\n` - ); - - expect(logSpy).toHaveBeenCalledWith('mock error message'); - }); - }); -}); diff --git a/src/__tests__/unit/util/json.test.ts b/src/__tests__/unit/util/json.test.ts deleted file mode 100644 index 1bb1cd451..000000000 --- a/src/__tests__/unit/util/json.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -jest.mock('fs/promises', () => require('../../../__mocks__/fs')); - -import {readAndParseJson} from '../../../util/json'; - -describe('util/json: ', () => { - describe('readAndParseJson(): ', () => { - it('returns file content from path.', async () => { - const path = 'mock/path/json'; - expect.assertions(1); - - const response = await readAndParseJson(path); - expect(response).toEqual(path); - }); - - it('throws error if path does not exist.', async () => { - const path = 'mock/path/json-reject'; - expect.assertions(1); - - try { - await readAndParseJson(path); - } catch (error) { - expect(error).toBeInstanceOf(Error); - } - }); - }); -}); diff --git a/src/builtins/README.md b/src/builtins/README.md deleted file mode 100644 index c37e143a1..000000000 --- a/src/builtins/README.md +++ /dev/null @@ -1,848 +0,0 @@ -# IF builtins - -There are three built-in features of IF: - -- time-sync -- CSV exporter -- groupby - -On this page, you can find the documentation for each of these three builtins. - -## Time-sync - -Time sync standardizes the start time, end time and temporal resolution of all output data across an entire tree. - -### Parameters - -### Plugin config - -The following should be defined in the plugin initialization: - -- `start-time`: global start time as ISO 8061 string -- `stop`: global end time as ISO 8061 string -- `interval`: temporal resolution in seconds -- `error-on-padding`: avoid zero/'zeroish' padding (if needed) and error out instead. `False` by defult. - -#### Inputs: - -- `inputs`: an array of observations - -#### Returns - -- `inputs`: time-synchronized version of the tree - - - - -#### Overview - -A manifest file for a tree might contain many nodes each representing some different part of an application's stack or even different applications running on different machines. It is therefore common to have time series data in each component that is not directly comparable to other components either because the temporal resolution of the data is different, they cover different periods, or there are gaps in some records (e.g. some apps might burst but then go dormant, while others run continuously). This makes post-hoc visualization, analysis and aggregation of data from groups of nodes difficult to achieve. To address this, we created a time synchronization plugin that takes in non-uniform times series and snaps them all to a regular timeline with uniform start time, end time and temporal resolution. - -We do this by implementing the following logic: - -- Shift readings to nearest whole seconds -- Upsample the time series to a base resolution (1s) -- Resample to desired resolution by batching 1s entries -- Extrapolate or trim to ensure all time series share global start and end dates - -The next section explains each stage in more detail. - -#### Details - -##### Upsampling rules - -A set of `inputs` is naturally a time series because all `observations` include a `timestamp` and a `duration`, measured in seconds. -For each `observation` in `inputs` we check whether the duration is greater than 1 second. If `duration` is greater than 1 second, we create N new `observation` objects, where N is equal to `duration`. This means we have an `observation` for every second between the initial timestamp and the end of the observation period. Each new object receives a timestamp incremented by one second. - -This looks as follows: - -```ts -[{timestamp: '2023-12-12T00:00:00.000Z', duration: 5}] - -# becomes -[ - {timestamp: '2023-12-12T00:00:01.000Z', duration: 1} - {timestamp: '2023-12-12T00:00:02.000Z', duration: 1} - {timestamp: '2023-12-12T00:00:03.000Z', duration: 1} - {timestamp: '2023-12-12T00:00:04.000Z', duration: 1} - {timestamp: '2023-12-12T00:00:05.000Z', duration: 1} -] -``` - -Each `observation` actually includes many key-value pairs. The precise content of the `observation` is not known until runtime because it depends on which plugins have been included in the pipeline. Different values have to be treated differently when we upsample in time. The method we use to upsample depends on the `aggregation-method` defined for each key in `units.yml`. - -If the right way to aggregate a value is to sum it, then the right way to upsample it is to divide by `duration`, effectively spreading the total out evenly across the new, higher resolution, `observations` so that the total across the same bucket of time is unchanged (i.e. if the total for some value is 10 when there is one entry with `duration = 10s`, then the total should still be 10 when there are 10 entries each witch `duration = 1s`). - -On the other hand, if the right way to aggregate a value is to take its average over some time period, the value should be copied unchanged into the newly upsampled `observations`. This is appropriate for values that are proportional or percentages, such as `cpu/utilization`. Treating these values as constants means the average over the `duration` for an observation is identical whether you consider the initial `observation` or the upsampled set of N `observation`s. - -Constants can simply be copied as-is, because they are constants. Examples might be the `grid/carbon-intensity` - this value does not change depending on how frequently you observe it. - -Therefore, we apply this logic and the resulting flow looks as follows (the `aggregation-method` for `carbon` and `energy` is `sum`, `grid/carbon-intensity` is a constant and `cpu/utilization` is expressed as a percentage): - -```ts -[{timestamp: '2023-12-12T00:00:00.000Z', duration: 5, 'cpu/utilization': 12, carbon: 5, energy: 10, 'grid/carbon-intensity': 471}] - -# becomes - -[ - {timestamp: '2023-12-12T00:00:00.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, - {timestamp: '2023-12-12T00:00:01.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, - {timestamp: '2023-12-12T00:00:02.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, - {timestamp: '2023-12-12T00:00:03.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, - {timestamp: '2023-12-12T00:00:04.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, - {timestamp: '2023-12-12T00:00:05.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471} -] -``` - -The end result is that for each `observation`, we upsample the time series to yield 1 second resolution data between `timestamp` and `timestamp + duration`. - -##### Gap-filling - -Sometimes there might be discontinuities in the time series between one `observation` and another. For example we might have two `observations` in a set of `inputs` that have timestamps spaced 10 seconds apart, but the `duration` of the first `observation` is only 5 seconds. in this case, 5 seconds of data are unaccounted for and create a discontinuity in the time series. - -To solve this problem, for all but the first `observation` in the `inputs` array, we grab the `timestamp` and `duration` from the previous `observation` and check that `timestamp[N] + duration[N] == timestamp[N+1]`. If this condition is not satisfied, we backfill the missing data with a "zero-observation" which is identical to the surrounding observations except any values whose `aggregation-method` is `sum` are set to zero. This is equivalent to assuming that when there is no data available, the app being monitored is switched off. - -The end result of this gap-filling is that we have continuous 1 second resolution data that can be resampled to a new temporal resolution. - -```ts -[ - {timestamp: '2023-12-12T00:00:00.000Z', duration: 5, 'cpu/utilization': 12, carbon: 5, energy: 10, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:08.000Z', duration: 2, 'cpu/utilization': 12, carbon: 5, energy: 10, grid/carbon-intensity: 471} -] - -# There are 2 seconds of missing data between the end of timestamp[0] + duration, and timestamp[1] -# After expansion and infilling, the array becomes: - -[ - {timestamp: '2023-12-12T00:00:00.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:01.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:02.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:03.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:04.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:05.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:06.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:07.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:08.000Z', duration: 1, 'cpu/utilization': 12, carbon: 2.5, energy: 5, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:09.000Z', duration: 1, 'cpu/utilization': 12, carbon: 2.5, energy: 5, grid/carbon-intensity: 471} -] -``` - -Note that when `error-on-padding` is `true` no gap-filling is performed and the plugin will error out instead. - -##### Trimming and padding - -To ensure parity across all the components in a tree, we need to synchronize the start and end times for all time series. To do this, we pass the `time-sync` plugin plugin some global config: `startTime`, `endTime` and `interval`. The `startTime` is the timestamp where _all_ input arrays across the entire tree should begin, and `endTime` is the timestamp where _all_ input arrays across the entire tree should end. `interval` is the time resolution we ultimately want to resample to. - -To synchronize the time series start and end we check the first element of `inputs` for each node in the tree and determine whether it is earlier, later or equal to the global start time. If it is equal then no action is required. If the `input` start time is earlier than the global start time, we simply discard entries from the front of the array until the start times are aligned. If the `input` start time is after the global start time, then we pad with our "zero-observation" object - one for every second separating the global start time from the `input` start time. The same process is repeated for the end time - we either trim away `input` data or pad it out with "zero-observation" objects. - -For example, for `startTime = 2023-12-12T00:00:00.000Z` and `endTime = 2023-12-12T00:00:15.000Z`: - -```ts -[ - {timestamp: '2023-12-12T00:00:05.000Z', duration: 5, 'cpu/utilization': 12, carbon: 5, energy: 10, 'grid/carbon-intensity': 471}, -] - -# There are 5 seconds missing from the start and end. After padding, the array becomes: - -[ - {timestamp: '2023-12-12T00:00:00.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:01.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:02.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:03.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:04.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:05.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:06.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:07.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:08.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:09.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:10.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:11.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:12.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:13.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - {timestamp: '2023-12-12T00:00:14.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, - -] -``` - -Note that when `error-on-padding` is `true` no padding is performed and the plugin will error out instead. - -##### Resampling rules - -Now we have synchronized, continuous, high resolution time series data, we can resample. To achieve this, we use `interval`, which sets the global temporal resolution for the final, processed time series. `intervalk` is expressed in units of seconds, which means we can simply batch `observations` together in groups of size `interval`. For each value in each object we either sum, average or copy the values into one single summary object representing each time bucket of size `interval` depending on their `aggregation-method` defined in `params.ts`. The returned array is the final, synchronized time series at the desired temporal resolution. - - -#### Assumptions and limitations - -To do time synchronization, we assume: - -- There is no environmental impact for an application when there is no data available. -- Evenly distributing the total for a `duration` across higher resolution `observations` is appropriate, as opposed to having some non-uniform distribution. - - -### Typescript implementation - - -To run the plugin, you must first create an instance of `TimeSync`. -Then, you can call `execute()`. - -```typescript -const globalConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:30.000Z', - interval: 10 -} -const timeSync = TimeSync(globalConfig); -const results = timeSync.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z' - duration: 10 - 'cpu/utilization': 10 - carbon: 100 - energy: 100 - requests: 300 - }, - { - timestamp: '2023-12-12T00:00:10.000Z' - duration: 10 - 'cpu/utilization': 20 - carbon: 100, - energy: 100, - requests: 380 - } -]) -``` - -### Example manifest - -IF users will typically call the plugin as part of a pipeline defined in an `manifest` -file. In this case, instantiating and configuring the plugin is handled by -`ie` and does not have to be done explicitly by the user. -The following is an example `manifest` that calls `time-sync`: - -```yaml -name: time-sync-demo -description: impl with 2 levels of nesting with non-uniform timing of observations -tags: -initialize: - plugins: - teads-curve: - method: TeadsCurve - path: '@grnsft/if-unofficial-plugins' - sci-e: - method: SciE - path: '@grnsft/if-plugins' - sci-embodied: - path: 'builtin' - method: SciEmbodied - sci-o: - method: SciO - path: '@grnsft/if-plugins' - time-sync: - method: TimeSync - path: builtin - global-config: - start-time: '2023-12-12T00:00:00.000Z' # ISO timestamp - end-time: '2023-12-12T00:01:00.000Z' # ISO timestamp - interval: 5 # seconds -tree: - children: - child: # an advanced grouping node - pipeline: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - config: - teads-curve: - cpu/thermal-design-power: 65 - sci-embodied: - device/emissions-embodied: 251000 # gCO2eq - time-reserved: 3600 # 1 hour in s - device/expected-lifespan: 126144000 # 4 years in seconds - resources-reserved: 1 - resources-total: 1 - sci-o: - grid/carbon-intensity: 457 # gCO2/kwh - children: - child-1: - inputs: - - timestamp: '2023-12-12T00:00:00.000Z' - duration: 10 - cpu/utilization: 10 - carbon: 100 - energy: 100 - requests: 300 - - timestamp: '2023-12-12T00:00:10.000Z' - duration: 10 - cpu/utilization: 20 - carbon: 200 - energy: 200 - requests: 380 -``` - - -## CSV Exporter - -IF supports exporting data to CSV files. This provides users with a data format that enables visualization and data analysis using standard data analysis tools. - -### Manifest config - -To export your data to a CSV file, you have to provide a small piece of config data to your manifest file: - -```yaml -initialize: - outputs: - - csv -``` - -You can also add `- yaml` if you want to export to both `yaml` and `csv` simultaneously. - -### CLI command - -Then, you must select the metric you want to export to CSV. The name of that metric must be added to the savepath provided to the `--output` command in the CLI, after a hashtag. - -For example, to export the `carbon` data from your tree to a CSV file: - -```sh -if-run --manifest example.yml --output example#carbon -``` - -This will save a CSV file called `example.csv`. The contents will look similar to the following: - -| | | | | | -| ---------------------------------------------- | ---------------- | ---------------------------- | ---------------------------- | ---------------------------- | -| **Path** | **Aggregated** | **2024-03-05T00:00:00.000Z** | **2024-03-05T00:05:00.000Z** | **2024-03-05T00:10:00.000Z** | -| tree.carbon | 425.289232008725 | 17.9269877157543 | 8.9024388783018 | 45.6021901509012 | -| tree.children.westus3.carbon | 104.696836722878 | 3.59973803197887 | 3.47438149032372 | 6.91318436533634 | -| tree.children.westus3.children.server-1.carbon | 104.696836722878 | 3.59973803197887 | 3.47438149032372 | 6.91318436533634 | -| tree.children.france.carbon | 320.592395285847 | 14.3272496837754 | 5.42805738797808 | 38.6890057855649 | -| tree.children.france.children.server-2.carbon | 320.592395285847 | 14.3272496837754 | 5.42805738797808 | 38.6890057855649 | - - -### Comparing CSV to Yaml - -The CSV above is generated from the following yaml. The `carbon` metric is extracted and added to the CSV. Otherwise, the CSV is an exact representation of the following yaml tree. You can see that the CSV representation is *much* easier to understand than the full yaml tree: - -```yaml -tree: - pipeline: - - mock-observations - - group-by - - cloud-metadata - - time-sync - - watttime - - teads-curve - - operational-carbon - defaults: - grid/carbon-intensity: 500 - config: - group-by: - group: - - cloud/region - - name - children: - westus3: - children: - server-1: - inputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 66 - grid/carbon-intensity: 500 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 4 - grid/carbon-intensity: 500 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 54 - grid/carbon-intensity: 500 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 19 - grid/carbon-intensity: 500 - outputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 65.78 - grid/carbon-intensity: 369.4947514218548 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: CAISO - cloud/region-em-zone-id: US-CAL-CISO - cloud/region-wt-id: CAISO_NORTH - cloud/region-location: US West (N. California) - cloud/region-geolocation: 34.0497,-118.1326 - geolocation: 34.0497,-118.1326 - cpu/energy: 0.018934842060004835 - carbon: 6.996324760173567 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 3.986666666666667 - grid/carbon-intensity: 369.38452029076234 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: CAISO - cloud/region-em-zone-id: US-CAL-CISO - cloud/region-wt-id: CAISO_NORTH - cloud/region-location: US West (N. California) - cloud/region-geolocation: 34.0497,-118.1326 - geolocation: 34.0497,-118.1326 - cpu/energy: 0.004545546617763956 - carbon: 1.6790545568620359 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 53.82 - grid/carbon-intensity: 372.58122309244305 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: CAISO - cloud/region-em-zone-id: US-CAL-CISO - cloud/region-wt-id: CAISO_NORTH - cloud/region-location: US West (N. California) - cloud/region-geolocation: 34.0497,-118.1326 - geolocation: 34.0497,-118.1326 - cpu/energy: 0.017357893372978016 - carbon: 6.467225143212361 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-1 - cloud/instance-type: Standard_E64_v3 - cloud/region: westus3 - cloud/vendor: azure - cpu/utilization: 18.936666666666667 - grid/carbon-intensity: 434.20042537311633 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: CAISO - cloud/region-em-zone-id: US-CAL-CISO - cloud/region-wt-id: CAISO_NORTH - cloud/region-location: US West (N. California) - cloud/region-geolocation: 34.0497,-118.1326 - geolocation: 34.0497,-118.1326 - cpu/energy: 0.010385485956624245 - carbon: 4.5093824200727735 - aggregated: - carbon: 19.651986880320734 - outputs: - - carbon: 6.996324760173567 - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - - carbon: 1.6790545568620359 - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - - carbon: 6.467225143212361 - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - - carbon: 4.5093824200727735 - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - aggregated: - carbon: 19.651986880320734 - france: - children: - server-2: - inputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 15 - grid/carbon-intensity: 500 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 78 - grid/carbon-intensity: 500 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 16 - grid/carbon-intensity: 500 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 6 - grid/carbon-intensity: 500 - outputs: - - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 14.95 - grid/carbon-intensity: 1719.1647205176753 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: France - cloud/region-em-zone-id: FR - cloud/region-wt-id: FR - cloud/region-location: Paris - cloud/region-geolocation: 48.8567,2.3522 - geolocation: 48.8567,2.3522 - cpu/energy: 0.00905914075141129 - carbon: 15.574155178030272 - - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 77.74 - grid/carbon-intensity: 1719.0544893865829 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: France - cloud/region-em-zone-id: FR - cloud/region-wt-id: FR - cloud/region-location: Paris - cloud/region-geolocation: 48.8567,2.3522 - geolocation: 48.8567,2.3522 - cpu/energy: 0.020379266251888902 - carbon: 35.0330691407141 - - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 15.946666666666667 - grid/carbon-intensity: 1718.8707708347622 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: France - cloud/region-em-zone-id: FR - cloud/region-wt-id: FR - cloud/region-location: Paris - cloud/region-geolocation: 48.8567,2.3522 - geolocation: 48.8567,2.3522 - cpu/energy: 0.009405866514354337 - carbon: 16.16746902589712 - - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - name: server-2 - cloud/instance-type: Standard_E64_v3 - cloud/region: france - cloud/vendor: azure - cpu/utilization: 5.98 - grid/carbon-intensity: 1718.6686804277592 - vcpus-allocated: 64 - vcpus-total: 64 - memory-available: 432 - physical-processor: >- - IntelĀ® XeonĀ® Platinum 8370C,IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® - XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-2673 v4 2.3 GHz - cpu/thermal-design-power: 269.1 - cloud/region-cfe: France - cloud/region-em-zone-id: FR - cloud/region-wt-id: FR - cloud/region-location: Paris - cloud/region-geolocation: 48.8567,2.3522 - geolocation: 48.8567,2.3522 - cpu/energy: 0.0054492484351820105 - carbon: 9.365452617417297 - aggregated: - carbon: 76.1401459620588 - outputs: - - carbon: 15.574155178030272 - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - - carbon: 35.0330691407141 - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - - carbon: 16.16746902589712 - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - - carbon: 9.365452617417297 - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - aggregated: - carbon: 76.1401459620588 - outputs: - - carbon: 22.57047993820384 - timestamp: '2024-03-05T00:00:00.000Z' - duration: 300 - - carbon: 36.71212369757613 - timestamp: '2024-03-05T00:05:00.000Z' - duration: 300 - - carbon: 22.63469416910948 - timestamp: '2024-03-05T00:10:00.000Z' - duration: 300 - - carbon: 13.87483503749007 - timestamp: '2024-03-05T00:15:00.000Z' - duration: 300 - aggregated: - carbon: 95.79213284237952 -``` - -### CSV and aggregation - -The CSV representation of the output data is helpful for intuiting how the aggregation procedure works. What we refer to as "horizontal" aggregation is really an aggregation of the *rows* of the CSV. You can replicate the IF aggregation function by summing the cells in each row of the CSV. Similarly, what we refer to as "vertical" aggregation can be replicated by summing the *columns* in the CSV representation (this is not *exactly* accurate because you have to skip summing both parent nodes and their children, both of which are represented in the CSV, but it is true conceptually). - - -## Groupby - -Groupby is an IF plugin that reorganizes a tree according to keys provided by the user. This allows users to regroup their observations according to various properties of their application. For example, the following manifest file contains a flat array of observations. This is how you might expect data to arrive from an importer plugin, maybe one that hits a metrics API for a cloud service. - - -```yaml -name: if-demo -description: demo pipeline -graph: - children: - my-app: - pipeline: - - group-by - - teads-curve - config: - group-by: - - cloud/region - - instance-type - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-west - cpu-util: 1 -``` - -However, each observation contains an `instance-type` field that varies between observations. There are two instance types being represented in this array of observations. This means there are duplicate entries for the same timestamp in this array. This is the problem that `group-by` solves. You provide `instance-type` as a key to the `group-by` plugin and it extracts the data belonging to the different instances and separates them into independent arrays. The above example would be restructured so that instance types `A1` and `B1` have their own data, as follows: - -```yaml -graph: - children: - my-app: - pipeline: - # - group-by - - teads-curve - config: - group-by: - groups: - - cloud/region - - instance-type - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 1 -``` - -### Using `group-by` - -To use `group-by`, you have to initialize it as a plugin and invoke it in a pipeline. - -The initialization looks as follows: - -```yaml -initialize: -plugins: -group-by: - path: 'builtin' - method: GroupBy -``` - -You then have to provide config defining which keys to group by in each component. This is done at the component level (i.e. not global config). -For example: - - -```yaml -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - group: - - cloud/region - - instance-type -``` - -In the example above, the plugin would regroup the input data for the specific component by `cloud/region` and by `instance-type`. - -Assuming the values `A1` and `B1` are found for `instance-type` and the values `uk-east` and `uk-west` are found for `cloud/region`, the result of `group-by` would look similar to the following: - -```yaml -tree: - children: - my-app: - pipeline: - - group-by - config: - group-by: - groups: - - cloud/region - - instance-type - children: - uk-west: - children: - A1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: A1 - cloud/region: uk-west - cpu-util: 12 - uk-east: - children: - B1: - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - instance-type: B1 - cloud/region: uk-east - cpu-util: 1 -``` - -This reorganized data can then be used to feed the rest of a computation pipeline. diff --git a/src/builtins/export-csv-raw.ts b/src/builtins/export-csv-raw.ts deleted file mode 100644 index 7afc21cb2..000000000 --- a/src/builtins/export-csv-raw.ts +++ /dev/null @@ -1,156 +0,0 @@ -import * as fs from 'fs/promises'; -import {ERRORS} from '@grnsft/if-core/utils'; - -import {STRINGS} from '../config'; - -import {Context} from '../types/manifest'; - -const {ExhaustOutputArgError, WriteFileError} = ERRORS; -const {OUTPUT_REQUIRED, WRITE_CSV_ERROR, EXPORTING_RAW_CSV_FILE} = STRINGS; - -export const ExportCSVRaw = () => { - /** - * handle a tree leaf, where there are no child nodes, by adding it as key->value pair to the flat map - * and capturing key as a header - */ - const handleLeafValue = ( - value: any, - fullPath: string, - key: any, - flatMap: {[key: string]: any}, - headers: Set - ) => { - if (fullPath.includes('outputs')) { - headers.add(key); - flatMap[fullPath] = value; - } - }; - - /** - * handle a tree node, recursively traverse the children and append their results to the flat map and captured headers - */ - const handleNodeValue = ( - value: any, - fullPath: string, - flatMap: Record, - headers: Set - ) => { - const [subFlatMap, subHeaders] = extractFlatMapAndHeaders(value, fullPath); - - if (Object.keys(subFlatMap).length > 0) { - Object.entries(subFlatMap).forEach(([subKey, value]) => { - flatMap[subKey] = value; - }); - - subHeaders.forEach(subHeader => { - headers.add(subHeader); - }); - } - }; - - /** - * Handles a key at the top level of the tree - */ - const handleKey = ( - value: any, - key: any, - prefix: string, - flatMap: Record, - headers: Set - ) => { - const fullPath = prefix ? `${prefix}.${key}` : key; - - if (value !== null && typeof value === 'object') { - return handleNodeValue(value, fullPath, flatMap, headers); - } - - return handleLeafValue(value, fullPath, key, flatMap, headers); - }; - - /** - * Recursively extract a flat map and headers from the hierarcial tree. - */ - const extractFlatMapAndHeaders = ( - tree: any, - prefix = '' - ): [Record, Set] => { - const headers: Set = new Set(); - const flatMap: Record = []; - - for (const key in tree) { - if (key in tree) { - handleKey(tree[key], key, prefix, flatMap, headers); - } - } - - return [flatMap, headers]; - }; - - /** - * extract the id of the key, that is removing the last token (which is the index). - * in this manner, multiple keys that identical besides their index share the same id. - */ - const extractIdHelper = (key: string): string => { - const parts = key.split('.'); - parts.pop(); - - return parts.join('.'); - }; - - /** - * generate a CSV formatted string based on a flat key->value map, headers and ids - */ - const getCsvString = ( - map: {[key: string]: any}, - headers: Set, - ids: Set - ): string => { - const csvRows: string[] = []; - csvRows.push(['id', ...headers].join(',')); - - ids.forEach(id => { - const rowData = [id]; - - headers.forEach(header => { - const value = map[`${id}.${header}`] ?? ''; - rowData.push(value.toString()); - }); - csvRows.push(rowData.join(',')); - }); - - return csvRows.join('\n'); - }; - - /** - * write the given string content to a file at the provided path - */ - const writeOutputFile = async (content: string, outputPath: string) => { - try { - await fs.writeFile(`${outputPath}.csv`, content); - } catch (error) { - throw new WriteFileError(WRITE_CSV_ERROR(outputPath, error)); - } - }; - - /** - * export the provided tree content to a CSV file, represented in a flat structure - */ - const execute = async (tree: any, _context: Context, outputPath: string) => { - if (!outputPath) { - throw new ExhaustOutputArgError(OUTPUT_REQUIRED); - } - - console.debug(EXPORTING_RAW_CSV_FILE(outputPath)); - - const [extractredFlatMap, extractedHeaders] = - extractFlatMapAndHeaders(tree); - const ids = new Set( - Object.keys(extractredFlatMap).map(key => extractIdHelper(key)) - ); - const csvString = getCsvString(extractredFlatMap, extractedHeaders, ids); - - await writeOutputFile(csvString, outputPath); - }; - - return {execute}; -}; diff --git a/src/builtins/export-csv.ts b/src/builtins/export-csv.ts deleted file mode 100644 index 53a105733..000000000 --- a/src/builtins/export-csv.ts +++ /dev/null @@ -1,117 +0,0 @@ -import {writeFile} from 'fs/promises'; - -import {stringify} from 'csv-stringify/sync'; -import {ERRORS} from '@grnsft/if-core/utils'; -import {PluginParams} from '@grnsft/if-core/types'; - -import {STRINGS} from '../config'; - -import {Context} from '../types/manifest'; - -const {ExhaustOutputArgError} = ERRORS; -const {CSV_EXPORT, OUTPUT_REQUIRED, EXPORTING_TO_CSV_FILE} = STRINGS; - -/** - * Extension to IF that outputs the tree in a CSV format. - */ -export const ExportCSV = () => { - const parseOutputAndField = (outputPath: string) => { - const validatedPath = validateOutputPath(outputPath); - - const paths = validatedPath.split('#'); - const criteria = paths[paths.length - 1]; - - if (paths.length <= 1 || !criteria) { - throw new ExhaustOutputArgError(CSV_EXPORT); - } - - const output = paths.slice(0, paths.length - 1).join(''); - console.debug(EXPORTING_TO_CSV_FILE(output)); - - return { - output, - criteria, - }; - }; - - /** - * Validates output path. - */ - const validateOutputPath = (outputPath: string) => { - if (!outputPath) { - throw new ExhaustOutputArgError(OUTPUT_REQUIRED); - } - - return outputPath; - }; - - /** - * Grabs output and criteria from cli args, then call tree walker to collect csv data. - */ - const execute = async (tree: any, context: Context, outputPath: string) => { - const columns = ['Path']; - const matrix = [columns]; - const {output, criteria} = parseOutputAndField(outputPath); - const aggregationIsEnabled = !!context.aggregation; - - /** - * Walks through all tree branches and leaves, collecting the data - */ - const treeWalker = (node: any, criteria: string, path = 'tree') => { - /** Hmm aggregated, then checks if it's the first one. If so adds column, pushes the value. */ - if (node.aggregated) { - if (path === 'tree') { - columns.push('Aggregated'); - } - - matrix.push([`${path}.${criteria}`, node.aggregated[criteria]]); - } - - /** So it has outputs, whats then? checks if timestamp is not there, adds one. Then appends values to it. */ - if (node.outputs) { - node.outputs.forEach((output: PluginParams) => { - const {timestamp} = output; - - if (!columns.includes(timestamp)) { - columns.push(output.timestamp); - } - - const lastRow = matrix[matrix.length - 1]; - - if (aggregationIsEnabled) { - lastRow.push(output[criteria]); - } else { - /** Handle without aggregation export strategy. */ - if (matrix.length === 1 || lastRow.length === columns.length) { - matrix.push([`${path}.${criteria}`, output[criteria]]); - } else { - lastRow.push(output[criteria]); - } - } - }); - } - - /** Ohh children? then call every one and execute. */ - if (node.children) { - for (const child in node.children) { - treeWalker( - node.children[child], - criteria, - `${path}.children.${child}` - ); - } - } - }; - - treeWalker(tree, criteria); - - await writeFile(`${output}.csv`, stringify(matrix, {columns})); - }; - - return { - execute, - metadata: { - kind: 'exhaust', - }, - }; -}; diff --git a/src/builtins/group-by.ts b/src/builtins/group-by.ts deleted file mode 100644 index 63c4cd5c5..000000000 --- a/src/builtins/group-by.ts +++ /dev/null @@ -1,95 +0,0 @@ -import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; -import { - GroupByPlugin, - PluginParams, - GroupByConfig, -} from '@grnsft/if-core/types'; - -import {STRINGS} from '../config'; - -import {validate} from '../util/validations'; - -const {InvalidGroupingError, GlobalConfigError} = ERRORS; - -const {INVALID_GROUP_BY, MISSING_GLOBAL_CONFIG} = STRINGS; - -/** - * Plugin for inputs grouping. - */ -export const GroupBy = (): GroupByPlugin => { - const metadata = { - kind: 'groupby', - }; - - /** - * Creates structure to insert inputs by groups. - */ - const appendGroup = ( - value: PluginParams, - object: any, - groups: string[] - ): any => { - if (groups.length > 0) { - const group = groups.shift() as string; - - object.children = object.children ?? {}; - object.children[group] = object.children[group] ?? {}; - - if (groups.length === 0) { - if ( - object.children[group].inputs && - object.children[group].inputs.length > 0 - ) { - object.children[group].inputs.push(value); - } else { - object.children[group].inputs = [value]; - } - } - - appendGroup(value, object.children[group], groups); - } - - return object; - }; - - /** - * Interates over inputs, grabs config-group types values for each one. - * Based on grouping types, initializes the structure grouped structure. - */ - const execute = (inputs: PluginParams[], config: GroupByConfig) => - inputs.reduce((acc, input) => { - const validatedConfig = validateConfig(config); - const groups = validatedConfig.group.map(groupType => { - if (!input[groupType]) { - throw new InvalidGroupingError(INVALID_GROUP_BY(groupType)); - } - - return input[groupType]; - }); - - acc = { - ...acc, - ...appendGroup(input, acc, groups), - }; - - return acc; - }, {} as any).children; - - /** - * Validates config parameter. - */ - const validateConfig = (config: GroupByConfig) => { - if (!config) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); - } - - const schema = z.object({ - group: z.array(z.string()).min(1), - }); - - return validate>(schema, config); - }; - - return {metadata, execute}; -}; diff --git a/src/common/config/index.ts b/src/common/config/index.ts new file mode 100644 index 000000000..3ee9fecbb --- /dev/null +++ b/src/common/config/index.ts @@ -0,0 +1 @@ +export {STRINGS} from './strings'; diff --git a/src/common/config/strings.ts b/src/common/config/strings.ts new file mode 100644 index 000000000..b68d5ce7c --- /dev/null +++ b/src/common/config/strings.ts @@ -0,0 +1,12 @@ +export const STRINGS = { + DISCLAIMER_MESSAGE: ` +Incubation Project + +This project is an incubation project being run inside the Green Software Foundation; as such, we DONā€™T recommend using it in any critical use case. +Incubation projects are experimental, offer no support guarantee, have minimal governance and process, and may be retired at any moment. This project may one day graduate, in which case this disclaimer will be removed.`, + SOURCE_IS_NOT_YAML: 'Given source file is not in yaml format.', + TARGET_IS_NOT_YAML: 'Given target file is not in yaml format.', + MANIFEST_NOT_FOUND: 'Manifest file not found.', + SUCCESS_MESSAGE: 'The environment is successfully setup!', + MANIFEST_IS_MISSING: 'Manifest is missing.', +}; diff --git a/src/common/lib/load.ts b/src/common/lib/load.ts new file mode 100644 index 000000000..ade17795d --- /dev/null +++ b/src/common/lib/load.ts @@ -0,0 +1,18 @@ +import {openYamlFileAsObject} from '../util/yaml'; + +import {STRINGS} from '../../if-run/config'; + +const {LOADING_MANIFEST} = STRINGS; + +/** + * Parses manifest file as an object. + */ +export const load = async (inputPath: string) => { + console.debug(LOADING_MANIFEST); + + const rawManifest = await openYamlFileAsObject(inputPath); + + return { + rawManifest, + }; +}; diff --git a/src/types/manifest.ts b/src/common/types/manifest.ts similarity index 75% rename from src/types/manifest.ts rename to src/common/types/manifest.ts index 2359a3433..1efab523d 100644 --- a/src/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -1,5 +1,7 @@ import {z} from 'zod'; +import {AggregationMethodTypes} from '../../if-run/types/aggregation'; + import {manifestSchema} from '../util/validations'; export type Manifest = z.infer; @@ -9,10 +11,12 @@ export type GlobalPlugins = Manifest['initialize']['plugins']; export type PluginOptions = GlobalPlugins[string]; export type AggregationParams = Manifest['aggregation']; +export type AggregationMetricsWithMethod = { + [key: string]: AggregationMethodTypes; +}; + export type AggregationParamsSure = Extract; export type Context = Omit; export type ContextWithExec = Omit; - -export type ManifestParameter = Extract[number]; diff --git a/src/util/debug-logger.ts b/src/common/util/debug-logger.ts similarity index 96% rename from src/util/debug-logger.ts rename to src/common/util/debug-logger.ts index f1f34ae0e..35dc77ea7 100644 --- a/src/util/debug-logger.ts +++ b/src/common/util/debug-logger.ts @@ -1,4 +1,4 @@ -import {STRINGS} from '../config/strings'; +import {STRINGS} from '../../if-run/config'; const logMessagesKeys: (keyof typeof STRINGS)[] = [ 'STARTING_IF', @@ -6,7 +6,6 @@ const logMessagesKeys: (keyof typeof STRINGS)[] = [ 'LOADING_MANIFEST', 'VALIDATING_MANIFEST', 'CAPTURING_RUNTIME_ENVIRONMENT_DATA', - 'SYNCING_PARAMETERS', 'CHECKING_AGGREGATION_METHOD', 'INITIALIZING_PLUGINS', 'INITIALIZING_PLUGIN', @@ -17,8 +16,6 @@ const logMessagesKeys: (keyof typeof STRINGS)[] = [ 'AGGREGATING_NODE', 'PREPARING_OUTPUT_DATA', 'EXPORTING_TO_YAML_FILE', - 'EXPORTING_TO_CSV_FILE', - 'EXPORTING_RAW_CSV_FILE', ]; enum LogLevel { diff --git a/src/util/fs.ts b/src/common/util/fs.ts similarity index 100% rename from src/util/fs.ts rename to src/common/util/fs.ts diff --git a/src/common/util/helpers.ts b/src/common/util/helpers.ts new file mode 100644 index 000000000..01b87c79d --- /dev/null +++ b/src/common/util/helpers.ts @@ -0,0 +1,65 @@ +import {createInterface} from 'node:readline/promises'; +import {exec} from 'child_process'; +import * as path from 'path'; +import {promisify} from 'util'; + +/** + * Promise version of Node's `exec` from `child-process`. + */ +export const execPromise = promisify(exec); + +/** + * Prepends process path to given `filePath`. + */ +export const prependFullFilePath = (filePath: string) => { + const processRunningPath = process.cwd(); + + if (path.isAbsolute(filePath)) { + return filePath; + } + + return path.normalize(`${processRunningPath}/${filePath}`); +}; + +/** + * Checks if there is data piped, then collects it. + * Otherwise returns empty string. + */ +const collectPipedData = async () => { + if (process.stdin.isTTY) { + return ''; + } + + const readline = createInterface({ + input: process.stdin, + }); + + const data: string[] = []; + + for await (const line of readline) { + data.push(line); + } + + return data.join('\n'); +}; + +/** + * Checks if there is piped data, tries to parse yaml from it. + * Returns empty string if haven't found anything. + */ +export const parseManifestFromStdin = async () => { + const pipedSourceManifest = await collectPipedData(); + + if (!pipedSourceManifest) { + return ''; + } + + const regex = /# start((?:.*\n)+?)# end/; + const match = regex.exec(pipedSourceManifest); + + if (!match) { + return ''; + } + + return match![1]; +}; diff --git a/src/util/logger.ts b/src/common/util/logger.ts similarity index 100% rename from src/util/logger.ts rename to src/common/util/logger.ts diff --git a/src/util/validations.ts b/src/common/util/validations.ts similarity index 75% rename from src/util/validations.ts rename to src/common/util/validations.ts index 920e7cbdb..5cdbec5db 100644 --- a/src/util/validations.ts +++ b/src/common/util/validations.ts @@ -1,10 +1,9 @@ import {ZodIssue, ZodIssueCode, ZodSchema, z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {STRINGS} from '../config/strings'; +import {STRINGS} from '../../if-run/config'; -import {AGGREGATION_METHODS} from '../types/aggregation'; -import {AGGREGATION_TYPES} from '../types/parameters'; +import {AGGREGATION_TYPES} from '../../if-run/types/aggregation'; const {ManifestValidationError, InputValidationError} = ERRORS; const {VALIDATING_MANIFEST} = STRINGS; @@ -22,6 +21,36 @@ export const atLeastOneDefined = ( export const allDefined = (obj: Record) => Object.values(obj).every(v => v !== undefined); +/** + * Schema for parameter metadata. + */ +const parameterMetadataSchema = z + .object({ + inputs: z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': z.string(), + }) + ) + .optional() + .nullable(), + outputs: z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': z.string(), + }) + ) + .optional() + .nullable(), + }) + .optional(); + /** * Validation schema for manifests. */ @@ -36,34 +65,27 @@ export const manifestSchema = z.object({ }) .optional() .nullable(), + explainer: z.boolean().optional(), + explain: z.record(z.string(), z.any()).optional(), aggregation: z .object({ metrics: z.array(z.string()), - type: z.enum(AGGREGATION_METHODS), + type: z.enum(AGGREGATION_TYPES), }) .optional() .nullable(), - params: z - .array( - z.object({ - name: z.string(), - description: z.string(), - aggregation: z.enum(AGGREGATION_TYPES), - unit: z.string(), - }) - ) - .optional() - .nullable(), initialize: z.object({ plugins: z.record( z.string(), - z.object({ - path: z.string(), - method: z.string(), - 'global-config': z.record(z.string(), z.any()).optional(), - }) + z + .object({ + path: z.string(), + method: z.string(), + 'global-config': z.record(z.string(), z.any()).optional(), + 'parameter-metadata': parameterMetadataSchema, + }) + .optional() ), - outputs: z.array(z.string()).optional(), }), execution: z .object({ diff --git a/src/util/yaml.ts b/src/common/util/yaml.ts similarity index 100% rename from src/util/yaml.ts rename to src/common/util/yaml.ts diff --git a/src/config/config.ts b/src/config/config.ts deleted file mode 100644 index b8f676924..000000000 --- a/src/config/config.ts +++ /dev/null @@ -1,156 +0,0 @@ -import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; - -import {STRINGS} from './strings'; - -import { - IFDiffArgs, - IEArgs, - IFEnvArgs, - IFCheckArgs, -} from '../types/process-args'; - -const {DISCLAIMER_MESSAGE} = STRINGS; - -export const CONFIG = { - IE: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the input file]', - }, - output: { - type: String, - optional: true, - alias: 'o', - description: '[path to the output file]', - }, - 'override-params': { - type: String, - optional: true, - alias: 'p', - description: '[path to a parameter file that overrides our defaults]', - }, - stdout: { - type: Boolean, - optional: true, - alias: 's', - description: '[prints out to the console]', - }, - help: { - type: Boolean, - optional: true, - alias: 'h', - description: '[prints out the above help instruction]', - }, - debug: { - type: Boolean, - optional: true, - alias: 'd', - description: '[prints out debug logs to the console]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - }, - IF_DIFF: { - ARGS: { - source: { - type: String, - optional: true, - alias: 's', - description: '[path to the source file]', - }, - target: { - type: String, - optional: false, - alias: 't', - description: '[path to the target file', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'IF-Diff Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - SUCCESS_MESSAGE: 'Files match!', - FAILURE_MESSAGE: 'Files do not match!', - }, - IF_ENV: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the manifest file]', - }, - install: { - type: Boolean, - optional: true, - alias: 'i', - description: '[command to install package.json]', - }, - cwd: { - type: Boolean, - optional: true, - alias: 'c', - description: - '[command to generate the package.json in the command working directory]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - SUCCESS_MESSAGE: 'The environment is successfully setup!', - FAILURE_MESSAGE: 'Faied to create the environment!', - FAILURE_MESSAGE_TEMPLATE: - 'Faied to create the environment with the template manifest!', - FAILURE_MESSAGE_DEPENDENCIES: 'Manifest dependencies are not available!', - }, - IF_CHECK: { - ARGS: { - manifest: { - type: String, - optional: true, - alias: 'm', - description: '[path to the manifest file]', - }, - directory: { - type: String, - optional: true, - alias: 'd', - description: '[path to the manifests directory]', - }, - } as ArgumentConfig, - HELP: { - helpArg: 'help', - headerContentSections: [ - {header: 'Impact Framework', content: 'IF-Check Helpful keywords:'}, - ], - footerContentSections: [ - {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, - ], - } as ParseOptions, - }, - GITHUB_PATH: 'https://github.com', - NATIVE_PLUGIN: 'if-plugins', - AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], -}; diff --git a/src/config/params.ts b/src/config/params.ts deleted file mode 100644 index 76732e57b..000000000 --- a/src/config/params.ts +++ /dev/null @@ -1,198 +0,0 @@ -import {Parameters} from '../types/parameters'; - -export const PARAMETERS: Parameters = { - carbon: { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'cpu/number-cores': { - description: 'number of cores available', - unit: 'cores', - aggregation: 'none', - }, - 'cpu/utilization': { - description: 'refers to CPU utilization.', - unit: 'percentage', - aggregation: 'avg', - }, - 'disk-io': { - description: 'refers to GB of data written/read from disk', - unit: 'GB', - aggregation: 'sum', - }, - duration: { - description: 'refers to the duration of the input', - unit: 'seconds', - aggregation: 'sum', - }, - energy: { - description: 'amount of energy utilised by the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'cpu/energy': { - description: 'Energy consumed by the CPU of the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'device/expected-lifespan': { - description: 'Total Expected Lifespan of the Component in Seconds', - unit: 'seconds', - aggregation: 'sum', - }, - 'memory/energy': { - description: 'Energy consumed by the Memory of the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'carbon-embodied': { - description: 'Embodied Emissions of the component', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'network/energy': { - description: 'Energy consumed by the Network of the component', - unit: 'kWh', - aggregation: 'sum', - }, - 'functional-unit': { - description: - 'the name of the functional unit in which the final SCI value should be expressed, e.g. requests, users', - unit: 'none', - aggregation: 'sum', - }, - 'gpu-util': { - description: 'refers to CPU utilization.', - unit: 'percentage', - aggregation: 'avg', - }, - 'grid/carbon-intensity': { - description: 'Carbon intensity for the grid', - unit: 'gCO2eq/kWh', - aggregation: 'avg', - }, - 'cloud/instance-type': { - description: 'Type of Cloud Instance name used in the cloud provider APIs', - unit: 'None', - aggregation: 'none', - }, - geolocation: { - description: - 'Geographic location of provider as string (for watt-time model it is provided as latitude and longitude, comma separated, in decimal degrees)', - unit: 'None (decimal degrees for watt-time model)', - aggregation: 'none', - }, - 'carbon-operational': { - description: 'Operational Emissions of the component', - unit: 'gCO2e', - aggregation: 'sum', - }, - 'cpu/name': { - description: 'Name of the physical processor', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region': { - description: 'region cloud instance runs in', - unit: 'None', - aggregation: 'none', - }, - 'cloud/vendor': { - description: - 'Name of the cloud service provider in the ccf model. Can be aws, gcp or azure', - unit: 'None', - aggregation: 'none', - }, - name: { - description: 'arbitrary name parameter.', - unit: 'None', - aggregation: 'none', - }, - 'ram-alloc': { - description: 'refers to GB of memory allocated.', - unit: 'GB', - aggregation: 'avg', - }, - 'ram-util': { - description: 'refers to percentage of memory utilized.', - unit: 'percentage', - aggregation: 'avg', - }, - 'resources-reserved': { - description: 'resources reserved for an application', - unit: 'count', - aggregation: 'none', - }, - 'cpu/thermal-design-power': { - description: 'thermal design power for a processor', - unit: 'kwh', - aggregation: 'avg', - }, - 'device/emissions-embodied': { - description: 'total embodied emissions of some component', - unit: 'gCO2e', - aggregation: 'sum', - }, - timestamp: { - description: 'refers to the time of occurrence of the input', - unit: 'RFC3339', - aggregation: 'none', - }, - 'time-reserved': { - description: 'time reserved for a component', - unit: 'seconds', - aggregation: 'avg', - }, - 'resources-total': { - description: 'total resources available', - unit: 'count', - aggregation: 'none', - }, - 'vcpus-allocated': { - description: 'number of vcpus allocated to particular resource', - unit: 'count', - aggregation: 'none', - }, - 'vcpus-total': { - description: 'total number of vcpus available on a particular resource', - unit: 'count', - aggregation: 'none', - }, - 'memory-available': { - description: 'total amount of memory available on a particular resource', - unit: 'GB', - aggregation: 'none', - }, - 'physical-processor': { - description: - 'name of the physical processor being used in a specific instance type', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-cfe': { - description: 'cloud region name in cfe format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-em-zone-id': { - description: 'cloud region name in electricity maps format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-wt-id': { - description: 'cloud region name in watt-time format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-location': { - description: 'cloud region name in our IF format', - unit: 'None', - aggregation: 'none', - }, - 'cloud/region-geolocation': { - description: 'location expressed as decimal coordinates (lat/lon)', - unit: 'decimal degrees', - aggregation: 'none', - }, -}; diff --git a/src/if-check/config/config.ts b/src/if-check/config/config.ts new file mode 100644 index 000000000..8d21da9f3 --- /dev/null +++ b/src/if-check/config/config.ts @@ -0,0 +1,33 @@ +import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; + +import {STRINGS} from '../../common/config'; + +import {IFCheckArgs} from '../types/process-args'; + +const {DISCLAIMER_MESSAGE} = STRINGS; + +export const CONFIG = { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + directory: { + type: String, + optional: true, + alias: 'd', + description: '[path to the manifests directory]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Check Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, +}; diff --git a/src/config/index.ts b/src/if-check/config/index.ts similarity index 64% rename from src/config/index.ts rename to src/if-check/config/index.ts index 03ec3e587..4972b390b 100644 --- a/src/config/index.ts +++ b/src/if-check/config/index.ts @@ -1,3 +1,2 @@ export {CONFIG} from './config'; export {STRINGS} from './strings'; -export {PARAMETERS} from './params'; diff --git a/src/if-check/config/strings.ts b/src/if-check/config/strings.ts new file mode 100644 index 000000000..1b6f1c3ef --- /dev/null +++ b/src/if-check/config/strings.ts @@ -0,0 +1,19 @@ +export const STRINGS = { + CHECKING: 'Checking...', + IF_CHECK_FLAGS_MISSING: + 'Either the `--manifest` or `--directory` command should be provided with a path', + DIRECTORY_NOT_FOUND: 'Directory not found.', + DIRECTORY_YAML_FILES_NOT_FOUND: + 'The directory does not contain any YAML/YML files.\n', + IF_CHECK_EXECUTING: (filename: string) => `Executing \`${filename}\``, + IF_CHECK_VERIFICATION_FAILURES: + '---------\nif-check verification failures:\n', + IF_CHECK_FAILED: (filename: string) => + `āœ– if-check could not verify ${filename}. The re-executed file does not match the original.\n`, + IF_CHECK_VERIFIED: (filename: string) => + `āœ” if-check successfully verified ${filename}\n`, + IF_CHECK_SUMMARY_ERROR_MESSAGE: (filename: string, message: string) => + `Executing \`${filename}\`\nāœ– ${message}`, + IF_CHECK_SUMMARY_LOG: (passedCount: number, totalCount: number) => + `---------\nCheck summary:\n${passedCount} of ${totalCount} files are passed.\n`, +}; diff --git a/src/check.ts b/src/if-check/index.ts similarity index 82% rename from src/check.ts rename to src/if-check/index.ts index f1e8f0cdd..56d476c2b 100644 --- a/src/check.ts +++ b/src/if-check/index.ts @@ -2,13 +2,15 @@ /* eslint-disable no-process-exit */ import * as path from 'path'; -import {logger} from './util/logger'; +import {getYamlFiles, removeFileIfExists} from '../common/util/fs'; +import {debugLogger} from '../common/util/debug-logger'; +import {logger} from '../common/util/logger'; + import {logStdoutFailMessage} from './util/helpers'; import {parseIfCheckArgs} from './util/args'; -import {getYamlFiles, removeFileIfExists} from './util/fs'; +import {executeCommands} from './util/npm'; import {STRINGS} from './config'; -import {executeCommands} from './util/npm'; const { CHECKING, @@ -19,6 +21,9 @@ const { } = STRINGS; const IfCheck = async () => { + // Call this function with false parameter to prevent log debug messages. + debugLogger.overrideConsoleMethods(false); + const commandArgs = await parseIfCheckArgs(); console.log(`${CHECKING}\n`); @@ -51,8 +56,11 @@ const IfCheck = async () => { } for await (const file of files) { - const fileName = path.basename(file); - console.log(IF_CHECK_EXECUTING(fileName)); + const fileRelativePath = path.relative( + process.env.CURRENT_DIR || process.cwd(), + file + ); + console.log(IF_CHECK_EXECUTING(fileRelativePath)); try { await executeCommands(file, true); diff --git a/src/if-check/types/process-args.ts b/src/if-check/types/process-args.ts new file mode 100644 index 000000000..72dd2305c --- /dev/null +++ b/src/if-check/types/process-args.ts @@ -0,0 +1,4 @@ +export interface IFCheckArgs { + manifest?: string; + directory?: string; +} diff --git a/src/if-check/util/args.ts b/src/if-check/util/args.ts new file mode 100644 index 000000000..ab6e3fd93 --- /dev/null +++ b/src/if-check/util/args.ts @@ -0,0 +1,72 @@ +import {parse} from 'ts-command-line-args'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {isDirectoryExists, isFileExists} from '../../common/util/fs'; +import {prependFullFilePath} from '../../common/util/helpers'; +import {checkIfFileIsYaml} from '../../common/util/yaml'; + +import {CONFIG, STRINGS} from '../config'; +import {STRINGS as COMMON_STRINGS} from '../../common/config'; + +import {IFCheckArgs} from '../types/process-args'; + +const { + ParseCliParamsError, + CliSourceFileError, + InvalidDirectoryError, + MissingCliFlagsError, +} = ERRORS; + +const {ARGS, HELP} = CONFIG; + +const {IF_CHECK_FLAGS_MISSING, DIRECTORY_NOT_FOUND} = STRINGS; +const {MANIFEST_NOT_FOUND, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; + +/** + * Parses `if-check` process arguments. + */ +const validateAndParseIfCheckArgs = () => { + try { + return parse(ARGS, HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * Checks if either `manifest` or `directory` command is provided. + */ +export const parseIfCheckArgs = async () => { + const {manifest, directory} = validateAndParseIfCheckArgs(); + + if (manifest) { + const response = prependFullFilePath(manifest); + const isManifestFileExists = await isFileExists(response); + + if (!isManifestFileExists) { + throw new ParseCliParamsError(MANIFEST_NOT_FOUND); + } + + if (checkIfFileIsYaml(manifest)) { + return {manifest}; + } + + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } else if (directory) { + const isDirExists = await isDirectoryExists(directory); + + if (!isDirExists) { + throw new InvalidDirectoryError(DIRECTORY_NOT_FOUND); + } + + const response = prependFullFilePath(directory); + + return {directory: response}; + } + + throw new MissingCliFlagsError(IF_CHECK_FLAGS_MISSING); +}; diff --git a/src/if-check/util/helpers.ts b/src/if-check/util/helpers.ts new file mode 100644 index 000000000..d94b99477 --- /dev/null +++ b/src/if-check/util/helpers.ts @@ -0,0 +1,17 @@ +import {STRINGS} from '../config'; + +const {IF_CHECK_FAILED, IF_CHECK_SUMMARY_ERROR_MESSAGE} = STRINGS; + +/** + * Logs the failure message from the stdout of an error. + */ +export const logStdoutFailMessage = (error: any, fileName: string) => { + console.log(IF_CHECK_FAILED(fileName)); + + const stdout = error.stdout; + const logs = stdout.split('\n\n'); + const failMessage = logs[logs.length - 1]; + + console.log(failMessage); + return IF_CHECK_SUMMARY_ERROR_MESSAGE(fileName, failMessage); +}; diff --git a/src/if-check/util/npm.ts b/src/if-check/util/npm.ts new file mode 100644 index 000000000..a6137a9d3 --- /dev/null +++ b/src/if-check/util/npm.ts @@ -0,0 +1,49 @@ +import * as path from 'node:path'; + +import {execPromise} from '../../common/util/helpers'; +import {getFileName, removeFileIfExists} from '../../common/util/fs'; + +import {STRINGS} from '../config'; + +const {IF_CHECK_VERIFIED} = STRINGS; + +/** + * Executes a series of npm commands based on the provided manifest file. + */ +export const executeCommands = async (manifest: string, cwd: boolean) => { + // TODO: After release remove isGlobal and appropriate checks + const isGlobal = !!process.env.npm_config_global; + const manifestDirPath = path.dirname(manifest); + const manifestFileName = getFileName(manifest); + const executedManifest = path.join(manifestDirPath, `re-${manifestFileName}`); + const prefixFlag = + process.env.CURRENT_DIR && process.env.CURRENT_DIR !== process.cwd() + ? `--prefix=${path.relative(process.env.CURRENT_DIR!, process.cwd())}` + : ''; + const ifEnv = `${ + isGlobal ? `if-env ${prefixFlag}` : `npm run if-env ${prefixFlag} --` + } -m ${manifest}`; + const ifEnvCommand = cwd ? `${ifEnv} -c` : ifEnv; + const ifRunCommand = `${ + isGlobal ? `if-run ${prefixFlag}` : `npm run if-run ${prefixFlag} --` + } -m ${manifest} -o ${executedManifest}`; + const ifDiffCommand = `${ + isGlobal ? `if-diff ${prefixFlag}` : `npm run if-diff ${prefixFlag} --` + } -s ${executedManifest}.yaml -t ${manifest}`; + const ttyCommand = " node -p 'Boolean(process.stdout.isTTY)'"; + + await execPromise( + `${ifEnvCommand} && ${ifRunCommand} && ${ttyCommand} | ${ifDiffCommand}`, + { + cwd: process.env.CURRENT_DIR || process.cwd(), + } + ); + + if (!cwd) { + await removeFileIfExists(`${manifestDirPath}/package.json`); + } + + await removeFileIfExists(`${executedManifest}.yaml`); + + console.log(IF_CHECK_VERIFIED(path.basename(manifest))); +}; diff --git a/src/if-csv/config/config.ts b/src/if-csv/config/config.ts new file mode 100644 index 000000000..8be9dba75 --- /dev/null +++ b/src/if-csv/config/config.ts @@ -0,0 +1,39 @@ +import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; + +import {STRINGS} from '../../common/config'; + +import {IFCsvArgs} from '../types/process-args'; + +const {DISCLAIMER_MESSAGE} = STRINGS; + +export const CONFIG = { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + output: { + type: String, + optional: true, + alias: 'o', + description: '[path to the csv output file]', + }, + params: { + type: String, + optional: false, + alias: 'p', + description: '[parameter to export]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Csv Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, +}; diff --git a/src/if-csv/config/index.ts b/src/if-csv/config/index.ts new file mode 100644 index 000000000..4972b390b --- /dev/null +++ b/src/if-csv/config/index.ts @@ -0,0 +1,2 @@ +export {CONFIG} from './config'; +export {STRINGS} from './strings'; diff --git a/src/if-csv/config/strings.ts b/src/if-csv/config/strings.ts new file mode 100644 index 000000000..717461c1b --- /dev/null +++ b/src/if-csv/config/strings.ts @@ -0,0 +1,4 @@ +export const STRINGS = { + PARAMS_NOT_PRESENT: 'Parameter not provided.', + FAILURE_MESSAGE_OUTPUTS: 'Manifest outputs are not available!', +}; diff --git a/src/if-csv/index.ts b/src/if-csv/index.ts new file mode 100644 index 000000000..602aabe1e --- /dev/null +++ b/src/if-csv/index.ts @@ -0,0 +1,48 @@ +#!/usr/bin/env node +/* eslint-disable no-process-exit */ +import * as YAML from 'js-yaml'; + +import {parseManifestFromStdin} from '../common/util/helpers'; +import {debugLogger} from '../common/util/debug-logger'; +import {Manifest} from '../common/types/manifest'; +import {logger} from '../common/util/logger'; + +import {generateCsv, getManifestData} from './util/helpers'; +import {parseIfCsvArgs} from './util/args'; +import {CsvOptions} from './types/csv'; + +const IfCsv = async () => { + // Call this function with false parameter to prevent log debug messages. + debugLogger.overrideConsoleMethods(false); + + const pipedManifest = await parseManifestFromStdin(); + const {manifest, output, params} = await parseIfCsvArgs(); + const resolvedManifest = manifest || pipedManifest; + + if (resolvedManifest) { + const manifestData = pipedManifest + ? ((await YAML.load(pipedManifest!)) as Manifest) + : await getManifestData(manifest!); + + const options: CsvOptions = { + tree: manifestData.tree, + context: manifestData, + outputPath: output, + params, + }; + const result = await generateCsv(options); + + if (!output && result) { + console.log(result); + } + } + + process.exit(0); +}; + +IfCsv().catch(error => { + if (error instanceof Error) { + logger.error(error); + process.exit(2); + } +}); diff --git a/src/if-csv/types/csv.ts b/src/if-csv/types/csv.ts new file mode 100644 index 000000000..3e2f969c6 --- /dev/null +++ b/src/if-csv/types/csv.ts @@ -0,0 +1,8 @@ +import {Context} from '../../common/types/manifest'; + +export type CsvOptions = { + tree: any; + context: Context; + outputPath?: string; + params: string; +}; diff --git a/src/if-csv/types/process-args.ts b/src/if-csv/types/process-args.ts new file mode 100644 index 000000000..f0bcb44ad --- /dev/null +++ b/src/if-csv/types/process-args.ts @@ -0,0 +1,5 @@ +export interface IFCsvArgs { + manifest?: string; + output?: string; + params: string; +} diff --git a/src/if-csv/util/args.ts b/src/if-csv/util/args.ts new file mode 100644 index 000000000..4b9674882 --- /dev/null +++ b/src/if-csv/util/args.ts @@ -0,0 +1,60 @@ +import {parse} from 'ts-command-line-args'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {checkIfFileIsYaml} from '../../common/util/yaml'; +import {prependFullFilePath} from '../../common/util/helpers'; + +import {CONFIG, STRINGS} from '../config'; +import {STRINGS as COMMON_STRINGS} from '../../common/config'; + +import {IFCsvArgs} from '../types/process-args'; +import {isFileExists} from '../../common/util/fs'; + +const {ParseCliParamsError, CliSourceFileError} = ERRORS; + +const {ARGS, HELP} = CONFIG; +const {PARAMS_NOT_PRESENT} = STRINGS; +const {SOURCE_IS_NOT_YAML, MANIFEST_NOT_FOUND} = COMMON_STRINGS; + +/** + * Parses `if-csv` process arguments. + */ +const validateAndParseIfCsvArgs = () => { + try { + return parse(ARGS, HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * Checks for `manifest` and `params` flags to be present. + */ +export const parseIfCsvArgs = async () => { + const {manifest, output, params} = validateAndParseIfCsvArgs(); + + if (!params) { + throw new ParseCliParamsError(PARAMS_NOT_PRESENT); + } + + if (manifest) { + const response = prependFullFilePath(manifest); + const isManifestFileExists = await isFileExists(response); + + if (!isManifestFileExists) { + throw new ParseCliParamsError(MANIFEST_NOT_FOUND); + } + + if (checkIfFileIsYaml(manifest)) { + return {manifest, output, params}; + } + + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } + + return {output, params}; +}; diff --git a/src/if-csv/util/helpers.ts b/src/if-csv/util/helpers.ts new file mode 100644 index 000000000..655611ff3 --- /dev/null +++ b/src/if-csv/util/helpers.ts @@ -0,0 +1,87 @@ +import * as fs from 'fs/promises'; +import {stringify} from 'csv-stringify/sync'; +import {PluginParams} from '@grnsft/if-core/types'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {load} from '../../common/lib/load'; + +import {CsvOptions} from '../types/csv'; +import {STRINGS} from '../config'; + +const {FAILURE_MESSAGE_OUTPUTS} = STRINGS; +const {ManifestValidationError} = ERRORS; + +/** + * Gets the manifest data if `outputs` is present in it. + */ +export const getManifestData = async (manifest: string) => { + const {rawManifest} = await load(manifest); + const {children} = rawManifest.tree; + + if (children?.child?.outputs || children?.['child-0']?.outputs) { + return rawManifest; + } + + throw new ManifestValidationError(FAILURE_MESSAGE_OUTPUTS); +}; + +/** + * Generates a CSV file based on the provided tree structure, context, output path, and params. + */ +export const generateCsv = async (options: CsvOptions) => { + const {tree, context, outputPath, params} = options; + const columns = ['Path']; + const matrix = [columns]; + const aggregationIsEnabled = !!context.aggregation; + + const traverseTree = (node: any, params: string, path = 'tree') => { + if (node.aggregated) { + if (path === 'tree') { + columns.push('Aggregated'); + } + matrix.push([`${path}.${params}`, node.aggregated[params]]); + } + + if (node.outputs) { + node.outputs.forEach((output: PluginParams) => { + const {timestamp} = output; + + if (!columns.includes(timestamp)) { + columns.push(timestamp); + } + + const lastRow = matrix[matrix.length - 1]; + + if (aggregationIsEnabled) { + lastRow.push(output[params]); + } else { + if (matrix.length === 1 || lastRow.length === columns.length) { + matrix.push([`${path}.${params}`, output[params]]); + } else { + lastRow.push(output[params]); + } + } + }); + } + + if (node.children) { + Object.keys(node.children).forEach(childKey => { + traverseTree( + node.children![childKey], + params, + `${path}.children.${childKey}` + ); + }); + } + }; + + traverseTree(tree, params); + + const csvString = stringify(matrix, {columns}); + + if (!outputPath) { + return csvString; + } + + return await fs.writeFile(`${outputPath}.csv`, csvString); +}; diff --git a/src/if-diff/config/config.ts b/src/if-diff/config/config.ts new file mode 100644 index 000000000..12f5ade09 --- /dev/null +++ b/src/if-diff/config/config.ts @@ -0,0 +1,33 @@ +import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; + +import {STRINGS} from '../../common/config'; + +import {IFDiffArgs} from '../types/process-args'; + +const {DISCLAIMER_MESSAGE} = STRINGS; + +export const CONFIG = { + ARGS: { + source: { + type: String, + optional: true, + alias: 's', + description: '[path to the source file]', + }, + target: { + type: String, + optional: false, + alias: 't', + description: '[path to the target file', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Diff Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, +}; diff --git a/src/if-diff/config/index.ts b/src/if-diff/config/index.ts new file mode 100644 index 000000000..4972b390b --- /dev/null +++ b/src/if-diff/config/index.ts @@ -0,0 +1,2 @@ +export {CONFIG} from './config'; +export {STRINGS} from './strings'; diff --git a/src/if-diff/config/strings.ts b/src/if-diff/config/strings.ts new file mode 100644 index 000000000..0222ca105 --- /dev/null +++ b/src/if-diff/config/strings.ts @@ -0,0 +1,6 @@ +export const STRINGS = { + INVALID_TARGET: 'Target is invalid.', + INVALID_SOURCE: 'Source is invalid.', + FAILURE_MESSAGE: 'Files do not match!', + SUCCESS_MESSAGE: 'Files match!', +}; diff --git a/src/diff.ts b/src/if-diff/index.ts similarity index 74% rename from src/diff.ts rename to src/if-diff/index.ts index aad02482c..aa4ae1871 100644 --- a/src/diff.ts +++ b/src/if-diff/index.ts @@ -4,16 +4,16 @@ import {loadIfDiffFiles} from './lib/load'; import {compare} from './lib/compare'; import {parseIfDiffArgs} from './util/args'; -import {formatNotMatchingLog, parseManifestFromStdin} from './util/helpers'; -import {validateManifest} from './util/validations'; +import {formatNotMatchingLog} from './util/helpers'; +import {parseManifestFromStdin} from '../common/util/helpers'; +import {validateManifest} from '../common/util/validations'; -import {logger} from './util/logger'; -import {debugLogger} from './util/debug-logger'; +import {logger} from '../common/util/logger'; +import {debugLogger} from '../common/util/debug-logger'; -import {CONFIG} from './config'; +import {STRINGS} from './config'; -const {IF_DIFF} = CONFIG; -const {SUCCESS_MESSAGE, FAILURE_MESSAGE} = IF_DIFF; +const {FAILURE_MESSAGE, SUCCESS_MESSAGE} = STRINGS; const IfDiff = async () => { const pipedSourceManifest = await parseManifestFromStdin(); diff --git a/src/lib/compare.ts b/src/if-diff/lib/compare.ts similarity index 96% rename from src/lib/compare.ts rename to src/if-diff/lib/compare.ts index 34d084488..2f4e9d442 100644 --- a/src/lib/compare.ts +++ b/src/if-diff/lib/compare.ts @@ -1,6 +1,6 @@ import {checkIfEqual, oneIsPrimitive} from '../util/helpers'; -import {Difference} from '../types/lib/compare'; +import {Difference} from '../types/compare'; /** * 1. If objects are not of the same type or are primitive types, compares directly. diff --git a/src/if-diff/lib/load.ts b/src/if-diff/lib/load.ts new file mode 100644 index 000000000..d11cd40e6 --- /dev/null +++ b/src/if-diff/lib/load.ts @@ -0,0 +1,38 @@ +import * as YAML from 'js-yaml'; + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {openYamlFileAsObject} from '../../common/util/yaml'; + +import {STRINGS} from '../config'; + +import {LoadDiffParams} from '../types/args'; +import {Manifest} from '../../common/types/manifest'; + +const {CliSourceFileError} = ERRORS; + +const {INVALID_SOURCE} = STRINGS; + +/** + * Loads files to compare. As a source file checks if data is piped and then decides which one to take. + */ +export const loadIfDiffFiles = async (params: LoadDiffParams) => { + const {sourcePath, targetPath, pipedSourceManifest} = params; + + if (!sourcePath && !pipedSourceManifest) { + throw new CliSourceFileError(INVALID_SOURCE); + } + + const loadFromSource = + sourcePath && (await openYamlFileAsObject(sourcePath!)); + const loadFromSTDIN = + pipedSourceManifest && (await YAML.load(pipedSourceManifest!)); + + const rawSourceManifest = loadFromSource || loadFromSTDIN; + const rawTargetManifest = await openYamlFileAsObject(targetPath); + + return { + rawSourceManifest, + rawTargetManifest, + }; +}; diff --git a/src/types/util/args.ts b/src/if-diff/types/args.ts similarity index 100% rename from src/types/util/args.ts rename to src/if-diff/types/args.ts diff --git a/src/types/lib/compare.ts b/src/if-diff/types/compare.ts similarity index 100% rename from src/types/lib/compare.ts rename to src/if-diff/types/compare.ts diff --git a/src/if-diff/types/process-args.ts b/src/if-diff/types/process-args.ts new file mode 100644 index 000000000..88bb6eb90 --- /dev/null +++ b/src/if-diff/types/process-args.ts @@ -0,0 +1,4 @@ +export interface IFDiffArgs { + source?: string; + target: string; +} diff --git a/src/if-diff/util/args.ts b/src/if-diff/util/args.ts new file mode 100644 index 000000000..e87dc540c --- /dev/null +++ b/src/if-diff/util/args.ts @@ -0,0 +1,61 @@ +import {parse} from 'ts-command-line-args'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {checkIfFileIsYaml} from '../../common/util/yaml'; +import {prependFullFilePath} from '../../common/util/helpers'; + +import {CONFIG, STRINGS} from '../config'; +import {STRINGS as COMMON_STRINGS} from '../../common/config'; + +import {IFDiffArgs} from '../types/process-args'; +import {LoadDiffParams} from '../types/args'; + +const {ParseCliParamsError, CliTargetFileError, CliSourceFileError} = ERRORS; + +const {ARGS, HELP} = CONFIG; +const {INVALID_TARGET} = STRINGS; +const {SOURCE_IS_NOT_YAML, TARGET_IS_NOT_YAML} = COMMON_STRINGS; + +/** + * Parses `if-diff` process arguments. + */ +const validateAndParseIfDiffArgs = () => { + try { + return parse(ARGS, HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * Checks for `source` and `target` flags to be present. + */ +export const parseIfDiffArgs = () => { + const {source, target} = validateAndParseIfDiffArgs(); + + if (target) { + if (source && !checkIfFileIsYaml(source)) { + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } + + if (checkIfFileIsYaml(target)) { + const response: LoadDiffParams = { + targetPath: prependFullFilePath(target), + }; + + if (source && checkIfFileIsYaml(source)) { + response.sourcePath = prependFullFilePath(source); + } + + return response; + } + + throw new CliTargetFileError(TARGET_IS_NOT_YAML); + } + + throw new ParseCliParamsError(INVALID_TARGET); +}; diff --git a/src/if-diff/util/helpers.ts b/src/if-diff/util/helpers.ts new file mode 100644 index 000000000..9552e8f5b --- /dev/null +++ b/src/if-diff/util/helpers.ts @@ -0,0 +1,84 @@ +import {Difference} from '../types/compare'; + +/** + * `If-diff` equality checker. + */ +export const checkIfEqual = (source: any, target: any) => { + if (source === target) { + return true; + } + + if (source === '*' || target === '*') { + return true; + } + + return false; +}; + +/** + * Checks if objects are primitive types. + */ +export const oneIsPrimitive = (source: any, target: any) => { + // eslint-disable-next-line eqeqeq + if (source == null || target == null) { + return true; + } + + return source !== Object(source) && target !== Object(target); +}; + +/** + * Converts given `value` to either `1` or `0`. + */ +const convertToXorable = (value: any) => { + if (typeof value === 'number') { + return value !== 0 ? 1 : 0; + } + + if (typeof value === 'boolean') { + return value ? 1 : 0; + } + + if (typeof value === 'string') { + return value.length > 0 ? 1 : 0; + } + + if (typeof value === 'object') { + return 1; + } + + return 0; +}; + +/** + * If one of the `valuesToCheck` values is undefined, then set `missing`, otherwise `exists`. + */ +const setValuesIfMissing = (response: Difference) => { + const source = convertToXorable(response.source); + const target = convertToXorable(response.target); + + if (source ^ target) { + ['source', 'target'].forEach(value => { + response[value] = response[value] ? 'exists' : 'missing'; + }); + + return response; + } + + return response; +}; + +/** + * Format not matching message for CLI logging. + */ +export const formatNotMatchingLog = (message: Difference) => { + const flattenMessage = setValuesIfMissing(message); + + Object.keys(flattenMessage).forEach(key => { + if (key === 'message' || key === 'path') { + console.log(message[key]); + } else { + console.log(`${key}: ${message[key]}`); + } + }); +}; diff --git a/src/if-env/config/config.ts b/src/if-env/config/config.ts new file mode 100644 index 000000000..f3f21b0c8 --- /dev/null +++ b/src/if-env/config/config.ts @@ -0,0 +1,40 @@ +import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; + +import {IFEnvArgs} from '../types/process-args'; + +import {STRINGS} from '../../common/config'; + +const {DISCLAIMER_MESSAGE} = STRINGS; + +export const CONFIG = { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the manifest file]', + }, + install: { + type: Boolean, + optional: true, + alias: 'i', + description: '[command to install package.json]', + }, + cwd: { + type: Boolean, + optional: true, + alias: 'c', + description: + '[command to generate the package.json in the command working directory]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'IF-Env Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, +}; diff --git a/src/config/env-template.yml b/src/if-env/config/env-template.yml similarity index 87% rename from src/config/env-template.yml rename to src/if-env/config/env-template.yml index f5fd65a6a..dc801cbc1 100644 --- a/src/config/env-template.yml +++ b/src/if-env/config/env-template.yml @@ -2,8 +2,6 @@ name: template manifest # rename me! description: auto-generated template # update description! tags: # add any tags that will help you to track your manifests initialize: - outputs: - - yaml # you can add - csv to export to csv plugins: # add more plugins for your use-case memory-energy-from-memory-util: # you can name this any way you like! method: Coefficient # the name of the function exported from the plugin @@ -16,7 +14,8 @@ tree: children: # add a chile for each distinct component you want to measure child: pipeline: # the pipeline is an ordered list of plugins you want to execute - - memory-energy-from-memory-util # must match the name in initialize! + compute: + - memory-energy-from-memory-util # must match the name in initialize! config: # any plugin specific, node-level config inputs: - timestamp: 2023-12-12T00:00:00.000Z # ISO 8061 string diff --git a/src/if-env/config/index.ts b/src/if-env/config/index.ts new file mode 100644 index 000000000..4972b390b --- /dev/null +++ b/src/if-env/config/index.ts @@ -0,0 +1,2 @@ +export {CONFIG} from './config'; +export {STRINGS} from './strings'; diff --git a/src/if-env/config/strings.ts b/src/if-env/config/strings.ts new file mode 100644 index 000000000..047ba89c0 --- /dev/null +++ b/src/if-env/config/strings.ts @@ -0,0 +1,8 @@ +export const STRINGS = { + FAILURE_MESSAGE: 'Faied to create the environment!', + FAILURE_MESSAGE_TEMPLATE: + 'Faied to create the environment with the template manifest!', + FAILURE_MESSAGE_DEPENDENCIES: 'Manifest dependencies are not available!', + INITIALIZING_PACKAGE_JSON: 'Initializing package.json.', + INSTALLING_NPM_PACKAGES: 'Installing npm packages...', +}; diff --git a/src/env.ts b/src/if-env/index.ts similarity index 79% rename from src/env.ts rename to src/if-env/index.ts index 48b0b76b9..001f769a3 100644 --- a/src/env.ts +++ b/src/if-env/index.ts @@ -1,21 +1,23 @@ #!/usr/bin/env node /* eslint-disable no-process-exit */ -import {parseIfEnvArgs} from './util/args'; -import {logger} from './util/logger'; - -import {CONFIG} from './config'; +import {debugLogger} from '../common/util/debug-logger'; +import {logger} from '../common/util/logger'; +import {STRINGS} from '../common/config'; -import {EnvironmentOptions} from './types/if-env'; import { addTemplateManifest, getOptionsFromArgs, initializeAndInstallLibs, } from './util/helpers'; +import {EnvironmentOptions} from './types/if-env'; +import {parseIfEnvArgs} from './util/args'; -const {IF_ENV} = CONFIG; -const {SUCCESS_MESSAGE} = IF_ENV; +const {SUCCESS_MESSAGE} = STRINGS; const IfEnv = async () => { + // Call this function with false parameter to prevent log debug messages. + debugLogger.overrideConsoleMethods(false); + const commandArgs = await parseIfEnvArgs(); const options: EnvironmentOptions = { folderPath: process.env.CURRENT_DIR || process.cwd(), diff --git a/src/types/if-env.ts b/src/if-env/types/if-env.ts similarity index 100% rename from src/types/if-env.ts rename to src/if-env/types/if-env.ts diff --git a/src/types/npm.ts b/src/if-env/types/npm.ts similarity index 100% rename from src/types/npm.ts rename to src/if-env/types/npm.ts diff --git a/src/if-env/types/process-args.ts b/src/if-env/types/process-args.ts new file mode 100644 index 000000000..7db2117a8 --- /dev/null +++ b/src/if-env/types/process-args.ts @@ -0,0 +1,5 @@ +export interface IFEnvArgs { + manifest?: string; + install?: boolean; + cwd?: boolean; +} diff --git a/src/if-env/util/args.ts b/src/if-env/util/args.ts new file mode 100644 index 000000000..525ae14f6 --- /dev/null +++ b/src/if-env/util/args.ts @@ -0,0 +1,55 @@ +import {parse} from 'ts-command-line-args'; + +import {ERRORS} from '@grnsft/if-core/utils'; + +import {isFileExists} from '../../common/util/fs'; +import {prependFullFilePath} from '../../common/util/helpers'; +import {checkIfFileIsYaml} from '../../common/util/yaml'; + +import {CONFIG} from '../config'; +import {STRINGS as COMMON_STRINGS} from '../../common/config'; + +import {IFEnvArgs} from '../types/process-args'; + +const {ParseCliParamsError, CliSourceFileError} = ERRORS; +const {ARGS, HELP} = CONFIG; +const {MANIFEST_NOT_FOUND, SOURCE_IS_NOT_YAML} = COMMON_STRINGS; + +/** + * Parses `if-env` process arguments. + */ +const validateAndParseIfEnvArgs = () => { + try { + return parse(ARGS, HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * Checks if the `manifest` command is provided and it is valid manifest file. + */ +export const parseIfEnvArgs = async () => { + const {manifest, install, cwd} = validateAndParseIfEnvArgs(); + + if (manifest) { + const response = prependFullFilePath(manifest); + const isManifestFileExists = await isFileExists(response); + + if (!isManifestFileExists) { + throw new ParseCliParamsError(MANIFEST_NOT_FOUND); + } + + if (checkIfFileIsYaml(manifest)) { + return {manifest: response, install, cwd}; + } + + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } + + return {install, cwd}; +}; diff --git a/src/if-env/util/helpers.ts b/src/if-env/util/helpers.ts new file mode 100644 index 000000000..e8ff24914 --- /dev/null +++ b/src/if-env/util/helpers.ts @@ -0,0 +1,92 @@ +/* eslint-disable no-process-exit */ +import * as path from 'path'; +import * as fs from 'fs/promises'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import { + extractPathsWithVersion, + initPackageJsonIfNotExists, + installDependencies, + updatePackageJsonDependencies, + updatePackageJsonProperties, +} from './npm'; +import {load} from '../../common/lib/load'; + +import {STRINGS} from '../config'; + +import {EnvironmentOptions} from '../types/if-env'; + +const {MissingManifestDependenciesError} = ERRORS; +const { + FAILURE_MESSAGE_DEPENDENCIES, + FAILURE_MESSAGE, + FAILURE_MESSAGE_TEMPLATE, +} = STRINGS; + +/** + * Gets the folder path of the manifest file, dependencies from manifest file and install argument from the given arguments. + */ +export const getOptionsFromArgs = async (commandArgs: { + manifest: string; + install: boolean | undefined; +}) => { + const {manifest, install} = commandArgs; + const folderPath = path.dirname(manifest); + const loadedManifest = await load(manifest); + const rawManifest = loadedManifest.rawManifest; + const plugins = rawManifest?.initialize?.plugins || {}; + const dependencies = rawManifest?.execution?.environment.dependencies || []; + + if (!dependencies.length) { + throw new MissingManifestDependenciesError(FAILURE_MESSAGE_DEPENDENCIES); + } + + const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); + + return { + folderPath, + dependencies: pathsWithVersion, + install, + }; +}; + +/** + * Creates folder if not exists, installs dependencies if required, update depenedencies. + */ +export const initializeAndInstallLibs = async (options: EnvironmentOptions) => { + try { + const {folderPath, install, cwd, dependencies} = options; + const packageJsonPath = await initPackageJsonIfNotExists(folderPath); + + await updatePackageJsonProperties(packageJsonPath, cwd); + + if (install) { + await installDependencies(folderPath, dependencies); + } else { + await updatePackageJsonDependencies(packageJsonPath, dependencies, cwd); + } + } catch (error) { + console.log(FAILURE_MESSAGE); + process.exit(2); + } +}; + +/** + * Adds a manifest template to the folder where the if-env CLI command runs. + */ +export const addTemplateManifest = async (destinationDir: string) => { + try { + const templateManifest = path.resolve( + __dirname, + '../config/env-template.yml' + ); + + const destinationPath = path.resolve(destinationDir, 'manifest.yml'); + const data = await fs.readFile(templateManifest, 'utf-8'); + + await fs.writeFile(destinationPath, data, 'utf-8'); + } catch (error) { + console.log(FAILURE_MESSAGE_TEMPLATE); + process.exit(1); + } +}; diff --git a/src/util/npm.ts b/src/if-env/util/npm.ts similarity index 65% rename from src/util/npm.ts rename to src/if-env/util/npm.ts index de27ae4de..c5f272255 100644 --- a/src/util/npm.ts +++ b/src/if-env/util/npm.ts @@ -3,22 +3,16 @@ import * as fs from 'fs/promises'; import * as path from 'path'; -import {execPromise} from './helpers'; -import { - isDirectoryExists, - getFileName, - isFileExists, - removeFileIfExists, -} from './fs'; -import {logger} from './logger'; +import {execPromise} from '../../common/util/helpers'; +import {isDirectoryExists, isFileExists} from '../../common/util/fs'; +import {logger} from '../../common/util/logger'; import {STRINGS} from '../config'; import {ManifestPlugin, PathWithVersion} from '../types/npm'; -const packageJson = require('../../package.json'); +const packageJson = require('../../../package.json'); -const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES, IF_CHECK_VERIFIED} = - STRINGS; +const {INITIALIZING_PACKAGE_JSON, INSTALLING_NPM_PACKAGES} = STRINGS; /** * Checks if the package.json is exists, if not, initializes it. @@ -85,6 +79,7 @@ export const updatePackageJsonDependencies = async ( packageJsonPath, JSON.stringify(parsedPackageJson, null, 2) ); + await fs.appendFile(packageJsonPath, '\n'); }; /** @@ -128,7 +123,7 @@ export const updatePackageJsonProperties = async ( const parsedPackageJsonContent = JSON.parse(packageJsonContent); const properties = { - name: 'if-environment', + name: packageJson.name || 'if-environment', description: packageJson.description, author: packageJson.author, bugs: packageJson.bugs, @@ -150,44 +145,3 @@ export const updatePackageJsonProperties = async ( JSON.stringify(newPackageJson, null, 2) ); }; - -/** - * Executes a series of npm commands based on the provided manifest file. - */ -export const executeCommands = async (manifest: string, cwd: boolean) => { - // TODO: After release remove isGlobal and appropriate checks - const isGlobal = !!process.env.npm_config_global; - const manifestDirPath = path.dirname(manifest); - const manifestFileName = getFileName(manifest); - const executedManifest = path.join(manifestDirPath, `re-${manifestFileName}`); - const prefixFlag = - process.env.CURRENT_DIR && process.env.CURRENT_DIR !== process.cwd() - ? `--prefix=${path.relative(process.env.CURRENT_DIR!, process.cwd())}` - : ''; - const ifEnv = `${ - isGlobal ? `if-env ${prefixFlag}` : `npm run if-env ${prefixFlag} --` - } -m ${manifest}`; - const ifEnvCommand = cwd ? `${ifEnv} -c` : ifEnv; - const ifRunCommand = `${ - isGlobal ? `if-run ${prefixFlag}` : `npm run if-run ${prefixFlag} --` - } -m ${manifest} -o ${executedManifest}`; - const ifDiffCommand = `${ - isGlobal ? `if-diff ${prefixFlag}` : `npm run if-diff ${prefixFlag} --` - } -s ${executedManifest}.yaml -t ${manifest}`; - const ttyCommand = " node -p 'Boolean(process.stdout.isTTY)'"; - - await execPromise( - `${ifEnvCommand} && ${ifRunCommand} && ${ttyCommand} | ${ifDiffCommand}`, - { - cwd: process.env.CURRENT_DIR || process.cwd(), - } - ); - - if (!cwd) { - await removeFileIfExists(`${manifestDirPath}/package.json`); - } - - await removeFileIfExists(`${executedManifest}.yaml`); - - console.log(IF_CHECK_VERIFIED(path.basename(manifest))); -}; diff --git a/src/builtins/coefficient/README.md b/src/if-run/builtins/coefficient/README.md similarity index 69% rename from src/builtins/coefficient/README.md rename to src/if-run/builtins/coefficient/README.md index 4579cd5b8..903a15496 100644 --- a/src/builtins/coefficient/README.md +++ b/src/if-run/builtins/coefficient/README.md @@ -16,6 +16,22 @@ Three parameters are required in global config: `input-parameter`, `coefficient` - `coefficient`: the value to multiply `input-parameter` by. - `output-parameter`: a string defining the name to use to add the product of the input parameters to the output array. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` +of the parameters of the inputs and outputs + +- `inputs`: describe parameters of the `input-parameter` of the global config. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe parameters of the `output-parameter` of the global config. Each parameter has: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs All of `input-parameters` must be available in the input array. @@ -53,15 +69,13 @@ const result = coeff.execute([ ## Example manifest -IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. The following is an example manifest that calls `coefficient`: +IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example manifest that calls `coefficient`: ```yaml name: coefficient-demo description: tags: initialize: - outputs: - - yaml plugins: coefficient: method: Coefficient @@ -70,13 +84,21 @@ initialize: input-parameter: 'carbon' coefficient: 3 output-parameter: 'carbon-product' + parameter-metadata: + inputs: + carbon: + description: "an amount of carbon emitted into the atmosphere" + unit: "gCO2e" + outputs: + carbon-product: + description: "a product of cabon property and the coefficient" + unit: "gCO2e" tree: children: child: pipeline: - - coefficient - config: - coefficient: + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -86,12 +108,11 @@ tree: You can run this example by saving it as `./examples/manifests/coefficient.yml` and executing the following command from the project root: ```sh -if-run --manifest ./examples/manifests/coefficient.yml --output ./examples/outputs/coefficient.yml +if-run --manifest ./examples/manifests/coefficient.yml --output ./examples/outputs/coefficient ``` The results will be saved to a new `yaml` file in `./examples/outputs` - ## Errors `Coefficient` exposes one of the IF error classes. @@ -101,10 +122,11 @@ The results will be saved to a new `yaml` file in `./examples/outputs` You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: + - `input-parameter`: this must be a string - `coefficient`: this must be a number - `output-parameter`: this must be a string You can fix this error by checking you are providing valid values for each parameter in the config. -For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors) diff --git a/src/builtins/coefficient/index.ts b/src/if-run/builtins/coefficient/index.ts similarity index 86% rename from src/builtins/coefficient/index.ts rename to src/if-run/builtins/coefficient/index.ts index 2d4f6a1c9..177eeece3 100644 --- a/src/builtins/coefficient/index.ts +++ b/src/if-run/builtins/coefficient/index.ts @@ -3,19 +3,25 @@ import {ERRORS} from '@grnsft/if-core/utils'; import { CoefficientConfig, ExecutePlugin, + PluginParametersMetadata, PluginParams, } from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; const {GlobalConfigError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; -export const Coefficient = (globalConfig: CoefficientConfig): ExecutePlugin => { +export const Coefficient = ( + globalConfig: CoefficientConfig, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/copy-param/README.md b/src/if-run/builtins/copy-param/README.md similarity index 70% rename from src/builtins/copy-param/README.md rename to src/if-run/builtins/copy-param/README.md index 78ab5f656..2f76c98da 100644 --- a/src/builtins/copy-param/README.md +++ b/src/if-run/builtins/copy-param/README.md @@ -39,6 +39,21 @@ Three parameters are required in config: `from` and `to` and `keep-existing`. `to`: a string defining the name to use to add the result of summing the input parameters to the output array. `keep-existing`: toggles whether to keep or delete the copied parameter (defined in `to`) +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameter of the `from` of the global config. The parameter has the following attributes: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameters of the `to` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs As with all plugins, `timestamp` and `duration` are required. The key passed to `from` must exist in the `input` data. @@ -47,24 +62,28 @@ As with all plugins, `timestamp` and `duration` are required. The key passed to The plugin adds a new parameter with the name defined in `to` to the `input` data. - ## Implementation To run the plugin, you must first create an instance of `Copy`. Then, you can call `execute()`. ```typescript -import { Copy } from "."; +import {Copy} from '.'; -const plugin = Copy({ 'keep-existing': true, from: 'from-param', to: 'to-param' }); +const plugin = Copy({ + 'keep-existing': true, + from: 'from-param', + to: 'to-param', +}); -const result = plugin.execute([{ - timestamp: "2023-12-12T00:00:13.000Z", +const result = plugin.execute([ + { + timestamp: '2023-12-12T00:00:13.000Z', duration: 30, 'from-param': 'hello', -}]) - -console.log(result) + }, +]); +console.log(result); ``` ## Example manifest @@ -88,11 +107,11 @@ tree: children: child-1: pipeline: - - copy-param + compute: + - copy-param inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' original: 'hello' - ``` You can run this example by saving it as `./manifests/examples/copy.yml` and executing the following command from the project root: diff --git a/src/builtins/copy-param/index.ts b/src/if-run/builtins/copy-param/index.ts similarity index 84% rename from src/builtins/copy-param/index.ts rename to src/if-run/builtins/copy-param/index.ts index 0602f6f75..7f10bf696 100644 --- a/src/builtins/copy-param/index.ts +++ b/src/if-run/builtins/copy-param/index.ts @@ -1,8 +1,12 @@ import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParametersMetadata, + PluginParams, +} from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; @@ -12,9 +16,14 @@ const {GlobalConfigError} = ERRORS; // from-param: the parameter you are copying from (e.g. cpu/name) // to-field: the parameter you are copying to (e.g. cpu/processor-name) -export const Copy = (globalConfig: Record): ExecutePlugin => { +export const Copy = ( + globalConfig: Record, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/csv-lookup/README.md b/src/if-run/builtins/csv-lookup/README.md similarity index 86% rename from src/builtins/csv-lookup/README.md rename to src/if-run/builtins/csv-lookup/README.md index 3663f4123..434e234a2 100644 --- a/src/builtins/csv-lookup/README.md +++ b/src/if-run/builtins/csv-lookup/README.md @@ -13,19 +13,17 @@ For example, for the following CSV: | 2022 | Google Cloud | asia-east2 | Hong Kong | HK | HK | Hong Kong | 22.3,114.2 | 0.28 | | | 0 | 453 | | | | 360 | | 2022 | Google Cloud | asia-northeast1 | Tokyo | JP-TK | JP-TK | Tokyo | 35.6897,139.692 | 0.28 | | | 0 | 463 | | | | 463 | - You could select all the data for the cloud provider `Google Cloud` in the region `asia-east2` using the following configuration: ```yaml filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: - cloud-provider: "cloud/provider" - cloud-region: "cloud/region" -output: "*" + cloud-provider: 'cloud/provider' + cloud-region: 'cloud/region' +output: '*' ``` -Notice that the query parameters are key/value pairs where the key is the column name in the target CSV and the value is a **reference to a value** in your `input` data (*not* an actual value - a reference). This is to enable you to chain CSV lookups together based on information from other plugins in your pipeline. - +Notice that the query parameters are key/value pairs where the key is the column name in the target CSV and the value is a **reference to a value** in your `input` data (_not_ an actual value - a reference). This is to enable you to chain CSV lookups together based on information from other plugins in your pipeline. ## Parameters @@ -33,28 +31,43 @@ Notice that the query parameters are key/value pairs where the key is the column - `filepath` - path to a csv file, either on the local filesystem or on the internet - `query` - an array of key/value pairs where the key is a column name in the target csv and the value is a parameter from inputs -- `output` - the columns to grab data from and add to output data - should support wildcard or multiple values. +- `output` - the columns to grab data from and add to output data - should support wildcard or multiple values. -The plugin also supports data renaming. This means you can grab data from a named column but push it into your manifest file data under another name, for example, maybe we want to grab data from the `processor-name` column int he target csv and add it to the manifest file data as `processor-id` because this is the name expected by some other plugin in your piepline. You can do this by passing comma separated values in arrays. +The plugin also supports data renaming. This means you can grab data from a named column but push it into your manifest file data under another name, for example, maybe we want to grab data from the `processor-name` column int he target csv and add it to the manifest file data as `processor-id` because this is the name expected by some other plugin in your piepline. You can do this by passing comma separated values in arrays. ```yaml -output: - ["processor-name": "processor-id"] +output: ['processor-name': 'processor-id'] ``` You can nest arrays to do this renaming for multiple columns. ```yaml output: - [["processor-name", "processor-model-id"],["tdp","thermal-design-power"]] + [['processor-name', 'processor-model-id'], ['tdp', 'thermal-design-power']] ``` All the following values are valid for the `output` field: + - `"*"` - `"tdp"` - `["processor-name", "processor-model-id"]` - `[["processor-name", "processor-model-id"],["tdp","thermal-design-power"]]` +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameters of the `inputs`. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameters in the `output` of the config block. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs There are no strict requirements on input for this plugin because they depend upon the contents of the target CSV and your input data at the time the CSV lookup is invoked. Please make sure you are requesting data from columns that exist in the target csv file and that your query values are available in your `input` data. @@ -86,7 +99,7 @@ const globalConfig = { }, output: ['cpu-tdp', 'tdp'], }; -const divide = CSVLookup(globalConfig); +const csvLookup = CSVLookup(globalConfig); const input = [ { @@ -108,8 +121,6 @@ name: csv-lookup-demo description: tags: initialize: - outputs: - - yaml plugins: cloud-metadata: method: CSVLookup @@ -117,14 +128,15 @@ initialize: global-config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: - cloud-provider: "cloud/provider" - cloud-region: "cloud/region" - output: "*" + cloud-provider: 'cloud/provider' + cloud-region: 'cloud/region' + output: '*' tree: children: child: pipeline: - - cloud-metadata + compute: + - cloud-metadata inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -141,7 +153,6 @@ if-run --manifest manifests/plugins/csv-lookup.yml --output manifests/outputs/cs The results will be saved to a new `yaml` file in `manifests/outputs`. - ## Errors Coefficient exposes six of the IF error classes. @@ -171,10 +182,11 @@ This error arises due to problems parsing CSV data into IF. This can occur when You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: + - `filepath`: This must be a path to a csv file - `query`: this must be an array of key-value pairs where the key is a string containing a column name an the value is a string containing the name of a value in `inputs` - `output`: this must be a string containing a name or a wildcard character (`"*"`) You can fix this error by checking you are providing valid values for each parameter in the config. -For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors) diff --git a/src/builtins/csv-lookup/index.ts b/src/if-run/builtins/csv-lookup/index.ts similarity index 94% rename from src/builtins/csv-lookup/index.ts rename to src/if-run/builtins/csv-lookup/index.ts index 8243a6391..e227eaadf 100644 --- a/src/builtins/csv-lookup/index.ts +++ b/src/if-run/builtins/csv-lookup/index.ts @@ -5,9 +5,13 @@ import axios from 'axios'; import {z} from 'zod'; import {parse} from 'csv-parse/sync'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParametersMetadata, + PluginParams, +} from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; @@ -28,9 +32,14 @@ const { CSVParseError, } = ERRORS; -export const CSVLookup = (globalConfig: any): ExecutePlugin => { +export const CSVLookup = ( + globalConfig: any, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/divide/README.md b/src/if-run/builtins/divide/README.md similarity index 77% rename from src/builtins/divide/README.md rename to src/if-run/builtins/divide/README.md index 9a4d04f36..abb534c8f 100644 --- a/src/builtins/divide/README.md +++ b/src/if-run/builtins/divide/README.md @@ -12,6 +12,21 @@ You provide the names of the values you want to divide, and a name to use to add - `denominator` - a parameter by a specific configured number or the number by which `numerator` is divided - `output` - the number to a configured output parameter +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameter of the `numerator` of the global config. The parameter has the following attributes: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameter of the `denominator` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs - `numerator` - as input parameter, must be available in the input array @@ -24,7 +39,7 @@ You provide the names of the values you want to divide, and a name to use to add The plugin throws an exception if the division result is not a number. ->Note: Plugin will warn and return `numerator` value in case if `denominator` is zero. +> Note: Plugin will warn and return `numerator` value in case if `denominator` is zero. ## Calculation @@ -42,7 +57,7 @@ const globalConfig = { denominator: 2, output: 'cpu/number-cores', }; -const divide = Divide(globalConfig); +const divide = Divide(globalConfig, parametersMetadata); const input = [ { @@ -62,8 +77,6 @@ name: divide-demo description: tags: initialize: - outputs: - - yaml plugins: divide: method: Divide @@ -76,9 +89,8 @@ tree: children: child: pipeline: - - divide - config: - divide: + compute: + - divide inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -103,6 +115,7 @@ The results will be saved to a new `yaml` file in `./examples/outputs`. You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: + - `numerator`: a string containing the name of the input parameter whose value should be divided by `denominator` - `denominator`: a number to use as the denominator - ``output`: a string containing the name to assign the result of the division @@ -113,9 +126,9 @@ You can fix this error by checking you are providing valid values for each param This error arises when a necessary piece of input data is missing from the `inputs` array. Every element in the ``inputs` array must contain: + - `timestamp` - `duration` - whatever value you passed to `numerator` - For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/divide/index.ts b/src/if-run/builtins/divide/index.ts similarity index 86% rename from src/builtins/divide/index.ts rename to src/if-run/builtins/divide/index.ts index 453a50921..3d09c130c 100644 --- a/src/builtins/divide/index.ts +++ b/src/if-run/builtins/divide/index.ts @@ -1,17 +1,27 @@ import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParams, + ConfigParams, + PluginParametersMetadata, +} from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; const {GlobalConfigError, MissingInputDataError} = ERRORS; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, ZERO_DIVISION} = STRINGS; -export const Divide = (globalConfig: ConfigParams): ExecutePlugin => { +export const Divide = ( + globalConfig: ConfigParams, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/exponent/README.md b/src/if-run/builtins/exponent/README.md similarity index 78% rename from src/builtins/exponent/README.md rename to src/if-run/builtins/exponent/README.md index 91d938cef..fddb55ffa 100644 --- a/src/builtins/exponent/README.md +++ b/src/if-run/builtins/exponent/README.md @@ -16,6 +16,21 @@ Three parameters are required in global config: `input-parameter`, `exponent` an `exponent`: a number defining the exponent. `output-parameter`: a string defining the name to use to add the result of the exponent to the output array. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameter of the `input-parameter` of the global config. The parameter has the following attributes: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes:: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs `input-parameter` and `exponent` must be available in the input array. @@ -63,8 +78,6 @@ name: exponent demo description: tags: initialize: - outputs: - - yaml plugins: exponent: method: Exponent @@ -77,9 +90,8 @@ tree: children: child: pipeline: - - exponent - config: - exponent: + compute: + - exponent inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -91,7 +103,7 @@ You can run this example by saving it as `manifests/examples/test/exponent.yml` ```sh npm i -g @grnsft/if -if-run --manifest manifests/examples/test/exponent.yml --output manifests/outputs/exponent.yml +if-run --manifest manifests/examples/test/exponent.yml --output manifests/outputs/exponent ``` The results will be saved to a new `yaml` file in `manifests/outputs`. @@ -109,9 +121,8 @@ Every element in the `inputs` array must contain: - `duration` - whatever value you passed to `input-parameter` - ### `InputValidationError` This error arises when an invalid value is passed to `Exponent`. Typically, this can occur when a non-numeric value (such as a string made of alphabetic characters) is passed where a number or numeric string is expected. Please check that the types are correct for all the relevant fields in your `inputs` array. -For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors) diff --git a/src/builtins/exponent/index.ts b/src/if-run/builtins/exponent/index.ts similarity index 85% rename from src/builtins/exponent/index.ts rename to src/if-run/builtins/exponent/index.ts index ce4e959ef..a821c3a14 100644 --- a/src/builtins/exponent/index.ts +++ b/src/if-run/builtins/exponent/index.ts @@ -3,13 +3,19 @@ import { ExecutePlugin, PluginParams, ExponentConfig, + PluginParametersMetadata, } from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; -export const Exponent = (globalConfig: ExponentConfig): ExecutePlugin => { +export const Exponent = ( + globalConfig: ExponentConfig, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/export-log.ts b/src/if-run/builtins/export-log.ts similarity index 86% rename from src/builtins/export-log.ts rename to src/if-run/builtins/export-log.ts index fee267dc2..0f1dad33a 100644 --- a/src/builtins/export-log.ts +++ b/src/if-run/builtins/export-log.ts @@ -1,6 +1,6 @@ import * as YAML from 'js-yaml'; -import {Context} from '../types/manifest'; +import {Context} from '../../common/types/manifest'; export const ExportLog = () => { /** diff --git a/src/builtins/export-yaml.ts b/src/if-run/builtins/export-yaml.ts similarity index 57% rename from src/builtins/export-yaml.ts rename to src/if-run/builtins/export-yaml.ts index 624e1298f..bcbc00804 100644 --- a/src/builtins/export-yaml.ts +++ b/src/if-run/builtins/export-yaml.ts @@ -1,18 +1,15 @@ import {ERRORS} from '@grnsft/if-core/utils'; -import {saveYamlFileAs} from '../util/yaml'; +import {saveYamlFileAs} from '../../common/util/yaml'; import {STRINGS} from '../config'; -import {Context} from '../types/manifest'; +import {Context} from '../../common/types/manifest'; const {ExhaustOutputArgError} = ERRORS; const {OUTPUT_REQUIRED, EXPORTING_TO_YAML_FILE} = STRINGS; export const ExportYaml = () => { - /** Takes string before hashtag. */ - const stripHashtag = (path: string) => path.split('#')[0]; - /** * Saves output file in YAML format. */ @@ -25,11 +22,14 @@ export const ExportYaml = () => { ...context, tree, }; - const path = stripHashtag(outputPath); + const pathWithoutExtension = + outputPath.split('.').length > 1 + ? outputPath.split('.').slice(0, -1).join('.') + : outputPath; - console.debug(EXPORTING_TO_YAML_FILE(path)); + console.debug(EXPORTING_TO_YAML_FILE(pathWithoutExtension)); - await saveYamlFileAs(outputFile, `${path}.yaml`); + await saveYamlFileAs(outputFile, `${pathWithoutExtension}.yaml`); }; return {execute}; diff --git a/src/builtins/index.ts b/src/if-run/builtins/index.ts similarity index 93% rename from src/builtins/index.ts rename to src/if-run/builtins/index.ts index d22491aca..36a414c95 100644 --- a/src/builtins/index.ts +++ b/src/if-run/builtins/index.ts @@ -1,5 +1,3 @@ -export {GroupBy} from './group-by'; -export {TimeSync} from './time-sync'; export {Interpolation} from './interpolation'; export {MockObservations} from './mock-observations'; export {Divide} from './divide'; @@ -14,3 +12,4 @@ export {CSVLookup} from './csv-lookup'; export {Shell} from './shell'; export {Regex} from './regex'; export {Copy} from './copy-param'; +export {TimeSync} from './time-sync'; diff --git a/src/builtins/interpolation/README.md b/src/if-run/builtins/interpolation/README.md similarity index 88% rename from src/builtins/interpolation/README.md rename to src/if-run/builtins/interpolation/README.md index 498c26c6e..243b70e70 100644 --- a/src/builtins/interpolation/README.md +++ b/src/if-run/builtins/interpolation/README.md @@ -25,6 +25,21 @@ To employ the `Interpolation` plugin, adhere to these steps: `x` and `y` arrays must be equal lengths. +## Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameter of the `input-parameter` of the global config. The parameter has the following attributes: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameters of the `output-parameter` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ## Input Parameters The plugin expects the following input parameters: @@ -102,8 +117,6 @@ name: interpolation-demo description: simple demo of interpolation plugin tags: initialize: - outputs: - - yaml plugins: interpolation: method: Interpolation @@ -118,7 +131,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 @@ -132,8 +146,6 @@ name: interpolation-demo description: simple demo of interpolation plugin tags: initialize: - outputs: - - yaml plugins: interpolation: method: Interpolation @@ -148,7 +160,8 @@ tree: children: child: pipeline: - - interpolation + compute: + - interpolation inputs: - timestamp: 2023-07-06T00:00 duration: 3600 @@ -178,6 +191,7 @@ if-run --manifest ./manifests/examples/interpolation.yml --output ./manifests/ou You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: + - `method`: a string containing either `linear`, `spline` or `polynomial` - `x`: an array of numbers - `y`: an array of numbers @@ -189,6 +203,7 @@ You can fix this error by checking you are providing valid values for each param ### Validation errors There are also several validation errors that can arise, including: + - if the lengths of `x` and `y` are not equal - if `x` or `y` are empty - if the requested point to interpolate at is outside the range of `x` diff --git a/src/builtins/interpolation/index.ts b/src/if-run/builtins/interpolation/index.ts similarity index 93% rename from src/builtins/interpolation/index.ts rename to src/if-run/builtins/interpolation/index.ts index 98a84ad8a..60fe90e5b 100644 --- a/src/builtins/interpolation/index.ts +++ b/src/if-run/builtins/interpolation/index.ts @@ -6,9 +6,10 @@ import { PluginParams, ConfigParams, Method, + PluginParametersMetadata, } from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; @@ -20,7 +21,16 @@ const { WITHIN_THE_RANGE, } = STRINGS; -export const Interpolation = (globalConfig: ConfigParams): ExecutePlugin => { +export const Interpolation = ( + globalConfig: ConfigParams, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { + const metadata = { + kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, + }; + /** * Executes the energy consumption calculation for an array of input parameters. */ @@ -189,9 +199,7 @@ export const Interpolation = (globalConfig: ConfigParams): ExecutePlugin => { }; return { - metadata: { - kind: 'execute', - }, + metadata, execute, }; }; diff --git a/src/builtins/mock-observations/README.md b/src/if-run/builtins/mock-observations/README.md similarity index 74% rename from src/builtins/mock-observations/README.md rename to src/if-run/builtins/mock-observations/README.md index 1af4cd6d6..53bd89924 100644 --- a/src/builtins/mock-observations/README.md +++ b/src/if-run/builtins/mock-observations/README.md @@ -17,6 +17,21 @@ The mode currently mocks 2 types of observation data: - `generators` define which fields to generate for each observation - `components` define the components for which to generate observations. The observations generated according to `timestamp-from`, `timestamp-to`, `duration` and `generators` will be duplicated for each component. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameters of the `inputs`. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the output parameters. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Authentication N/A @@ -47,15 +62,13 @@ const result = await mockObservations.execute([]); ### manifest Example -IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `mock-observation`: +IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `mock-observation`: ```yaml name: mock-observation-demo description: example invoking mock-observation plugin tags: initialize: - outputs: - - yaml plugins: mock-observations: kind: plugin @@ -83,7 +96,8 @@ tree: children: child: pipeline: - - mock-observations + observe: + - mock-observations inputs: ``` diff --git a/src/builtins/mock-observations/helpers/common-generator.ts b/src/if-run/builtins/mock-observations/helpers/common-generator.ts similarity index 100% rename from src/builtins/mock-observations/helpers/common-generator.ts rename to src/if-run/builtins/mock-observations/helpers/common-generator.ts diff --git a/src/builtins/mock-observations/helpers/rand-int-generator.ts b/src/if-run/builtins/mock-observations/helpers/rand-int-generator.ts similarity index 100% rename from src/builtins/mock-observations/helpers/rand-int-generator.ts rename to src/if-run/builtins/mock-observations/helpers/rand-int-generator.ts diff --git a/src/builtins/mock-observations/index.ts b/src/if-run/builtins/mock-observations/index.ts similarity index 93% rename from src/builtins/mock-observations/index.ts rename to src/if-run/builtins/mock-observations/index.ts index c7488de0f..facc78353 100644 --- a/src/builtins/mock-observations/index.ts +++ b/src/if-run/builtins/mock-observations/index.ts @@ -5,18 +5,24 @@ import { PluginParams, ConfigParams, ObservationParams, + PluginParametersMetadata, } from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {CommonGenerator} from './helpers/common-generator'; import {RandIntGenerator} from './helpers/rand-int-generator'; import {Generator} from './interfaces/index'; -export const MockObservations = (globalConfig: ConfigParams): ExecutePlugin => { +export const MockObservations = ( + globalConfig: ConfigParams, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** @@ -55,7 +61,7 @@ export const MockObservations = (globalConfig: ConfigParams): ExecutePlugin => { const schema = z.object({ 'timestamp-from': z.string(), 'timestamp-to': z.string(), - duration: z.number(), + duration: z.number().gt(0), components: z.array(z.record(z.string())), generators: z.object({ common: z.record(z.string().or(z.number())), diff --git a/src/builtins/mock-observations/interfaces/index.ts b/src/if-run/builtins/mock-observations/interfaces/index.ts similarity index 100% rename from src/builtins/mock-observations/interfaces/index.ts rename to src/if-run/builtins/mock-observations/interfaces/index.ts diff --git a/src/builtins/multiply/README.md b/src/if-run/builtins/multiply/README.md similarity index 76% rename from src/builtins/multiply/README.md rename to src/if-run/builtins/multiply/README.md index 7e9c2dfbd..dc6cde7c7 100644 --- a/src/builtins/multiply/README.md +++ b/src/if-run/builtins/multiply/README.md @@ -15,6 +15,21 @@ Two parameters are required in global config: `input-parameters` and `output-par `input-parameters`: an array of strings. Each string should match an existing key in the `inputs` array `output-parameter`: a string defining the name to use to add the product of the input parameters to the output array. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe parameters of the `input-parameters` of the global config. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs All of `input-parameters` must be available in the input array. @@ -41,8 +56,8 @@ const config = { outputParameter: 'energy-product', }; -const mult = Multiply(config); -const result = await mult.execute([ +const multiply = Multiply(config, parametersMetadata); +const result = await multiply.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -61,8 +76,6 @@ name: multiply-demo description: tags: initialize: - outputs: - - yaml plugins: multiply: method: Multiply @@ -74,9 +87,8 @@ tree: children: child: pipeline: - - multiply - config: - multiply: + compute: + - multiply inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -93,7 +105,6 @@ if-run --manifest ./examples/manifests/test/multiply.yml --output ./examples/out The results will be saved to a new `yaml` file in `./examples/outputs` - ## Errors `Multiply` uses one of the IF error classes. @@ -102,6 +113,7 @@ The results will be saved to a new `yaml` file in `./examples/outputs` This error arises when a necessary piece of input data is missing from the `inputs` array. Every element in the `inputs` array must contain: + - `timestamp` - `duration` - whatever values you passed to `input-parameters` diff --git a/src/builtins/multiply/index.ts b/src/if-run/builtins/multiply/index.ts similarity index 85% rename from src/builtins/multiply/index.ts rename to src/if-run/builtins/multiply/index.ts index cd2a74837..b51c1a426 100644 --- a/src/builtins/multiply/index.ts +++ b/src/if-run/builtins/multiply/index.ts @@ -3,13 +3,19 @@ import { ExecutePlugin, PluginParams, MultiplyConfig, + PluginParametersMetadata, } from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; -export const Multiply = (globalConfig: MultiplyConfig): ExecutePlugin => { +export const Multiply = ( + globalConfig: MultiplyConfig, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/regex/README.md b/src/if-run/builtins/regex/README.md similarity index 76% rename from src/builtins/regex/README.md rename to src/if-run/builtins/regex/README.md index a0fa5d624..2c6a4cd88 100644 --- a/src/builtins/regex/README.md +++ b/src/if-run/builtins/regex/README.md @@ -16,20 +16,34 @@ IntelĀ® XeonĀ® Platinum 8272CL,IntelĀ® XeonĀ® 8171M 2.1 GHz,IntelĀ® XeonĀ® E5-26 - `match` - a regex by which needs to match the `parameter` - `output` - output parameter name in the input +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameter of the `parameter` value of the global config. The parameter has the following attributes: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameters of the `output` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs - `parameter` - as input parameter, must be available in the input array ## Returns -- `output`: the first match of `parameter` with the parameter name with `match` defined in global config. +- `output`: The match of the `parameter` value using the `match` regex defined in the global config. If the `match` regex includes the global flag (`g`), a string containing all matches separated by spaces. ## Implementation To run the plugin, you must first create an instance of `Regex`. Then, you can call `execute()`. ```typescript - const globalConfig = { parameter: 'physical-processor', match: '^[^,]+', @@ -56,8 +70,6 @@ name: regex-demo description: tags: initialize: - outputs: - - yaml plugins: regex: method: Regex @@ -70,9 +82,8 @@ tree: children: child: pipeline: - - regex - config: - regex: + compute: + - regex inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -96,11 +107,11 @@ The results will be saved to a new `yaml` file in `manifests/outputs`. This error arises when a necessary piece of input data is missing from the `inputs` array. Every element in the `inputs` array must contain: + - `timestamp` - `duration` - whatever value you passed to `parameter` - ### `GlobalConfigError` You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. @@ -113,7 +124,6 @@ The required parameters are: You can fix this error by checking you are providing valid values for each parameter in the config. - ### `RegexMismatchError` This error arises when the requested regex cannot find any matches in the given data. If there are multiple matches, the plugin returns the first, but if there are none, it throws this error. diff --git a/src/builtins/regex/index.ts b/src/if-run/builtins/regex/index.ts similarity index 79% rename from src/builtins/regex/index.ts rename to src/if-run/builtins/regex/index.ts index 8d382ec74..0076b6cfe 100644 --- a/src/builtins/regex/index.ts +++ b/src/if-run/builtins/regex/index.ts @@ -1,17 +1,27 @@ import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParams, + ConfigParams, + PluginParametersMetadata, +} from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; const {MissingInputDataError, GlobalConfigError, RegexMismatchError} = ERRORS; const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; -export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { +export const Regex = ( + globalConfig: ConfigParams, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** @@ -80,13 +90,13 @@ export const Regex = (globalConfig: ConfigParams): ExecutePlugin => { } const regex = eval(match); - const matchedItem = input[parameter].match(regex); + const matchedItems = input[parameter].match(regex); - if (!matchedItem || !matchedItem[0]) { + if (!matchedItems || matchedItems.length === 0) { throw new RegexMismatchError(REGEX_MISMATCH(input[parameter], match)); } - return matchedItem[0]; + return matchedItems.join(' '); }; return { diff --git a/src/builtins/sci-embodied/README.md b/src/if-run/builtins/sci-embodied/README.md similarity index 86% rename from src/builtins/sci-embodied/README.md rename to src/if-run/builtins/sci-embodied/README.md index 97b6ce4af..a4c304c6d 100644 --- a/src/builtins/sci-embodied/README.md +++ b/src/if-run/builtins/sci-embodied/README.md @@ -10,6 +10,21 @@ Read more on [embodied carbon](https://github.com/Green-Software-Foundation/sci/ Not Needed +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameters of the `inputs`. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the `carbon-embodied` parameter. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs - `device/emissions-embodied`: the sum of Life Cycle Assessment (LCA) emissions for the component @@ -79,8 +94,6 @@ name: sci-embodied description: simple demo invoking sci-embodied tags: initialize: - outputs: - - yaml plugins: sci-embodied: method: SciEmbodied @@ -89,7 +102,8 @@ tree: children: child: pipeline: - - sci-embodied # duration & config -> embodied + compute: + - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq device/expected-lifespan: 3 # 3 years in seconds @@ -109,7 +123,6 @@ if-run --manifest manifests/plugins/sci-embodied.yml --output manifests/outputs/ The results will be saved to a new `yaml` file in `./examples/outputs`. - ## Errors `SciEmbodied` uses one of IF's error classes @@ -120,5 +133,4 @@ This error class is used to describe a problem with one of the input values to ` You will receive a specific error message explaining which parameter is problematic, and you can check and replace where appropriate. - For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/builtins/sci-embodied/index.ts b/src/if-run/builtins/sci-embodied/index.ts similarity index 64% rename from src/builtins/sci-embodied/index.ts rename to src/if-run/builtins/sci-embodied/index.ts index a88017e9f..b304d4ae1 100644 --- a/src/builtins/sci-embodied/index.ts +++ b/src/if-run/builtins/sci-embodied/index.ts @@ -1,15 +1,65 @@ import {z} from 'zod'; -import {ExecutePlugin, PluginParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + ParameterMetadata, + PluginParametersMetadata, + PluginParams, +} from '@grnsft/if-core/types'; -import {validate, allDefined} from '../../util/validations'; +import {validate, allDefined} from '../../../common/util/validations'; import {STRINGS} from '../../config'; const {SCI_EMBODIED_ERROR} = STRINGS; -export const SciEmbodied = (): ExecutePlugin => { +export const SciEmbodied = ( + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: { + ...({ + 'device/emissions-embodied': { + description: 'total embodied emissions of some component', + unit: 'gCO2e', + 'aggregation-method': 'sum', + }, + 'device/expected-lifespan': { + description: 'Total Expected Lifespan of the Component in Seconds', + unit: 'seconds', + 'aggregation-method': 'sum', + }, + 'resources-reserved': { + description: 'resources reserved for an application', + unit: 'count', + 'aggregation-method': 'none', + }, + 'resources-total': { + description: 'total resources available', + unit: 'count', + 'aggregation-method': 'none', + }, + 'vcpus-allocated': { + description: 'number of vcpus allocated to particular resource', + unit: 'count', + 'aggregation-method': 'none', + }, + 'vcpus-total': { + description: + 'total number of vcpus available on a particular resource', + unit: 'count', + 'aggregation-method': 'none', + }, + } as ParameterMetadata), + ...parametersMetadata?.inputs, + }, + outputs: parametersMetadata?.outputs || { + 'carbon-embodied': { + description: 'embodied emissions of the component', + unit: 'gCO2e', + 'aggregation-method': 'sum', + }, + }, }; const METRICS = [ diff --git a/src/builtins/sci/README.md b/src/if-run/builtins/sci/README.md similarity index 75% rename from src/builtins/sci/README.md rename to src/if-run/builtins/sci/README.md index 592bd1ed0..31b15afed 100644 --- a/src/builtins/sci/README.md +++ b/src/if-run/builtins/sci/README.md @@ -8,9 +8,22 @@ - `functional-unit`: the name of the functional unit in which to express the carbon impact (required) +### Plugin parameter metadata -### Inputs +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe parameters of the `inputs`. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the `sci` parameter which has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) +### Inputs - `carbon`: total carbon in gCO2eq (required) - `functional-unit`: whatever `functional-unit` you define in global config also has to be present in each input, for example if you provide `functional-unit: requests` in global config, `requests` must be present in your input data. @@ -19,7 +32,7 @@ - `sci`: carbon expressed in terms of the given functional unit ->Note: Plugin will warn and return `carbon` value in case if `functional-unit`'s value is zero. +> Note: Plugin will warn and return `carbon` value in case if `functional-unit`'s value is zero. ## Calculation @@ -29,7 +42,6 @@ SCI is calculated as: sci = carbon / functional unit ``` - ## IF Implementation To run the plugin, you must first create an instance of `Sci`. Then, you can call `execute()` to return `sci`. @@ -60,8 +72,6 @@ name: sci-demo description: example invoking sci plugin tags: initialize: - outputs: - - yaml plugins: sci: method: Sci @@ -72,8 +82,8 @@ tree: children: child: pipeline: - - sci - config: + compute: + - sci inputs: - timestamp: 2023-07-06T00:00 carbon: 5 @@ -99,14 +109,14 @@ The results will be saved to a new `yaml` file. This error arises when a necessary piece of input data is missing from the `inputs` array. Every element in the `inputs` array must contain: + - `timestamp` - `duration` - `carbon`: a numeric value named `carbon` must exist in the inputs array - whatever value you passed to `functional-unit` - ### Validation errors There is also a validation step that checks that the `functional-unit` was provided in the plugin config. If you see an error reporting this value as missing, please check you have provided it. -For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors) diff --git a/src/builtins/sci/config.ts b/src/if-run/builtins/sci/config.ts similarity index 100% rename from src/builtins/sci/config.ts rename to src/if-run/builtins/sci/config.ts diff --git a/src/builtins/sci/index.ts b/src/if-run/builtins/sci/index.ts similarity index 65% rename from src/builtins/sci/index.ts rename to src/if-run/builtins/sci/index.ts index 47d495351..1106f3725 100644 --- a/src/builtins/sci/index.ts +++ b/src/if-run/builtins/sci/index.ts @@ -1,8 +1,14 @@ import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParams, + ConfigParams, + PluginParametersMetadata, + ParameterMetadata, +} from '@grnsft/if-core/types'; -import {validate, allDefined} from '../../util/validations'; +import {validate, allDefined} from '../../../common/util/validations'; import {STRINGS} from '../../config'; @@ -14,9 +20,35 @@ const { ZERO_DIVISION, } = STRINGS; -export const Sci = (globalConfig: ConfigParams): ExecutePlugin => { +export const Sci = ( + globalConfig: ConfigParams, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: { + ...({ + carbon: { + description: 'an amount of carbon emitted into the atmosphere', + unit: 'gCO2e', + 'aggregation-method': 'sum', + }, + 'functional-unit': { + description: + 'the name of the functional unit in which the final SCI value should be expressed, e.g. requests, users', + unit: 'none', + 'aggregation-method': 'sum', + }, + } as ParameterMetadata), + ...parametersMetadata?.inputs, + }, + outputs: parametersMetadata?.outputs || { + sci: { + description: 'carbon expressed in terms of the given functional unit', + unit: 'gCO2e', + 'aggregation-method': 'sum', + }, + }, }; /** diff --git a/src/builtins/shell/README.md b/src/if-run/builtins/shell/README.md similarity index 87% rename from src/builtins/shell/README.md rename to src/if-run/builtins/shell/README.md index bacba1255..d114adb3b 100644 --- a/src/builtins/shell/README.md +++ b/src/if-run/builtins/shell/README.md @@ -22,6 +22,21 @@ The `shell` plugin interface requires a path to the plugin command. This path is - `command`: the path to the plugin executable along with the execution command as it would be entered into a shell. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe the parameters of the `inputs`. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the output parameter. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs The parameters included in the `inputs` field in the `manifest` depend entirely on the plugin itself. A typical plugin might expect the following common data to be provided as `inputs`: @@ -66,8 +81,6 @@ name: shell-demo description: tags: initialize: - outputs: - - yaml plugins: sampler: method: Shell @@ -78,7 +91,8 @@ tree: children: child: pipeline: - - sampler + compute: + - sampler inputs: - timestamp: 2023-07-06T00:00 duration: 1 # Secs @@ -94,8 +108,6 @@ name: shell-demo description: tags: initialize: - outputs: - - yaml plugins: sampler: method: Shell @@ -106,7 +118,8 @@ tree: children: child: pipeline: - - sampler + compute: + - sampler inputs: - timestamp: 2023-07-06T00:00 duration: 1 # Secs @@ -129,7 +142,6 @@ if-run --manifest manifests/plugins/shell.yml --output manifests/outputs/shell.y The results will be saved to a new `yaml` file. - ## Errors `Shell` uses one of the error classes provided by IF diff --git a/src/builtins/shell/index.ts b/src/if-run/builtins/shell/index.ts similarity index 81% rename from src/builtins/shell/index.ts rename to src/if-run/builtins/shell/index.ts index 352ee6bcd..9abe8363c 100644 --- a/src/builtins/shell/index.ts +++ b/src/if-run/builtins/shell/index.ts @@ -3,15 +3,25 @@ import {spawnSync, SpawnSyncReturns} from 'child_process'; import {loadAll, dump} from 'js-yaml'; import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParams, + ConfigParams, + PluginParametersMetadata, +} from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; const {ProcessExecutionError} = ERRORS; -export const Shell = (globalConfig: ConfigParams): ExecutePlugin => { +export const Shell = ( + globalConfig: ConfigParams, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/subtract/README.md b/src/if-run/builtins/subtract/README.md similarity index 79% rename from src/builtins/subtract/README.md rename to src/if-run/builtins/subtract/README.md index 8ff9a922f..776526c7e 100644 --- a/src/builtins/subtract/README.md +++ b/src/if-run/builtins/subtract/README.md @@ -15,6 +15,21 @@ Two parameters are required in global config: `input-parameters` and `output-par `input-parameters`: an array of strings. Each string should match an existing key in the `inputs` array `output-parameter`: a string defining the name to use to add the result of the diff to the output array. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe parameters of the `input-parameters` of the global config. Each parameter has the following attributes: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs All of `input-parameters` must be available in the input array. @@ -61,8 +76,6 @@ name: subtract demo description: tags: initialize: - outputs: - - yaml plugins: subtract: method: Subtract @@ -74,9 +87,8 @@ tree: children: child: pipeline: - - subtract - config: - subtract: + compute: + - subtract inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -93,7 +105,6 @@ if-run --manifest /manifests/plugins/subtract.yml --output manifests/outputs/sub The results will be saved to a new `yaml` file in `manifests/outputs`. - ## Errors `Subtract` uses one of IF's error classes: diff --git a/src/builtins/subtract/index.ts b/src/if-run/builtins/subtract/index.ts similarity index 87% rename from src/builtins/subtract/index.ts rename to src/if-run/builtins/subtract/index.ts index 797b259c1..2598ecb8a 100644 --- a/src/builtins/subtract/index.ts +++ b/src/if-run/builtins/subtract/index.ts @@ -1,15 +1,21 @@ import {z} from 'zod'; import { ExecutePlugin, + PluginParametersMetadata, PluginParams, SubtractConfig, } from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; -export const Subtract = (globalConfig: SubtractConfig): ExecutePlugin => { +export const Subtract = ( + globalConfig: SubtractConfig, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/builtins/sum/README.md b/src/if-run/builtins/sum/README.md similarity index 71% rename from src/builtins/sum/README.md rename to src/if-run/builtins/sum/README.md index ea0f0113d..14ad1c336 100644 --- a/src/builtins/sum/README.md +++ b/src/if-run/builtins/sum/README.md @@ -15,6 +15,21 @@ Two parameters are required in global config: `input-parameters` and `output-par `input-parameters`: an array of strings. Each string should match an existing key in the `inputs` array `output-parameter`: a string defining the name to use to add the result of summing the input parameters to the output array. +### Plugin parameter metadata + +The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs + +- `inputs`: describe parameters of the `input-parameters` of the global config. Each parameter has: + + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + +- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: + - `description`: description of the parameter + - `unit`: unit of the parameter + - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + ### Inputs All of `input-parameters` must be available in the input array. @@ -39,7 +54,7 @@ const config = { outputParameter: 'energy', }; -const sum = Sum(config); +const sum = Sum(config, parametersMetadata); const result = sum.execute([ { timestamp: '2021-01-01T00:00:00Z', @@ -59,8 +74,6 @@ name: sum demo description: tags: initialize: - outputs: - - yaml plugins: sum: method: Sum @@ -68,13 +81,27 @@ initialize: global-config: input-parameters: ['cpu/energy', 'network/energy'] output-parameter: 'energy' + parameter-metadata: + inputs: + cpu/energy: + description: energy consumed by the cpu + unit: kWh + aggregation-method: sum + network/energy: + description: energy consumed by data ingress and egress + unit: kWh + aggregation-method: sum + outputs: + energy: + description: sum of energy components + unit: kWh + aggregation-method: sum tree: children: child: pipeline: - - sum - config: - sum: + compute: + - sum inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -90,7 +117,6 @@ if-run --manifest ./examples/manifests/sum.yml --output ./examples/outputs/sum.y The results will be saved to a new `yaml` file in `./examples/outputs`. - ## Errors `Sum` exposes two of the IF error classes. @@ -100,6 +126,7 @@ The results will be saved to a new `yaml` file in `./examples/outputs`. You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: + - `input-parameters`: this must be an array of strings, each being the name of a value in the `inputs` array - `output-parameter`: this must be a string @@ -114,5 +141,4 @@ Every element in the ``inputs` array must contain: - `duration` - whatever values you passed to `input-parameters` - For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors). diff --git a/src/builtins/sum/index.ts b/src/if-run/builtins/sum/index.ts similarity index 83% rename from src/builtins/sum/index.ts rename to src/if-run/builtins/sum/index.ts index 57437ba31..da642dd13 100644 --- a/src/builtins/sum/index.ts +++ b/src/if-run/builtins/sum/index.ts @@ -1,17 +1,27 @@ import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import {ExecutePlugin, PluginParams, SumConfig} from '@grnsft/if-core/types'; +import { + ExecutePlugin, + PluginParams, + SumConfig, + PluginParametersMetadata, +} from '@grnsft/if-core/types'; -import {validate} from '../../util/validations'; +import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; const {GlobalConfigError} = ERRORS; const {MISSING_GLOBAL_CONFIG} = STRINGS; -export const Sum = (globalConfig: SumConfig): ExecutePlugin => { +export const Sum = ( + globalConfig: SumConfig, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: parametersMetadata?.inputs, + outputs: parametersMetadata?.outputs, }; /** diff --git a/src/if-run/builtins/time-sync/README.md b/src/if-run/builtins/time-sync/README.md new file mode 100644 index 000000000..335866546 --- /dev/null +++ b/src/if-run/builtins/time-sync/README.md @@ -0,0 +1,267 @@ +## Time-sync + +Time sync standardizes the start time, end time and temporal resolution of all output data across an entire tree. + +### Parameters + +### Plugin config + +The following should be defined in the plugin initialization: + +- `start-time`: global start time as ISO 8061 string +- `stop`: global end time as ISO 8061 string +- `interval`: temporal resolution in seconds +- `error-on-padding`: avoid zero/'zeroish' padding (if needed) and error out instead. `False` by defult. + +#### Inputs: + +- `inputs`: an array of observations + +#### Returns + +- `inputs`: time-synchronized version of the tree + +#### Overview + +A manifest file for a tree might contain many nodes each representing some different part of an application's stack or even different applications running on different machines. It is therefore common to have time series data in each component that is not directly comparable to other components either because the temporal resolution of the data is different, they cover different periods, or there are gaps in some records (e.g. some apps might burst but then go dormant, while others run continuously). This makes post-hoc visualization, analysis and aggregation of data from groups of nodes difficult to achieve. To address this, we created a time synchronization plugin that takes in non-uniform times series and snaps them all to a regular timeline with uniform start time, end time and temporal resolution. + +We do this by implementing the following logic: + +- Shift readings to nearest whole seconds +- Upsample the time series to a base resolution (1s) +- Resample to desired resolution by batching 1s entries +- Extrapolate or trim to ensure all time series share global start and end dates + +The next section explains each stage in more detail. + +#### Details + +##### Upsampling rules + +A set of `inputs` is naturally a time series because all `observations` include a `timestamp` and a `duration`, measured in seconds. +For each `observation` in `inputs` we check whether the duration is greater than 1 second. If `duration` is greater than 1 second, we create N new `observation` objects, where N is equal to `duration`. This means we have an `observation` for every second between the initial timestamp and the end of the observation period. Each new object receives a timestamp incremented by one second. + +This looks as follows: + +```ts +[{timestamp: '2023-12-12T00:00:00.000Z', duration: 5}] + +# becomes +[ + {timestamp: '2023-12-12T00:00:01.000Z', duration: 1} + {timestamp: '2023-12-12T00:00:02.000Z', duration: 1} + {timestamp: '2023-12-12T00:00:03.000Z', duration: 1} + {timestamp: '2023-12-12T00:00:04.000Z', duration: 1} + {timestamp: '2023-12-12T00:00:05.000Z', duration: 1} +] +``` + +Each `observation` actually includes many key-value pairs. The precise content of the `observation` is not known until runtime because it depends on which plugins have been included in the pipeline. Different values have to be treated differently when we upsample in time. The method we use to upsample depends on the `aggregation-method` defined for each key in `units.yml`. + +If the right way to aggregate a value is to sum it, then the right way to upsample it is to divide by `duration`, effectively spreading the total out evenly across the new, higher resolution, `observations` so that the total across the same bucket of time is unchanged (i.e. if the total for some value is 10 when there is one entry with `duration = 10s`, then the total should still be 10 when there are 10 entries each witch `duration = 1s`). + +On the other hand, if the right way to aggregate a value is to take its average over some time period, the value should be copied unchanged into the newly upsampled `observations`. This is appropriate for values that are proportional or percentages, such as `cpu/utilization`. Treating these values as constants means the average over the `duration` for an observation is identical whether you consider the initial `observation` or the upsampled set of N `observation`s. + +Constants can simply be copied as-is, because they are constants. Examples might be the `grid/carbon-intensity` - this value does not change depending on how frequently you observe it. + +Therefore, we apply this logic and the resulting flow looks as follows (the `aggregation-method` for `carbon` and `energy` is `sum`, `grid/carbon-intensity` is a constant and `cpu/utilization` is expressed as a percentage): + +```ts +[{timestamp: '2023-12-12T00:00:00.000Z', duration: 5, 'cpu/utilization': 12, carbon: 5, energy: 10, 'grid/carbon-intensity': 471}] + +# becomes + +[ + {timestamp: '2023-12-12T00:00:00.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, + {timestamp: '2023-12-12T00:00:01.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, + {timestamp: '2023-12-12T00:00:02.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, + {timestamp: '2023-12-12T00:00:03.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, + {timestamp: '2023-12-12T00:00:04.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471}, + {timestamp: '2023-12-12T00:00:05.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, 'grid/carbon-intensity': 471} +] +``` + +The end result is that for each `observation`, we upsample the time series to yield 1 second resolution data between `timestamp` and `timestamp + duration`. + +##### Gap-filling + +Sometimes there might be discontinuities in the time series between one `observation` and another. For example we might have two `observations` in a set of `inputs` that have timestamps spaced 10 seconds apart, but the `duration` of the first `observation` is only 5 seconds. in this case, 5 seconds of data are unaccounted for and create a discontinuity in the time series. + +To solve this problem, for all but the first `observation` in the `inputs` array, we grab the `timestamp` and `duration` from the previous `observation` and check that `timestamp[N] + duration[N] == timestamp[N+1]`. If this condition is not satisfied, we backfill the missing data with a "zero-observation" which is identical to the surrounding observations except any values whose `aggregation-method` is `sum` are set to zero. This is equivalent to assuming that when there is no data available, the app being monitored is switched off. + +The end result of this gap-filling is that we have continuous 1 second resolution data that can be resampled to a new temporal resolution. + +```ts +[ + {timestamp: '2023-12-12T00:00:00.000Z', duration: 5, 'cpu/utilization': 12, carbon: 5, energy: 10, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:08.000Z', duration: 2, 'cpu/utilization': 12, carbon: 5, energy: 10, grid/carbon-intensity: 471} +] + +# There are 2 seconds of missing data between the end of timestamp[0] + duration, and timestamp[1] +# After expansion and infilling, the array becomes: + +[ + {timestamp: '2023-12-12T00:00:00.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:01.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:02.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:03.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:04.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:05.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:06.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:07.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:08.000Z', duration: 1, 'cpu/utilization': 12, carbon: 2.5, energy: 5, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:09.000Z', duration: 1, 'cpu/utilization': 12, carbon: 2.5, energy: 5, grid/carbon-intensity: 471} +] +``` + +Note that when `error-on-padding` is `true` no gap-filling is performed and the plugin will error out instead. + +##### Trimming and padding + +To ensure parity across all the components in a tree, we need to synchronize the start and end times for all time series. To do this, we pass the `time-sync` plugin plugin some global config: `startTime`, `endTime` and `interval`. The `startTime` is the timestamp where _all_ input arrays across the entire tree should begin, and `endTime` is the timestamp where _all_ input arrays across the entire tree should end. `interval` is the time resolution we ultimately want to resample to. + +To synchronize the time series start and end we check the first element of `inputs` for each node in the tree and determine whether it is earlier, later or equal to the global start time. If it is equal then no action is required. If the `input` start time is earlier than the global start time, we simply discard entries from the front of the array until the start times are aligned. If the `input` start time is after the global start time, then we pad with our "zero-observation" object - one for every second separating the global start time from the `input` start time. The same process is repeated for the end time - we either trim away `input` data or pad it out with "zero-observation" objects. + +For example, for `startTime = 2023-12-12T00:00:00.000Z` and `endTime = 2023-12-12T00:00:15.000Z`: + +```ts +[ + {timestamp: '2023-12-12T00:00:05.000Z', duration: 5, 'cpu/utilization': 12, carbon: 5, energy: 10, 'grid/carbon-intensity': 471}, +] + +# There are 5 seconds missing from the start and end. After padding, the array becomes: + +[ + {timestamp: '2023-12-12T00:00:00.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:01.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:02.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:03.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:04.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:05.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:06.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:07.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:08.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:09.000Z', duration: 1, 'cpu/utilization': 12, carbon: 1, energy: 2, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:10.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:11.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:12.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:13.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + {timestamp: '2023-12-12T00:00:14.000Z', duration: 1, 'cpu/utilization': 0, carbon: 0, energy: 0, grid/carbon-intensity: 471}, + +] +``` + +Note that when `error-on-padding` is `true` no padding is performed and the plugin will error out instead. + +##### Resampling rules + +Now we have synchronized, continuous, high resolution time series data, we can resample. To achieve this, we use `interval`, which sets the global temporal resolution for the final, processed time series. `interval` is expressed in units of seconds, which means we can simply batch `observations` together in groups of size `interval`. For each value in each object we either sum, average or copy the values into one single summary object representing each time bucket of size `interval` depending on their `aggregation-method` defined in `aggregation` section in the manifest file. The returned array is the final, synchronized time series at the desired temporal resolution. + +#### Assumptions and limitations + +To do time synchronization, we assume: + +- There is no environmental impact for an application when there is no data available. +- Evenly distributing the total for a `duration` across higher resolution `observations` is appropriate, as opposed to having some non-uniform distribution. + +### Typescript implementation + +To run the plugin, you must first create an instance of `TimeSync`. +Then, you can call `execute()`. + +```typescript +const globalConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:30.000Z', + interval: 10 +} +const timeSync = TimeSync(globalConfig); +const results = timeSync.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z' + duration: 10 + 'cpu/utilization': 10 + carbon: 100 + energy: 100 + requests: 300 + }, + { + timestamp: '2023-12-12T00:00:10.000Z' + duration: 10 + 'cpu/utilization': 20 + carbon: 100, + energy: 100, + requests: 380 + } +]) +``` + +### Example manifest + +IF users will typically call the plugin as part of a pipeline defined in an `manifest` +file. In this case, instantiating and configuring the plugin is handled by +`if-run` and does not have to be done explicitly by the user. +The following is an example `manifest` that calls `time-sync`: + +```yaml +name: time-sync-demo +description: impl with 2 levels of nesting with non-uniform timing of observations +tags: +initialize: + plugins: + teads-curve: + method: TeadsCurve + path: '@grnsft/if-unofficial-plugins' + sci-e: + method: SciE + path: '@grnsft/if-plugins' + sci-embodied: + path: 'builtin' + method: SciEmbodied + sci-o: + method: SciO + path: '@grnsft/if-plugins' + time-sync: + method: TimeSync + path: builtin + global-config: + start-time: '2023-12-12T00:00:00.000Z' # ISO timestamp + end-time: '2023-12-12T00:01:00.000Z' # ISO timestamp + interval: 5 # seconds +tree: + children: + child: # an advanced grouping node + pipeline: + compute: + - teads-curve + - sci-e + - sci-embodied + - sci-o + - time-sync + config: + teads-curve: + cpu/thermal-design-power: 65 + sci-embodied: + device/emissions-embodied: 251000 # gCO2eq + time-reserved: 3600 # 1 hour in s + device/expected-lifespan: 126144000 # 4 years in seconds + resources-reserved: 1 + resources-total: 1 + sci-o: + grid/carbon-intensity: 457 # gCO2/kwh + children: + child-1: + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 10 + cpu/utilization: 10 + carbon: 100 + energy: 100 + requests: 300 + - timestamp: '2023-12-12T00:00:10.000Z' + duration: 10 + cpu/utilization: 20 + carbon: 200 + energy: 200 + requests: 380 +``` diff --git a/src/builtins/time-sync.ts b/src/if-run/builtins/time-sync/index.ts similarity index 86% rename from src/builtins/time-sync.ts rename to src/if-run/builtins/time-sync/index.ts index 8b37241b8..27f0aac8e 100644 --- a/src/builtins/time-sync.ts +++ b/src/if-run/builtins/time-sync/index.ts @@ -9,13 +9,14 @@ import { PaddingReceipt, TimeNormalizerConfig, TimeParams, + PluginParametersMetadata, + ParameterMetadata, } from '@grnsft/if-core/types'; -import {parameterize} from '../lib/parameterize'; +import {validate} from '../../../common/util/validations'; -import {validate} from '../util/validations'; - -import {STRINGS} from '../config'; +import {STRINGS} from '../../config'; +import {getAggregationMethod} from '../../lib/aggregate'; Settings.defaultZone = 'utc'; @@ -36,9 +37,43 @@ const { INVALID_DATETIME, } = STRINGS; -export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { +/** + * Time synchronization plugin converted into framework integrated tool. + * It can't be requested in `initialize.plugins` section anymore. Instead describe configuration in context. + * @example + * ```yaml + * name: time-sync + * description: sample in time sync lib + * tags: sample, time, sync + * time-sync: + * start-time: '2023-12-12T00:00:00.000Z' + * end-time: '2023-12-12T00:01:00.000Z' + * interval: 5 + * allow-padding: true + * ``` + */ +export const TimeSync = ( + globalConfig: TimeNormalizerConfig, + parametersMetadata: PluginParametersMetadata +): ExecutePlugin => { const metadata = { kind: 'execute', + inputs: { + ...({ + timestamp: { + description: 'refers to the time of occurrence of the input', + unit: 'RFC3339', + 'aggregation-method': 'none', + }, + duration: { + description: 'refers to the duration of the input', + unit: 'seconds', + 'aggregation-method': 'sum', + }, + } as ParameterMetadata), + ...parametersMetadata?.inputs, + }, + outputs: parametersMetadata?.outputs, }; /** @@ -115,14 +150,16 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return resampleInputs(sortedInputs, timeParams) as PluginParams[]; }; + /** + * Dates are passed to `time-sync` both in ISO 8601 format + * and as a Date object (from the deserialization of a YAML file). + * If the YAML parser fails to identify as a date, it passes as a string. + */ const parseDate = (date: Date | string) => { if (!date) { return DateTime.invalid('Invalid date'); } - // dates are passed to time-sync.ts both in ISO 8601 format - // and as a Date object (from the deserialization of a YAML file) - // if the YAML parser fails to identify as a date, it passes as a string if (isDate(date)) { return DateTime.fromJSDate(date); } @@ -200,7 +237,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const inputKeys = Object.keys(input); return inputKeys.reduce((acc, key) => { - const method = parameterize.getAggregationMethod(key); + const method = getAggregationMethod(key); if (key === 'timestamp') { const perSecond = normalizeTimePerSecond(input.timestamp, i); @@ -254,7 +291,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return acc; } - const method = parameterize.getAggregationMethod(metric); + const method = getAggregationMethod(metric); if (method === 'avg' || method === 'sum') { acc[metric] = 0; @@ -313,8 +350,7 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { const metrics = Object.keys(input); metrics.forEach(metric => { - const method = parameterize.getAggregationMethod(metric); - acc[metric] = acc[metric] ?? 0; + let method = getAggregationMethod(metric); if (metric === 'timestamp') { acc[metric] = inputs[0][metric]; @@ -322,6 +358,16 @@ export const TimeSync = (globalConfig: TimeNormalizerConfig): ExecutePlugin => { return; } + if (metric === 'duration') { + method = 'sum'; + } + + if (!method) { + return; + } + + acc[metric] = acc[metric] ?? 0; + if (method === 'sum') { acc[metric] += input[metric]; diff --git a/src/if-run/config/config.ts b/src/if-run/config/config.ts new file mode 100644 index 000000000..9bd39d89f --- /dev/null +++ b/src/if-run/config/config.ts @@ -0,0 +1,75 @@ +import {ArgumentConfig, ParseOptions} from 'ts-command-line-args'; + +import {STRINGS} from '../../common/config'; + +import {IfRunArgs} from '../types/process-args'; + +const {DISCLAIMER_MESSAGE} = STRINGS; + +export const CONFIG = { + ARGS: { + manifest: { + type: String, + optional: true, + alias: 'm', + description: '[path to the input file]', + }, + output: { + type: String, + optional: true, + alias: 'o', + description: '[path to the output file]', + }, + 'override-params': { + type: String, + optional: true, + alias: 'p', + description: '[path to a parameter file that overrides our defaults]', + }, + 'no-output': { + type: Boolean, + optional: true, + alias: 'n', + description: '[prevent output to the console]', + }, + help: { + type: Boolean, + optional: true, + alias: 'h', + description: '[prints out the above help instruction]', + }, + debug: { + type: Boolean, + optional: true, + alias: 'd', + description: '[prints out debug logs to the console]', + }, + observe: { + type: Boolean, + optional: true, + description: '[executes only observe pipeline of the manifest]', + }, + regroup: { + type: Boolean, + optional: true, + description: '[executes only regroup pipeline of the manifest]', + }, + compute: { + type: Boolean, + optional: true, + description: '[executes only compute pipeline of the manifest]', + }, + } as ArgumentConfig, + HELP: { + helpArg: 'help', + headerContentSections: [ + {header: 'Impact Framework', content: 'Helpful keywords:'}, + ], + footerContentSections: [ + {header: 'Green Software Foundation', content: DISCLAIMER_MESSAGE}, + ], + } as ParseOptions, + GITHUB_PATH: 'https://github.com', + NATIVE_PLUGIN: 'if-plugins', + AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], +}; diff --git a/src/if-run/config/index.ts b/src/if-run/config/index.ts new file mode 100644 index 000000000..4972b390b --- /dev/null +++ b/src/if-run/config/index.ts @@ -0,0 +1,2 @@ +export {CONFIG} from './config'; +export {STRINGS} from './strings'; diff --git a/src/config/strings.ts b/src/if-run/config/strings.ts similarity index 61% rename from src/config/strings.ts rename to src/if-run/config/strings.ts index b6f3410b4..6231e9ff7 100644 --- a/src/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -1,21 +1,11 @@ -import {ManifestParameter} from '../types/manifest'; - export const STRINGS = { - MANIFEST_IS_MISSING: 'Manifest is missing.', MISSING_METHOD: "Initalization param 'method' is missing.", MISSING_PATH: "Initalization param 'path' is missing.", UNSUPPORTED_PLUGIN: "Plugin interface doesn't implement 'execute' or 'metadata' methods.", - OVERRIDE_WARNING: - '\n**WARNING**: You are overriding the IF default parameters file. Please be extremely careful of unintended side-effects in your plugin pipeline!\n', - DISCLAIMER_MESSAGE: ` -Incubation Project - -This project is an incubation project being run inside the Green Software Foundation; as such, we DONā€™T recommend using it in any critical use case. -Incubation projects are experimental, offer no support guarantee, have minimal governance and process, and may be retired at any moment. This project may one day graduate, in which case this disclaimer will be removed.`, NOT_NATIVE_PLUGIN: (path: string) => ` -You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`, + You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`, INVALID_MODULE_PATH: (path: string, error?: any) => `Provided module \`${path}\` is invalid or not found. ${error ?? ''} `, @@ -33,34 +23,66 @@ You are using plugin ${path} which is not part of the Impact Framework standard `Aggregation is not possible for given ${metric} since method is 'none'.`, METRIC_MISSING: (metric: string, index: number) => `Aggregation metric ${metric} is not found in inputs[${index}].`, - INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`, - REJECTING_OVERRIDE: (param: ManifestParameter) => - `Rejecting overriding of canonical parameter: ${param.name}.`, + INVALID_GROUP_KEY: (key: string) => `Invalid group ${key}.`, + REGROUP_ERROR: 'not an array or should contain at least one key', INVALID_EXHAUST_PLUGIN: (pluginName: string) => `Invalid exhaust plugin: ${pluginName}.`, UNKNOWN_PARAM: (name: string) => - `Unknown parameter: ${name}. Using 'sum' aggregation method.`, + `Unknown parameter: ${name}. Omitting from the output.`, NOT_INITALIZED_PLUGIN: (name: string) => `Not initalized plugin: ${name}. Check if ${name} is in 'manifest.initalize.plugins'.`, NO_OUTPUT: ` -You have not selected an output method. To see your output data, you can choose from: ---stdout: this will print your output data to the console ---output : this will save your output data to the given filepath (do not provide file extension) +You have not added an output command: +--output : will save your output data to the given filepath (do not provide file extension) Note that for the '--output' option you also need to define the output type in your manifest file. See https://if.greensoftware.foundation/major-concepts/manifest-file#initialize`, - SOURCE_IS_NOT_YAML: 'Given source file is not in yaml format.', - TARGET_IS_NOT_YAML: 'Given target file is not in yaml format.', - INVALID_TARGET: 'Target is invalid.', - INVALID_SOURCE: 'Source is invalid.', UNSUPPORTED_ERROR: (errorName: string) => `UnsupportedErrorClass: plugin threw error class: ${errorName} that is not recognized by Impact Framework`, - /** Plugin messages */ - MISSING_GLOBAL_CONFIG: 'Global config is not provided.', - MISSING_INPUT_DATA: (param: string) => - `${param} is missing from the input array, or has nullish value.`, - MANIFEST_NOT_FOUND: 'Manifest file not found.', - INITIALIZING_PACKAGE_JSON: 'Initializing package.json.', - INSTALLING_NPM_PACKAGES: 'Installing npm packages...', - NOT_NUMERIC_VALUE: (str: any) => `${str} is not numberic.`, + /** Debugging logs */ + STARTING_IF: 'Starting IF', + EXITING_IF: 'Exiting IF', + LOADING_MANIFEST: 'Loading manifest', + VALIDATING_MANIFEST: 'Validating manifest', + CAPTURING_RUNTIME_ENVIRONMENT_DATA: 'Capturing runtime environment data', + CHECKING_AGGREGATION_METHOD: (unitName: string) => + `Checking aggregation method for ${unitName}`, + INITIALIZING_PLUGINS: 'Initializing plugins', + INITIALIZING_PLUGIN: (pluginName: string) => `Initializing ${pluginName}`, + LOADING_PLUGIN_FROM_PATH: (pluginName: string, path: string) => + `Loading ${pluginName} from ${path}`, + COMPUTING_PIPELINE_FOR_NODE: (nodeName: string) => + `Computing pipeline for \`${nodeName}\``, + MERGING_DEFAULTS_WITH_INPUT_DATA: 'Merging defaults with input data', + AGGREGATING_OUTPUTS: 'Aggregating outputs', + AGGREGATING_NODE: (nodeName: string) => `Aggregating node ${nodeName}`, + PREPARING_OUTPUT_DATA: 'Preparing output data', + EXPORTING_TO_YAML_FILE: (savepath: string) => + `Exporting to yaml file: ${savepath}`, + EMPTY_PIPELINE: `You're using an old style manifest. Please update for phased execution. More information can be found here: +https://if.greensoftware.foundation/major-concepts/manifest-file`, + /** Exhaust messages */ + OUTPUT_REQUIRED: + 'Output path is required, please make sure output is configured properly.', + /** Plugins messages */ + INVALID_NAME: + '`name` global config parameter is empty or contains all spaces', + START_LOWER_END: '`start-time` should be lower than `end-time`', + TIMESTAMP_REQUIRED: (index: number) => `required in input[${index}]`, + INVALID_DATETIME: (index: number) => `invalid datetime in input[${index}]`, + X_Y_EQUAL: 'The length of `x` and `y` should be equal', + ARRAY_LENGTH_NON_EMPTY: + 'the length of the input arrays must be greater than 1', + WITHIN_THE_RANGE: + 'The target x value must be within the range of the given x values', + MISSING_CSV_COLUMN: (columnName: string) => + `There is no column with the name: ${columnName}.`, + NO_QUERY_DATA: + 'One or more of the given query parameters are not found in the target CSV file column headers.', + INVALID_DATE_TYPE: (date: any) => + `Unexpected date datatype: ${typeof date}: ${date}`, + INVALID_OBSERVATION_OVERLAP: + 'Observation timestamps overlap, please check inputs.', + SCI_MISSING_FN_UNIT: (functionalUnit: string) => + `'carbon' and ${functionalUnit} should be present in your input data.`, MISSING_FUNCTIONAL_UNIT_CONFIG: '`functional-unit` should be provided in your global config', MISSING_FUNCTIONAL_UNIT_INPUT: @@ -82,75 +104,9 @@ ${message}`, error: string ) => `Failed reading the file: ${filepath}. ${error}`, - MISSING_CSV_COLUMN: (columnName: string) => - `There is no column with the name: ${columnName}.`, - NO_QUERY_DATA: - 'One or more of the given query parameters are not found in the target CSV file column headers.', - INVALID_DATE_TYPE: (date: any) => - `Unexpected date datatype: ${typeof date}: ${date}`, - INVALID_OBSERVATION_OVERLAP: - 'Observation timestamps overlap, please check inputs.', - SCI_MISSING_FN_UNIT: (functionalUnit: string) => - `'carbon' and ${functionalUnit} should be present in your input data.`, - /** Exhaust messages */ - OUTPUT_REQUIRED: - 'Output path is required, please make sure output is configured properly.', - CSV_EXPORT: - 'CSV export criteria is not found in output path. Please append it after --output #.', - WRITE_CSV_ERROR: (outputPath: string, error: any) => - `Failed to write CSV file to ${outputPath}: ${error}`, - INVALID_NAME: - '`name` global config parameter is empty or contains all spaces', - START_LOWER_END: '`start-time` should be lower than `end-time`', - TIMESTAMP_REQUIRED: (index: number) => `required in input[${index}]`, - INVALID_DATETIME: (index: number) => `invalid datetime in input[${index}]`, - X_Y_EQUAL: 'The length of `x` and `y` should be equal', - ARRAY_LENGTH_NON_EMPTY: - 'the length of the input arrays must be greater than 1', - WITHIN_THE_RANGE: - 'The target x value must be within the range of the given x values', - /** Debugging logs */ - STARTING_IF: 'Starting IF', - EXITING_IF: 'Exiting IF', - LOADING_MANIFEST: 'Loading manifest', - VALIDATING_MANIFEST: 'Validating manifest', - CAPTURING_RUNTIME_ENVIRONMENT_DATA: 'Capturing runtime environment data', - SYNCING_PARAMETERS: 'Syncing parameters', - CHECKING_AGGREGATION_METHOD: (unitName: string) => - `Checking aggregation method for ${unitName}`, - INITIALIZING_PLUGINS: 'Initializing plugins', - INITIALIZING_PLUGIN: (pluginName: string) => `Initializing ${pluginName}`, - LOADING_PLUGIN_FROM_PATH: (pluginName: string, path: string) => - `Loading ${pluginName} from ${path}`, - COMPUTING_PIPELINE_FOR_NODE: (nodeName: string) => - `Computing pipeline for \`${nodeName}\``, - MERGING_DEFAULTS_WITH_INPUT_DATA: 'Merging defaults with input data', - AGGREGATING_OUTPUTS: 'Aggregating outputs', - AGGREGATING_NODE: (nodeName: string) => `Aggregating node ${nodeName}`, - PREPARING_OUTPUT_DATA: 'Preparing output data', - EXPORTING_TO_YAML_FILE: (savepath: string) => - `Exporting to yaml file: ${savepath}`, - EXPORTING_TO_CSV_FILE: (savepath: string) => - `Exporting to csv file: ${savepath}`, - EXPORTING_RAW_CSV_FILE: (savepath: string) => - `Exporting raw csv file: ${savepath}`, - CHECKING: 'Checking...', - IF_CHECK_FLAGS_MISSING: - 'Either the `--manifest` or `--directory` command should be provided with a path', - DIRECTORY_NOT_FOUND: 'Directory not found.', - DIRECTORY_YAML_FILES_NOT_FOUND: - 'The directory does not contain any YAML/YML files.\n', - IF_CHECK_EXECUTING: (filename: string) => `Executing \`${filename}\``, - IF_CHECK_VERIFICATION_FAILURES: - '---------\nif-check verification failures:\n', - IF_CHECK_FAILED: (filename: string) => - `āœ– if-check could not verify ${filename}. The re-executed file does not match the original.\n`, - IF_CHECK_VERIFIED: (filename: string) => - `āœ” if-check successfully verified ${filename}\n`, - IF_CHECK_SUMMARY_ERROR_MESSAGE: (filename: string, message: string) => - `Executing \`${filename}\`\nāœ– ${message}`, - IF_CHECK_SUMMARY_LOG: (passedCount: number, totalCount: number) => - `---------\nCheck summary:\n${passedCount} of ${totalCount} files are passed.\n`, ZERO_DIVISION: (moduleName: string, index: number) => `-- SKIPPING -- DivisionByZero: you are attempting to divide by zero in ${moduleName} plugin : inputs[${index}]\n`, + MISSING_GLOBAL_CONFIG: 'Global config is not provided.', + MISSING_INPUT_DATA: (param: string) => + `${param} is missing from the input array, or has nullish value.`, }; diff --git a/src/if-run/index.ts b/src/if-run/index.ts new file mode 100644 index 000000000..6d9f61a78 --- /dev/null +++ b/src/if-run/index.ts @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import {STRINGS as COMMON_STRINGS} from '../common/config'; +import {validateManifest} from '../common/util/validations'; +import {debugLogger} from '../common/util/debug-logger'; +import {logger} from '../common/util/logger'; +import {load} from '../common/lib/load'; + +import {aggregate, storeAggregationMetrics} from './lib/aggregate'; +import {injectEnvironment} from './lib/environment'; +import {initialize} from './lib/initialize'; +import {compute} from './lib/compute'; +import {exhaust} from './lib/exhaust'; +import {explain} from './lib/explain'; + +import {AGGREGATION_METHODS} from './types/aggregation'; + +import {parseIfRunProcessArgs} from './util/args'; +import {andHandle} from './util/helpers'; + +import {STRINGS} from './config'; + +const {EXITING_IF, STARTING_IF} = STRINGS; +const {DISCLAIMER_MESSAGE} = COMMON_STRINGS; + +const impactEngine = async () => { + const options = parseIfRunProcessArgs(); + const { + inputPath, + outputOptions, + debug, + observe, + regroup, + compute: computeFlag, + } = options; + + debugLogger.overrideConsoleMethods(!!debug); + + logger.info(DISCLAIMER_MESSAGE); + console.info(STARTING_IF); + + const {rawManifest} = await load(inputPath); + const envManifest = await injectEnvironment(rawManifest); + + try { + const {tree, ...context} = validateManifest(envManifest); + + if (context.aggregation) { + const convertMetrics = context.aggregation?.metrics.map( + (metric: string) => ({ + [metric]: AGGREGATION_METHODS[2], + }) + ); + + storeAggregationMetrics(...convertMetrics); + } + + const pluginStorage = await initialize(context); + const computedTree = await compute(tree, { + context, + pluginStorage, + observe, + regroup, + compute: computeFlag, + }); + + const aggregatedTree = aggregate(computedTree, context.aggregation); + envManifest.explainer && (context.explain = explain()); + + await exhaust(aggregatedTree, context, outputOptions); + } catch (error) { + if (error instanceof Error) { + envManifest.execution!.status = 'fail'; + envManifest.execution!.error = error.toString(); + logger.error(error); + const {tree, ...context} = envManifest; + + if (error.name !== 'ExhaustError') { + exhaust(tree, context, outputOptions); + } + } + } + console.info(EXITING_IF); +}; + +impactEngine().catch(andHandle); diff --git a/src/lib/aggregate.ts b/src/if-run/lib/aggregate.ts similarity index 61% rename from src/lib/aggregate.ts rename to src/if-run/lib/aggregate.ts index c6eef11cc..508d7df25 100644 --- a/src/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -1,12 +1,26 @@ import {PluginParams} from '@grnsft/if-core/types'; +import {debugLogger} from '../../common/util/debug-logger'; +import {logger} from '../../common/util/logger'; +import { + AggregationParams, + AggregationParamsSure, + AggregationMetricsWithMethod, +} from '../../common/types/manifest'; + import {aggregateInputsIntoOne} from '../util/aggregation-helper'; +import {memoizedLog} from '../util/log-memoize'; -import {STRINGS} from '../config/strings'; +import {AggregationMetric} from '../types/aggregation'; -import {AggregationParams, AggregationParamsSure} from '../types/manifest'; +import {STRINGS} from '../config/strings'; -const {AGGREGATING_NODE, AGGREGATING_OUTPUTS} = STRINGS; +const { + AGGREGATING_NODE, + AGGREGATING_OUTPUTS, + UNKNOWN_PARAM, + CHECKING_AGGREGATION_METHOD, +} = STRINGS; /** * Gets `i`th element from all children outputs and collects them in single array. @@ -25,7 +39,7 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: string[]) => { +const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); @@ -49,7 +63,9 @@ const temporalAggregation = (node: any, metrics: string[]) => { * 5. Now a grouping node has it's own outputs, it can horizotnally aggregate them. */ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { - const metrics = aggregationParams!.metrics; + const metrics: AggregationMetric[] = aggregationParams!.metrics.map( + metric => ({[metric]: 'none'}) + ); const type = aggregationParams!.type; if (node.children) { @@ -89,3 +105,57 @@ export const aggregate = (tree: any, aggregationParams: AggregationParams) => { return copyOfTree; }; + +/** + * Gets or stores aggregation metrics. + */ +export const storeAggregationMetrics = ( + aggregationMetrics?: AggregationMetricsWithMethod +) => { + if (aggregationMetrics) { + metricManager.metrics = { + ...metricManager.metrics, + ...aggregationMetrics, + }; + } + + return metricManager.metrics; +}; + +/** + * Creates an encapsulated object to retrieve the metrics. + */ +const metricManager = (() => { + let metric: AggregationMetric; + + const manager = { + get metrics() { + return metric; + }, + set metrics(value: AggregationMetric) { + metric = value; + }, + }; + + return manager; +})(); + +/** + * Returns aggregation method for given `unitName`. If doesn't exist then returns value `sum`. + */ +export const getAggregationMethod = (unitName: string) => { + debugLogger.setExecutingPluginName(); + memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); + const aggregationMetricsStorage = storeAggregationMetrics(); + + if ( + aggregationMetricsStorage && + Object.keys(aggregationMetricsStorage).includes(unitName) + ) { + return aggregationMetricsStorage[unitName]; + } + + memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); + + return undefined; +}; diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts new file mode 100644 index 000000000..710e15373 --- /dev/null +++ b/src/if-run/lib/compute.ts @@ -0,0 +1,163 @@ +import {PluginParams} from '@grnsft/if-core/types'; + +import {Regroup} from './regroup'; +import {addExplainData} from './explain'; + +import {mergeObjects} from '../util/helpers'; +import {debugLogger} from '../../common/util/debug-logger'; +import {logger} from '../../common/util/logger'; + +import {STRINGS} from '../config/strings'; + +import {ComputeParams, Node, PhasedPipeline} from '../types/compute'; +import {isExecute} from '../types/interface'; + +const {MERGING_DEFAULTS_WITH_INPUT_DATA, EMPTY_PIPELINE} = STRINGS; + +/** + * Traverses all child nodes based on children grouping. + */ +const traverse = async (children: any, params: ComputeParams) => { + for (const child in children) { + await computeNode(children[child], params); + } +}; + +/** + * Appends `default` values to `inputs`. + */ +const mergeDefaults = ( + inputs: PluginParams[], + defaults: PluginParams | undefined +) => { + if (inputs) { + const response = defaults + ? inputs.map(input => mergeObjects(defaults, input)) + : inputs; + + return response; + } + + console.debug(MERGING_DEFAULTS_WITH_INPUT_DATA); + + return defaults ? [defaults] : []; +}; + +/** + * 1. If the node has it's own pipeline, defaults or config then use that, + * otherwise use whatever has been passed down from further up the tree. + * 2. If it's a grouping node, then first of all computes all it's children. + * This is doing a depth first traversal. + * 3. Otherwise merges the defaults into the inputs. + * 4. Iterates over pipeline phases (observe, regroup, compute). + * 5. Observe plugins are used to insert input values + * (isolated execution can be achived by passing `--observe` flag to CLI command). + * 6. Regroup plugin is used to group existing inputs by criteria + * (isolated execution can be achived by passing `--regroup` flag to CLI command). + * Since it creates new children for node, existing inputs and outputs are dropped and recursive traversal is called + * for newbord child component. + * 7. Compute plugins are used to do desired computations and appending the result to outputs + * (isolated execution can be achived by passing `--compute` flag to CLI command). + */ +const computeNode = async (node: Node, params: ComputeParams): Promise => { + const pipeline = node.pipeline || (params.pipeline as PhasedPipeline); + const config = node.config || params.config; + const defaults = node.defaults || params.defaults; + const noFlags = !params.observe && !params.regroup && !params.compute; + + if (node.children) { + return traverse(node.children, { + ...params, + pipeline, + defaults, + config, + }); + } + + let inputStorage = structuredClone(node.inputs) as PluginParams[]; + inputStorage = mergeDefaults(inputStorage, defaults); + const pipelineCopy = structuredClone(pipeline) || {}; + + /** Checks if pipeline is not an array or empty object. */ + if ( + Array.isArray(pipelineCopy) || + (typeof pipelineCopy === 'object' && + pipelineCopy !== null && + Object.keys(pipelineCopy).length === 0) + ) { + logger.warn(EMPTY_PIPELINE); + } + + /** + * If iteration is on observe pipeline, then executes observe plugins and sets the inputs value. + */ + if ((noFlags || params.observe) && pipelineCopy.observe) { + while (pipelineCopy.observe.length !== 0) { + const pluginName = pipelineCopy.observe.shift() as string; + const plugin = params.pluginStorage.get(pluginName); + const nodeConfig = config && config[pluginName]; + + if (isExecute(plugin)) { + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.inputs = inputStorage; + + if (params.context.explainer) { + addExplainData({ + pluginName, + metadata: plugin.metadata, + pluginData: params.context.initialize!.plugins[pluginName], + }); + } + + node.outputs = inputStorage; + } + } + } + + /** + * If regroup is requested, execute regroup strategy, delete child's inputs, outputs and empty regroup array. + */ + if ((noFlags || params.regroup) && pipelineCopy.regroup) { + node.children = Regroup(inputStorage, pipelineCopy.regroup); + delete node.inputs; + delete node.outputs; + + return traverse(node.children, { + ...params, + pipeline: { + ...pipelineCopy, + regroup: undefined, + }, + defaults, + config, + }); + } + + /** + * If iteration is on compute plugin, then executes compute plugins and sets the outputs value. + */ + if ((noFlags || params.compute) && pipelineCopy.compute) { + while (pipelineCopy.compute.length !== 0) { + const pluginName = pipelineCopy.compute.shift() as string; + const plugin = params.pluginStorage.get(pluginName); + const nodeConfig = config && config[pluginName]; + + if (isExecute(plugin)) { + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.outputs = inputStorage; + debugLogger.setExecutingPluginName(); + } + } + } +}; + +/** + * Creates copy of existing tree, then applies computing strategy. + */ +export const compute = async (tree: any, params: ComputeParams) => { + const copyOfTree = structuredClone(tree); + + await computeNode(copyOfTree, params); + + return copyOfTree; +}; diff --git a/src/lib/environment.ts b/src/if-run/lib/environment.ts similarity index 94% rename from src/lib/environment.ts rename to src/if-run/lib/environment.ts index 52e2b0c9c..21808cc20 100644 --- a/src/lib/environment.ts +++ b/src/if-run/lib/environment.ts @@ -1,14 +1,13 @@ import {DateTime} from 'luxon'; -import {execPromise} from '../util/helpers'; - -import {Manifest} from '../types/manifest'; -import {NpmListResponse, PackageDependency} from '../types/environment'; import {osInfo} from '../util/os-checker'; +import {execPromise} from '../../common/util/helpers'; +import {Manifest} from '../../common/types/manifest'; import {STRINGS} from '../config/strings'; +import {NpmListResponse, PackageDependency} from '../types/environment'; -const packageJson = require('../../package.json'); +const packageJson = require('../../../package.json'); const {CAPTURING_RUNTIME_ENVIRONMENT_DATA} = STRINGS; diff --git a/src/if-run/lib/exhaust.ts b/src/if-run/lib/exhaust.ts new file mode 100644 index 000000000..bed36f405 --- /dev/null +++ b/src/if-run/lib/exhaust.ts @@ -0,0 +1,36 @@ +/** + * @todo This is temporary solution, will be refactored to support dynamic plugins. + */ +import {ExportLog} from '../builtins/export-log'; +import {ExportYaml} from '../builtins/export-yaml'; + +import {STRINGS} from '../config'; + +import {Options} from '../types/process-args'; +import {Context} from '../../common/types/manifest'; + +const {PREPARING_OUTPUT_DATA} = STRINGS; + +/** + * Output manager - Exhaust. + * Grabs output plugins from context, executes every. + */ +export const exhaust = async ( + tree: any, + context: Context, + outputOptions: Options +) => { + console.debug(PREPARING_OUTPUT_DATA); + + if (!outputOptions.noOutput && !outputOptions.outputPath) { + ExportLog().execute(tree, context); + } + + if (!outputOptions.outputPath) { + return; + } + + const exportYaml = ExportYaml(); + + await exportYaml.execute(tree, context, outputOptions.outputPath); +}; diff --git a/src/if-run/lib/explain.ts b/src/if-run/lib/explain.ts new file mode 100644 index 000000000..dce49b75a --- /dev/null +++ b/src/if-run/lib/explain.ts @@ -0,0 +1,44 @@ +import {ExplainParams} from '../types/explain'; + +/** + * Retrieves stored explain data. + */ +export const explain = () => storeExplainData.plugins; + +/** + * Manages the storage of explain data. + */ +const storeExplainData = (() => { + let plugin = {}; + + const pluginManager = { + get plugins() { + return plugin; + }, + set plugins(value: object) { + plugin = value; + }, + }; + + return pluginManager; +})(); + +/** + * Adds new explain data to the stored explain data. + */ +export const addExplainData = (params: ExplainParams) => { + const {pluginName, pluginData, metadata} = params; + const plugin = { + [pluginName]: { + method: pluginData!.method, + path: pluginData!.path, + inputs: metadata?.inputs || 'undefined', + outputs: metadata?.outputs || 'undefined', + }, + }; + + storeExplainData.plugins = { + ...storeExplainData.plugins, + ...plugin, + }; +}; diff --git a/src/lib/initialize.ts b/src/if-run/lib/initialize.ts similarity index 80% rename from src/lib/initialize.ts rename to src/if-run/lib/initialize.ts index 212d98e37..181e88d15 100644 --- a/src/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -2,15 +2,16 @@ import * as path from 'node:path'; import {ERRORS} from '@grnsft/if-core/utils'; -import {logger} from '../util/logger'; +import {logger} from '../../common/util/logger'; import {memoizedLog} from '../util/log-memoize'; import {pluginStorage} from '../util/plugin-storage'; import {CONFIG, STRINGS} from '../config'; import {PluginInterface} from '../types/interface'; -import {GlobalPlugins, PluginOptions} from '../types/manifest'; +import {Context, PluginOptions} from '../../common/types/manifest'; import {PluginStorageInterface} from '../types/plugin-storage'; +import {storeAggregationMetrics} from './aggregate'; const { PluginInitializationError, @@ -79,7 +80,12 @@ const handModule = (method: string, pluginPath: string) => { const initPlugin = async ( initPluginParams: PluginOptions ): Promise => { - const {method, path, 'global-config': globalConfig} = initPluginParams; + const { + method, + path, + 'global-config': globalConfig, + 'parameter-metadata': parameterMetadata, + } = initPluginParams!; console.debug(INITIALIZING_PLUGIN(method)); @@ -93,21 +99,29 @@ const initPlugin = async ( const plugin = await handModule(method, path); - return plugin(globalConfig); + return plugin(globalConfig, parameterMetadata); }; /** * Registers all plugins from `manifest`.`initialize` property. */ export const initialize = async ( - plugins: GlobalPlugins + context: Context ): Promise => { console.debug(INITIALIZING_PLUGINS); - + const {plugins} = context.initialize; const storage = pluginStorage(); for await (const pluginName of Object.keys(plugins)) { const plugin = await initPlugin(plugins[pluginName]); + const parameters = {...plugin.metadata.inputs, ...plugin.metadata.outputs}; + + Object.keys(parameters).forEach(key => { + storeAggregationMetrics({ + [key]: parameters[key]['aggregation-method'], + }); + }); + storage.set(pluginName, plugin); } diff --git a/src/if-run/lib/regroup.ts b/src/if-run/lib/regroup.ts new file mode 100644 index 000000000..affd51307 --- /dev/null +++ b/src/if-run/lib/regroup.ts @@ -0,0 +1,80 @@ +import {z} from 'zod'; +import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginParams} from '@grnsft/if-core/types'; + +import {validate} from '../../common/util/validations'; + +import {STRINGS} from '../config'; + +const {InvalidGroupingError} = ERRORS; + +const {INVALID_GROUP_KEY, REGROUP_ERROR} = STRINGS; + +/** + * Grouping strategy. + */ +export const Regroup = (inputs: PluginParams[], groups: string[]) => { + /** + * Creates structure to insert inputs by groups. + */ + const appendGroup = (value: PluginParams, object: any, groups: string[]) => { + if (groups.length > 0) { + const group = groups.shift() as string; + + object.children = object.children ?? {}; + object.children[group] = object.children[group] ?? {}; + + if (groups.length === 0) { + if ( + object.children[group].inputs && + object.children[group].inputs.length > 0 + ) { + object.children[group].inputs.push(value); + } else { + object.children[group].inputs = [value]; + } + } + + appendGroup(value, object.children[group], groups); + } + + return object; + }; + + /** + * Validates groups array. + */ + const validateGroups = (regroup: string[]) => { + const inputData = {regroup}; + const validationSchema = z.record( + z.string(), + z.array(z.string(), {message: REGROUP_ERROR}).min(1) + ); + + validate(validationSchema, inputData); + + return groups; + }; + + /** + * Interates over inputs, grabs group values for each one. + * Based on grouping, initializes the structure. + */ + return inputs.reduce((acc, input) => { + const validtedGroups = validateGroups(groups); + const groupsWithData = validtedGroups.map(groupType => { + if (!input[groupType]) { + throw new InvalidGroupingError(INVALID_GROUP_KEY(groupType)); + } + + return input[groupType]; + }); + + acc = { + ...acc, + ...appendGroup(input, acc, groupsWithData), + }; + + return acc; + }, {} as any).children; +}; diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts new file mode 100644 index 000000000..c3b143a1f --- /dev/null +++ b/src/if-run/types/aggregation.ts @@ -0,0 +1,7 @@ +export type AggregationResult = Record; + +export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; +export const AGGREGATION_METHODS = ['sum', 'avg', 'none'] as const; + +export type AggregationMethodTypes = 'sum' | 'avg' | 'none'; +export type AggregationMetric = Record; diff --git a/src/types/compute.ts b/src/if-run/types/compute.ts similarity index 63% rename from src/types/compute.ts rename to src/if-run/types/compute.ts index 2de3e6e33..6777bf553 100644 --- a/src/types/compute.ts +++ b/src/if-run/types/compute.ts @@ -1,30 +1,34 @@ import {PluginParams} from '@grnsft/if-core/types'; -import {Context} from './manifest'; import {PluginStorageInterface} from './plugin-storage'; +import {Context} from '../../common/types/manifest'; export type NodeConfig = { [key: string]: Record; }; -export type Params = { +export type PhasedPipeline = { + observe?: string[]; + regroup?: string[]; + compute?: string[]; +}; + +export type ComputeParams = { pluginStorage: PluginStorageInterface; context: Context; - pipeline?: string[]; + pipeline?: PhasedPipeline; config?: NodeConfig; defaults?: PluginParams; + observe?: Boolean; + regroup?: Boolean; + compute?: Boolean; }; export type Node = { children?: any; - pipeline?: string[]; + pipeline?: PhasedPipeline; config?: NodeConfig; defaults?: PluginParams; inputs?: PluginParams[]; outputs?: PluginParams[]; }; - -export type ComputeParams = { - context: Context; - pluginStorage: PluginStorageInterface; -}; diff --git a/src/types/environment.ts b/src/if-run/types/environment.ts similarity index 100% rename from src/types/environment.ts rename to src/if-run/types/environment.ts diff --git a/src/types/exhaust-plugin-interface.ts b/src/if-run/types/exhaust-plugin-interface.ts similarity index 80% rename from src/types/exhaust-plugin-interface.ts rename to src/if-run/types/exhaust-plugin-interface.ts index 3dc00182a..de6183842 100644 --- a/src/types/exhaust-plugin-interface.ts +++ b/src/if-run/types/exhaust-plugin-interface.ts @@ -1,4 +1,4 @@ -import {Context} from './manifest'; +import {Context} from '../../common/types/manifest'; export interface ExhaustPluginInterface { /** diff --git a/src/if-run/types/explain.ts b/src/if-run/types/explain.ts new file mode 100644 index 000000000..c7f1a6d38 --- /dev/null +++ b/src/if-run/types/explain.ts @@ -0,0 +1,9 @@ +import {ParameterMetadata} from '@grnsft/if-core/types'; + +import {PluginOptions} from '../../common/types/manifest'; + +export type ExplainParams = { + pluginName: string; + pluginData: PluginOptions; + metadata: {inputs?: ParameterMetadata; outputs?: ParameterMetadata}; +}; diff --git a/src/if-run/types/interface.ts b/src/if-run/types/interface.ts new file mode 100644 index 000000000..3884305a2 --- /dev/null +++ b/src/if-run/types/interface.ts @@ -0,0 +1,6 @@ +import {ExecutePlugin} from '@grnsft/if-core/types'; + +export type PluginInterface = ExecutePlugin; + +export const isExecute = (plugin: ExecutePlugin): plugin is ExecutePlugin => + (plugin as ExecutePlugin).metadata.kind === 'execute'; diff --git a/src/types/plugin-storage.ts b/src/if-run/types/plugin-storage.ts similarity index 100% rename from src/types/plugin-storage.ts rename to src/if-run/types/plugin-storage.ts diff --git a/src/if-run/types/process-args.ts b/src/if-run/types/process-args.ts new file mode 100644 index 000000000..298cadda6 --- /dev/null +++ b/src/if-run/types/process-args.ts @@ -0,0 +1,27 @@ +export interface IfRunArgs { + manifest?: string; + output?: string; + 'no-output'?: boolean; + debug?: boolean; + observe?: boolean; + regroup?: boolean; + compute?: boolean; +} + +export interface ProcessArgsOutputs { + inputPath: string; + outputOptions: { + outputPath?: string; + noOutput?: boolean; + }; + paramPath?: string; + debug?: boolean; + observe?: boolean; + regroup?: boolean; + compute?: boolean; +} + +export interface Options { + outputPath?: string; + noOutput?: boolean; +} diff --git a/src/types/time-sync.ts b/src/if-run/types/time-sync.ts similarity index 100% rename from src/types/time-sync.ts rename to src/if-run/types/time-sync.ts diff --git a/src/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts similarity index 60% rename from src/util/aggregation-helper.ts rename to src/if-run/util/aggregation-helper.ts index 408a4e9f7..e8fe63de5 100644 --- a/src/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -1,31 +1,15 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {PluginParams} from '@grnsft/if-core/types'; -import {parameterize} from '../lib/parameterize'; - import {CONFIG, STRINGS} from '../config'; -import {AggregationResult} from '../types/aggregation'; - -const {InvalidAggregationMethodError, MissingAggregationParamError} = ERRORS; -const {INVALID_AGGREGATION_METHOD, METRIC_MISSING} = STRINGS; -const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; +import {AggregationMetric, AggregationResult} from '../types/aggregation'; -/** - * Validates metrics array before applying aggregator. - * If aggregation method is `none`, then throws error. - */ -const checkIfMetricsAreValid = (metrics: string[]) => { - metrics.forEach(metric => { - const method = parameterize.getAggregationMethod(metric); +import {getAggregationMethod} from '../lib/aggregate'; - if (method === 'none') { - throw new InvalidAggregationMethodError( - INVALID_AGGREGATION_METHOD(metric) - ); - } - }); -}; +const {MissingAggregationParamError} = ERRORS; +const {METRIC_MISSING} = STRINGS; +const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; /** * Aggregates child node level metrics. Validates if metric aggregation type is `none`, then rejects with error. @@ -33,11 +17,11 @@ const checkIfMetricsAreValid = (metrics: string[]) => { */ export const aggregateInputsIntoOne = ( inputs: PluginParams[], - metrics: string[], + metrics: AggregationMetric[], isTemporal?: boolean ) => { - checkIfMetricsAreValid(metrics); - const extendedMetrics = [...metrics, ...AGGREGATION_ADDITIONAL_PARAMS]; + const metricsKeys: string[] = metrics.map(metric => Object.keys(metric)[0]); + const extendedMetrics = [...metricsKeys, ...AGGREGATION_ADDITIONAL_PARAMS]; return inputs.reduce((acc, input, index) => { for (const metric of extendedMetrics) { @@ -51,12 +35,18 @@ export const aggregateInputsIntoOne = ( acc[metric] = input[metric]; } } else { + const method = getAggregationMethod(metric); + + if (!method) { + return acc; + } + acc[metric] = acc[metric] ?? 0; acc[metric] += parseFloat(input[metric]); /** Checks for the last iteration. */ if (index === inputs.length - 1) { - if (parameterize.getAggregationMethod(metric) === 'avg') { + if (method === 'avg') { acc[metric] /= inputs.length; } } diff --git a/src/if-run/util/args.ts b/src/if-run/util/args.ts new file mode 100644 index 000000000..538d37a36 --- /dev/null +++ b/src/if-run/util/args.ts @@ -0,0 +1,76 @@ +import {parse} from 'ts-command-line-args'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {checkIfFileIsYaml} from '../../common/util/yaml'; +import {prependFullFilePath} from '../../common/util/helpers'; + +import {logger} from '../../common/util/logger'; + +import {CONFIG, STRINGS} from '../config'; +import {STRINGS as COMMON_STRINGS} from '../../common/config'; + +import {IfRunArgs, ProcessArgsOutputs} from '../types/process-args'; + +const {ParseCliParamsError, CliSourceFileError} = ERRORS; + +const {ARGS, HELP} = CONFIG; +const {NO_OUTPUT} = STRINGS; +const {SOURCE_IS_NOT_YAML, MANIFEST_IS_MISSING} = COMMON_STRINGS; + +/** + * Validates `if-run` process arguments. + */ +const validateAndParseProcessArgs = () => { + try { + return parse(ARGS, HELP); + } catch (error) { + if (error instanceof Error) { + throw new ParseCliParamsError(error.message); + } + + throw error; + } +}; + +/** + * 1. Parses process arguments for `if-run`. + * 2. If output params are missing, warns user about it. + * 3. Otherwise checks if `manifest` param is there, then processes with checking if it's a yaml file. + * If it is, then returns object containing full path. + * 4. If params are missing or invalid, then rejects with `ParseCliParamsError`. + */ +export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { + const { + manifest, + output, + 'no-output': noOutput, + debug, + observe, + regroup, + compute, + } = validateAndParseProcessArgs(); + + if (!output && noOutput) { + logger.warn(NO_OUTPUT); + } + + if (manifest) { + if (checkIfFileIsYaml(manifest)) { + return { + inputPath: prependFullFilePath(manifest), + outputOptions: { + ...(output && {outputPath: prependFullFilePath(output)}), + ...(noOutput && {noOutput}), + }, + debug, + observe, + regroup, + compute, + }; + } + + throw new CliSourceFileError(SOURCE_IS_NOT_YAML); + } + + throw new CliSourceFileError(MANIFEST_IS_MISSING); +}; diff --git a/src/if-run/util/helpers.ts b/src/if-run/util/helpers.ts new file mode 100644 index 000000000..2939022e2 --- /dev/null +++ b/src/if-run/util/helpers.ts @@ -0,0 +1,73 @@ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {logger} from '../../common/util/logger'; +import {GlobalPlugins} from '../../common/types/manifest'; +import {PluginStorageInterface} from '../types/plugin-storage'; +import {storeAggregationMetrics} from '../lib/aggregate'; + +import {STRINGS} from '../config'; + +const {UNSUPPORTED_ERROR} = STRINGS; + +/** + * Impact engine error handler. Logs errors and appends issue template if error is unknown. + */ +export const andHandle = (error: Error) => { + const knownErrors = Object.keys(ERRORS); + + logger.error(error); + + if (!knownErrors.includes(error.name)) { + logger.error(UNSUPPORTED_ERROR(error.name)); + // eslint-disable-next-line no-process-exit + process.exit(2); + } +}; + +/** + * Append entries from defaults which are missing from inputs. + */ +export const mergeObjects = (defaults: any, input: any) => { + const merged: Record = structuredClone(input); + + for (const key in defaults) { + if (!(key in input)) { + merged[key] = defaults[key]; + } + + if (merged[key] === undefined || merged[key] === null) { + merged[key] = defaults[key]; + } + } + + return merged; +}; + +/** + * Stores `'aggregation-method'` of the plugins in the pipeline. + */ +export const storeAggregationMethods = ( + plugins: GlobalPlugins, + pluginStorage: PluginStorageInterface +) => { + Object.keys(plugins).forEach(pluginName => { + const plugin = pluginStorage.get(pluginName); + + if ('inputs' in plugin.metadata || 'outputs' in plugin.metadata) { + const pluginParameters = + {...plugin.metadata.inputs, ...plugin.metadata.outputs} || {}; + + Object.entries(pluginParameters).forEach( + ([parameterName, parameterMetadata]) => { + const {'aggregation-method': aggregationMethod} = parameterMetadata; + + if (aggregationMethod) { + const metrics = {[parameterName]: aggregationMethod}; + + storeAggregationMetrics(metrics); + } + } + ); + } + }); +}; diff --git a/src/util/log-memoize.ts b/src/if-run/util/log-memoize.ts similarity index 100% rename from src/util/log-memoize.ts rename to src/if-run/util/log-memoize.ts diff --git a/src/util/os-checker.ts b/src/if-run/util/os-checker.ts similarity index 98% rename from src/util/os-checker.ts rename to src/if-run/util/os-checker.ts index c795ecc61..1614d5a4b 100644 --- a/src/util/os-checker.ts +++ b/src/if-run/util/os-checker.ts @@ -1,6 +1,6 @@ import {release, platform} from 'os'; -import {execPromise} from './helpers'; +import {execPromise} from '../../common/util/helpers'; /** * Executes `lsb_release -a` command in terminal. diff --git a/src/util/plugin-storage.ts b/src/if-run/util/plugin-storage.ts similarity index 100% rename from src/util/plugin-storage.ts rename to src/if-run/util/plugin-storage.ts index 1f9711943..ca2907fcc 100644 --- a/src/util/plugin-storage.ts +++ b/src/if-run/util/plugin-storage.ts @@ -2,8 +2,8 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../config'; -import {PluginInterface} from '../types/interface'; import {PluginStorage} from '../types/plugin-storage'; +import {PluginInterface} from '../types/interface'; const {PluginInitializationError} = ERRORS; const {NOT_INITALIZED_PLUGIN} = STRINGS; diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index e0d87009c..000000000 --- a/src/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env node -import {aggregate} from './lib/aggregate'; -import {compute} from './lib/compute'; -import {injectEnvironment} from './lib/environment'; -import {exhaust} from './lib/exhaust'; -import {initialize} from './lib/initialize'; -import {load} from './lib/load'; -import {parameterize} from './lib/parameterize'; - -import {debugLogger} from './util/debug-logger'; -import {parseIEProcessArgs} from './util/args'; -import {andHandle} from './util/helpers'; -import {logger} from './util/logger'; -import {validateManifest} from './util/validations'; - -import {STRINGS} from './config'; - -const {DISCLAIMER_MESSAGE, EXITING_IF, STARTING_IF} = STRINGS; - -const impactEngine = async () => { - const options = parseIEProcessArgs(); - const {inputPath, paramPath, outputOptions, debug} = options; - - debugLogger.overrideConsoleMethods(!!debug); - - logger.info(DISCLAIMER_MESSAGE); - console.info(STARTING_IF); - - const {rawManifest, parameters} = await load(inputPath, paramPath); - const envManifest = await injectEnvironment(rawManifest); - - try { - const {tree, ...context} = validateManifest(envManifest); - parameterize.combine(context.params, parameters); - const pluginStorage = await initialize(context.initialize.plugins); - const computedTree = await compute(tree, {context, pluginStorage}); - const aggregatedTree = aggregate(computedTree, context.aggregation); - await exhaust(aggregatedTree, context, outputOptions); - } catch (error) { - if (error instanceof Error) { - envManifest.execution!.status = 'fail'; - envManifest.execution!.error = error.toString(); - logger.error(error); - const {tree, ...context} = envManifest; - - if (error.name !== 'ExhaustError') { - exhaust(tree, context, outputOptions); - } - } - } - console.info(EXITING_IF); -}; - -impactEngine().catch(andHandle); diff --git a/src/lib/compute.ts b/src/lib/compute.ts deleted file mode 100644 index b83a22d2e..000000000 --- a/src/lib/compute.ts +++ /dev/null @@ -1,117 +0,0 @@ -import {PluginParams, GroupByConfig} from '@grnsft/if-core/types'; - -import {debugLogger} from '../util/debug-logger'; -import {mergeObjects} from '../util/helpers'; - -import {ComputeParams, Node, Params} from '../types/compute'; -import {isExecute, isGroupBy} from '../types/interface'; - -import {STRINGS} from '../config/strings'; - -const {MERGING_DEFAULTS_WITH_INPUT_DATA, COMPUTING_PIPELINE_FOR_NODE} = STRINGS; - -/** - * Traverses all child nodes based on children grouping. - */ -const traverse = async (children: any, params: Params) => { - for (const child in children) { - await computeNode(children[child], params); - } -}; - -/** - * Appends `default` values to `inputs`. - */ -const mergeDefaults = ( - inputs: PluginParams[], - defaults: PluginParams | undefined -) => { - if (inputs) { - const response = defaults - ? inputs.map(input => mergeObjects(defaults, input)) - : inputs; - - return response; - } - - console.debug(MERGING_DEFAULTS_WITH_INPUT_DATA); - - return defaults ? [defaults] : []; -}; - -/** - * 1. If the node has it's own pipeline, defaults or config then use that, - * otherwise use whatever has been passed down from further up the tree. - * 2. If it's a grouping node, then first of all computes all it's children. - * This is doing a depth first traversal. - * 3. Otherwise merges the defaults into the inputs. - * 4. Goes through the pipeline plugins, by checking if it's `execute` plugin. If so sets outputs. - * If is a `groupby` plugin, it will return child components rather than outputs. - * 5. Since after `groupby`, there are new child components, then computes them. - * Note: `pipeline` now equals the remaining plu.gins to apply to each child - */ -const computeNode = async (node: Node, params: Params): Promise => { - const pipeline = (node.pipeline || params.pipeline) as string[]; - const config = node.config || params.config; - const defaults = node.defaults || params.defaults; - - if (node.children) { - return traverse(node.children, { - ...params, - pipeline, - defaults, - config, - }); - } - - let inputStorage = structuredClone(node.inputs) as PluginParams[]; - inputStorage = mergeDefaults(inputStorage, defaults); - const pipelineCopy = structuredClone(pipeline); - - while (pipelineCopy.length !== 0) { - const pluginName = pipelineCopy.shift() as string; - const plugin = params.pluginStorage.get(pluginName); - const nodeConfig = config && config[pluginName]; - - console.debug(COMPUTING_PIPELINE_FOR_NODE(pluginName)); - debugLogger.setExecutingPluginName(pluginName); - - if (isExecute(plugin)) { - inputStorage = await plugin.execute(inputStorage, nodeConfig); - debugLogger.setExecutingPluginName(); - - node.outputs = inputStorage; - } - - if (isGroupBy(plugin)) { - node.children = await plugin.execute( - inputStorage, - nodeConfig as GroupByConfig - ); - delete node.inputs; - delete node.outputs; - - await traverse(node.children, { - ...params, - pipeline: pipelineCopy, - defaults, - config, - }); - - debugLogger.setExecutingPluginName(); - - break; - } - } -}; - -/** - * Creates copy of existing tree, then applies computing strategy. - */ -export const compute = async (tree: any, params: ComputeParams) => { - const copyOfTree = structuredClone(tree); - - await computeNode(copyOfTree, params); - - return copyOfTree; -}; diff --git a/src/lib/exhaust.ts b/src/lib/exhaust.ts deleted file mode 100644 index 1b65ca898..000000000 --- a/src/lib/exhaust.ts +++ /dev/null @@ -1,68 +0,0 @@ -import {ERRORS} from '@grnsft/if-core/utils'; - -/** - * @todo This is temporary solution, will be refactored to support dynamic plugins. - */ -import {ExportCSV} from '../builtins/export-csv'; -import {ExportCSVRaw} from '../builtins/export-csv-raw'; -import {ExportLog} from '../builtins/export-log'; -import {ExportYaml} from '../builtins/export-yaml'; - -import {STRINGS} from '../config'; - -import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface'; -import {Context} from '../types/manifest'; -import {Options} from '../types/process-args'; - -const {InvalidExhaustPluginError} = ERRORS; -const {INVALID_EXHAUST_PLUGIN, PREPARING_OUTPUT_DATA} = STRINGS; - -/** - * Initialize exhaust plugins based on the provided config - */ -const initializeExhaustPlugins = (plugins: string[]) => - plugins.map(initializeExhaustPlugin); - -/** - * Factory method for exhaust plugins. - */ -const initializeExhaustPlugin = (name: string): ExhaustPluginInterface => { - switch (name) { - case 'yaml': - return ExportYaml(); - case 'csv': - return ExportCSV(); - case 'csv-raw': - return ExportCSVRaw(); - default: - throw new InvalidExhaustPluginError(INVALID_EXHAUST_PLUGIN(name)); - } -}; - -/** - * Output manager - Exhaust. - * Grabs output plugins from context, executes every. - */ -export const exhaust = async ( - tree: any, - context: Context, - outputOptions: Options -) => { - console.debug(PREPARING_OUTPUT_DATA); - - const outputPlugins = context.initialize.outputs; - - if (outputOptions.stdout) { - ExportLog().execute(tree, context); - } - - if (!outputPlugins) { - return; - } - - const exhaustPlugins = initializeExhaustPlugins(outputPlugins); - - for await (const plugin of exhaustPlugins) { - await plugin.execute(tree, context, outputOptions.outputPath); - } -}; diff --git a/src/lib/load.ts b/src/lib/load.ts deleted file mode 100644 index 61f2b6cf5..000000000 --- a/src/lib/load.ts +++ /dev/null @@ -1,61 +0,0 @@ -import * as YAML from 'js-yaml'; -import {ERRORS} from '@grnsft/if-core/utils'; - -import {openYamlFileAsObject} from '../util/yaml'; -import {readAndParseJson} from '../util/json'; - -import {PARAMETERS} from '../config'; -import {STRINGS} from '../config'; - -import {Parameters} from '../types/parameters'; -import {LoadDiffParams} from '../types/util/args'; -import {Manifest} from '../types/manifest'; - -const {CliSourceFileError} = ERRORS; - -const {INVALID_SOURCE, LOADING_MANIFEST} = STRINGS; - -/** - * Parses manifest file as an object. Checks if parameter file is passed via CLI, then loads it too. - * Returns context, tree and parameters (either the default one, or from CLI). - */ -export const load = async (inputPath: string, paramPath?: string) => { - console.debug(LOADING_MANIFEST); - - const rawManifest = await openYamlFileAsObject(inputPath); - const parametersFromCli = - paramPath && - (await readAndParseJson(paramPath)); /** @todo validate json */ - const parameters = - parametersFromCli || - PARAMETERS; /** @todo PARAMETERS should be specified in parameterize only */ - - return { - rawManifest, - parameters, - }; -}; - -/** - * Loads files to compare. As a source file checks if data is piped and then decides which one to take. - */ -export const loadIfDiffFiles = async (params: LoadDiffParams) => { - const {sourcePath, targetPath, pipedSourceManifest} = params; - - if (!sourcePath && !pipedSourceManifest) { - throw new CliSourceFileError(INVALID_SOURCE); - } - - const loadFromSource = - sourcePath && (await openYamlFileAsObject(sourcePath!)); - const loadFromSTDIN = - pipedSourceManifest && (await YAML.load(pipedSourceManifest!)); - - const rawSourceManifest = loadFromSource || loadFromSTDIN; - const rawTargetManifest = await openYamlFileAsObject(targetPath); - - return { - rawSourceManifest, - rawTargetManifest, - }; -}; diff --git a/src/lib/parameterize.ts b/src/lib/parameterize.ts deleted file mode 100644 index 591ef781e..000000000 --- a/src/lib/parameterize.ts +++ /dev/null @@ -1,77 +0,0 @@ -import {logger} from '../util/logger'; -import {memoizedLog} from '../util/log-memoize'; -import {debugLogger} from '../util/debug-logger'; - -import {STRINGS, PARAMETERS} from '../config'; - -import {ManifestParameter} from '../types/manifest'; -import {Parameters} from '../types/parameters'; - -const { - REJECTING_OVERRIDE, - UNKNOWN_PARAM, - SYNCING_PARAMETERS, - CHECKING_AGGREGATION_METHOD, -} = STRINGS; - -/** - * Parameters manager. Provides get aggregation method and combine functionality. - */ -const Parameterize = () => { - let parametersStorage = PARAMETERS; - - /** - * Returns aggregation method for given `unitName`. If doesn't exist then returns value `sum`. - */ - const getAggregationMethod = (unitName: string) => { - debugLogger.setExecutingPluginName(); - memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); - - if (`${unitName}` in parametersStorage) { - return parametersStorage[unitName as keyof typeof PARAMETERS].aggregation; - } - - memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); - - return 'sum'; - }; - - /** - * Checks if additional parameters are provided in context. - * If so, then checks if they are coincident with default ones and exits with warning message. - * Otherwise appends context based parameters to defaults. - */ - const combine = ( - contextParameters: ManifestParameter[] | null | undefined, - parameters: Parameters - ) => { - console.debug(SYNCING_PARAMETERS); - - if (contextParameters) { - contextParameters.forEach(param => { - if (`${param.name}` in parameters) { - logger.warn(REJECTING_OVERRIDE(param)); - - return; - } - - const {description, unit, aggregation, name} = param; - - parameters[name] = { - description, - unit, - aggregation, - }; - }); - } - - parametersStorage = parameters; - }; - - return { - combine, - getAggregationMethod, - }; -}; - -export const parameterize = Parameterize(); diff --git a/src/types/aggregation.ts b/src/types/aggregation.ts deleted file mode 100644 index d15322d29..000000000 --- a/src/types/aggregation.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type AggregationResult = Record; - -export const AGGREGATION_METHODS = ['horizontal', 'vertical', 'both'] as const; diff --git a/src/types/interface.ts b/src/types/interface.ts deleted file mode 100644 index 6834a3ebb..000000000 --- a/src/types/interface.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {ExecutePlugin, GroupByPlugin} from '@grnsft/if-core/types'; - -export type PluginInterface = ExecutePlugin | GroupByPlugin; - -export const isExecute = (plugin: PluginInterface): plugin is ExecutePlugin => - (plugin as ExecutePlugin).metadata.kind === 'execute'; - -export const isGroupBy = (plugin: PluginInterface): plugin is GroupByPlugin => - (plugin as GroupByPlugin).metadata.kind === 'groupby'; diff --git a/src/types/parameters.ts b/src/types/parameters.ts deleted file mode 100644 index 7434777ce..000000000 --- a/src/types/parameters.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {ManifestParameter} from './manifest'; - -export const AGGREGATION_TYPES = ['sum', 'none', 'avg'] as const; - -type ParameterProps = Omit; - -export type Parameters = Record; diff --git a/src/types/process-args.ts b/src/types/process-args.ts deleted file mode 100644 index 0f3799f5a..000000000 --- a/src/types/process-args.ts +++ /dev/null @@ -1,38 +0,0 @@ -export interface IEArgs { - manifest?: string; - output?: string; - 'override-params'?: string; - stdout?: boolean; - debug?: boolean; -} - -export interface IFDiffArgs { - source?: string; - target: string; -} - -export interface IFEnvArgs { - manifest?: string; - install?: boolean; - cwd?: boolean; -} - -export interface IFCheckArgs { - manifest?: string; - directory?: string; -} - -export interface Options { - outputPath?: string; - stdout?: boolean; -} - -export interface ProcessArgsOutputs { - inputPath: string; - outputOptions: { - outputPath?: string; - stdout?: boolean; - }; - paramPath?: string; - debug?: boolean; -} diff --git a/src/util/args.ts b/src/util/args.ts deleted file mode 100644 index 31da90461..000000000 --- a/src/util/args.ts +++ /dev/null @@ -1,248 +0,0 @@ -import * as path from 'path'; - -import {parse} from 'ts-command-line-args'; -import {ERRORS} from '@grnsft/if-core/utils'; - -import {checkIfFileIsYaml} from './yaml'; - -import {isDirectoryExists, isFileExists} from './fs'; - -import {logger} from './logger'; - -import {CONFIG, STRINGS} from '../config'; - -import { - IFDiffArgs, - IEArgs, - ProcessArgsOutputs, - IFEnvArgs, - IFCheckArgs, -} from '../types/process-args'; -import {LoadDiffParams} from '../types/util/args'; - -const { - ParseCliParamsError, - CliTargetFileError, - CliSourceFileError, - InvalidDirectoryError, - MissingCliFlagsError, -} = ERRORS; - -const {IE, IF_DIFF, IF_ENV, IF_CHECK} = CONFIG; - -const { - MANIFEST_IS_MISSING, - MANIFEST_NOT_FOUND, - NO_OUTPUT, - SOURCE_IS_NOT_YAML, - TARGET_IS_NOT_YAML, - INVALID_TARGET, - IF_CHECK_FLAGS_MISSING, - DIRECTORY_NOT_FOUND, -} = STRINGS; - -/** - * Validates `ie` process arguments. - */ -const validateAndParseProcessArgs = () => { - try { - return parse(IE.ARGS, IE.HELP); - } catch (error) { - if (error instanceof Error) { - throw new ParseCliParamsError(error.message); - } - - throw error; - } -}; - -/** - * Prepends process path to given `filePath`. - */ -const prependFullFilePath = (filePath: string) => { - const processRunningPath = process.cwd(); - - if (path.isAbsolute(filePath)) { - return filePath; - } - - return path.normalize(`${processRunningPath}/${filePath}`); -}; - -/** - * 1. Parses process arguments like `manifest`, `output`, `override-params`, `help` and `debug`. - * 2. Checks if `help` param is provided, then logs help message and exits. - * 3. If output params are missing, warns user about it. - * 3. Otherwise checks if `manifest` param is there, then processes with checking if it's a yaml file. - * If it is, then returns object containing full path. - * 4. If params are missing or invalid, then rejects with `ParseCliParamsError`. - */ -export const parseIEProcessArgs = (): ProcessArgsOutputs => { - const { - manifest, - output, - 'override-params': overrideParams, - stdout, - debug, - } = validateAndParseProcessArgs(); - - if (!output && !stdout) { - logger.warn(NO_OUTPUT); - } - - if (manifest) { - if (checkIfFileIsYaml(manifest)) { - return { - inputPath: prependFullFilePath(manifest), - outputOptions: { - ...(output && {outputPath: prependFullFilePath(output)}), - ...(stdout && {stdout}), - }, - ...(overrideParams && {paramPath: overrideParams}), - debug, - }; - } - - throw new CliSourceFileError(SOURCE_IS_NOT_YAML); - } - - throw new CliSourceFileError(MANIFEST_IS_MISSING); -}; - -/** -- IF Diff -- */ - -/** - * Parses `if-diff` process arguments. - */ -const validateAndParseIfDiffArgs = () => { - try { - return parse(IF_DIFF.ARGS, IF_DIFF.HELP); - } catch (error) { - if (error instanceof Error) { - throw new ParseCliParamsError(error.message); - } - - throw error; - } -}; - -/** - * Checks for `source` and `target` flags to be present. - */ -export const parseIfDiffArgs = () => { - const {source, target} = validateAndParseIfDiffArgs(); - - if (target) { - if (source && !checkIfFileIsYaml(source)) { - throw new CliSourceFileError(SOURCE_IS_NOT_YAML); - } - - if (checkIfFileIsYaml(target)) { - const response: LoadDiffParams = { - targetPath: prependFullFilePath(target), - }; - - if (source && checkIfFileIsYaml(source)) { - response.sourcePath = prependFullFilePath(source); - } - - return response; - } - - throw new CliTargetFileError(TARGET_IS_NOT_YAML); - } - - throw new ParseCliParamsError(INVALID_TARGET); -}; - -/** -- IF Env -- */ - -/** - * Parses `if-env` process arguments. - */ -const validateAndParseIfEnvArgs = () => { - try { - return parse(IF_ENV.ARGS, IF_ENV.HELP); - } catch (error) { - if (error instanceof Error) { - throw new ParseCliParamsError(error.message); - } - - throw error; - } -}; - -/** - * Checks if the `manifest` command is provided and it is valid manifest file. - */ -export const parseIfEnvArgs = async () => { - const {manifest, install, cwd} = validateAndParseIfEnvArgs(); - - if (manifest) { - const response = prependFullFilePath(manifest); - const isManifestFileExists = await isFileExists(response); - - if (!isManifestFileExists) { - throw new ParseCliParamsError(MANIFEST_NOT_FOUND); - } - - if (checkIfFileIsYaml(manifest)) { - return {manifest: response, install, cwd}; - } - - throw new CliSourceFileError(SOURCE_IS_NOT_YAML); - } - - return {install, cwd}; -}; - -/** -- IF Check -- */ - -/** - * Parses `if-check` process arguments. - */ -const validateAndParseIfCheckArgs = () => { - try { - return parse(IF_CHECK.ARGS, IF_CHECK.HELP); - } catch (error) { - if (error instanceof Error) { - throw new ParseCliParamsError(error.message); - } - - throw error; - } -}; - -/** - * Checks if either `manifest` or `directory` command is provided. - */ -export const parseIfCheckArgs = async () => { - const {manifest, directory} = validateAndParseIfCheckArgs(); - - if (manifest) { - const response = prependFullFilePath(manifest); - const isManifestFileExists = await isFileExists(response); - - if (!isManifestFileExists) { - throw new ParseCliParamsError(MANIFEST_NOT_FOUND); - } - - if (checkIfFileIsYaml(manifest)) { - return {manifest}; - } - - throw new CliSourceFileError(SOURCE_IS_NOT_YAML); - } else if (directory) { - const isDirExists = await isDirectoryExists(directory); - - if (!isDirExists) { - throw new InvalidDirectoryError(DIRECTORY_NOT_FOUND); - } - - const response = prependFullFilePath(directory); - - return {directory: response}; - } - - throw new MissingCliFlagsError(IF_CHECK_FLAGS_MISSING); -}; diff --git a/src/util/helpers.ts b/src/util/helpers.ts deleted file mode 100644 index c55b43724..000000000 --- a/src/util/helpers.ts +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env node -/* eslint-disable no-process-exit */ -import {createInterface} from 'node:readline/promises'; -import {exec} from 'node:child_process'; -import {promisify} from 'node:util'; - -import * as fs from 'fs/promises'; -import * as path from 'path'; - -import {ERRORS} from '@grnsft/if-core/utils'; - -import {STRINGS, CONFIG} from '../config'; - -import {Difference} from '../types/lib/compare'; - -import {load} from '../lib/load'; - -import { - installDependencies, - initPackageJsonIfNotExists, - updatePackageJsonDependencies, - extractPathsWithVersion, - updatePackageJsonProperties, -} from './npm'; - -import {logger} from './logger'; -import {EnvironmentOptions} from '../types/if-env'; - -const {IF_ENV} = CONFIG; -const { - FAILURE_MESSAGE, - FAILURE_MESSAGE_TEMPLATE, - FAILURE_MESSAGE_DEPENDENCIES, -} = IF_ENV; - -const {UNSUPPORTED_ERROR, IF_CHECK_FAILED, IF_CHECK_SUMMARY_ERROR_MESSAGE} = - STRINGS; -const {MissingPluginDependenciesError} = ERRORS; - -/** - * Impact engine error handler. Logs errors and appends issue template if error is unknown. - */ -export const andHandle = (error: Error) => { - const knownErrors = Object.keys(ERRORS); - - logger.error(error); - - if (!knownErrors.includes(error.name)) { - logger.error(UNSUPPORTED_ERROR(error.name)); - // eslint-disable-next-line no-process-exit - process.exit(2); - } -}; - -/** - * Append entries from defaults which are missing from inputs. - */ -export const mergeObjects = (defaults: any, input: any) => { - const merged: Record = structuredClone(input); - - for (const key in defaults) { - if (!(key in input)) { - merged[key] = defaults[key]; - } - - if (merged[key] === undefined || merged[key] === null) { - merged[key] = defaults[key]; - } - } - - return merged; -}; - -/** - * Promise version of Node's `exec` from `child-process`. - */ -export const execPromise = promisify(exec); - -/** - * `If-diff` equality checker. - */ -export const checkIfEqual = (source: any, target: any) => { - if (source === target) { - return true; - } - - if (source === '*' || target === '*') { - return true; - } - - return false; -}; - -/** - * Converts given `value` to either `1` or `0`. - */ -const convertToXorable = (value: any) => { - if (typeof value === 'number') { - return value !== 0 ? 1 : 0; - } - - if (typeof value === 'boolean') { - return value ? 1 : 0; - } - - if (typeof value === 'string') { - return value.length > 0 ? 1 : 0; - } - - if (typeof value === 'object') { - return 1; - } - - return 0; -}; - -/** - * If one of the `valuesToCheck` values is undefined, then set `missing`, otherwise `exists`. - */ -const setValuesIfMissing = (response: Difference) => { - const source = convertToXorable(response.source); - const target = convertToXorable(response.target); - - if (source ^ target) { - ['source', 'target'].forEach(value => { - response[value] = response[value] ? 'exists' : 'missing'; - }); - - return response; - } - - return response; -}; - -/** - * Checks if objects are primitive types. - */ -export const oneIsPrimitive = (source: any, target: any) => { - // eslint-disable-next-line eqeqeq - if (source == null || target == null) { - return true; - } - - return source !== Object(source) && target !== Object(target); -}; - -/** - * Format not matching message for CLI logging. - */ -export const formatNotMatchingLog = (message: Difference) => { - const flattenMessage = setValuesIfMissing(message); - - Object.keys(flattenMessage).forEach(key => { - if (key === 'message' || key === 'path') { - console.log(message[key]); - } else { - console.log(`${key}: ${message[key]}`); - } - }); -}; - -/** - * Checks if there is data piped, then collects it. - * Otherwise returns empty string. - */ -const collectPipedData = async () => { - if (process.stdin.isTTY) { - return ''; - } - - const readline = createInterface({ - input: process.stdin, - }); - - const data: string[] = []; - - for await (const line of readline) { - data.push(line); - } - - return data.join('\n'); -}; - -/** - * Checks if there is piped data, tries to parse yaml from it. - * Returns empty string if haven't found anything. - */ -export const parseManifestFromStdin = async () => { - const pipedSourceManifest = await collectPipedData(); - - if (!pipedSourceManifest) { - return ''; - } - - const regex = /# start((?:.*\n)+?)# end/; - const match = regex.exec(pipedSourceManifest); - - if (!match) { - return ''; - } - - return match![1]; -}; - -/** - * Gets the folder path of the manifest file, dependencies from manifest file and install argument from the given arguments. - */ -export const getOptionsFromArgs = async (commandArgs: { - manifest: string; - install: boolean | undefined; -}) => { - const {manifest, install} = commandArgs; - const folderPath = path.dirname(manifest); - const loadedManifest = await load(manifest); - const rawManifest = loadedManifest.rawManifest; - const plugins = rawManifest?.initialize?.plugins || {}; - const dependencies = rawManifest?.execution?.environment.dependencies || []; - - if (!dependencies.length) { - throw new MissingPluginDependenciesError(FAILURE_MESSAGE_DEPENDENCIES); - } - - const pathsWithVersion = extractPathsWithVersion(plugins, dependencies); - - return { - folderPath, - dependencies: pathsWithVersion, - install, - }; -}; - -/** - * Creates folder if not exists, installs dependencies if required, update depenedencies. - */ -export const initializeAndInstallLibs = async (options: EnvironmentOptions) => { - try { - const {folderPath, install, cwd, dependencies} = options; - const packageJsonPath = await initPackageJsonIfNotExists(folderPath); - - await updatePackageJsonProperties(packageJsonPath, cwd); - - if (install) { - await installDependencies(folderPath, dependencies); - } else { - await updatePackageJsonDependencies(packageJsonPath, dependencies, cwd); - } - } catch (error) { - console.log(FAILURE_MESSAGE); - process.exit(2); - } -}; - -/** - * Adds a manifest template to the folder where the if-env CLI command runs. - */ -export const addTemplateManifest = async (destinationDir: string) => { - try { - const templateManifest = path.resolve( - __dirname, - '../config/env-template.yml' - ); - - const destinationPath = path.resolve(destinationDir, 'manifest.yml'); - const data = await fs.readFile(templateManifest, 'utf-8'); - - await fs.writeFile(destinationPath, data, 'utf-8'); - } catch (error) { - console.log(FAILURE_MESSAGE_TEMPLATE); - process.exit(1); - } -}; - -/** - * Logs the failure message from the stdout of an error. - */ -export const logStdoutFailMessage = (error: any, fileName: string) => { - console.log(IF_CHECK_FAILED(fileName)); - - const stdout = error.stdout; - const logs = stdout.split('\n\n'); - const failMessage = logs[logs.length - 1]; - - console.log(failMessage); - return IF_CHECK_SUMMARY_ERROR_MESSAGE(fileName, failMessage); -}; diff --git a/src/util/json.ts b/src/util/json.ts deleted file mode 100644 index c9180a788..000000000 --- a/src/util/json.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as fs from 'fs/promises'; - -/** - * Reads and parses json file. - */ -export const readAndParseJson = async (paramPath: string): Promise => { - const file = await fs.readFile(paramPath, 'utf-8'); - - return JSON.parse(file) as T; -};