Phased execution without builtin time-sync

.github/workflows/dispatch-integration-test.yml

@@ -0,0 +1,23 @@
+name: Run if-check on requested branch
+
+on: 
+  workflow_dispatch: 
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run integration tests
+        run: npm run if-check -- -d manifests/outputs

manifests/examples/builtins/time-sync/success.yml

@@ -1,23 +1,16 @@
 name: time-sync
 description: successful path
 tags:
+'time-sync':
+  start-time: '2023-12-12T00:00:00.000Z'
+  end-time: '2023-12-12T00:01:00.000Z'
+  interval: 5
+  allow-padding: true
 initialize:
-  output:
-    - yaml
-  plugins:
-    'time-sync':
-      method: TimeSync
-      path: "builtin"
-      global-config: 
-        start-time: '2023-12-12T00:00:00.000Z'
-        end-time: '2023-12-12T00:01:00.000Z'
-        interval: 5
-        allow-padding: true
+  plugins: {}
 tree:
   children:
     child:
-      pipeline:
-        - time-sync
       config:
       inputs:
         - timestamp: '2023-12-12T00:00:00.000Z'

manifests/examples/pipelines/nesting.yml

@@ -185,17 +185,18 @@ tree:
         vcpus-allocated: 1
         vcpus-total: 8
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       inputs:
         - timestamp: "2023-12-12T00:00:00.000Z"
           cloud/instance-type: A1
@@ -235,17 +236,18 @@ tree:
         vcpus-allocated: 1
         vcpus-total: 8
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       inputs:
         - timestamp: "2023-12-12T00:00:00.000Z"
           cloud/instance-type: A1
@@ -287,17 +289,18 @@ tree:
             vcpus-allocated: 1
             vcpus-total: 8
           pipeline:
-            - interpolate
-            - cpu-factor-to-wattage
-            - wattage-times-duration
-            - wattage-to-energy-kwh
-            - calculate-vcpu-ratio
-            - correct-cpu-energy-for-vcpu-ratio
-            - sci-embodied
-            - operational-carbon
-            - sum-carbon
-            - time-sync
-            - sci
+            compute:
+              - interpolate
+              - cpu-factor-to-wattage
+              - wattage-times-duration
+              - wattage-to-energy-kwh
+              - calculate-vcpu-ratio
+              - correct-cpu-energy-for-vcpu-ratio
+              - sci-embodied
+              - operational-carbon
+              - sum-carbon
+              - time-sync
+              - sci
           inputs:
             - timestamp: "2023-12-12T00:00:00.000Z"
               cloud/instance-type: A1
@@ -337,17 +340,18 @@ tree:
             vcpus-allocated: 1
             vcpus-total: 8
           pipeline:
-            - interpolate
-            - cpu-factor-to-wattage
-            - wattage-times-duration
-            - wattage-to-energy-kwh
-            - calculate-vcpu-ratio
-            - correct-cpu-energy-for-vcpu-ratio
-            - sci-embodied
-            - operational-carbon
-            - sum-carbon
-            - time-sync
-            - sci
+            compute:
+              - interpolate
+              - cpu-factor-to-wattage
+              - wattage-times-duration
+              - wattage-to-energy-kwh
+              - calculate-vcpu-ratio
+              - correct-cpu-energy-for-vcpu-ratio
+              - sci-embodied
+              - operational-carbon
+              - sum-carbon
+              - time-sync
+              - sci
           inputs:
             - timestamp: "2023-12-12T00:00:00.000Z"
               cloud/instance-type: A1

manifests/examples/pipelines/pipeline-with-mocks.yml

@@ -13,9 +13,9 @@ initialize:
       method: MockObservations
       path: "builtin"
       global-config:
-        timestamp-from: 2023-12-12T00:00
-        timestamp-to: 2023-12-12T00:10
-        duration: 60
+        timestamp-from: "2023-12-12T00:00:00.000Z"
+        timestamp-to: "2023-12-12T00:00:13.000Z"
+        duration: 30
         components:
           - cloud/instance-type: A1
         generators:
@@ -238,52 +238,27 @@ initialize:
         end-time: "2023-12-12T00:01:00.000Z"
         interval: 5
         allow-padding: true
-      parameter-metadata:
-        inputs:
-          time-reserved:
-            description: time reserved for a component
-            unit: seconds
-            aggregation-method: avg
-        outputs:
-          synced-time:
-            description: Synced time
-            unit: none
-            aggregation-method: none
-    group-by:
-      path: builtin
-      method: GroupBy
-      parameter-metadata:
-        inputs:
-          group:
-            description: Group by fields
-            unit: none
-            aggregation-method: none
-        outputs:
-          grouped-data:
-            description: Grouped data
-            unit: none
-            aggregation-method: none
 tree:
   children:
     child-1:
       pipeline:
-        - mock-observations
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - instance-type
+        observe:
+          - mock-observations
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -293,49 +268,25 @@ tree:
         vcpus-total: 8
         vcpus-allocated: 1
       inputs:
-        - timestamp: "2023-12-12T00:00:00.000Z"
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          duration: 1
-          cpu/utilization: 10
-          requests: 30
-        - timestamp: "2023-12-12T00:00:01.000Z"
-          duration: 5
-          cpu/utilization: 20
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 40
-        - timestamp: "2023-12-12T00:00:06.000Z"
-          duration: 7
-          cpu/utilization: 15
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 30
-        - timestamp: "2023-12-12T00:00:13.000Z"
-          duration: 30
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          cpu/utilization: 15
-          requests: 50
     child-2:
       pipeline:
-        - mock-observations
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - cloud/instance-type
+        observe:
+          - mock-observations
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -345,27 +296,3 @@ tree:
         vcpus-total: 8
         vcpus-allocated: 1
       inputs:
-        - timestamp: "2023-12-12T00:00:00.000Z"
-          duration: 1
-          cpu/utilization: 30
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 30
-        - timestamp: "2023-12-12T00:00:01.000Z"
-          duration: 5
-          cpu/utilization: 28
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 40
-        - timestamp: "2023-12-12T00:00:06.000Z"
-          duration: 7
-          cpu/utilization: 40
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 50
-        - timestamp: "2023-12-12T00:00:13.000Z"
-          duration: 30
-          cpu/utilization: 33
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 60

manifests/examples/pipelines/scenario-1.yml

@@ -0,0 +1,34 @@
+name: demo
+description: demo for observe feat
+tags:
+initialize:
+  plugins:
+    mock-observations:
+      kind: plugin
+      method: MockObservations
+      path: "builtin"
+      global-config:
+        timestamp-from: 2023-07-06T00:00
+        timestamp-to: 2023-07-06T00:01
+        duration: 60
+        components:
+          - cloud/instance-type: A1
+          - cloud/instance-type: B1
+        generators:
+          common:
+            region: uk-west
+            common-key: common-val
+          randint:
+            cpu/utilization:
+              min: 1
+              max: 99
+            memory/utilization:
+              min: 1
+              max: 99
+tree:
+  children:
+    child:
+      pipeline:
+        observe:
+          - mock-observations
+      inputs: null