Merge branch 'main' into timeline-comparison

* main: (42 commits) Added exception handling to client.py Fix and better display of SCI values in badged and dashboard Increased the waiting time for powermetrics to shut down Bugfix for non-string types replacement Docker prune is now the default for jobs.py Bump psycopg[binary] from 3.1.9 to 3.1.10 (#413) Bump fastapi from 0.100.1 to 0.101.0 (#414) Bump orjson from 3.9.2 to 3.9.3 (#415) Ignore filesystem paths Bugfix: Phases ordering was wrong way round eco-ci will now post the energy reading in the PR conversation (#397) SCI metric (#412) Fixes the github security errors (#411) API does not block returning machines, but will now return if they are available or not Enables empty services in the usage_scenario (#409) wrong import jobs.py now appends date Fixes the error on mac on which /tmp is a symlink to /private/tmp (#410) Fixes the metric output file becoming corrupted (#393) Only available machines may be listed ...
green-coding-solutions · Aug 8, 2023 · 74ed9eb · 74ed9eb
2 parents 5da1b03 + 0a99a8c
commit 74ed9eb
Show file tree

Hide file tree

Showing 41 changed files with 849 additions and 700 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,16 +7,16 @@ version: 2
 updates:
   - package-ecosystem: "pip"
     directory: "/"
-    target-branch: "dev"
+    target-branch: "main"
     schedule:
       interval: "daily"
   - package-ecosystem: "docker"
     directory: "/"
-    target-branch: "dev"
+    target-branch: "main"
     schedule:
       interval: "weekly"
   - package-ecosystem: "github-actions"
     directory: "/"
-    target-branch: "dev"
+    target-branch: "main"
     schedule:
       interval: "daily"
diff --git a/.github/workflows/tests-bare-metal-dev.yml b/.github/workflows/tests-bare-metal-dev.yml
diff --git a/.github/workflows/tests-eco-ci-energy-estimation.yaml b/.github/workflows/tests-eco-ci-energy-estimation.yaml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  run-tests-dev:
+  run-tests-main:
     runs-on: ubuntu-latest
     permissions:
       packages: write
@@ -13,7 +13,7 @@ jobs:
       - name: 'Checkout repository'
         uses: actions/checkout@v3
         with:
-          ref: 'dev'
+          ref: 'main'
           submodules: 'true'
 
       - name: Eco CI Energy Estimation - Initialize
@@ -37,13 +37,13 @@ jobs:
         uses: green-coding-berlin/eco-ci-energy-estimation@testing
         with:
           task: get-measurement
-          branch: dev
+          branch: main
 
       - name: Eco CI Energy Estimation - End Measurement
         uses: green-coding-berlin/eco-ci-energy-estimation@testing
         with:
           task: display-results
-          branch: dev
+          branch: main
 
 
 
diff --git a/.github/workflows/tests-vm-dev.yml b/.github/workflows/tests-vm-dev.yml
diff --git a/.github/workflows/tests-vm-main.yml b/.github/workflows/tests-vm-main.yml
@@ -26,7 +26,7 @@ jobs:
           submodules: 'true'
 
       - name: Eco CI Energy Estimation - Initialize
-        uses: green-coding-berlin/eco-ci-energy-estimation@v2
+        uses: green-coding-berlin/eco-ci-energy-estimation@701b5f2f4ba601be587823cd0786f07cb6ae2ee6
         with:
           task: start-measurement
 
@@ -38,13 +38,13 @@ jobs:
          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Eco CI Energy Estimation - Get Measurement
-        uses: green-coding-berlin/eco-ci-energy-estimation@v2
+        uses: green-coding-berlin/eco-ci-energy-estimation@701b5f2f4ba601be587823cd0786f07cb6ae2ee6
         with:
           task: get-measurement
           branch: main
 
       - name: Eco CI Energy Estimation - End Measurement
-        uses: green-coding-berlin/eco-ci-energy-estimation@v2
+        uses: green-coding-berlin/eco-ci-energy-estimation@701b5f2f4ba601be587823cd0786f07cb6ae2ee6
         with:
           task: display-results
           branch: main
diff --git a/.github/workflows/tests-vm-pr.yml b/.github/workflows/tests-vm-pr.yml
@@ -9,6 +9,7 @@ jobs:
     permissions:
       packages: write
       contents: read
+      pull-requests: write
     steps:
       - name: 'Checkout repository'
         uses: actions/checkout@v3
@@ -31,13 +32,12 @@ jobs:
         uses: green-coding-berlin/eco-ci-energy-estimation@v2
         with:
           task: get-measurement
-          branch: dev
+          branch: main
 
       - name: Eco CI Energy Estimation - End Measurement
         uses: green-coding-berlin/eco-ci-energy-estimation@v2
         with:
           task: display-results
-          branch: dev
-
-
-
+          branch: main
+          pr-comment: true
+
diff --git a/README.md b/README.md
@@ -1,6 +1,5 @@
 [![Tests Status - Main](https://github.com/green-coding-berlin/green-metrics-tool/actions/workflows/tests-vm-main.yml/badge.svg)](https://github.com/green-coding-berlin/green-metrics-tool/actions/workflows/tests-vm-main.yml)
 
-[![Tests Status - Dev](https://github.com/green-coding-berlin/green-metrics-tool/actions/workflows/tests-vm-dev.yml/badge.svg)](https://github.com/green-coding-berlin/green-metrics-tool/actions/workflows/tests-vm-dev.yml)
 
 [![Energy Used](https://api.green-coding.berlin/v1/ci/badge/get/?repo=green-coding-berlin/green-metrics-tool&branch=dev&workflow=45267392)](https://metrics.green-coding.berlin/ci.html?repo=green-coding-berlin/green-metrics-tool&branch=dev&workflow=45267392) (This is the energy cost of running our CI-Pipelines on Github. [Find out more about Eco-CI](https://www.green-coding.berlin/projects/eco-ci/))
 

diff --git a/api/api.py b/api/api.py
@@ -8,6 +8,7 @@
 import sys
 import os
 
+from xml.sax.saxutils import escape as xml_escape
 from fastapi import FastAPI, Request, Response, status
 from fastapi.responses import ORJSONResponse
 from fastapi.encoders import jsonable_encoder
@@ -31,7 +32,6 @@
                          sanitize, get_phase_stats, get_phase_stats_object,
                          is_valid_uuid, rescale_energy_value)
 
-
 # It seems like FastAPI already enables faulthandler as it shows stacktrace on SEGFAULT
 # Is the redundant call problematic
 faulthandler.enable()  # will catch segfaults and write to STDERR
@@ -133,7 +133,7 @@ async def get_notes(project_id):
 @app.get('/v1/machines/')
 async def get_machines():
     query = """
-            SELECT id, description
+            SELECT id, description, available
             FROM machines
             ORDER BY description ASC
             """
@@ -146,14 +146,28 @@ async def get_machines():
 
 # A route to return all of the available entries in our catalog.
 @app.get('/v1/projects')
-async def get_projects():
+async def get_projects(repo: str, filename: str):
     query = """
             SELECT a.id, a.name, a.uri, COALESCE(a.branch, 'main / master'), a.end_measurement, a.last_run, a.invalid_project, a.filename, b.description, a.commit_hash
             FROM projects as a
             LEFT JOIN machines as b on a.machine_id = b.id
-            ORDER BY a.created_at DESC  -- important to order here, the charting library in JS cannot do that automatically!
+            WHERE 1=1
             """
-    data = DB().fetch_all(query)
+    params = []
+
+    filename = filename.strip()
+    if filename not in ('', 'null'):
+        query = f"{query} AND a.filename LIKE %s  \n"
+        params.append(f"%{filename}%")
+
+    repo = repo.strip()
+    if repo not in ('', 'null'):
+        query = f"{query} AND a.uri LIKE %s \n"
+        params.append(f"%{repo}%")
+
+    query = f"{query} ORDER BY a.created_at DESC  -- important to order here, the charting library in JS cannot do that automatically!"
+
+    data = DB().fetch_all(query, params=tuple(params))
     if data is None or data == []:
         return Response(status_code=204) # No-Content
 
@@ -343,42 +357,46 @@ async def get_badge_single(project_id: str, metric: str = 'ml-estimated'):
         return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400)
 
     query = '''
-        WITH times AS (
-            SELECT start_measurement, end_measurement FROM projects WHERE id = %s
-        ) SELECT
-            (SELECT start_measurement FROM times), (SELECT end_measurement FROM times),
-            SUM(measurements.value), measurements.unit
-        FROM measurements
+        SELECT
+            SUM(value), MAX(unit)
+        FROM
+            phase_stats
         WHERE
-            measurements.project_id = %s
-            AND measurements.time >= (SELECT start_measurement FROM times)
-            AND measurements.time <= (SELECT end_measurement FROM times)
-            AND measurements.metric LIKE %s
-        GROUP BY measurements.unit
+            project_id = %s
+            AND metric LIKE %s
+            AND phase LIKE '%%_[RUNTIME]'
     '''
 
     value = None
+    label = 'Energy Cost'
+    via = ''
     if metric == 'ml-estimated':
         value = 'psu_energy_ac_xgboost_machine'
+        via = 'via XGBoost ML'
     elif metric == 'RAPL':
-        value = '%_rapl_%'
+        value = '%_energy_rapl_%'
+        via = 'via RAPL'
     elif metric == 'AC':
         value = 'psu_energy_ac_%'
+        via = 'via PSU (AC)'
+    elif metric == 'SCI':
+        label = 'SCI'
+        value = 'software_carbon_intensity_global'
     else:
         return ORJSONResponse({'success': False, 'err': f"Unknown metric '{metric}' submitted"}, status_code=400)
 
-    params = (project_id, project_id, value)
+    params = (project_id, value)
     data = DB().fetch_one(query, params=params)
 
-    if data is None or data == []:
+    if data is None or data == [] or not data[1] :
         badge_value = 'No energy data yet'
     else:
-        [energy_value, energy_unit] = rescale_energy_value(data[2], data[3])
-        badge_value= f"{energy_value:.2f} {energy_unit} via {metric}"
+        [energy_value, energy_unit] = rescale_energy_value(data[0], data[1])
+        badge_value= f"{energy_value:.2f} {energy_unit} {via}"
 
     badge = anybadge.Badge(
-        label='Energy cost',
-        value=badge_value,
+        label=xml_escape(label),
+        value=xml_escape(badge_value),
         num_value_padding_chars=1,
         default_color='cornflowerblue')
     return Response(content=str(badge), media_type="image/svg+xml")
@@ -580,7 +598,7 @@ async def get_ci_badge_get(repo: str, branch: str, workflow:str):
 
     badge = anybadge.Badge(
         label='Energy Used',
-        value=badge_value,
+        value=xml_escape(badge_value),
         num_value_padding_chars=1,
         default_color='green')
     return Response(content=str(badge), media_type="image/svg+xml")