Merge branch 'main' into network-proxy

* main: Ignore filesystem paths Bugfix: Phases ordering was wrong way round eco-ci will now post the energy reading in the PR conversation (#397) SCI metric (#412) Fixes the github security errors (#411) API does not block returning machines, but will now return if they are available or not Enables empty services in the usage_scenario (#409) wrong import jobs.py now appends date Fixes the error on mac on which /tmp is a symlink to /private/tmp (#410) Fixes the metric output file becoming corrupted (#393) Only available machines may be listed Index.js now can filter by repo and filename (#408) Python requirements are now freshly updated with every install Gunicorn container now on python3-slim (Debian) instead of Ubuntu 22.04
green-coding-solutions · Aug 4, 2023 · 4f2993c · 4f2993c
2 parents 9fa39b3 + 736571d
commit 4f2993c
Show file tree

Hide file tree

Showing 24 changed files with 368 additions and 113 deletions.
diff --git a/.github/workflows/tests-vm-pr.yml b/.github/workflows/tests-vm-pr.yml
@@ -9,6 +9,7 @@ jobs:
     permissions:
       packages: write
       contents: read
+      pull-requests: write
     steps:
       - name: 'Checkout repository'
         uses: actions/checkout@v3
@@ -38,6 +39,5 @@ jobs:
         with:
           task: display-results
           branch: main
-
-
-
+          pr-comment: true
+
diff --git a/api/api.py b/api/api.py
@@ -8,6 +8,7 @@
 import sys
 import os
 
+from xml.sax.saxutils import escape as xml_escape
 from fastapi import FastAPI, Request, Response, status
 from fastapi.responses import ORJSONResponse
 from fastapi.encoders import jsonable_encoder
@@ -31,7 +32,6 @@
                          sanitize, get_phase_stats, get_phase_stats_object,
                          is_valid_uuid, rescale_energy_value)
 
-
 # It seems like FastAPI already enables faulthandler as it shows stacktrace on SEGFAULT
 # Is the redundant call problematic
 faulthandler.enable()  # will catch segfaults and write to STDERR
@@ -152,7 +152,7 @@ async def get_network(project_id):
 @app.get('/v1/machines/')
 async def get_machines():
     query = """
-            SELECT id, description
+            SELECT id, description, available
             FROM machines
             ORDER BY description ASC
             """
@@ -165,14 +165,28 @@ async def get_machines():
 
 # A route to return all of the available entries in our catalog.
 @app.get('/v1/projects')
-async def get_projects():
+async def get_projects(repo: str, filename: str):
     query = """
             SELECT a.id, a.name, a.uri, COALESCE(a.branch, 'main / master'), a.end_measurement, a.last_run, a.invalid_project, a.filename, b.description, a.commit_hash
             FROM projects as a
             LEFT JOIN machines as b on a.machine_id = b.id
-            ORDER BY a.created_at DESC  -- important to order here, the charting library in JS cannot do that automatically!
+            WHERE 1=1
             """
-    data = DB().fetch_all(query)
+    params = []
+
+    filename = filename.strip()
+    if filename not in ('', 'null'):
+        query = f"{query} AND a.filename LIKE %s  \n"
+        params.append(f"%{filename}%")
+
+    repo = repo.strip()
+    if repo not in ('', 'null'):
+        query = f"{query} AND a.uri LIKE %s \n"
+        params.append(f"%{repo}%")
+
+    query = f"{query} ORDER BY a.created_at DESC  -- important to order here, the charting library in JS cannot do that automatically!"
+
+    data = DB().fetch_all(query, params=tuple(params))
     if data is None or data == []:
         return Response(status_code=204) # No-Content
 
@@ -316,42 +330,46 @@ async def get_badge_single(project_id: str, metric: str = 'ml-estimated'):
         return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400)
 
     query = '''
-        WITH times AS (
-            SELECT start_measurement, end_measurement FROM projects WHERE id = %s
-        ) SELECT
-            (SELECT start_measurement FROM times), (SELECT end_measurement FROM times),
-            SUM(measurements.value), measurements.unit
-        FROM measurements
+        SELECT
+            SUM(value), MAX(unit)
+        FROM
+            phase_stats
         WHERE
-            measurements.project_id = %s
-            AND measurements.time >= (SELECT start_measurement FROM times)
-            AND measurements.time <= (SELECT end_measurement FROM times)
-            AND measurements.metric LIKE %s
-        GROUP BY measurements.unit
+            project_id = %s
+            AND metric LIKE %s
+            AND phase LIKE '%%_[RUNTIME]'
     '''
 
     value = None
+    label = 'Energy Cost'
+    via = ''
     if metric == 'ml-estimated':
         value = 'psu_energy_ac_xgboost_machine'
+        via = 'via XGBoost ML'
     elif metric == 'RAPL':
-        value = '%_rapl_%'
+        value = '%_energy_rapl_%'
+        via = 'via RAPL'
     elif metric == 'AC':
         value = 'psu_energy_ac_%'
+        via = 'via PSU (AC)'
+    elif metric == 'SCI':
+        label = 'SCI'
+        value = 'software_carbon_intensity_global'
     else:
         return ORJSONResponse({'success': False, 'err': f"Unknown metric '{metric}' submitted"}, status_code=400)
 
-    params = (project_id, project_id, value)
+    params = (project_id, value)
     data = DB().fetch_one(query, params=params)
 
-    if data is None or data == []:
+    if data is None or data == [] or not data[1] :
         badge_value = 'No energy data yet'
     else:
-        [energy_value, energy_unit] = rescale_energy_value(data[2], data[3])
-        badge_value= f"{energy_value:.2f} {energy_unit} via {metric}"
+        [energy_value, energy_unit] = rescale_energy_value(data[0], data[1])
+        badge_value= f"{energy_value:.2f} {energy_unit} {via}"
 
     badge = anybadge.Badge(
-        label='Energy cost',
-        value=badge_value,
+        label=xml_escape(label),
+        value=xml_escape(badge_value),
         num_value_padding_chars=1,
         default_color='cornflowerblue')
     return Response(content=str(badge), media_type="image/svg+xml")
@@ -553,7 +571,7 @@ async def get_ci_badge_get(repo: str, branch: str, workflow:str):
 
     badge = anybadge.Badge(
         label='Energy Used',
-        value=badge_value,
+        value=xml_escape(badge_value),
         num_value_padding_chars=1,
         default_color='green')
     return Response(content=str(badge), media_type="image/svg+xml")

diff --git a/api/api_helpers.py b/api/api_helpers.py
@@ -21,6 +21,17 @@
 
 METRIC_MAPPINGS = {
 
+
+    'embodied_carbon_share_machine': {
+        'clean_name': 'Embodied Carbon',
+        'source': 'formula',
+        'explanation': 'Embodied carbon attributed by time share of the life-span and total embodied carbon',
+    },
+    'software_carbon_intensity_global': {
+        'clean_name': 'SCI',
+        'source': 'formula',
+        'explanation': 'SCI metric by the Green Software Foundation',
+    },
     'phase_time_syscall_system': {
         'clean_name': 'Phase Duration',
         'source': 'Syscall',
@@ -223,17 +234,21 @@
 
 def rescale_energy_value(value, unit):
     # We only expect values to be mJ for energy!
-    if unit != 'mJ':
-        raise RuntimeError('Unexpected unit occured for energy rescaling: ', unit)
+    if unit in ['mJ', 'ug'] or unit.startswith('ugCO2e/'):
+        unit_type = unit[1:]
 
-    energy_rescaled = [value, unit]
+        energy_rescaled = [value, unit]
+
+        # pylint: disable=multiple-statements
+        if value > 1_000_000_000: energy_rescaled = [value/(10**12), f"G{unit_type}"]
+        elif value > 1_000_000_000: energy_rescaled = [value/(10**9), f"M{unit_type}"]
+        elif value > 1_000_000: energy_rescaled = [value/(10**6), f"k{unit_type}"]
+        elif value > 1_000: energy_rescaled = [value/(10**3), f"{unit_type}"]
+        elif value < 0.001: energy_rescaled = [value*(10**3), f"n{unit_type}"]
+
+    else:
+        raise RuntimeError('Unexpected unit occured for energy rescaling: ', unit)
 
-    # pylint: disable=multiple-statements
-    if value > 1_000_000_000: energy_rescaled = [value/(10**12), 'GJ']
-    elif value > 1_000_000_000: energy_rescaled = [value/(10**9), 'MJ']
-    elif value > 1_000_000: energy_rescaled = [value/(10**6), 'kJ']
-    elif value > 1_000: energy_rescaled = [value/(10**3), 'J']
-    elif value < 0.001: energy_rescaled = [value*(10**3), 'nJ']
 
     return energy_rescaled
 

diff --git a/config.yml.example b/config.yml.example
@@ -27,6 +27,7 @@ admin:
   notify_admin_for_own_project_ready: False
 
 
+
 cluster:
   api_url: __API_URL__
   metrics_url: __METRICS_URL__
@@ -122,3 +123,31 @@ measurement:
 #        HW_MemAmountGB: 16
 #        Hardware_Availability_Year: 2011
   #--- END
+
+
+sci:
+    # https://github.com/Green-Software-Foundation/sci/blob/main/Software_Carbon_Intensity/Software_Carbon_Intensity_Specification.md
+
+    # The values specific to the machine will be set here. The values that are specific to the
+    # software, like R – Functional unit, will be set in the usage_scenario.yml
+
+    # EL Expected Lifespan; the anticipated time that the equipment will be installed. Value is in years
+    # The number 3.5 comes from a typical developer machine (Apple Macbook 16" 2023 - https://dataviz.boavizta.org/manufacturerdata?lifetime=3.5&name=14-inch%20MacBook%20Pro%20with%2064GB)
+    EL: 3.5
+    # RS Resource-share; the share of the total available resources of the hardware reserved for use by the software.
+    # This ratio is typically 1 with the Green Metrics Tool unless you use a custom distributed orchestrator
+    RS: 1
+    # TE Total Embodied Emissions; the sum of Life Cycle Assessment (LCA) emissions for all hardware components.
+    # Value is in gCO2eq
+    # The value has to be identified from vendor datasheets. Here are some example sources:
+    # https://dataviz.boavizta.org/manufacturerdata
+    # https://tco.exploresurface.com/sustainability/calculator
+    # https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/Full_LCA_Dell_R740.pdf
+    # The default is the value for a developer machine (Apple Macbook 16" 2023 - https://dataviz.boavizta.org/manufacturerdata?lifetime=3.5&name=14-inch%20MacBook%20Pro%20with%2064GB)
+    TE: 194000
+    # I is the Carbon Intensity at the location of this machine
+    # The value can either be a number in gCO2e/kWh or a carbon intensity provider that fetches this number dynamically
+    # https://docs.green-coding.berlin/docs/measuring/carbon-intensity-providers/carbon-intensity-providers-overview/
+    # For fixed values get the number from https://ember-climate.org/insights/research/global-electricity-review-2022/
+    # The number 475 that comes as default is for Germany from 2022
+    I: 475
diff --git a/docker/Dockerfile-gunicorn b/docker/Dockerfile-gunicorn
@@ -1,14 +1,8 @@
 # syntax=docker/dockerfile:1
-FROM ubuntu:22.04
+FROM python:3.11.4-slim-bookworm
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN rm -rf /var/lib/apt/lists/*
-RUN apt update && \
-    apt install python3 python3-pip gunicorn -y
-
 COPY requirements.txt requirements.txt
 RUN pip3 install -r requirements.txt
 
-RUN rm -rf /var/lib/apt/lists/*
-
-ENTRYPOINT ["/bin/gunicorn", "--workers=2", "--access-logfile=-", "--error-logfile=-", "--worker-tmp-dir=/dev/shm", "--threads=4", "--worker-class=gthread",  "--bind", "unix:/tmp/green-coding-api.sock", "-m", "007", "--user", "www-data", "--chdir", "/var/www/green-metrics-tool/api", "-k", "uvicorn.workers.UvicornWorker", "api:app"]
+ENTRYPOINT ["/usr/local/bin/gunicorn", "--workers=2", "--access-logfile=-", "--error-logfile=-", "--worker-tmp-dir=/dev/shm", "--threads=4", "--worker-class=gthread",  "--bind", "unix:/tmp/green-coding-api.sock", "-m", "007", "--user", "www-data", "--chdir", "/var/www/green-metrics-tool/api", "-k", "uvicorn.workers.UvicornWorker", "api:app"]
diff --git a/docker/requirements.txt b/docker/requirements.txt
@@ -7,7 +7,5 @@ PyYAML==6.0.1
 anybadge==1.14.0
 uvicorn==0.23.2
 orjson==3.9.2
-pyserial==3.5
-psutil==5.9.5
 scipy==1.11.1
-schema==0.7.5
+schema==0.7.5
diff --git a/frontend/css/green-coding.css b/frontend/css/green-coding.css
@@ -160,6 +160,7 @@ a,
     text-overflow: ellipsis;
     overflow-wrap: normal;
     overflow: hidden;
+    white-space: nowrap;
 }
 
 .si-unit {

diff --git a/frontend/index.html b/frontend/index.html
@@ -53,6 +53,13 @@ <h1 class="ui header float left">
             <div class="results"></div>
         </div>
 -->
+        <div class="ui warning message hidden">
+            <div class="header">
+                <i class="notched warning icon"></i> Filters are active
+            </div>
+            <ul></ul>
+        </div>
+
         <table id="projects-table" class="ui sortable celled striped table">
             <thead>
                 <tr>

diff --git a/frontend/js/ci.js b/frontend/js/ci.js
@@ -171,7 +171,7 @@ const getChartOptions = (runs, chart_element) => {
 
 const displayGraph = (runs) => {
     const element = createChartContainer("#chart-container", "run-energy", runs);
-    
+
     const options = getChartOptions(runs, element);
 
     const chart_instance = echarts.init(element);
@@ -241,26 +241,26 @@ const displayCITable = (runs, url_params) => {
 
         var run_link = ''
         if(source == 'github') {
-            run_link = `https://github.com/${url_params.get('repo')}/actions/runs/${run_id}`;
+            run_link = `https://github.com/${escapeString(url_params.get('repo'))}/actions/runs/${escapeString(run_id)}`;
         }
         else if (source == 'gitlab') {
-            run_link = `https://gitlab.com/${url_params.get('repo')}/-/pipelines/${run_id}`
+            run_link = `https://gitlab.com/${escapeString(url_params.get('repo'))}/-/pipelines/${escapeString(run_id)}`
         }
 
-        const run_link_node = `<a href="${run_link}" target="_blank">${run_id}</a>`
+        const run_link_node = `<a href="${run_link}" target="_blank">${escapeString(run_id)}</a>`
 
         const created_at = el[3]
 
         const label = el[4]
         const duration = el[7]
 
-        li_node.innerHTML = `<td class="td-index">${value}</td>\
-                            <td class="td-index">${label}</td>\
+        li_node.innerHTML = `<td class="td-index">${escapeString(value)}</td>\
+                            <td class="td-index">${escapeString(label)}</td>\
                             <td class="td-index">${run_link_node}</td>\
-                            <td class="td-index"><span title="${created_at}">${dateToYMD(new Date(created_at))}</span></td>\
-                            <td class="td-index" ${tooltip}>${short_hash}</td>\
-                            <td class="td-index">${cpu}</td>\
-                            <td class="td-index">${duration} seconds</td>`;
+                            <td class="td-index"><span title="${escapeString(created_at)}">${dateToYMD(new Date(created_at))}</span></td>\
+                            <td class="td-index" ${escapeString(tooltip)}>${escapeString(short_hash)}</td>\
+                            <td class="td-index">${escapeString(cpu)}</td>\
+                            <td class="td-index">${escapeString(duration)} seconds</td>`;
         document.querySelector("#ci-table").appendChild(li_node);
     });
     $('table').tablesort();
@@ -318,17 +318,17 @@ $(document).ready((e) => {
         let repo_link = ''
 
         if(badges_data.data[0][8] == 'github') {
-            repo_link = `https://github.com/${url_params.get('repo')}`;
+            repo_link = `https://github.com/${escapeString(url_params.get('repo'))}`;
         }
         else if(badges_data.data[0][8] == 'gitlab') {
-            repo_link = `https://gitlab.com/${url_params.get('repo')}`;
+            repo_link = `https://gitlab.com/${escapeString(url_params.get('repo'))}`;
         }
         //${repo_link}
-        const repo_link_node = `<a href="${repo_link}" target="_blank">${url_params.get('repo')}</a>`
+        const repo_link_node = `<a href="${repo_link}" target="_blank">${escapeString(url_params.get('repo'))}</a>`
         document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `<tr><td><strong>Repository:</strong></td><td>${repo_link_node}</td></tr>`)
-        document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `<tr><td><strong>Branch:</strong></td><td>${url_params.get('branch')}</td></tr>`)
-        document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `<tr><td><strong>Workflow:</strong></td><td>${url_params.get('workflow')}</td></tr>`)
-        
+        document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `<tr><td><strong>Branch:</strong></td><td>${escapeString(url_params.get('branch'))}</td></tr>`)
+        document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `<tr><td><strong>Workflow:</strong></td><td>${escapeString(url_params.get('workflow'))}</td></tr>`)
+
         displayCITable(badges_data.data, url_params);
         chart_instance = displayGraph(badges_data.data)
         displayAveragesTable(badges_data.data)

diff --git a/frontend/js/helpers/config.js.example b/frontend/js/helpers/config.js.example
@@ -60,4 +60,15 @@ const machine_power_metric_condition = (metric) => {
 const machine_energy_metric_condition = (metric) => {
     if(metric.match(/^.*_energy_.*_machine$/) !== null) return true;
     return false;
-}
+}
+
+const sci_metric_condition = (metric) => {
+    if(metric == 'software_carbon_intensity_global') return true;
+    return false;
+}
+
+const embodied_carbon_share_metric_condition = (metric) => {
+    if(metric == 'embodied_carbon_share_machine') return true;
+    return false;
+}
+