Merge branch 'main' into user-zero

* main:
  pytest.skip() [skip ci]
  Adding saving for image and volume sizes (#1027)
  Added average function to badges (#1029)
  Bump psutil from 6.1.0 to 6.1.1 (#1028)
  Removing href for non actual links [skip ci] (#1026)
  Badges for CI now include carbon and also Totals (#998)
  Removes dead link
  Bump aiohttp from 3.11.10 to 3.11.11 (#1025)
  Bump pydantic from 2.10.3 to 2.10.4 (#1024)
  Bump deepdiff from 8.0.1 to 8.1.1 (#1020)
  Updated cloud energy
  (fix): Unselect button broke repositories view
  Noise reduced run start (#1023)
  (improvement): More strict check
  Added unselect button [skip ci] (#1022)

api/api_helpers.py

@@ -59,15 +59,15 @@ def store_artifact(artifact_type: Enum, key:str, data, ex=2592000):
 # Use this function never in the phase_stats. The metrics must always be on
 # The same unit for proper comparison!
 #
-def rescale_energy_value(value, unit):
+def rescale_metric_value(value, unit):
     if unit == 'mJ':
         value = value * 1_000
         unit = 'uJ'
 
     # We only expect values to be uJ for energy in the future. Changing values now temporarily.
     # TODO: Refactor this once all data in the DB is uJ
-    if unit != 'uJ' and not unit.startswith('ugCO2e/'):
-        raise ValueError('Unexpected unit occured for energy rescaling: ', unit)
+    if unit not in ('uJ', 'ug') and not unit.startswith('ugCO2e/'):
+        raise ValueError('Unexpected unit occured for metric rescaling: ', unit)
 
     unit_type = unit[1:]
 
@@ -690,7 +690,7 @@ def __init__(
 header_scheme = APIKeyHeader(
     name='X-Authentication',
     scheme_name='Header',
-    description='Authentication key - See https://docs.green-coding.io/authentication',
+    description='Authentication key',
     auto_error=False
 )
 

api/eco_ci.py

@@ -3,8 +3,9 @@
 from fastapi import APIRouter
 from fastapi import Request, Response, Depends
 from fastapi.responses import ORJSONResponse
+from fastapi.exceptions import RequestValidationError
 
-from api.api_helpers import authenticate, html_escape_multi, get_connecting_ip, rescale_energy_value
+from api.api_helpers import authenticate, html_escape_multi, get_connecting_ip, rescale_metric_value
 from api.object_specifications import CI_Measurement_Old, CI_Measurement
 
 import anybadge
@@ -178,6 +179,64 @@ async def get_ci_measurements(repo: str, branch: str, workflow: str, start_date:
 
     return ORJSONResponse({'success': True, 'data': data})
 
+@router.get('/v1/ci/stats')
+async def get_ci_stats(repo: str, branch: str, workflow: str, start_date: date, end_date: date):
+
+
+    query = '''
+        WITH my_table as (
+            SELECT
+                SUM(energy_uj) as a,
+                SUM(duration_us) as b,
+                SUM(cpu_util_avg * duration_us) / NULLIF(SUM(duration_us), 0)  as c, -- weighted average
+                SUM(carbon_intensity_g * duration_us) / NULLIF(SUM(duration_us), 0) as d,-- weighted average
+                SUM(carbon_ug) as e
+            FROM ci_measurements
+            WHERE
+                repo = %s AND branch = %s AND workflow_id = %s
+                AND DATE(created_at) >= TO_DATE(%s, 'YYYY-MM-DD') AND DATE(created_at) <= TO_DATE(%s, 'YYYY-MM-DD')
+            GROUP BY run_id
+        ) SELECT
+            -- Cast is to avoid DECIMAL which ORJJSON cannot handle
+            AVG(a)::float, SUM(a)::float, STDDEV(a)::float, (STDDEV(a) / NULLIF(AVG(a), 0))::float * 100,
+            AVG(b)::float, SUM(b)::float, STDDEV(b)::float, (STDDEV(b) / NULLIF(AVG(b), 0))::float * 100,
+            AVG(c)::float, NULL, STDDEV(c)::float, (STDDEV(c) / NULLIF(AVG(c), 0))::float * 100, -- SUM of cpu_util_avg makes no sense
+            AVG(d)::float, NULL, STDDEV(d)::float, (STDDEV(d) / NULLIF(AVG(d), 0))::float * 100, -- SUM of carbon_intensity_g makes no sense
+            AVG(e)::float, SUM(e)::float, STDDEV(e)::float, (STDDEV(e) / NULLIF(AVG(e), 0))::float * 100,
+            COUNT(*)
+        FROM my_table;
+    '''
+    params = (repo, branch, workflow, str(start_date), str(end_date))
+    totals_data = DB().fetch_one(query, params=params)
+
+    if totals_data is None or totals_data[0] is None: # aggregate query always returns row
+        return Response(status_code=204)  # No-Content
+
+    query = '''
+        SELECT
+            -- Cast is to avoid DECIMAL which ORJJSON cannot handle
+            -- Here we do not need a weighted average, even if the times differ, because we specifically want to look per step and duration is not relevant
+            AVG(energy_uj)::float, SUM(energy_uj)::float, STDDEV(energy_uj)::float, (STDDEV(energy_uj) / NULLIF(AVG(energy_uj), 0))::float * 100,
+            AVG(duration_us)::float, SUM(duration_us)::float, STDDEV(duration_us)::float, (STDDEV(duration_us) / NULLIF(AVG(duration_us), 0))::float * 100,
+            AVG(cpu_util_avg)::float, NULL, STDDEV(cpu_util_avg)::float, (STDDEV(cpu_util_avg) / NULLIF(AVG(cpu_util_avg), 0))::float * 100, -- SUM of cpu_util_avg makes no sense
+            AVG(carbon_intensity_g)::float, NULL, STDDEV(carbon_intensity_g)::float, (STDDEV(carbon_intensity_g) / NULLIF(AVG(carbon_intensity_g), 0))::float * 100, -- SUM of carbon_intensity_g makes no sense
+            AVG(carbon_ug)::float, SUM(carbon_ug)::float, STDDEV(carbon_ug)::float, (STDDEV(carbon_ug) / NULLIF(AVG(carbon_ug), 0))::float * 100,
+            COUNT(*), label
+        FROM ci_measurements
+        WHERE
+            repo = %s AND branch = %s AND workflow_id = %s
+            AND DATE(created_at) >= TO_DATE(%s, 'YYYY-MM-DD') AND DATE(created_at) <= TO_DATE(%s, 'YYYY-MM-DD')
+        GROUP BY label
+    '''
+    params = (repo, branch, workflow, str(start_date), str(end_date))
+    per_label_data = DB().fetch_all(query, params=params)
+
+    if per_label_data is None or per_label_data[0] is None:
+        return Response(status_code=204)  # No-Content
+
+    return ORJSONResponse({'success': True, 'data': {'totals': totals_data, 'per_label': per_label_data}})
+
+
 @router.get('/v1/ci/repositories')
 async def get_ci_repositories(repo: str | None = None, sort_by: str = 'name'):
 
@@ -238,30 +297,74 @@ async def get_ci_runs(repo: str, sort_by: str = 'name'):
     return ORJSONResponse({'success': True, 'data': data}) # no escaping needed, as it happend on ingest
 
 @router.get('/v1/ci/badge/get')
-async def get_ci_badge_get(repo: str, branch: str, workflow:str):
-    query = """
-        SELECT SUM(energy_uj), MAX(run_id)
+async def get_ci_badge_get(repo: str, branch: str, workflow:str, mode: str = 'last', metric: str = 'energy', duration_days: int | None = None):
+    if metric == 'energy':
+        metric = 'energy_uj'
+        metric_unit = 'uJ'
+        label = 'energy used'
+        default_color = 'orange'
+    elif metric == 'carbon':
+        metric = 'carbon_ug'
+        metric_unit = 'ug'
+        label = 'carbon emitted'
+        default_color = 'black'
+    # Do not easily add values like cpu_util or carbon_intensity_g here. They need a weighted average in the SQL query later!
+    else:
+        raise RequestValidationError('Unsupported metric requested')
+
+
+    if duration_days and (duration_days < 1 or duration_days > 365):
+        raise RequestValidationError('Duration days must be between 1 and 365 days')
+
+    params = [repo, branch, workflow]
+
+
+    query = f"""
+        SELECT SUM({metric})
         FROM ci_measurements
         WHERE repo = %s AND branch = %s AND workflow_id = %s
-        GROUP BY run_id
-        ORDER BY MAX(created_at) DESC
-        LIMIT 1
     """
 
-    params = (repo, branch, workflow)
+    if mode == 'avg':
+        if not duration_days:
+            raise RequestValidationError('Duration days must be set for average')
+        query = f"""
+            WITH my_table as (
+                SELECT SUM({metric}) my_sum
+                FROM ci_measurements
+                WHERE repo = %s AND branch = %s AND workflow_id = %s AND DATE(created_at) > NOW() - make_interval(days => %s)
+                GROUP BY run_id
+            ) SELECT AVG(my_sum) FROM my_table;
+        """
+        params.append(duration_days)
+        label = f"Per run moving average ({duration_days} days) {label}"
+    elif mode == 'last':
+        query = f"{query} GROUP BY run_id ORDER BY MAX(created_at) DESC LIMIT 1"
+        label = f"Last run {label}"
+    elif mode == 'totals' and duration_days:
+        query = f"{query} AND DATE(created_at) > NOW() - make_interval(days => %s)"
+        params.append(duration_days)
+        label = f"Last {duration_days} days total {label}"
+    elif mode == 'totals':
+        label = f"All runs total {label}"
+    else:
+        raise RuntimeError('Unknown mode')
+
+
     data = DB().fetch_one(query, params=params)
 
-    if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result
+    if data is None or data == [] or data[0] is None: # special check for SUM element as this is aggregate query which always returns result
         return Response(status_code=204) # No-Content
 
-    energy_value = data[0]
+    metric_value = data[0]
 
-    [energy_value, energy_unit] = rescale_energy_value(energy_value, 'uJ')
-    badge_value= f"{energy_value:.2f} {energy_unit}"
+    [metric_value, metric_unit] = rescale_metric_value(metric_value, metric_unit)
+    badge_value= f"{metric_value:.2f} {metric_unit}"
 
     badge = anybadge.Badge(
-        label='Energy Used',
+        label=label,
         value=xml_escape(badge_value),
         num_value_padding_chars=1,
-        default_color='green')
+        default_color=default_color)
+
     return Response(content=str(badge), media_type="image/svg+xml")

api/main.py

@@ -25,7 +25,7 @@
 from api.api_helpers import (ORJSONResponseObjKeep, add_phase_stats_statistics,
                          determine_comparison_case,get_comparison_details,
                          html_escape_multi, get_phase_stats, get_phase_stats_object,
-                         is_valid_uuid, rescale_energy_value, get_timeline_query,
+                         is_valid_uuid, rescale_metric_value, get_timeline_query,
                          get_run_info, get_machine_list, get_artifact, store_artifact,
                          authenticate)
 
@@ -514,10 +514,10 @@ async def get_badge_single(run_id: str, metric: str = 'ml-estimated'):
     data = DB().fetch_one(query, params=params)
 
     if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result
-        badge_value = 'No energy data yet'
+        badge_value = 'No metric data yet'
     else:
-        [energy_value, energy_unit] = rescale_energy_value(data[0], data[1])
-        badge_value= f"{energy_value:.2f} {energy_unit} {via}"
+        [metric_value, energy_unit] = rescale_metric_value(data[0], data[1])
+        badge_value= f"{metric_value:.2f} {energy_unit} {via}"
 
     badge = anybadge.Badge(
         label=xml_escape(label),

docker/requirements.txt

@@ -10,7 +10,7 @@ anybadge==1.14.0
 orjson==3.10.12
 scipy==1.14.1
 schema==0.7.7
-deepdiff==8.0.1
+deepdiff==8.1.1
 redis==5.2.1
 hiredis==3.1.0
 requests==2.32.3

frontend/authentication.html

@@ -27,7 +27,7 @@
     <gmt-menu></gmt-menu>
     <div class="main ui container" id="main">
         <h1 class="ui header float left">
-            <a href="#" id="menu-toggle" class="opened"><i class="bars bordered inverted left icon openend"></i></a>
+            <a id="menu-toggle" class="opened"><i class="bars bordered inverted left icon openend"></i></a>
             Authentication
         </h1>
         <div class="ui full-width-card segment card">

frontend/ci-index.html

@@ -31,7 +31,7 @@
     <gmt-menu></gmt-menu>
     <div class="main ui container" id="main">
         <h1 class="ui header float left">
-            <a href="#" id="menu-toggle" class="opened"><i class="bars bordered inverted left icon openend"></i></a>
+            <a id="menu-toggle" class="opened"><i class="bars bordered inverted left icon openend"></i></a>
             CI Projects
         </h1>
 <!--

frontend/ci.html

@@ -32,10 +32,10 @@
 <body class="preload">
     <gmt-menu></gmt-menu>
     <div class="main ui container" id="main">
-        <h1 class="ui header float left"><a href="#" id="menu-toggle" class="opened"><i class="bars bordered inverted left icon openend"></i></a> CI Run Info</h1>
+        <h1 class="ui header float left"><a id="menu-toggle" class="opened"><i class="bars bordered inverted left icon openend"></i></a> CI Run Info</h1>
         <div class="ui full-width-card card">
             <div class="content">
-                <div class="header"><a class="ui red ribbon label" href="#">
+                <div class="header"><a class="ui red ribbon label">
                     <h3>General Info</h3>
                 </a></div>
                 <div class="description">
@@ -44,9 +44,35 @@ <h3>General Info</h3>
                             <tr>
                                 <td><strong>Last Run Badge:</strong></td>
                                 <td>
-                                    <span class="energy-badge-container" data-metric="ml-estimated"></span>
+                                    <span id="energy-badge-container-last" class="badge-container"></span>
+                                    <a class="copy-badge"><i class="copy icon"></i></a>
+                                </td>
+                                <td>
+                                    <span id="carbon-badge-container-last" class="badge-container"></span>
+                                    <a class="copy-badge"><i class="copy icon"></i></a>
+                                </td>
+                            </tr>
+                            <tr>
+                                <td><strong>Totals Badge:</strong></td>
+                                <td>
+                                    <span id="energy-badge-container-totals" class="badge-container"></span>
                                     <a href="#" class="copy-badge"><i class="copy icon"></i></a>
                                 </td>
+                                <td>
+                                    <span id="carbon-badge-container-totals" class="badge-container"></span>
+                                    <a class="copy-badge"><i class="copy icon"></i></a>
+                                </td>
+                            </tr>
+                            <tr>
+                                <td data-position="bottom left" data-inverted="" data-tooltip="Modify the badge source code to show alternative durations like 60 days, 365 days etc."><strong>Monthly Badge <i class="question circle icon"></i>: </strong></td>
+                                <td>
+                                    <span id="energy-badge-container-totals-monthly" class="badge-container"></span>
+                                    <a class="copy-badge"><i class="copy icon"></i></a>
+                                </td>
+                                <td>
+                                    <span id="carbon-badge-container-totals-monthly" class="badge-container"></span>
+                                    <a class="copy-badge"><i class="copy icon"></i></a>
+                                </td>
                             </tr>
                         </table>
                     </div>
@@ -93,7 +119,7 @@ <h3>General Info</h3>
 
         </div>
         <div class = "ui segment" id="stats-container">
-            <div class="header"><a class="ui teal ribbon label" href="#">
+            <div class="header"><a class="ui teal ribbon label">
                     <h3>Pipeline stats</h3>
                 </a></div>
                 <br/>
@@ -132,8 +158,34 @@ <h3>Pipeline stats</h3>
                 </table>
             </div>
         </div>
-        <div class="ui segment" id="runs-table">
-            <div class="header"><a class="ui teal ribbon label" href="#">
+
+       <div id="loader-question" class="ui icon info message blue">
+            <i class="info circle icon"></i>
+
+            <div class="content">
+                <div class="header">
+                    Runs detail table is not displayed automatically
+                </div>
+                <p>Please click the button below to fetch data.</p>
+                <p>You can change the default behaviour under <a href="/settings.html" style="text-decoration: underline; font-weight: bold;">Settings</a></p>
+                <button id="display-run-details-table" class="blue ui button">Display runs table</button>
+            </div>
+            <ul></ul>
+        </div>
+
+        <div class="ui one cards" id="api-loader" style="display:none;">
+            <div class="card" style="min-height: 300px">
+                <div class="ui active dimmer">
+                    <div class="ui indeterminate text loader">Building table ...</div>
+                </div>
+                <p></p>
+            </div>
+        </div>
+        <div id="chart-container"></div>
+
+
+        <div class="ui segment" id="run-details-table" style="display: none">
+            <div class="header"><a class="ui teal ribbon label">
                     <h3 data-tooltip="The runs table shows all measurements your pipeline has made in the selected timeframe" data-position="top left">Runs Table <i class="question circle icon "></i> </h3>
                 </a></div>
             <table class="ui sortable celled striped table">