diff --git a/.github/workflows/tests-bare-metal-main.yml b/.github/workflows/tests-bare-metal-main.yml
index 824e056ec..92bd8cc2f 100644
--- a/.github/workflows/tests-bare-metal-main.yml
+++ b/.github/workflows/tests-bare-metal-main.yml
@@ -35,7 +35,7 @@ jobs:
uses: actions/checkout@v4
with:
ref: 'main'
- submodules: 'true'
+ submodules: 'false'
- name: Eco CI Energy Estimation - Initialize
uses: green-coding-solutions/eco-ci-energy-estimation@main
diff --git a/.github/workflows/tests-eco-ci-energy-estimation.yaml b/.github/workflows/tests-eco-ci-energy-estimation.yaml
index 517437343..95af24b58 100644
--- a/.github/workflows/tests-eco-ci-energy-estimation.yaml
+++ b/.github/workflows/tests-eco-ci-energy-estimation.yaml
@@ -14,7 +14,7 @@ jobs:
uses: actions/checkout@v4
with:
ref: 'main'
- submodules: 'true'
+ submodules: 'false'
- name: Eco CI Energy Estimation - Initialize
uses: green-coding-solutions/eco-ci-energy-estimation@testing
diff --git a/.github/workflows/tests-vm-mac.yml b/.github/workflows/tests-vm-mac.yml
index 1b856060b..84dff4d93 100644
--- a/.github/workflows/tests-vm-mac.yml
+++ b/.github/workflows/tests-vm-mac.yml
@@ -21,7 +21,7 @@ jobs:
uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- submodules: 'true'
+ submodules: 'false'
- if: ${{ github.event_name == 'workflow_dispatch' || steps.check-date.outputs.should_run == 'true'}}
name: 'Setup, Run, and Teardown Tests'
diff --git a/.github/workflows/tests-vm-main.yml b/.github/workflows/tests-vm-main.yml
index d2e49a976..9cd63ec97 100644
--- a/.github/workflows/tests-vm-main.yml
+++ b/.github/workflows/tests-vm-main.yml
@@ -36,7 +36,7 @@ jobs:
uses: actions/checkout@v4
with:
ref: 'main'
- submodules: 'true'
+ submodules: 'false'
- if: ${{ github.event_name == 'workflow_dispatch' || steps.check-date.outputs.should_run == 'true'}}
name: 'Setup, Run, and Teardown Tests'
diff --git a/.github/workflows/tests-vm-pr.yml b/.github/workflows/tests-vm-pr.yml
index 8da9eaa67..fcb8fa912 100644
--- a/.github/workflows/tests-vm-pr.yml
+++ b/.github/workflows/tests-vm-pr.yml
@@ -23,7 +23,7 @@ jobs:
uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- submodules: 'true'
+ submodules: 'false'
- name: 'Setup, Run, and Teardown Tests'
diff --git a/.gitignore b/.gitignore
index 39638a75e..7a2e1adc0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,4 @@ static-binary
/docker/test-compose.yml
/tests/structure.sql
tools/sgx_enable
+/tests/test-config.yml
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
index aed73413c..3af8fb727 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,6 @@
[submodule "metric_providers/psu/energy/ac/xgboost/machine/model"]
path = metric_providers/psu/energy/ac/xgboost/machine/model
url = https://github.com/green-coding-solutions/spec-power-model
+[submodule "ee"]
+ path = ee
+ url = git@github.com:green-coding-solutions/gmt-enterprise.git
diff --git a/api/api_helpers.py b/api/api_helpers.py
index b98effc7f..16ff15741 100644
--- a/api/api_helpers.py
+++ b/api/api_helpers.py
@@ -2,19 +2,18 @@
import faulthandler
faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr
+from urllib.parse import urlparse
+
from functools import cache
from html import escape as html_escape
-import re
-import math
import typing
-import ipaddress
-import json
import uuid
from starlette.background import BackgroundTask
from fastapi.responses import ORJSONResponse
+from fastapi import Depends, Request, HTTPException
+from fastapi.security import APIKeyHeader
import numpy as np
-import requests
import scipy.stats
from pydantic import BaseModel
@@ -22,6 +21,8 @@
from lib.global_config import GlobalConfig
from lib.db import DB
from lib import error_helpers
+from lib.user import User, UserAuthenticationError
+from lib.secure_variable import SecureVariable
import redis
from enum import Enum
@@ -632,191 +633,33 @@ def __init__(
self.content = content
super().__init__(content, status_code, headers, media_type, background)
-# The decorator will not work between requests, so we are not prone to stale data over time
-@cache
-def get_geo(ip):
-
- ip_obj = ipaddress.ip_address(ip) # may raise a ValueError
- if ip_obj.is_private:
- error_helpers.log_error(f"Private IP was submitted to get_geo {ip}. This is normal in development, but should not happen in production.")
- return('52.53721666833642', '13.424863870661927')
-
- query = "SELECT ip_address, data FROM ip_data WHERE created_at > NOW() - INTERVAL '24 hours' AND ip_address=%s;"
- db_data = DB().fetch_all(query, (ip,))
-
- if db_data is not None and len(db_data) != 0:
- return (db_data[0][1].get('latitude'), db_data[0][1].get('longitude'))
-
- latitude, longitude = get_geo_ip_api_com(ip)
-
- if not latitude:
- latitude, longitude = get_geo_ipapi_co(ip)
- if not latitude:
- latitude, longitude = get_geo_ip_ipinfo(ip)
- if not latitude:
- raise RuntimeError(f"Could not get Geo-IP for {ip} after 3 tries")
-
- return (latitude, longitude)
-
-
-def get_geo_ipapi_co(ip):
-
- print(f"Accessing https://ipapi.co/{ip}/json/")
- try:
- response = requests.get(f"https://ipapi.co/{ip}/json/", timeout=10)
- except Exception as exc: #pylint: disable=broad-exception-caught
- error_helpers.log_error('API request to ipapi.co failed ...', exception=exc)
- return (False, False)
-
- if response.status_code == 200:
- resp_data = response.json()
-
- if 'error' in resp_data or 'latitude' not in resp_data or 'longitude' not in resp_data:
- return (None, None)
-
- resp_data['source'] = 'ipapi.co'
-
- query = "INSERT INTO ip_data (ip_address, data) VALUES (%s, %s)"
- DB().query(query=query, params=(ip, json.dumps(resp_data)))
- return (resp_data.get('latitude'), resp_data.get('longitude'))
+header_scheme = APIKeyHeader(
+ name='X-Authentication',
+ scheme_name='Header',
+ description='Authentication key - See https://docs.green-coding.io/authentication',
+ auto_error=False
+)
- error_helpers.log_error(f"Could not get Geo-IP from ipapi.co for {ip}. Trying next ...", response=response)
-
- return (False, False)
-
-def get_geo_ip_api_com(ip):
-
- print(f"Accessing http://ip-api.com/json/{ip}")
- try:
- response = requests.get(f"http://ip-api.com/json/{ip}", timeout=10)
- except Exception as exc: #pylint: disable=broad-exception-caught
- error_helpers.log_error('API request to ip-api.com failed ...', exception=exc)
- return (False, False)
-
- if response.status_code == 200:
- resp_data = response.json()
-
- if ('status' in resp_data and resp_data.get('status') == 'fail') or 'lat' not in resp_data or 'lon' not in resp_data:
- return (None, None)
-
- resp_data['latitude'] = resp_data.get('lat')
- resp_data['longitude'] = resp_data.get('lon')
- resp_data['source'] = 'ip-api.com'
-
- query = "INSERT INTO ip_data (ip_address, data) VALUES (%s, %s)"
- DB().query(query=query, params=(ip, json.dumps(resp_data)))
-
- return (resp_data.get('latitude'), resp_data.get('longitude'))
-
- error_helpers.log_error(f"Could not get Geo-IP from ip-api.com for {ip}. Trying next ...", response=response)
-
- return (False, False)
-
-def get_geo_ip_ipinfo(ip):
-
- print(f"Accessing https://ipinfo.io/{ip}/json")
+def authenticate(authentication_token=Depends(header_scheme), request: Request = None):
+ parsed_url = urlparse(str(request.url))
try:
- response = requests.get(f"https://ipinfo.io/{ip}/json", timeout=10)
- except Exception as exc: #pylint: disable=broad-exception-caught
- error_helpers.log_error('API request to ipinfo.io failed ...', exception=exc)
- return (False, False)
-
- if response.status_code == 200:
- resp_data = response.json()
-
- if 'bogon' in resp_data or 'loc' not in resp_data:
- return (None, None)
-
- lat_lng = resp_data.get('loc').split(',')
+ if not authentication_token or authentication_token.strip() == '': # Note that if no token is supplied this will authenticate as the DEFAULT user, which in FOSS systems has full capabilities
+ authentication_token = 'DEFAULT'
- resp_data['latitude'] = lat_lng[0]
- resp_data['longitude'] = lat_lng[1]
- resp_data['source'] = 'ipinfo.io'
+ user = User.authenticate(SecureVariable(authentication_token))
- query = "INSERT INTO ip_data (ip_address, data) VALUES (%s, %s)"
- DB().query(query=query, params=(ip, json.dumps(resp_data)))
+ if not user.can_use_route(parsed_url.path):
+ raise HTTPException(status_code=401, detail="Route not allowed") from UserAuthenticationError
- return (resp_data.get('latitude'), resp_data.get('longitude'))
+ if not user.has_api_quota(parsed_url.path):
+ raise HTTPException(status_code=401, detail="Quota exceeded") from UserAuthenticationError
- error_helpers.log_error(f"Could not get Geo-IP from ipinfo.io for {ip}. Trying next ...", response=response)
-
- return (False, False)
-
-# The decorator will not work between requests, so we are not prone to stale data over time
-@cache
-def get_carbon_intensity(latitude, longitude):
-
- if latitude is None or longitude is None:
- return None
-
- query = "SELECT latitude, longitude, data FROM carbon_intensity WHERE created_at > NOW() - INTERVAL '1 hours' AND latitude=%s AND longitude=%s;"
- db_data = DB().fetch_all(query, (latitude, longitude))
-
- if db_data is not None and len(db_data) != 0:
- return db_data[0][2].get('carbonIntensity')
-
- if not (electricitymaps_token := GlobalConfig().config.get('electricity_maps_token')):
- raise ValueError('You need to specify an electricitymap token in the config!')
-
- if electricitymaps_token == 'testing':
- # If we are running tests we always return 1000
- return 1000
-
- headers = {'auth-token': electricitymaps_token }
- params = {'lat': latitude, 'lon': longitude }
-
- response = requests.get('https://api.electricitymap.org/v3/carbon-intensity/latest', params=params, headers=headers, timeout=10)
- print(f"Accessing electricitymap with {latitude} {longitude}")
- if response.status_code == 200:
- resp_data = response.json()
- query = "INSERT INTO carbon_intensity (latitude, longitude, data) VALUES (%s, %s, %s)"
- DB().query(query=query, params=(latitude, longitude, json.dumps(resp_data)))
-
- return resp_data.get('carbonIntensity')
-
- error_helpers.log_error(f"Could not get carbon intensity from Electricitymaps.org for {params}", response=response)
-
- return None
-
-def carbondb_add(connecting_ip, data, source, user_id):
-
- query = '''
- INSERT INTO carbondb_data_raw
- ("type", "project", "machine", "source", "tags","time","energy_kwh","carbon_kg","carbon_intensity_g","latitude","longitude","ip_address","user_id","created_at")
- VALUES
- (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())
- '''
-
- used_client_ip = data.get('ip', None) # An ip has been given with the data. We prioritize that
- if used_client_ip is None:
- used_client_ip = connecting_ip
-
- carbon_intensity_g_per_kWh = data.get('carbon_intensity_g', None)
-
- if carbon_intensity_g_per_kWh is not None: # we need this check explicitely as we want to allow 0 as possible value
- latitude = None # no use to derive if we get supplied data. We rather indicate with NULL that user supplied
- longitude = None # no use to derive if we get supplied data. We rather indicate with NULL that user supplied
- else:
- latitude, longitude = get_geo(used_client_ip) # cached
- carbon_intensity_g_per_kWh = get_carbon_intensity(latitude, longitude) # cached
-
- energy_J = float(data['energy_uj']) / 1e6
- energy_kWh = energy_J / (3_600*1_000)
- carbon_kg = (energy_kWh * carbon_intensity_g_per_kWh)/1_000
-
- DB().query(
- query=query,
- params=(
- data['type'],
- data['project'], data['machine'], source, data['tags'], data['time'], energy_kWh, carbon_kg, carbon_intensity_g_per_kWh, latitude, longitude, used_client_ip, user_id))
-
-
-def validate_carbondb_params(param, elements: list):
- for el in elements:
- if not re.fullmatch(r'[A-Za-z0-9\._-]+', el):
- raise ValueError(f"Parameter for '{param}' may only contain A-Za-z0-9._- characters and no spaces. Was: {el}")
+ user.deduct_api_quota(parsed_url.path, 1)
+ except UserAuthenticationError:
+ raise HTTPException(status_code=401, detail="Invalid token") from UserAuthenticationError
+ return user
def get_connecting_ip(request):
connecting_ip = request.headers.get("x-forwarded-for")
@@ -825,64 +668,3 @@ def get_connecting_ip(request):
return connecting_ip.split(",")[0]
return request.client.host
-
-
-def replace_nan_with_zero(obj):
- if isinstance(obj, dict):
- for k, v in obj.items():
- if isinstance(v, (dict, list)):
- replace_nan_with_zero(v)
- elif isinstance(v, float) and math.isnan(v):
- obj[k] = 0
- elif isinstance(obj, list):
- for i, item in enumerate(obj):
- if isinstance(item, (dict, list)):
- replace_nan_with_zero(item)
- elif isinstance(item, float) and math.isnan(item):
- obj[i] = 0
- return obj
-
-# Refactor have this in the Pydantic model?
-# https://github.com/green-coding-solutions/green-metrics-tool/issues/907
-def validate_hog_measurement_data(data):
- required_top_level_fields = [
- 'coalitions', 'all_tasks', 'elapsed_ns', 'processor', 'thermal_pressure'
- ]
- for field in required_top_level_fields:
- if field not in data:
- raise ValueError(f"Missing required field: {field}")
-
- # Validate 'coalitions' structure
- if not isinstance(data['coalitions'], list):
- raise ValueError("Expected 'coalitions' to be a list")
-
- for coalition in data['coalitions']:
- required_coalition_fields = [
- 'name', 'tasks', 'energy_impact_per_s', 'cputime_ms_per_s',
- 'diskio_bytesread', 'diskio_byteswritten', 'intr_wakeups', 'idle_wakeups'
- ]
- for field in required_coalition_fields:
- if field not in coalition:
- raise ValueError(f"Missing required coalition field: {field}")
- if field == 'tasks' and not isinstance(coalition['tasks'], list):
- raise ValueError(f"Expected 'tasks' to be a list in coalition: {coalition['name']}")
-
- # Validate 'all_tasks' structure
- if 'energy_impact_per_s' not in data['all_tasks']:
- raise ValueError("Missing 'energy_impact_per_s' in 'all_tasks'")
-
- # Validate 'processor' structure based on the processor type
- processor_fields = data['processor'].keys()
- if 'ane_energy' in processor_fields:
- required_processor_fields = ['combined_power', 'cpu_energy', 'gpu_energy', 'ane_energy']
- elif 'package_joules' in processor_fields:
- required_processor_fields = ['package_joules', 'cpu_joules', 'igpu_watts']
- else:
- raise ValueError("Unknown processor type")
-
- for field in required_processor_fields:
- if field not in processor_fields:
- raise ValueError(f"Missing required processor field: {field}")
-
- # All checks passed
- return True
diff --git a/api/eco_ci.py b/api/eco_ci.py
new file mode 100644
index 000000000..b6815915b
--- /dev/null
+++ b/api/eco_ci.py
@@ -0,0 +1,267 @@
+from datetime import date
+
+from fastapi import APIRouter
+from fastapi import Request, Response, Depends
+from fastapi.responses import ORJSONResponse
+
+from api.api_helpers import authenticate, html_escape_multi, get_connecting_ip, rescale_energy_value
+from api.object_specifications import CI_Measurement_Old, CI_Measurement
+
+import anybadge
+
+from xml.sax.saxutils import escape as xml_escape
+
+from lib import error_helpers
+from lib.user import User
+from lib.db import DB
+
+router = APIRouter()
+
+
+@router.post('/v1/ci/measurement/add')
+async def post_ci_measurement_add_deprecated(
+ request: Request,
+ measurement: CI_Measurement_Old,
+ user: User = Depends(authenticate) # pylint: disable=unused-argument
+ ):
+
+ measurement = html_escape_multi(measurement)
+
+ used_client_ip = get_connecting_ip(request)
+
+ co2i_transformed = int(measurement.co2i) if measurement.co2i else None
+
+ co2eq_transformed = int(float(measurement.co2eq)*1000000) if measurement.co2eq else None
+
+ query = '''
+ INSERT INTO
+ ci_measurements (energy_uj,
+ repo,
+ branch,
+ workflow_id,
+ run_id,
+ label,
+ source,
+ cpu,
+ commit_hash,
+ duration_us,
+ cpu_util_avg,
+ workflow_name,
+ lat,
+ lon,
+ city,
+ carbon_intensity_g,
+ carbon_ug,
+ filter_type,
+ filter_project,
+ filter_machine,
+ filter_tags,
+ user_id,
+ ip_address
+ )
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+ '''
+
+ params = ( measurement.energy_value*1000, measurement.repo, measurement.branch,
+ measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu,
+ measurement.commit_hash, measurement.duration*1000000, measurement.cpu_util_avg, measurement.workflow_name,
+ measurement.lat, measurement.lon, measurement.city, co2i_transformed, co2eq_transformed,
+ 'machine.ci', 'CI/CD', 'unknown', [],
+ user._id, used_client_ip)
+
+
+ DB().query(query=query, params=params)
+
+ if measurement.energy_value <= 1 or (measurement.co2eq and co2eq_transformed <= 1):
+ error_helpers.log_error(
+ 'Extremely small energy budget was submitted to old Eco-CI API',
+ measurement=measurement
+ )
+
+ return Response(status_code=204)
+
+
+@router.post('/v2/ci/measurement/add')
+async def post_ci_measurement_add(
+ request: Request,
+ measurement: CI_Measurement,
+ user: User = Depends(authenticate) # pylint: disable=unused-argument
+ ):
+
+ measurement = html_escape_multi(measurement)
+
+ params = [measurement.energy_uj, measurement.repo, measurement.branch,
+ measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu,
+ measurement.commit_hash, measurement.duration_us, measurement.cpu_util_avg, measurement.workflow_name,
+ measurement.lat, measurement.lon, measurement.city, measurement.carbon_intensity_g, measurement.carbon_ug,
+ measurement.filter_type, measurement.filter_project, measurement.filter_machine]
+
+ tags_replacer = ' ARRAY[]::text[] '
+ if measurement.filter_tags:
+ tags_replacer = f" ARRAY[{','.join(['%s']*len(measurement.filter_tags))}] "
+ params = params + measurement.filter_tags
+
+ used_client_ip = measurement.ip # If an ip has been given with the data. We prioritize that
+ if used_client_ip is None:
+ used_client_ip = get_connecting_ip(request)
+
+ params.append(used_client_ip)
+ params.append(user._id)
+
+ query = f"""
+ INSERT INTO
+ ci_measurements (energy_uj,
+ repo,
+ branch,
+ workflow_id,
+ run_id,
+ label,
+ source,
+ cpu,
+ commit_hash,
+ duration_us,
+ cpu_util_avg,
+ workflow_name,
+ lat,
+ lon,
+ city,
+ carbon_intensity_g,
+ carbon_ug,
+ filter_type,
+ filter_project,
+ filter_machine,
+ filter_tags,
+ ip_address,
+ user_id
+ )
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
+ %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
+ {tags_replacer},
+ %s, %s)
+
+ """
+
+ DB().query(query=query, params=params)
+
+ if measurement.energy_uj <= 1 or (measurement.carbon_ug and measurement.carbon_ug <= 1):
+ error_helpers.log_error(
+ 'Extremely small energy budget was submitted to Eco-CI API',
+ measurement=measurement
+ )
+
+ return Response(status_code=204)
+
+@router.get('/v1/ci/measurements')
+async def get_ci_measurements(repo: str, branch: str, workflow: str, start_date: date, end_date: date):
+
+ query = """
+ SELECT energy_uj, run_id, created_at, label, cpu, commit_hash, duration_us, source, cpu_util_avg,
+ (SELECT workflow_name FROM ci_measurements AS latest_workflow
+ WHERE latest_workflow.repo = ci_measurements.repo
+ AND latest_workflow.branch = ci_measurements.branch
+ AND latest_workflow.workflow_id = ci_measurements.workflow_id
+ ORDER BY latest_workflow.created_at DESC
+ LIMIT 1) AS workflow_name,
+ lat, lon, city, carbon_intensity_g, carbon_ug
+ FROM ci_measurements
+ WHERE
+ repo = %s AND branch = %s AND workflow_id = %s
+ AND DATE(created_at) >= TO_DATE(%s, 'YYYY-MM-DD')
+ AND DATE(created_at) <= TO_DATE(%s, 'YYYY-MM-DD')
+ ORDER BY run_id ASC, created_at ASC
+ """
+ params = (repo, branch, workflow, str(start_date), str(end_date))
+ data = DB().fetch_all(query, params=params)
+
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponse({'success': True, 'data': data})
+
+@router.get('/v1/ci/repositories')
+async def get_ci_repositories(repo: str | None = None, sort_by: str = 'name'):
+
+ params = []
+ query = """
+ SELECT repo, source, MAX(created_at) as last_run
+ FROM ci_measurements
+ WHERE 1=1
+ """
+
+ if repo: # filter is currently not used, but may be a feature in the future
+ query = f"{query} AND ci_measurements.repo = %s \n"
+ params.append(repo)
+
+ query = f"{query} GROUP BY repo, source"
+
+ if sort_by == 'date':
+ query = f"{query} ORDER BY last_run DESC"
+ else:
+ query = f"{query} ORDER BY repo ASC"
+
+ data = DB().fetch_all(query, params=tuple(params))
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponse({'success': True, 'data': data}) # no escaping needed, as it happend on ingest
+
+
+@router.get('/v1/ci/runs')
+async def get_ci_runs(repo: str, sort_by: str = 'name'):
+
+ params = []
+ query = """
+ SELECT repo, branch, workflow_id, source, MAX(created_at) as last_run,
+ (SELECT workflow_name FROM ci_measurements AS latest_workflow
+ WHERE latest_workflow.repo = ci_measurements.repo
+ AND latest_workflow.branch = ci_measurements.branch
+ AND latest_workflow.workflow_id = ci_measurements.workflow_id
+ ORDER BY latest_workflow.created_at DESC
+ LIMIT 1) AS workflow_name
+ FROM ci_measurements
+ WHERE 1=1
+ """
+
+ query = f"{query} AND ci_measurements.repo = %s \n"
+ params.append(repo)
+ query = f"{query} GROUP BY repo, branch, workflow_id, source"
+
+ if sort_by == 'date':
+ query = f"{query} ORDER BY last_run DESC"
+ else:
+ query = f"{query} ORDER BY repo ASC"
+
+ data = DB().fetch_all(query, params=tuple(params))
+ if data is None or data == []:
+ return Response(status_code=204) # No-Content
+
+ return ORJSONResponse({'success': True, 'data': data}) # no escaping needed, as it happend on ingest
+
+@router.get('/v1/ci/badge/get')
+async def get_ci_badge_get(repo: str, branch: str, workflow:str):
+ query = """
+ SELECT SUM(energy_uj), MAX(run_id)
+ FROM ci_measurements
+ WHERE repo = %s AND branch = %s AND workflow_id = %s
+ GROUP BY run_id
+ ORDER BY MAX(created_at) DESC
+ LIMIT 1
+ """
+
+ params = (repo, branch, workflow)
+ data = DB().fetch_one(query, params=params)
+
+ if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result
+ return Response(status_code=204) # No-Content
+
+ energy_value = data[0]
+
+ [energy_value, energy_unit] = rescale_energy_value(energy_value, 'uJ')
+ badge_value= f"{energy_value:.2f} {energy_unit}"
+
+ badge = anybadge.Badge(
+ label='Energy Used',
+ value=xml_escape(badge_value),
+ num_value_padding_chars=1,
+ default_color='green')
+ return Response(content=str(badge), media_type="image/svg+xml")
diff --git a/api/main.py b/api/main.py
index ad78a4efc..12206e59e 100644
--- a/api/main.py
+++ b/api/main.py
@@ -4,43 +4,36 @@
import faulthandler
faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr
-import zlib
-import base64
import orjson
-from typing import List
from xml.sax.saxutils import escape as xml_escape
-from urllib.parse import urlparse
from datetime import date
-from fastapi import FastAPI, Request, Response, Depends, HTTPException
+from fastapi import FastAPI, Request, Response, Depends
from fastapi.responses import ORJSONResponse
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
-from fastapi.security import APIKeyHeader
from starlette.responses import RedirectResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.datastructures import Headers as StarletteHeaders
-from pydantic import ValidationError
-
import anybadge
-from api.object_specifications import Measurement, CI_Measurement_Old, CI_Measurement, HogMeasurement, Software, EnergyData
-from api.api_helpers import (ORJSONResponseObjKeep, add_phase_stats_statistics, carbondb_add, determine_comparison_case,
+from api import eco_ci
+from api.object_specifications import Software
+from api.api_helpers import (ORJSONResponseObjKeep, add_phase_stats_statistics, determine_comparison_case,
html_escape_multi, get_phase_stats, get_phase_stats_object,
is_valid_uuid, rescale_energy_value, get_timeline_query,
- get_run_info, get_machine_list, get_artifact, store_artifact, get_connecting_ip,
- validate_hog_measurement_data, replace_nan_with_zero)
+ get_run_info, get_machine_list, get_artifact, store_artifact,
+ authenticate)
from lib.global_config import GlobalConfig
from lib.db import DB
from lib.diff import get_diffable_row, diff_rows
from lib import error_helpers
from lib.job.base import Job
-from lib.user import User, UserAuthenticationError
-from lib.secure_variable import SecureVariable
+from lib.user import User
from lib.timeline_project import TimelineProject
from lib import utils
@@ -122,39 +115,12 @@ async def catch_exceptions_middleware(request: Request, call_next):
allow_headers=['*'],
)
-header_scheme = APIKeyHeader(
- name='X-Authentication',
- scheme_name='Header',
- description='Authentication key - See https://docs.green-coding.io/authentication',
- auto_error=False
-)
-
def obfuscate_authentication_token(headers: StarletteHeaders):
headers_mut = headers.mutablecopy()
if 'X-Authentication' in headers_mut:
headers_mut['X-Authentication'] = '****OBFUSCATED****'
return headers_mut
-def authenticate(authentication_token=Depends(header_scheme), request: Request = None):
- parsed_url = urlparse(str(request.url))
- try:
- if not authentication_token or authentication_token.strip() == '': # Note that if no token is supplied this will authenticate as the DEFAULT user, which in FOSS systems has full capabilities
- authentication_token = 'DEFAULT'
-
- user = User.authenticate(SecureVariable(authentication_token))
-
- if not user.can_use_route(parsed_url.path):
- raise HTTPException(status_code=401, detail="Route not allowed") from UserAuthenticationError
-
- if not user.has_api_quota(parsed_url.path):
- raise HTTPException(status_code=401, detail="Quota exceeded") from UserAuthenticationError
-
- user.deduct_api_quota(parsed_url.path, 1)
-
- except UserAuthenticationError:
- raise HTTPException(status_code=401, detail="Invalid token") from UserAuthenticationError
- return user
-
@app.get('/')
async def home():
@@ -630,375 +596,6 @@ async def get_jobs(machine_id: int | None = None, state: str | None = None):
return ORJSONResponse({'success': True, 'data': data})
-
-
-@app.post('/v1/hog/add')
-async def hog_add(
- measurements: List[HogMeasurement],
- user: User = Depends(authenticate), # pylint: disable=unused-argument
- ):
-
- for measurement in measurements:
- decoded_data = base64.b64decode(measurement.data)
- decompressed_data = zlib.decompress(decoded_data)
- measurement_data = orjson.loads(decompressed_data.decode()) # pylint: disable=no-member
-
- # For some reason we sometimes get NaN in the data.
- measurement_data = replace_nan_with_zero(measurement_data)
-
- #Check if the data is valid, if not this will throw an exception and converted into a request by the middleware
- try:
- _ = Measurement(**measurement_data)
- except (ValidationError, RequestValidationError) as exc:
- print('Caught Exception in Measurement()', exc.__class__.__name__, exc)
- print('Hog parsing error. Missing expected, but non critical key', str(exc))
- # Output is extremely verbose. Please only turn on if debugging manually
- # print(f"Errors are: {exc.errors()}")
-
-
- try:
- validate_hog_measurement_data(measurement_data)
- except ValueError as exc:
- print(f"Caught Exception in validate_hog_measurement_data() {exc.__class__.__name__} {exc}")
- raise exc
-
- coalitions = []
- for coalition in measurement_data['coalitions']:
- if coalition['name'] == 'com.googlecode.iterm2' or \
- coalition['name'] == 'com.apple.Terminal' or \
- coalition['name'] == 'com.vix.cron' or \
- coalition['name'].strip() == '':
- tmp = coalition['tasks']
- for tmp_el in tmp:
- tmp_el['tasks'] = []
- coalitions.extend(tmp)
- else:
- coalitions.append(coalition)
-
- # We remove the coalitions as we don't want to save all the data in hog_measurements
- del measurement_data['coalitions']
- del measurement.data
-
- cpu_energy_data = {}
- energy_impact = round(measurement_data['all_tasks'].get('energy_impact_per_s') * measurement_data['elapsed_ns'] / 1_000_000_000)
- if 'ane_energy' in measurement_data['processor']:
- cpu_energy_data = {
- 'combined_energy': round(measurement_data['processor'].get('combined_power', 0) * measurement_data['elapsed_ns'] / 1_000_000_000.0),
- 'cpu_energy': round(measurement_data['processor'].get('cpu_energy', 0)),
- 'gpu_energy': round(measurement_data['processor'].get('gpu_energy', 0)),
- 'ane_energy': round(measurement_data['processor'].get('ane_energy', 0)),
- 'energy_impact': energy_impact,
- }
- elif 'package_joules' in measurement_data['processor']:
- # Intel processors report in joules/ watts and not mJ
- cpu_energy_data = {
- 'combined_energy': round(measurement_data['processor'].get('package_joules', 0) * 1_000),
- 'cpu_energy': round(measurement_data['processor'].get('cpu_joules', 0) * 1_000),
- 'gpu_energy': round(measurement_data['processor'].get('igpu_watts', 0) * measurement_data['elapsed_ns'] / 1_000_000_000.0 * 1_000),
- 'ane_energy': 0,
- 'energy_impact': energy_impact,
- }
- else:
- raise RequestValidationError("input not valid")
-
- query = """
- INSERT INTO
- hog_measurements (
- time,
- machine_uuid,
- elapsed_ns,
- combined_energy,
- cpu_energy,
- gpu_energy,
- ane_energy,
- energy_impact,
- thermal_pressure,
- settings,
- user_id)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
- RETURNING id
- """
- params = (
- measurement.time,
- measurement.machine_uuid,
- measurement_data['elapsed_ns'],
- cpu_energy_data['combined_energy'],
- cpu_energy_data['cpu_energy'],
- cpu_energy_data['gpu_energy'],
- cpu_energy_data['ane_energy'],
- cpu_energy_data['energy_impact'],
- measurement_data['thermal_pressure'],
- measurement.settings,
- user._id,
- )
-
- measurement_db_id = DB().fetch_one(query=query, params=params)[0]
-
-
- # Save hog_measurements
- for coalition in coalitions:
-
- if coalition['energy_impact'] < 1.0:
- # If the energy_impact is too small we just skip the coalition.
- continue
-
- c_tasks = coalition['tasks'].copy()
- del coalition['tasks']
-
- c_energy_impact = round((coalition['energy_impact_per_s'] / 1_000_000_000) * measurement_data['elapsed_ns'])
- c_cputime_ns = ((coalition['cputime_ms_per_s'] * 1_000_000) / 1_000_000_000) * measurement_data['elapsed_ns']
-
- query = """
- INSERT INTO
- hog_coalitions (
- measurement,
- name,
- cputime_ns,
- cputime_per,
- energy_impact,
- diskio_bytesread,
- diskio_byteswritten,
- intr_wakeups,
- idle_wakeups)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
- RETURNING id
- """
- params = (
- measurement_db_id,
- coalition['name'],
- c_cputime_ns,
- int(c_cputime_ns / measurement_data['elapsed_ns'] * 100),
- c_energy_impact,
- coalition['diskio_bytesread'],
- coalition['diskio_byteswritten'],
- coalition['intr_wakeups'],
- coalition['idle_wakeups'],
- )
-
- coaltion_db_id = DB().fetch_one(query=query, params=params)[0]
-
- for task in c_tasks:
- t_energy_impact = round((task['energy_impact_per_s'] / 1_000_000_000) * measurement_data['elapsed_ns'])
- t_cputime_ns = ((task['cputime_ms_per_s'] * 1_000_000) / 1_000_000_000) * measurement_data['elapsed_ns']
-
- query = """
- INSERT INTO
- hog_tasks (
- coalition,
- name,
- cputime_ns,
- cputime_per,
- energy_impact,
- bytes_received,
- bytes_sent,
- diskio_bytesread,
- diskio_byteswritten,
- intr_wakeups,
- idle_wakeups)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
- RETURNING id
- """
- params = (
- coaltion_db_id,
- task['name'],
- t_cputime_ns,
- int(t_cputime_ns / measurement_data['elapsed_ns'] * 100),
- t_energy_impact,
- task.get('bytes_received', 0),
- task.get('bytes_sent', 0),
- task.get('diskio_bytesread', 0),
- task.get('diskio_byteswritten', 0),
- task.get('intr_wakeups', 0),
- task.get('idle_wakeups', 0),
- )
- DB().fetch_one(query=query, params=params)
-
- return Response(status_code=204) # No-Content
-
-
-@app.get('/v1/hog/top_processes')
-async def hog_get_top_processes():
- query = """
- SELECT
- name,
- (SUM(energy_impact)::bigint) AS total_energy_impact
- FROM
- hog_coalitions
- GROUP BY
- name
- ORDER BY
- total_energy_impact DESC
- LIMIT 100;
- """
- data = DB().fetch_all(query)
-
- if data is None:
- data = []
-
- query = """
- SELECT COUNT(DISTINCT machine_uuid) FROM hog_measurements;
- """
-
- machine_count = DB().fetch_one(query)[0]
-
- return ORJSONResponse({'success': True, 'process_data': data, 'machine_count': machine_count})
-
-
-@app.get('/v1/hog/machine_details/{machine_uuid}')
-async def hog_get_machine_details(machine_uuid: str):
-
- if machine_uuid is None or not is_valid_uuid(machine_uuid):
- return ORJSONResponse({'success': False, 'err': 'machine_uuid is empty or malformed'}, status_code=422)
-
- query = """
- SELECT
- time,
- combined_energy,
- cpu_energy,
- gpu_energy,
- ane_energy,
- energy_impact::bigint,
- id
- FROM
- hog_measurements
- WHERE
- machine_uuid = %s
- ORDER BY
- time
- """
-
- data = DB().fetch_all(query, (machine_uuid,))
-
- return ORJSONResponse({'success': True, 'data': data})
-
-
-@app.get('/v1/hog/coalitions_tasks/{machine_uuid}/{measurements_id_start}/{measurements_id_end}')
-async def hog_get_coalitions_tasks(machine_uuid: str, measurements_id_start: int, measurements_id_end: int):
-
- if machine_uuid is None or not is_valid_uuid(machine_uuid):
- return ORJSONResponse({'success': False, 'err': 'machine_uuid is empty'}, status_code=422)
-
- if measurements_id_start is None:
- return ORJSONResponse({'success': False, 'err': 'measurements_id_start is empty'}, status_code=422)
-
- if measurements_id_end is None:
- return ORJSONResponse({'success': False, 'err': 'measurements_id_end is empty'}, status_code=422)
-
-
- coalitions_query = """
- SELECT
- name,
- (SUM(hc.energy_impact)::bigint) AS total_energy_impact,
- (SUM(hc.diskio_bytesread)::bigint) AS total_diskio_bytesread,
- (SUM(hc.diskio_byteswritten)::bigint) AS total_diskio_byteswritten,
- (SUM(hc.intr_wakeups)::bigint) AS total_intr_wakeups,
- (SUM(hc.idle_wakeups)::bigint) AS total_idle_wakeups,
- (AVG(hc.cputime_per)::integer) AS avg_cpu_per
- FROM
- hog_coalitions AS hc
- JOIN
- hog_measurements AS hm ON hc.measurement = hm.id
- WHERE
- hc.measurement BETWEEN %s AND %s
- AND hm.machine_uuid = %s
- GROUP BY
- name
- ORDER BY
- total_energy_impact DESC
- LIMIT 100;
- """
-
- measurements_query = """
- SELECT
- (SUM(combined_energy)::bigint) AS total_combined_energy,
- (SUM(cpu_energy)::bigint) AS total_cpu_energy,
- (SUM(gpu_energy)::bigint) AS total_gpu_energy,
- (SUM(ane_energy)::bigint) AS total_ane_energy,
- (SUM(energy_impact)::bigint) AS total_energy_impact
- FROM
- hog_measurements
- WHERE
- id BETWEEN %s AND %s
- AND machine_uuid = %s
-
- """
-
- coalitions_data = DB().fetch_all(coalitions_query, (measurements_id_start, measurements_id_end, machine_uuid))
-
- energy_data = DB().fetch_one(measurements_query, (measurements_id_start, measurements_id_end, machine_uuid))
-
- return ORJSONResponse({'success': True, 'data': coalitions_data, 'energy_data': energy_data})
-
-@app.get('/v1/hog/tasks_details/{machine_uuid}/{measurements_id_start}/{measurements_id_end}/{coalition_name}')
-async def hog_get_task_details(machine_uuid: str, measurements_id_start: int, measurements_id_end: int, coalition_name: str):
-
- if machine_uuid is None or not is_valid_uuid(machine_uuid):
- return ORJSONResponse({'success': False, 'err': 'machine_uuid is empty'}, status_code=422)
-
- if measurements_id_start is None:
- return ORJSONResponse({'success': False, 'err': 'measurements_id_start is empty'}, status_code=422)
-
- if measurements_id_end is None:
- return ORJSONResponse({'success': False, 'err': 'measurements_id_end is empty'}, status_code=422)
-
- if coalition_name is None or not coalition_name.strip():
- return ORJSONResponse({'success': False, 'err': 'coalition_name is empty'}, status_code=422)
-
- tasks_query = """
- SELECT
- t.name,
- COUNT(t.id)::bigint AS number_of_tasks,
- SUM(t.energy_impact)::bigint AS total_energy_impact,
- SUM(t.cputime_ns)::bigint AS total_cputime_ns,
- SUM(t.bytes_received)::bigint AS total_bytes_received,
- SUM(t.bytes_sent)::bigint AS total_bytes_sent,
- SUM(t.diskio_bytesread)::bigint AS total_diskio_bytesread,
- SUM(t.diskio_byteswritten)::bigint AS total_diskio_byteswritten,
- SUM(t.intr_wakeups)::bigint AS total_intr_wakeups,
- SUM(t.idle_wakeups)::bigint AS total_idle_wakeups
- FROM
- hog_tasks t
- JOIN hog_coalitions c ON t.coalition = c.id
- JOIN hog_measurements m ON c.measurement = m.id
- WHERE
- c.name = %s
- AND c.measurement BETWEEN %s AND %s
- AND m.machine_uuid = %s
- GROUP BY
- t.name
- ORDER BY
- total_energy_impact DESC;
- """
-
- coalitions_query = """
- SELECT
- c.name,
- (SUM(c.energy_impact)::bigint) AS total_energy_impact,
- (SUM(c.diskio_bytesread)::bigint) AS total_diskio_bytesread,
- (SUM(c.diskio_byteswritten)::bigint) AS total_diskio_byteswritten,
- (SUM(c.intr_wakeups)::bigint) AS total_intr_wakeups,
- (SUM(c.idle_wakeups)::bigint) AS total_idle_wakeups
- FROM
- hog_coalitions c
- JOIN hog_measurements m ON c.measurement = m.id
- WHERE
- c.name = %s
- AND c.measurement BETWEEN %s AND %s
- AND m.machine_uuid = %s
- GROUP BY
- c.name
- ORDER BY
- total_energy_impact DESC
- LIMIT 100;
- """
-
- tasks_data = DB().fetch_all(tasks_query, (coalition_name, measurements_id_start,measurements_id_end, machine_uuid))
- coalitions_data = DB().fetch_one(coalitions_query, (coalition_name, measurements_id_start, measurements_id_end, machine_uuid))
-
- return ORJSONResponse({'success': True, 'tasks_data': tasks_data, 'coalitions_data': coalitions_data})
-
-
-
@app.post('/v1/software/add')
async def software_add(software: Software, user: User = Depends(authenticate)):
@@ -1123,414 +720,6 @@ async def robots_txt():
return Response(content=data, media_type='text/plain')
-
-@app.post('/v1/ci/measurement/add')
-async def post_ci_measurement_add_deprecated(
- request: Request,
- measurement: CI_Measurement_Old,
- user: User = Depends(authenticate) # pylint: disable=unused-argument
- ):
-
- measurement = html_escape_multi(measurement)
-
- used_client_ip = get_connecting_ip(request)
-
- co2i_transformed = int(measurement.co2i) if measurement.co2i else None
-
- co2eq_transformed = int(float(measurement.co2eq)*1000000) if measurement.co2eq else None
-
- query = '''
- INSERT INTO
- ci_measurements (energy_uj,
- repo,
- branch,
- workflow_id,
- run_id,
- label,
- source,
- cpu,
- commit_hash,
- duration_us,
- cpu_util_avg,
- workflow_name,
- lat,
- lon,
- city,
- carbon_intensity_g,
- carbon_ug,
- filter_type,
- filter_project,
- filter_machine,
- filter_tags,
- user_id,
- ip_address
- )
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
- '''
-
- params = ( measurement.energy_value*1000, measurement.repo, measurement.branch,
- measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu,
- measurement.commit_hash, measurement.duration*1000000, measurement.cpu_util_avg, measurement.workflow_name,
- measurement.lat, measurement.lon, measurement.city, co2i_transformed, co2eq_transformed,
- 'machine.ci', 'CI/CD', 'unknown', [],
- user._id, used_client_ip)
-
-
- DB().query(query=query, params=params)
-
- if measurement.energy_value <= 1 or (measurement.co2eq and co2eq_transformed <= 1):
- error_helpers.log_error(
- 'Extremely small energy budget was submitted to old Eco-CI API',
- measurement=measurement
- )
-
- return Response(status_code=204)
-
-
-@app.post('/v2/ci/measurement/add')
-async def post_ci_measurement_add(
- request: Request,
- measurement: CI_Measurement,
- user: User = Depends(authenticate) # pylint: disable=unused-argument
- ):
-
- measurement = html_escape_multi(measurement)
-
- params = [measurement.energy_uj, measurement.repo, measurement.branch,
- measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu,
- measurement.commit_hash, measurement.duration_us, measurement.cpu_util_avg, measurement.workflow_name,
- measurement.lat, measurement.lon, measurement.city, measurement.carbon_intensity_g, measurement.carbon_ug,
- measurement.filter_type, measurement.filter_project, measurement.filter_machine]
-
- tags_replacer = ' ARRAY[]::text[] '
- if measurement.filter_tags:
- tags_replacer = f" ARRAY[{','.join(['%s']*len(measurement.filter_tags))}] "
- params = params + measurement.filter_tags
-
- used_client_ip = measurement.ip # If an ip has been given with the data. We prioritize that
- if used_client_ip is None:
- used_client_ip = get_connecting_ip(request)
-
- params.append(used_client_ip)
- params.append(user._id)
-
- query = f"""
- INSERT INTO
- ci_measurements (energy_uj,
- repo,
- branch,
- workflow_id,
- run_id,
- label,
- source,
- cpu,
- commit_hash,
- duration_us,
- cpu_util_avg,
- workflow_name,
- lat,
- lon,
- city,
- carbon_intensity_g,
- carbon_ug,
- filter_type,
- filter_project,
- filter_machine,
- filter_tags,
- ip_address,
- user_id
- )
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
- %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
- {tags_replacer},
- %s, %s)
-
- """
-
- DB().query(query=query, params=params)
-
- if measurement.energy_uj <= 1 or (measurement.carbon_ug and measurement.carbon_ug <= 1):
- error_helpers.log_error(
- 'Extremely small energy budget was submitted to Eco-CI API',
- measurement=measurement
- )
-
- return Response(status_code=204)
-
-@app.get('/v1/ci/measurements')
-async def get_ci_measurements(repo: str, branch: str, workflow: str, start_date: date, end_date: date):
-
- query = """
- SELECT energy_uj, run_id, created_at, label, cpu, commit_hash, duration_us, source, cpu_util_avg,
- (SELECT workflow_name FROM ci_measurements AS latest_workflow
- WHERE latest_workflow.repo = ci_measurements.repo
- AND latest_workflow.branch = ci_measurements.branch
- AND latest_workflow.workflow_id = ci_measurements.workflow_id
- ORDER BY latest_workflow.created_at DESC
- LIMIT 1) AS workflow_name,
- lat, lon, city, carbon_intensity_g, carbon_ug
- FROM ci_measurements
- WHERE
- repo = %s AND branch = %s AND workflow_id = %s
- AND DATE(created_at) >= TO_DATE(%s, 'YYYY-MM-DD')
- AND DATE(created_at) <= TO_DATE(%s, 'YYYY-MM-DD')
- ORDER BY run_id ASC, created_at ASC
- """
- params = (repo, branch, workflow, str(start_date), str(end_date))
- data = DB().fetch_all(query, params=params)
-
- if data is None or data == []:
- return Response(status_code=204) # No-Content
-
- return ORJSONResponse({'success': True, 'data': data})
-
-@app.get('/v1/ci/repositories')
-async def get_ci_repositories(repo: str | None = None, sort_by: str = 'name'):
-
- params = []
- query = """
- SELECT repo, source, MAX(created_at) as last_run
- FROM ci_measurements
- WHERE 1=1
- """
-
- if repo: # filter is currently not used, but may be a feature in the future
- query = f"{query} AND ci_measurements.repo = %s \n"
- params.append(repo)
-
- query = f"{query} GROUP BY repo, source"
-
- if sort_by == 'date':
- query = f"{query} ORDER BY last_run DESC"
- else:
- query = f"{query} ORDER BY repo ASC"
-
- data = DB().fetch_all(query, params=tuple(params))
- if data is None or data == []:
- return Response(status_code=204) # No-Content
-
- return ORJSONResponse({'success': True, 'data': data}) # no escaping needed, as it happend on ingest
-
-
-@app.get('/v1/ci/runs')
-async def get_ci_runs(repo: str, sort_by: str = 'name'):
-
- params = []
- query = """
- SELECT repo, branch, workflow_id, source, MAX(created_at) as last_run,
- (SELECT workflow_name FROM ci_measurements AS latest_workflow
- WHERE latest_workflow.repo = ci_measurements.repo
- AND latest_workflow.branch = ci_measurements.branch
- AND latest_workflow.workflow_id = ci_measurements.workflow_id
- ORDER BY latest_workflow.created_at DESC
- LIMIT 1) AS workflow_name
- FROM ci_measurements
- WHERE 1=1
- """
-
- query = f"{query} AND ci_measurements.repo = %s \n"
- params.append(repo)
- query = f"{query} GROUP BY repo, branch, workflow_id, source"
-
- if sort_by == 'date':
- query = f"{query} ORDER BY last_run DESC"
- else:
- query = f"{query} ORDER BY repo ASC"
-
- data = DB().fetch_all(query, params=tuple(params))
- if data is None or data == []:
- return Response(status_code=204) # No-Content
-
- return ORJSONResponse({'success': True, 'data': data}) # no escaping needed, as it happend on ingest
-
-@app.get('/v1/ci/badge/get')
-async def get_ci_badge_get(repo: str, branch: str, workflow:str):
- query = """
- SELECT SUM(energy_uj), MAX(run_id)
- FROM ci_measurements
- WHERE repo = %s AND branch = %s AND workflow_id = %s
- GROUP BY run_id
- ORDER BY MAX(created_at) DESC
- LIMIT 1
- """
-
- params = (repo, branch, workflow)
- data = DB().fetch_one(query, params=params)
-
- if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result
- return Response(status_code=204) # No-Content
-
- energy_value = data[0]
-
- [energy_value, energy_unit] = rescale_energy_value(energy_value, 'uJ')
- badge_value= f"{energy_value:.2f} {energy_unit}"
-
- badge = anybadge.Badge(
- label='Energy Used',
- value=xml_escape(badge_value),
- num_value_padding_chars=1,
- default_color='green')
- return Response(content=str(badge), media_type="image/svg+xml")
-
-
-@app.post('/v1/carbondb/add')
-async def add_carbondb_deprecated():
- return Response("This endpoint is not supported anymore. Please migrate to /v2/carbondb/add !", status_code=410)
-
-@app.post('/v2/carbondb/add')
-async def add_carbondb(
- request: Request,
- energydata: EnergyData,
- user: User = Depends(authenticate) # pylint: disable=unused-argument
- ):
-
- try:
- carbondb_add(get_connecting_ip(request), energydata.dict(), 'CUSTOM', user._id)
- except ValueError as exc:
- raise RequestValidationError(str(exc)) from exc
-
- return Response(status_code=204)
-
-
-@app.get('/v1/carbondb/')
-async def get_carbondb_deprecated():
- return Response("This endpoint is not supported anymore. Please migrate to /v2/carbondb/ !", status_code=410)
-
-@app.get('/v2/carbondb')
-async def carbondb_get(
- user: User = Depends(authenticate),
- start_date: date | None = None, end_date: date | None = None,
- tags_include: str | None = None, tags_exclude: str | None = None,
- types_include: str | None = None, types_exclude: str | None = None,
- projects_include: str | None = None, projects_exclude: str | None = None,
- machines_include: str | None = None, machines_exclude: str | None = None,
- sources_include: str | None = None, sources_exclude: str | None = None
- ):
-
- params = [user._id,]
-
- start_date_condition = ''
- if start_date is not None:
- start_date_condition = "AND DATE(cedd.date) >= %s"
- params.append(start_date)
-
- end_date_condition = ''
- if end_date is not None:
- end_date_condition = "AND DATE(cedd.date) <= %s"
- params.append(end_date)
-
- tags_include_condition = ''
- if tags_include:
- tags_include_list = tags_include.split(',')
- tags_include_condition = f" AND cedd.tags @> ARRAY[{','.join(['%s::integer']*len(tags_include_list))}]"
- params = params + tags_include_list
-
- tags_exclude_condition = ''
- if tags_exclude:
- tags_exclude_list = tags_exclude.split(',')
- tags_exclude_condition = f" AND cedd.tags NOT @> ARRAY[{','.join(['%s::integer']*len(tags_exclude_list))}]"
- params = params + tags_exclude_list
-
- machines_include_condition = ''
- if machines_include:
- machines_include_list = machines_include.split(',')
- machines_include_condition = f" AND cedd.machine IN ({','.join(['%s']*len(machines_include_list))})"
- params = params + machines_include_list
-
- machines_exclude_condition = ''
- if machines_exclude:
- machines_exclude_list = machines_exclude.split(',')
- machines_exclude_condition = f" AND cedd.machine NOT IN ({','.join(['%s']*len(machines_exclude_list))})"
- params = params + machines_exclude_list
-
- types_include_condition = ''
- if types_include:
- types_include_list = types_include.split(',')
- types_include_condition = f" AND cedd.type IN ({','.join(['%s']*len(types_include_list))})"
- params = params + types_include_list
-
- types_exclude_condition = ''
- if types_exclude:
- types_exclude_list = types_exclude.split(',')
- types_exclude_condition = f" AND cedd.type NOT IN ({','.join(['%s']*len(types_exclude_list))})"
- params = params + types_exclude_list
-
- projects_include_condition = ''
- if projects_include:
- projects_include_list = projects_include.split(',')
- projects_include_condition = f" AND cedd.project IN ({','.join(['%s']*len(projects_include_list))})"
- params = params + projects_include_list
-
- projects_exclude_condition = ''
- if projects_exclude:
- projects_exclude_list = projects_exclude.split(',')
- projects_exclude_condition = f" AND cedd.project NOT IN ({','.join(['%s']*len(projects_exclude_list))})"
- params = params + projects_exclude_list
-
- sources_include_condition = ''
- if sources_include:
- sources_include_list = sources_include.split(',')
- sources_include_condition = f" AND cedd.source IN ({','.join(['%s']*len(sources_include_list))})"
- params = params + sources_include_list
-
- sources_exclude_condition = ''
- if sources_exclude:
- sources_exclude_list = sources_exclude.split(',')
- sources_exclude_condition = f" AND cedd.source NOT IN ({','.join(['%s']*len(sources_exclude_list))})"
- params = params + sources_exclude_list
-
- query = f"""
- SELECT
- type, project, machine, source, tags, date, energy_kwh_sum, carbon_kg_sum, carbon_intensity_g_avg, record_count
- FROM
- carbondb_data as cedd
- WHERE
- user_id = %s
- {start_date_condition}
- {end_date_condition}
- {tags_include_condition}
- {tags_exclude_condition}
- {machines_include_condition}
- {machines_exclude_condition}
- {types_include_condition}
- {types_exclude_condition}
- {projects_include_condition}
- {projects_exclude_condition}
- {sources_include_condition}
- {sources_exclude_condition}
-
- ORDER BY
- date ASC
- ;
- """
- data = DB().fetch_all(query, params)
-
- return ORJSONResponse({'success': True, 'data': data})
-
-
-@app.get('/v2/carbondb/filters')
-async def carbondb_get_filters(
- user: User = Depends(authenticate)
- ):
-
- query = 'SELECT jsonb_object_agg(id, type) FROM carbondb_types WHERE user_id = %s'
- carbondb_types = DB().fetch_one(query, (user._id, ))[0]
-
- query = 'SELECT jsonb_object_agg(id, tag) FROM carbondb_tags WHERE user_id = %s'
- carbondb_tags = DB().fetch_one(query, (user._id, ))[0]
-
- query = 'SELECT jsonb_object_agg(id, machine) FROM carbondb_machines WHERE user_id = %s'
- carbondb_machines = DB().fetch_one(query, (user._id, ))[0]
-
- query = 'SELECT jsonb_object_agg(id, project) FROM carbondb_projects WHERE user_id = %s'
- carbondb_projects = DB().fetch_one(query, (user._id, ))[0]
-
- query = 'SELECT jsonb_object_agg(id, source) FROM carbondb_sources WHERE user_id = %s'
- carbondb_sources = DB().fetch_one(query, (user._id, ))[0]
-
- return ORJSONResponse({'success': True, 'data': {'types': carbondb_types, 'tags': carbondb_tags, 'machines': carbondb_machines, 'projects': carbondb_projects, 'sources': carbondb_sources}})
-
-
# @app.get('/v1/authentication/new')
# This will fail if the DB insert fails but still report 'success': True
# Must be reworked if we want to allow API based token generation
@@ -1543,5 +732,13 @@ async def carbondb_get_filters(
async def read_authentication_token(user: User = Depends(authenticate)):
return ORJSONResponse({'success': True, 'data': user.to_dict()})
+app.include_router(eco_ci.router)
+
+# include enterprise functionality if activated
+if GlobalConfig().config.get('ee_token', False):
+ from ee.api import carbondb, power_hog
+ app.include_router(carbondb.router)
+ app.include_router(power_hog.router)
+
if __name__ == '__main__':
app.run() # pylint: disable=no-member
diff --git a/api/object_specifications.py b/api/object_specifications.py
index efea4aad6..41698473a 100644
--- a/api/object_specifications.py
+++ b/api/object_specifications.py
@@ -1,92 +1,23 @@
-from typing import List, Dict, Optional
-from pydantic import BaseModel, ConfigDict, field_validator, Field
-from fastapi.exceptions import RequestValidationError
-
-
-###### HOG
-
-class HogMeasurement(BaseModel):
- time: int
- data: str
- settings: str
- machine_uuid: str
- row_id: Optional[int] = -1 # we use this only for debugging
-
- model_config = ConfigDict(extra='forbid')
-
-
-class Task(BaseModel):
- # We need to set the optional to a value as otherwise the key is required in the input
- # https://docs.pydantic.dev/latest/migration/#required-optional-and-nullable-fields
- name: str
- cputime_ns: int
- timer_wakeups: List
- diskio_bytesread: Optional[int] = 0
- diskio_byteswritten: Optional[int] = 0
- packets_received: int
- packets_sent: int
- bytes_received: int
- bytes_sent: int
- energy_impact: float
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from typing import Optional
- model_config = ConfigDict(extra='forbid')
+from fastapi.exceptions import RequestValidationError
+### Software Add
-class Coalition(BaseModel):
+class Software(BaseModel):
name: str
- cputime_ns: int
- diskio_bytesread: int = 0
- diskio_byteswritten: int = 0
- energy_impact: float
- tasks: List[Task]
-
- model_config = ConfigDict(extra='forbid')
-
-class Processor(BaseModel):
- # https://docs.pydantic.dev/latest/migration/#required-optional-and-nullable-fields
- clusters: Optional[List] = None
- cpu_power_zones_engaged: Optional[float] = None
- cpu_energy: Optional[int] = None
- cpu_power: Optional[float] = None
- gpu_energy: Optional[int] = None
- gpu_power: Optional[float] = None
- ane_energy: Optional[int] = None
- ane_power: Optional[float] = None
- combined_power: Optional[float] = None
- package_joules: Optional[float] = None
- cpu_joules: Optional[float] = None
- igpu_watts: Optional[float] = None
-
- model_config = ConfigDict(extra='forbid')
-
-class GPU(BaseModel):
- gpu_energy: Optional[int] = None
-
- model_config = ConfigDict(extra='forbid')
-
-
-class Measurement(BaseModel):
- is_delta: bool
- elapsed_ns: int
- timestamp: int
- coalitions: List[Coalition]
- all_tasks: Dict
- network: Optional[Dict] = None # network is optional when system is in flight mode / network turned off
- disk: Optional[Dict] = None # No idea what system would not have a disk but we are seeing this in production
- interrupts: List
- processor: Processor
- thermal_pressure: str
- sfi: Dict
- gpu: Optional[GPU] = None
+ url: str
+ email: str
+ filename: str
+ branch: str
+ machine_id: int
+ schedule_mode: str
model_config = ConfigDict(extra='forbid')
### Eco-CI
-
-
-
-
# pylint: disable=invalid-name
class CI_Measurement_Old(BaseModel):
energy_value: int
@@ -172,48 +103,3 @@ def check_empty_elements(cls, value):
if any(not item or item.strip() == '' for item in value):
raise ValueError("The list contains empty elements.")
return value
-
-
-
-### Software Add
-
-class Software(BaseModel):
- name: str
- url: str
- email: str
- filename: str
- branch: str
- machine_id: int
- schedule_mode: str
-
- model_config = ConfigDict(extra='forbid')
-
-
-### CarbonDB
-
-class EnergyData(BaseModel):
- tags: Optional[list] = Field(default_factory=list) # never do a reference object as default as it will be shared
- project: str
- machine: str
- type: str
- time: int # value is in us as UTC timestamp
- energy_uj: int # is in uJ
- carbon_intensity_g: Optional[int] = None # Will be populated if not transmitted, so we never have NULL in DB
- ip: Optional[str] = None # Will be populated if not transmitted, so we never have NULL in DB
-
- model_config = ConfigDict(extra='forbid')
-
-
- @field_validator('ip', 'project', 'machine','type')
- @classmethod
- def empty_str_to_none(cls, values, _):
- if not values or values.strip() == '':
- raise ValueError('Value is empty')
- return values
-
- @field_validator('tags')
- @classmethod
- def check_empty_elements(cls, value):
- if any(not item or item.strip() == '' for item in value):
- raise ValueError("The list contains empty elements.")
- return value
diff --git a/config.yml.example b/config.yml.example
index f7b99e1e8..02e8c3d2f 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -222,3 +222,7 @@ sci:
# You can get this under https://api-portal.electricitymaps.com/
# This is a free service please note that you need to pay if you want to use this commercially!
#electricity_maps_token: '123'
+
+# GMT can activate additional enterprise only functionality
+# If you have a subscription insert your token here
+ee_token: False
\ No newline at end of file
diff --git a/cron/carbondb_compress.py b/cron/carbondb_compress.py
deleted file mode 100644
index bb5ae9034..000000000
--- a/cron/carbondb_compress.py
+++ /dev/null
@@ -1,135 +0,0 @@
-import sys
-import faulthandler
-faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr
-
-import os
-
-from lib.global_config import GlobalConfig
-from lib.db import DB
-from lib import error_helpers
-
-# The main job of the compress script is to take all the data from the carbondb_data_raw table
-# and compress it to daily sums.
-# During this process we also transform all text fields and transform them to integers and drop them into normalized
-# joined tables.
-
-########### Remove NULL values from tags
-# UPDATE carbondb_data_raw
-# SET tags = array_remove(tags, NULL)
-# WHERE array_position(tags, NULL) IS NOT NULL;
-
-
-def compress_carbondb_raw():
- query = '''
-
- INSERT INTO carbondb_types (type, user_id)
- SELECT DISTINCT type, user_id
- FROM carbondb_data_raw
- ON CONFLICT (type, user_id) DO NOTHING;
-
- INSERT INTO carbondb_machines (machine, user_id)
- SELECT DISTINCT machine, user_id
- FROM carbondb_data_raw
- ON CONFLICT (machine, user_id) DO NOTHING;
-
- INSERT INTO carbondb_tags (tag, user_id)
- SELECT DISTINCT unnest(tags), user_id
- FROM carbondb_data_raw
- ON CONFLICT (tag, user_id) DO NOTHING;
-
- INSERT INTO carbondb_sources (source, user_id)
- SELECT DISTINCT source, user_id
- FROM carbondb_data_raw
- ON CONFLICT (source, user_id) DO NOTHING;
-
- INSERT INTO carbondb_projects (project, user_id)
- SELECT DISTINCT project, user_id
- FROM carbondb_data_raw
- ON CONFLICT (project, user_id) DO NOTHING;
-
- DROP TABLE IF EXISTS carbondb_data_raw_tmp;
-
- CREATE TEMPORARY TABLE carbondb_data_raw_tmp AS
- SELECT * FROM carbondb_data_raw;
-
- UPDATE carbondb_data_raw_tmp AS cdrt
- SET "type" = s.id
- FROM carbondb_types AS s
- WHERE cdrt.type = s.type AND cdrt.user_id = s.user_id;
-
- UPDATE carbondb_data_raw_tmp AS cdrt
- SET "source" = s.id
- FROM carbondb_sources AS s
- WHERE cdrt.source = s.source AND cdrt.user_id = s.user_id;
-
- UPDATE carbondb_data_raw_tmp AS cdrt
- SET "machine" = s.id
- FROM carbondb_machines AS s
- WHERE cdrt.machine = s.machine AND cdrt.user_id = s.user_id;
-
- UPDATE carbondb_data_raw_tmp AS cdrt
- SET "project" = s.id
- FROM carbondb_projects AS s
- WHERE cdrt.project = s.project AND cdrt.user_id = s.user_id;
-
- UPDATE carbondb_data_raw_tmp
- SET tags = COALESCE(
- (SELECT ARRAY_AGG(t2.id)
- FROM UNNEST(carbondb_data_raw_tmp.tags) AS elem
- LEFT JOIN carbondb_tags AS t2 ON t2.tag = elem and t2.user_id = carbondb_data_raw_tmp.user_id)
- , ARRAY[]::int[]
- );
-
- INSERT INTO carbondb_data (
- type,
- machine,
- project,
- source,
- tags,
- date,
- energy_kwh_sum,
- carbon_kg_sum,
- carbon_intensity_g_avg,
- record_count,
- user_id
- )
- SELECT
- cdr.type::int,
- cdr.machine::int,
- cdr.project::int,
- cdr.source::int,
- cdr.tags::int[],
- DATE_TRUNC('day', TO_TIMESTAMP(cdr.time / 1000000)),
- SUM(cdr.energy_kwh),
- SUM(cdr.carbon_kg),
- COALESCE(SUM(cdr.carbon_kg)*1e3 / NULLIF(SUM(cdr.energy_kwh), 0), 0), -- weighted average instead of just averaging carbon_intensity. Since the solar panel might not be producing power at all for a day, which results in 0, we need to COALESCE and insert 0 in this case
- COUNT(*),
- cdr.user_id
- FROM
- carbondb_data_raw_tmp AS cdr
- GROUP BY
- cdr.type,
- cdr.source,
- cdr.machine,
- cdr.project,
- cdr.tags,
- DATE_TRUNC('day', TO_TIMESTAMP(cdr.time / 1000000)),
- cdr.user_id
- ON CONFLICT (type, source, machine, project, tags, date, user_id) DO UPDATE
- SET
- -- excluded will take the fields positional from the insert query. The names are not the actual column values
- -- but the calculation planned for these columns through the SELECT statement
- energy_kwh_sum = EXCLUDED.energy_kwh_sum,
- carbon_kg_sum = EXCLUDED.carbon_kg_sum,
- carbon_intensity_g_avg = EXCLUDED.carbon_intensity_g_avg,
- record_count = EXCLUDED.record_count;
- '''
- DB().query(query)
-
-
-if __name__ == '__main__':
- try:
- GlobalConfig().override_config(config_location=f"{os.path.dirname(os.path.realpath(__file__))}/../manager-config.yml")
- compress_carbondb_raw()
- except Exception as exc: # pylint: disable=broad-except
- error_helpers.log_error(f'Processing in {__file__} failed.', exception=exc, machine=GlobalConfig().config['machine']['description'])
diff --git a/cron/carbondb_compress.py b/cron/carbondb_compress.py
new file mode 120000
index 000000000..f2f07f690
--- /dev/null
+++ b/cron/carbondb_compress.py
@@ -0,0 +1 @@
+../ee/cron/carbondb_compress.py
\ No newline at end of file
diff --git a/cron/carbondb_copy_over_and_remove_duplicates.py b/cron/carbondb_copy_over_and_remove_duplicates.py
deleted file mode 100644
index b05f7a078..000000000
--- a/cron/carbondb_copy_over_and_remove_duplicates.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import faulthandler
-faulthandler.enable() # will catch segfaults and write to stderr
-
-import os
-
-from lib.global_config import GlobalConfig
-from lib.db import DB
-from lib import error_helpers
-
-def copy_over_eco_ci():
- DB().query('''
- INSERT INTO carbondb_data_raw
- ("type", "project", "machine", "source", "tags","time","energy_kwh","carbon_kg","carbon_intensity_g","latitude","longitude","ip_address","user_id","created_at")
-
- SELECT
- filter_type,
- filter_project,
- filter_machine,
- 'Eco-CI',
- filter_tags,
- EXTRACT(EPOCH FROM created_at) * 1e6,
- (energy_uj::DOUBLE PRECISION)/1e6/3600/1000, -- to get to kWh
- (carbon_ug::DOUBLE PRECISION)/1e9, -- to get to kg
- 0, -- (carbon_intensity_g) there is no need for this column for further processing
- 0.0, -- (latitude) there is no need for this column for further processing
- 0.0, -- (longitude) there is no need for this column for further processing
- ip_address,
- user_id,
- created_at
- FROM ci_measurements
- WHERE
- created_at >= CURRENT_DATE - INTERVAL '1 DAYS';
- ''')
-
-def copy_over_gmt():
- DB().query('''
-
- INSERT INTO carbondb_data_raw
- ("type", "project", "machine", "source", "tags","time","energy_kwh","carbon_kg","carbon_intensity_g","latitude","longitude","ip_address","user_id","created_at")
- SELECT
- 'machine.server' as type,
- 'Energy-ID' as project,
- m.description,
- 'Green Metrics Tool',
- ARRAY[]::text[] as tags ,
- EXTRACT(EPOCH FROM r.created_at) * 1e6 as time,
-
- -- we do these two queries as subselects as if they were left joins they will blow up the table whenever we relax the condition that only one metric with same name may exist
- (SELECT SUM(value::DOUBLE PRECISION) FROM phase_stats as p WHERE p.run_id = r.id AND p.unit = 'mJ' AND p.metric LIKE '%_energy_%_machine')/1e3/3600/1000 as energy_kwh,
- (SELECT SUM(value::DOUBLE PRECISION) FROM phase_stats as p2 WHERE p2.run_id = r.id AND p2.unit = 'ug' AND p2.metric LIKE '%_carbon_%')/1e9 as carbon_kg,
-
- 0, -- there is no need for this column for further processing
- 0.0, -- there is no need for this column for further processing
- 0.0, -- there is no need for this column for further processing
- NULL,
- r.user_id,
- r.created_at
- FROM runs as r
- -- we do LEFT JOIN as we do not want to silent skip data. If a column gets NULL it will fail
- LEFT JOIN machines as m ON m.id = r.machine_id
-
- WHERE
- r.user_id IS NOT NULL
- AND r.created_at >= CURRENT_DATE - INTERVAL '30 DAYS'
- GROUP BY
- r.id, m.description;
-
- ''')
-
-def validate_table_constraints():
- data = DB().fetch_all('''
- SELECT
- column_name,
- is_nullable
- FROM
- information_schema.columns
- WHERE
- table_name = 'carbondb_data_raw'
- AND column_name IN ('user_id', 'time', 'energy_kwh', 'carbon_kg', 'carbon_intensity_g', 'type', 'project', 'machine', 'source', 'tags')
- ''')
-
- for row in data:
- if row[1] == 'YES':
- raise RuntimeError(f"{row[0]} was NULL-able: {row[1]}. CarbonDB cannot remove duplicates.")
-
-
-def remove_duplicates():
- validate_table_constraints() # since the query works only if columns are not null
- DB().query('''
- DELETE FROM carbondb_data_raw a
- USING carbondb_data_raw b
- WHERE
- a.ctid < b.ctid
- AND a.time = b.time
- AND a.machine = b.machine
- AND a.type = b.type
- AND a.project = b.project
- AND a.source = b.source
- AND a.tags = b.tags
- AND a.energy_kwh = b.energy_kwh
- AND a.carbon_kg = b.carbon_kg
- AND a.user_id = b.user_id;
- ''')
-
-
-if __name__ == '__main__':
- try:
- GlobalConfig().override_config(config_location=f"{os.path.dirname(os.path.realpath(__file__))}/../manager-config.yml")
- copy_over_eco_ci()
- copy_over_gmt()
- remove_duplicates()
- except Exception as exc: # pylint: disable=broad-except
- error_helpers.log_error(f'Processing in {__file__} failed.', exception=exc, machine=GlobalConfig().config['machine']['description'])
diff --git a/cron/carbondb_copy_over_and_remove_duplicates.py b/cron/carbondb_copy_over_and_remove_duplicates.py
new file mode 120000
index 000000000..8aea64a20
--- /dev/null
+++ b/cron/carbondb_copy_over_and_remove_duplicates.py
@@ -0,0 +1 @@
+../ee/cron/carbondb_copy_over_and_remove_duplicates.py
\ No newline at end of file
diff --git a/docker/requirements.txt b/docker/requirements.txt
index 5e0f0bd18..2cf6a0fab 100644
--- a/docker/requirements.txt
+++ b/docker/requirements.txt
@@ -15,3 +15,5 @@ redis==5.2.0
hiredis==3.0.0
requests==2.32.3
uvicorn-worker==0.2.0
+
+pytest==8.3.3 # needed because we need to exit in db.py if tests run with wrong config
diff --git a/ee b/ee
new file mode 160000
index 000000000..2a6febf2b
--- /dev/null
+++ b/ee
@@ -0,0 +1 @@
+Subproject commit 2a6febf2b9ef5d134c11b71c070a92a1b1f7672d
diff --git a/frontend/carbondb-details.html b/frontend/carbondb-details.html
deleted file mode 100644
index df387732b..000000000
--- a/frontend/carbondb-details.html
+++ /dev/null
@@ -1,100 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool - CarbonDB
-
-
-
-
Your machine stats
-
-
-
-
-
-
-
- Sum Energy (J)
-
-
-
-
-
-
-
- Sum CO2eq (g)
-
-
-
-
-
-
-
- Avg. Carbon Intensity (gCO2e/kWh)
-
-
-
-
-
-
-
- Records received
-
-
-
-
-
-
Your machine details
-
-
-
-
-
-
-
-
- Why can I only see days?
-
-
- This is the free version of the Green Metrics Tool. Because saving all the values would be quite expensive we
- only offer daily statistic for free. If you want detailed data please check out our paid plans.
- You can see how this would look like on our demo machine.
-
-
-
-
- Overall energy statistics
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/frontend/carbondb-lists.html b/frontend/carbondb-lists.html
deleted file mode 100644
index 45418c557..000000000
--- a/frontend/carbondb-lists.html
+++ /dev/null
@@ -1,77 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
The idea behind CarbonDB is that a lot of components in your stack produce different amounts of CO2 based on how much energy they consume, how long the hardware is used, what time and where certain operations are computed and many more. For companies it is vital to record all this data so they can a) improve their stack and also b) report the data as will be required by new EU laws.
-
The solution is a central database in which all the different services can report their usage and then there is a central point where analytics and further operations can be performed
Maybe your user ID is not correctly set in the Dashboard? Go to Authentication and enter your token.
-
If you are using the hosted version of GMT please note that CarbonDB is a premium feature and you need to get a token first.
-
-
-
-
- Show Filters
- Filters are active! Open Dropdown for details
-
-
-
-
-
-
Filters
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- --
-
-
- Total Carbon [kg]
-
-
-
-
-
-
-
-
- --
-
-
- Total Energy [kWh]
-
-
-
-
-
-
-
-
- --
-
-
- Total Machines
-
-
-
-
-
-
-
-
- --
-
-
- Carbon per machine [kg/Unit]
-
-
-
-
-
-
-
-
- --
-
-
- AVG carbon intensity [g]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/frontend/carbondb.html b/frontend/carbondb.html
new file mode 120000
index 000000000..10fdaf52d
--- /dev/null
+++ b/frontend/carbondb.html
@@ -0,0 +1 @@
+../ee/frontend/carbondb.html
\ No newline at end of file
diff --git a/frontend/hog-details.html b/frontend/hog-details.html
deleted file mode 100644
index ea416754c..000000000
--- a/frontend/hog-details.html
+++ /dev/null
@@ -1,87 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool - Power HOG
-
-
-
-
Your machine stats
-
-
You can click on a bar that you want more details about. Please note that we don't show this one bar
- as the surrounding processes might also influence the results. Please use the zoom to look at one specific bar.
-
-
-
-
-
-
-
-
Loading and processing data
-
-
-
- Overall energy statistics
-
-
-
-
-
-
-
-
- Per process energy statistics
-
-
-
-
-
-
-
Details
-
-
-
-
-
-
-
-
-
-
Ok
-
-
-
-
-
\ No newline at end of file
diff --git a/frontend/hog-details.html b/frontend/hog-details.html
new file mode 120000
index 000000000..8a42d63a5
--- /dev/null
+++ b/frontend/hog-details.html
@@ -0,0 +1 @@
+../ee/frontend/hog-details.html
\ No newline at end of file
diff --git a/frontend/hog.html b/frontend/hog.html
deleted file mode 100644
index 7f41baa60..000000000
--- a/frontend/hog.html
+++ /dev/null
@@ -1,51 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Green Metrics Tool - Power HOG
-
-
-
-
Status overview
-
-
This list shows you all processes we have monitored on the machines that run the power hog.
-
If you are looking for the data from your install please click on "Details" in the power hog desktop app.
-
Number of Machines this data is based on: 0
-
-
-
-
-
Processes
-
-
-
-
\ No newline at end of file
diff --git a/frontend/hog.html b/frontend/hog.html
new file mode 120000
index 000000000..8993891df
--- /dev/null
+++ b/frontend/hog.html
@@ -0,0 +1 @@
+../ee/frontend/hog.html
\ No newline at end of file
diff --git a/frontend/js/carbondb-details.js b/frontend/js/carbondb-details.js
deleted file mode 100644
index 1b938ceef..000000000
--- a/frontend/js/carbondb-details.js
+++ /dev/null
@@ -1,129 +0,0 @@
-$(document).ready(function () {
- function getURLParameter(name) {
- return new URLSearchParams(window.location.search).get(name);
- }
-
- (async () => {
- const machine_uuid = getURLParameter('machine_uuid')
- try {
-
- var measurements = await makeAPICall(`/v1/carbondb/machine/day/${machine_uuid}`);
- } catch (err) {
- showNotification('Could not get data from API', err);
- return;
- }
- if (measurements.data.length == 0){
- showNotification('No data', 'We could not find any data. Did you follow a correct URL?')
- return
- }
-
- let types = new Set();
- let companies = new Set();
- let machines = new Set();
- let projects = new Set();
- let tags = new Set();
-
- measurements.data.forEach(item => {
- types.add(item[1]);
- companies.add(item[2]);
- machines.add(item[3]);
- projects.add(item[4]);
- item[5]?.forEach(tag => tags.add(tag));
- });
-
- types = Array.from(types);
- companies = Array.from(companies);
- machines = Array.from(machines);
- projects = Array.from(projects);
- tags = Array.from(tags);
-
- let info_string = ``
-
- if (types.length > 0){
- info_string += `
-