diff --git a/api/api_helpers.py b/api/api_helpers.py
index 92fa4a555..c957e6ac1 100644
--- a/api/api_helpers.py
+++ b/api/api_helpers.py
@@ -608,13 +608,14 @@ def __init__(
self.content = content
super().__init__(content, status_code, headers, media_type, background)
+# The decorator will not work between requests, so we are not prone to stale data over time
+@cache
def get_geo(ip):
- try:
- ip_obj = ipaddress.ip_address(ip)
- if ip_obj.is_private:
- return('52.53721666833642', '13.424863870661927')
- except ValueError:
- return (None, None)
+
+ ip_obj = ipaddress.ip_address(ip) # may raise a ValueError
+ if ip_obj.is_private:
+ error_helpers.log_error(f"Private IP was submitted to get_geo {ip}. This is normal in development, but should not happen in production.")
+ return('52.53721666833642', '13.424863870661927')
query = "SELECT ip_address, data FROM ip_data WHERE created_at > NOW() - INTERVAL '24 hours' AND ip_address=%s;"
db_data = DB().fetch_all(query, (ip,))
@@ -624,19 +625,20 @@ def get_geo(ip):
latitude, longitude = get_geo_ipapi_co(ip)
- if latitude is False:
+ if not latitude:
latitude, longitude = get_geo_ip_api_com(ip)
- if latitude is False:
+ if not latitude:
latitude, longitude = get_geo_ip_ipinfo(ip)
+ if not latitude:
+ raise RuntimeError(f"Could not get Geo-IP for {ip} after 3 tries")
- #If all 3 fail there is something bigger wrong
return (latitude, longitude)
def get_geo_ipapi_co(ip):
response = requests.get(f"https://ipapi.co/{ip}/json/", timeout=10)
- print(f"Accessing https://ipapi.co/{ip}/json/")
+
if response.status_code == 200:
resp_data = response.json()
@@ -650,6 +652,8 @@ def get_geo_ipapi_co(ip):
return (resp_data.get('latitude'), resp_data.get('longitude'))
+ error_helpers.log_error(f"Could not get Geo-IP from ipapi.co for {ip}. Trying next ...", response=response)
+
return (False, False)
def get_geo_ip_api_com(ip):
@@ -671,6 +675,8 @@ def get_geo_ip_api_com(ip):
return (resp_data.get('latitude'), resp_data.get('longitude'))
+ error_helpers.log_error(f"Could not get Geo-IP from ip-api.com for {ip}. Trying next ...", response=response)
+
return (False, False)
def get_geo_ip_ipinfo(ip):
@@ -694,8 +700,12 @@ def get_geo_ip_ipinfo(ip):
return (resp_data.get('latitude'), resp_data.get('longitude'))
+ error_helpers.log_error(f"Could not get Geo-IP from ipinfo.io for {ip}. Trying next ...", response=response)
+
return (False, False)
+# The decorator will not work between requests, so we are not prone to stale data over time
+@cache
def get_carbon_intensity(latitude, longitude):
if latitude is None or longitude is None:
@@ -726,12 +736,11 @@ def get_carbon_intensity(latitude, longitude):
return resp_data.get('carbonIntensity')
- return None
+ error_helpers.log_error(f"Could not get carbon intensity from Electricitymaps.org for {params}", response=response)
-def carbondb_add(client_ip, energydatas):
+ return None
- latitude, longitude = get_geo(client_ip)
- carbon_intensity = get_carbon_intensity(latitude, longitude)
+def carbondb_add(client_ip, energydatas, user_id):
data_rows = []
@@ -752,10 +761,12 @@ def carbondb_add(client_ip, energydatas):
if field_value is None or str(field_value).strip() == '':
raise RequestValidationError(f"{field_name.capitalize()} is empty. Ignoring everything!")
- if 'ip' in e:
- # An ip has been given with the data. Let's use this:
- latitude, longitude = get_geo(e['ip'])
- carbon_intensity = get_carbon_intensity(latitude, longitude)
+ if 'ip' in e: # An ip has been given with the data. We prioritize that
+ latitude, longitude = get_geo(e['ip']) # cached
+ carbon_intensity = get_carbon_intensity(latitude, longitude) # cached
+ else:
+ latitude, longitude = get_geo(client_ip) # cached
+ carbon_intensity = get_carbon_intensity(latitude, longitude) # cached
energy_kwh = float(e['energy_value']) * 2.77778e-7 # kWh
co2_value = energy_kwh * carbon_intensity # results in g
@@ -764,12 +775,12 @@ def carbondb_add(client_ip, energydatas):
project_uuid = e['project'] if e['project'] is not None else ''
tags_clean = "{" + ",".join([f'"{tag.strip()}"' for tag in e['tags'].split(',') if e['tags']]) + "}" if e['tags'] is not None else ''
- row = f"{e['type']}|{company_uuid}|{e['machine']}|{project_uuid}|{tags_clean}|{int(e['time_stamp'])}|{e['energy_value']}|{co2_value}|{carbon_intensity}|{latitude}|{longitude}|{client_ip}"
+ row = f"{e['type']}|{company_uuid}|{e['machine']}|{project_uuid}|{tags_clean}|{int(e['time_stamp'])}|{e['energy_value']}|{co2_value}|{carbon_intensity}|{latitude}|{longitude}|{client_ip}|{user_id}"
data_rows.append(row)
data_str = "\n".join(data_rows)
data_file = io.StringIO(data_str)
- columns = ['type', 'company', 'machine', 'project', 'tags', 'time_stamp', 'energy_value', 'co2_value', 'carbon_intensity', 'latitude', 'longitude', 'ip_address']
+ columns = ['type', 'company', 'machine', 'project', 'tags', 'time_stamp', 'energy_value', 'co2_value', 'carbon_intensity', 'latitude', 'longitude', 'ip_address', 'user_id']
DB().copy_from(file=data_file, table='carbondb_energy_data', columns=columns, sep='|')
diff --git a/api/main.py b/api/main.py
index 226c2d3a5..e2e672188 100644
--- a/api/main.py
+++ b/api/main.py
@@ -11,15 +11,20 @@
from typing import List
from xml.sax.saxutils import escape as xml_escape
import math
-from fastapi import FastAPI, Request, Response
+from urllib.parse import urlparse
+
+from fastapi import FastAPI, Request, Response, Depends, HTTPException
from fastapi.responses import ORJSONResponse
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
+from fastapi.security import APIKeyHeader
+
from datetime import date
from starlette.responses import RedirectResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
+from starlette.datastructures import Headers as StarletteHeaders
from pydantic import BaseModel, ValidationError, field_validator
from typing import Optional
@@ -38,6 +43,8 @@
from lib.diff import get_diffable_row, diff_rows
from lib import error_helpers
from lib.job.base import Job
+from lib.user import User, UserAuthenticationError
+from lib.secure_variable import SecureVariable
from tools.timeline_projects import TimelineProject
from enum import Enum
@@ -53,7 +60,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
url=request.url,
query_params=request.query_params,
client=request.client,
- headers=request.headers,
+ headers=obfuscate_authentication_token(request.headers),
body=exc.body,
details=exc.errors(),
exception=exc
@@ -71,7 +78,7 @@ async def http_exception_handler(request, exc):
url=request.url,
query_params=request.query_params,
client=request.client,
- headers=request.headers,
+ headers=obfuscate_authentication_token(request.headers),
body=body,
details=exc.detail,
exception=exc
@@ -84,6 +91,7 @@ async def http_exception_handler(request, exc):
async def catch_exceptions_middleware(request: Request, call_next):
#pylint: disable=broad-except
body = None
+
try:
body = await request.body()
return await call_next(request)
@@ -93,7 +101,7 @@ async def catch_exceptions_middleware(request: Request, call_next):
url=request.url,
query_params=request.query_params,
client=request.client,
- headers=request.headers,
+ headers=obfuscate_authentication_token(request.headers),
body=body,
exception=exc
)
@@ -117,6 +125,40 @@ async def catch_exceptions_middleware(request: Request, call_next):
allow_headers=['*'],
)
+header_scheme = APIKeyHeader(
+ name='X-Authentication',
+ scheme_name='Header',
+ description='Authentication key - See https://docs.green-coding.io/authentication',
+ auto_error=False
+)
+
+def obfuscate_authentication_token(headers: StarletteHeaders):
+ headers_mut = headers.mutablecopy()
+ if 'X-Authentication' in headers_mut:
+ headers_mut['X-Authentication'] = '****OBFUSCATED****'
+ return headers_mut
+
+def authenticate(authentication_token=Depends(header_scheme), request: Request = None):
+ parsed_url = urlparse(str(request.url))
+ try:
+
+ if not authentication_token: # Note that if no token is supplied this will authenticate as the DEFAULT user, which in FOSS systems has full capabilities
+ authentication_token = 'DEFAULT'
+
+ user = User.authenticate(SecureVariable(authentication_token))
+
+ if not user.can_use_route(parsed_url.path):
+ raise HTTPException(status_code=401, detail="Route not allowed") from UserAuthenticationError
+
+ if not user.has_api_quota(parsed_url.path):
+ raise HTTPException(status_code=401, detail="Quota exceeded") from UserAuthenticationError
+
+ user.deduct_api_quota(parsed_url.path, 1)
+
+ except UserAuthenticationError:
+ raise HTTPException(status_code=401, detail="Invalid token") from UserAuthenticationError
+ return user
+
@app.get('/')
async def home():
@@ -215,6 +257,7 @@ async def get_repositories(uri: str | None = None, branch: str | None = None, ma
return ORJSONResponse({'success': True, 'data': escaped_data})
+
# A route to return all of the available entries in our catalog.
@app.get('/v1/runs')
async def get_runs(uri: str | None = None, branch: str | None = None, machine_id: int | None = None, machine: str | None = None, filename: str | None = None, limit: int | None = None, uri_mode = 'none'):
@@ -590,7 +633,6 @@ async def get_jobs(machine_id: int | None = None, state: str | None = None):
return ORJSONResponse({'success': True, 'data': data})
-####
class HogMeasurement(BaseModel):
time: int
@@ -658,7 +700,10 @@ def validate_measurement_data(data):
return True
@app.post('/v1/hog/add')
-async def hog_add(measurements: List[HogMeasurement]):
+async def hog_add(
+ measurements: List[HogMeasurement],
+ user: User = Depends(authenticate), # pylint: disable=unused-argument
+ ):
for measurement in measurements:
decoded_data = base64.b64decode(measurement.data)
@@ -735,8 +780,9 @@ async def hog_add(measurements: List[HogMeasurement]):
ane_energy,
energy_impact,
thermal_pressure,
- settings)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+ settings,
+ user_id)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
"""
params = (
@@ -750,6 +796,7 @@ async def hog_add(measurements: List[HogMeasurement]):
cpu_energy_data['energy_impact'],
measurement_data['thermal_pressure'],
measurement.settings,
+ user._id,
)
measurement_db_id = DB().fetch_one(query=query, params=params)[0]
@@ -1018,9 +1065,6 @@ async def hog_get_task_details(machine_uuid: str, measurements_id_start: int, me
return ORJSONResponse({'success': True, 'tasks_data': tasks_data, 'coalitions_data': coalitions_data})
-
-####
-
class Software(BaseModel):
name: str
url: str
@@ -1031,7 +1075,7 @@ class Software(BaseModel):
schedule_mode: str
@app.post('/v1/software/add')
-async def software_add(software: Software):
+async def software_add(software: Software, user: User = Depends(authenticate)):
software = html_escape_multi(software)
@@ -1057,22 +1101,26 @@ async def software_add(software: Software):
if not DB().fetch_one('SELECT id FROM machines WHERE id=%s AND available=TRUE', params=(software.machine_id,)):
raise RequestValidationError('Machine does not exist')
+ if not user.can_use_machine(software.machine_id):
+ raise RequestValidationError('Your user does not have the permissions to use that machine.')
if software.schedule_mode not in ['one-off', 'daily', 'weekly', 'commit', 'variance']:
raise RequestValidationError(f"Please select a valid measurement interval. ({software.schedule_mode}) is unknown.")
- # notify admin of new add
- if notification_email := GlobalConfig().config['admin']['notification_email']:
- Job.insert('email', name='New run added from Web Interface', message=str(software), email=notification_email)
-
+ if not user.can_schedule_job(software.schedule_mode):
+ raise RequestValidationError('Your user does not have the permissions to use that schedule mode.')
if software.schedule_mode in ['daily', 'weekly', 'commit']:
- TimelineProject.insert(software.name, software.url, software.branch, software.filename, software.machine_id, software.schedule_mode)
+ TimelineProject.insert(name=software.name, url=software.url, branch=software.branch, filename=software.filename, machine_id=software.machine_id, user_id=user._id, schedule_mode=software.schedule_mode)
# even for timeline projects we do at least one run
amount = 10 if software.schedule_mode == 'variance' else 1
for _ in range(0,amount):
- Job.insert('run', name=software.name, url=software.url, email=software.email, branch=software.branch, filename=software.filename, machine_id=software.machine_id)
+ Job.insert('run', user_id=user._id, name=software.name, url=software.url, email=software.email, branch=software.branch, filename=software.filename, machine_id=software.machine_id)
+
+ # notify admin of new add
+ if notification_email := GlobalConfig().config['admin']['notification_email']:
+ Job.insert('email', user_id=user._id, name='New run added from Web Interface', message=str(software), email=notification_email)
return ORJSONResponse({'success': True}, status_code=202)
@@ -1165,7 +1213,11 @@ class CI_Measurement(BaseModel):
co2eq: Optional[str] = ''
@app.post('/v1/ci/measurement/add')
-async def post_ci_measurement_add(request: Request, measurement: CI_Measurement):
+async def post_ci_measurement_add(
+ request: Request,
+ measurement: CI_Measurement,
+ user: User = Depends(authenticate) # pylint: disable=unused-argument
+ ):
for key, value in measurement.model_dump().items():
match key:
case 'unit':
@@ -1206,14 +1258,15 @@ async def post_ci_measurement_add(request: Request, measurement: CI_Measurement)
lon,
city,
co2i,
- co2eq
+ co2eq,
+ user_id
)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = (measurement.energy_value, measurement.energy_unit, measurement.repo, measurement.branch,
measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu,
measurement.commit_hash, measurement.duration, measurement.cpu_util_avg, measurement.workflow_name,
- measurement.lat, measurement.lon, measurement.city, measurement.co2i, measurement.co2eq)
+ measurement.lat, measurement.lon, measurement.city, measurement.co2i, measurement.co2eq, user._id)
DB().query(query=query, params=params)
@@ -1239,8 +1292,16 @@ async def post_ci_measurement_add(request: Request, measurement: CI_Measurement)
'tags': f"{measurement.label},{measurement.repo},{measurement.branch},{measurement.workflow}"
}
- # If there is an error the function will raise an Error
- carbondb_add(client_ip, [energydata])
+ try:
+ carbondb_add(client_ip, [energydata], user._id)
+ #pylint: disable=broad-except
+ except Exception as exc:
+ error_helpers.log_error('CI Measurement was successfully added, but CarbonDB did failed', exception=exc)
+ return ORJSONResponse({
+ 'success': False,
+ 'err': f"CI Measurement was successfully added, but CarbonDB did respond with exception: {str(exc)}"},
+ status_code=207
+ )
return ORJSONResponse({'success': True}, status_code=201)
@@ -1377,7 +1438,11 @@ def empty_str_to_none(cls, values, _):
return values
@app.post('/v1/carbondb/add')
-async def add_carbondb(request: Request, energydatas: List[EnergyData]):
+async def add_carbondb(
+ request: Request,
+ energydatas: List[EnergyData],
+ user: User = Depends(authenticate) # pylint: disable=unused-argument
+ ):
client_ip = request.headers.get("x-forwarded-for")
if client_ip:
@@ -1385,7 +1450,7 @@ async def add_carbondb(request: Request, energydatas: List[EnergyData]):
else:
client_ip = request.client.host
- carbondb_add(client_ip, energydatas)
+ carbondb_add(client_ip, energydatas, user._id)
return Response(status_code=204)
@@ -1441,5 +1506,19 @@ async def carbondb_get_company_project_details(cptype: str, uuid: str):
return ORJSONResponse({'success': True, 'data': data})
+# @app.get('/v1/authentication/new')
+# This will fail if the DB insert fails but still report 'success': True
+# Must be reworked if we want to allow API based token generation
+# async def get_authentication_token(name: str = None):
+# if name is not None and name.strip() == '':
+# name = None
+# return ORJSONResponse({'success': True, 'data': User.get_new(name)})
+
+@app.get('/v1/authentication/data')
+async def read_authentication_token(user: User = Depends(authenticate)):
+ return ORJSONResponse({'success': True, 'data': user.__dict__})
+
+
+
if __name__ == '__main__':
app.run() # pylint: disable=no-member
diff --git a/config.yml.example b/config.yml.example
index 02937615d..20075f87d 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -26,7 +26,6 @@ admin:
email_bcc: False
-
cluster:
api_url: __API_URL__
metrics_url: __METRICS_URL__
@@ -65,8 +64,6 @@ measurement:
pre-test-sleep: 5
idle-duration: 5
baseline-duration: 5
- flow-process-duration: 1800 # half hour
- total-duration: 3600 # one hour
post-test-sleep: 5
phase-transition-time: 1
boot:
diff --git a/docker/structure.sql b/docker/structure.sql
index 65ecdb477..da4e67684 100644
--- a/docker/structure.sql
+++ b/docker/structure.sql
@@ -1,9 +1,35 @@
CREATE DATABASE "green-coding";
\c green-coding;
+CREATE SCHEMA IF NOT EXISTS "public";
+
CREATE EXTENSION "uuid-ossp";
CREATE EXTENSION "moddatetime";
+CREATE TABLE users (
+ id SERIAL PRIMARY KEY,
+ name text,
+ token text NOT NULL,
+ capabilities JSONB NOT NULL,
+ created_at timestamp with time zone DEFAULT now(),
+ updated_at timestamp with time zone
+);
+
+CREATE UNIQUE INDEX name_unique ON users(name text_ops);
+CREATE UNIQUE INDEX token_unique ON users(token text_ops);
+
+CREATE TRIGGER users_moddatetime
+ BEFORE UPDATE ON users
+ FOR EACH ROW
+ EXECUTE PROCEDURE moddatetime (updated_at);
+
+-- Default password for authentication is DEFAULT
+INSERT INTO "public"."users"("name","token","capabilities","created_at","updated_at")
+VALUES
+(E'DEFAULT',E'89dbf71048801678ca4abfbaa3ea8f7c651aae193357a3e23d68e21512cd07f5',E'{"api":{"quotas":{},"routes":["/v1/carbondb/add","/v1/ci/measurement/add","/v1/software/add","/v1/hog/add","/v1/authentication/data"]},"data":{"runs":{"retention":2678400},"hog_tasks":{"retention":2678400},"measurements":{"retention":2678400},"hog_coalitions":{"retention":2678400},"ci_measurements":{"retention":2678400},"hog_measurements":{"retention":2678400}},"jobs":{"schedule_modes":["one-off","daily","weekly","commit","variance"]},"machines":[1],"measurement":{"quotas":{},"settings":{"total-duration":86400,"flow-process-duration":86400}},"optimizations":["container_memory_utilization","container_cpu_utilization","message_optimization","container_build_time","container_boot_time","container_image_size"]}',E'2024-08-22 11:28:24.937262+00',NULL);
+
+
+
CREATE TABLE machines (
id SERIAL PRIMARY KEY,
description text,
@@ -24,6 +50,12 @@ CREATE TRIGGER machines_moddatetime
FOR EACH ROW
EXECUTE PROCEDURE moddatetime (updated_at);
+-- Default password for authentication is DEFAULT
+INSERT INTO "public"."machines"("description", "available")
+VALUES
+(E'Local machine', true);
+
+
CREATE TABLE jobs (
id SERIAL PRIMARY KEY,
type text,
@@ -36,6 +68,7 @@ CREATE TABLE jobs (
categories int[],
machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE,
message text,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
@@ -67,6 +100,7 @@ CREATE TABLE runs (
logs text,
invalid_run text,
failed boolean DEFAULT false,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
@@ -183,6 +217,7 @@ CREATE TABLE ci_measurements (
city text,
co2i text,
co2eq text,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
@@ -217,6 +252,7 @@ CREATE TABLE timeline_projects (
machine_id integer REFERENCES machines(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL,
schedule_mode text NOT NULL,
last_scheduled timestamp with time zone,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
@@ -238,6 +274,7 @@ CREATE TABLE hog_measurements (
thermal_pressure text,
settings jsonb,
data jsonb,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
@@ -287,7 +324,6 @@ CREATE TABLE hog_tasks (
diskio_byteswritten bigint,
intr_wakeups bigint,
idle_wakeups bigint,
-
data jsonb,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
@@ -336,6 +372,7 @@ CREATE TABLE carbondb_energy_data (
latitude DOUBLE PRECISION,
longitude DOUBLE PRECISION,
ip_address INET,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
@@ -365,6 +402,7 @@ CREATE TABLE carbondb_energy_data_day (
co2_sum FLOAT,
carbon_intensity_avg FLOAT,
record_count INT,
+ user_id integer REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
created_at timestamp with time zone DEFAULT now(),
updated_at timestamp with time zone
);
diff --git a/frontend/authentication.html b/frontend/authentication.html
new file mode 100644
index 000000000..9d5384285
--- /dev/null
+++ b/frontend/authentication.html
@@ -0,0 +1,95 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ Green Metrics Tool
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The Green Metrics Tool supports restricting certain functionalites for fair-use or premium access.
+
In case you already have a token you can input it here to use it with all API calls in this Dashboard and / or see your current capabilities and quotas.
+
If you want to acquire a new authentication token to access certain premium features that are not distributed with the open-source version contact us at info@green-coding.io
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/frontend/carbondb-details.html b/frontend/carbondb-details.html
index 8dde252a1..df387732b 100644
--- a/frontend/carbondb-details.html
+++ b/frontend/carbondb-details.html
@@ -49,7 +49,7 @@