diff --git a/.devcontainer/codespace-setup.sh b/.devcontainer/codespace-setup.sh new file mode 100755 index 000000000..5314dd3d6 --- /dev/null +++ b/.devcontainer/codespace-setup.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -euxo pipefail + +# we have to rename this makefile as it doesn't compile in Codespaces +if [ -f /workspaces/green-metrics-tool/metric_providers/lm_sensors/Makefile ]; then + mv /workspaces/green-metrics-tool/metric_providers/lm_sensors/Makefile /workspaces/green-metrics-tool/metric_providers/lm_sensors/Makefile.bak + git update-index --assume-unchanged /workspaces/green-metrics-tool/metric_providers/lm_sensors/Makefile +fi + +/workspaces/green-metrics-tool/install_linux.sh -p testpw -a "https://${CODESPACE_NAME}-9142.app.github.dev" -m "https://${CODESPACE_NAME}-9143.app.github.dev" -t -i -s -l +source venv/bin/activate + +# Also add XGBoost, as we need it +python3 -m pip install -r /workspaces/green-metrics-tool/metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt + +# make edits to ports so we can use 9143 to access front end +sed -i 's/listen \[::\]:9142;/listen [::]:9143;/; s/listen 9142;/listen 9143;/' /workspaces/green-metrics-tool/docker/nginx/frontend.conf +sed -i 's/- 9142:9142/- 9142:9142\n - 9143:9143/' /workspaces/green-metrics-tool/docker/compose.yml +sed -i 's|- ./nginx/block.conf|#- ./nginx/block.conf|' /workspaces/green-metrics-tool/docker/compose.yml + +# activate XGBoost provider with sane values for GitHub Codespaces +sed -i 's/common:/common:\n psu.energy.ac.xgboost.machine.provider.PsuEnergyAcXgboostMachineProvider:\n resolution: 99\n CPUChips: 1\n HW_CPUFreq: 2800\n CPUCores: 32\n CPUThreads: 64\n TDP: 270\n HW_MemAmountGB: 256\n VHost_Ratio: 0.03125\n/' /workspaces/green-metrics-tool/config.yml + + +git clone https://github.com/green-coding-solutions/example-applications.git --depth=1 --single-branch /workspaces/green-metrics-tool/example-applications || true + +source venv/bin/activate + +docker compose -f /workspaces/green-metrics-tool/docker/compose.yml down + +docker compose -f /workspaces/green-metrics-tool/docker/compose.yml up -d + + +gh codespace ports visibility 9142:public -c $CODESPACE_NAME + +gh codespace ports visibility 9143:public -c $CODESPACE_NAME \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..9fa8d8350 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,31 @@ +{ + "postStartCommand": "/workspaces/green-metrics-tool/.devcontainer/on-start.sh", + "forwardPorts": [9142, 9143], + "portsAttributes": { + "9143": { + "label": "metrics page" + }, + "9142": { + "label": "api" + } + }, + "customizations": { + "codespaces": { + "openFiles": [ + ".devcontainer/splash.md" + ] + }, + "vscode": { + "settings": { + "workbench.editorAssociations": { + "*.md": "vscode.markdown.preview.editor" // Open markdown files in preview mode by default + } + }, + "extensions": + [ + "ms-python.python", + "ms-azuretools.vscode-docker" + ] + } + } +} diff --git a/.devcontainer/on-start.sh b/.devcontainer/on-start.sh new file mode 100755 index 000000000..8f0c1f449 --- /dev/null +++ b/.devcontainer/on-start.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -euo pipefail + +etc_hosts_line_1="127.0.0.1 green-coding-postgres-container" + +echo "Writing to /etc/hosts file..." + +# Entry 1 is needed for the local resolution of the containers through the jobs.py and runner.py +if ! sudo grep -Fxq "$etc_hosts_line_1" /etc/hosts; then + echo "$etc_hosts_line_1" | sudo tee -a /etc/hosts +else + echo "Entry was already present..." +fi + +# Ensure that after a restart of the Codespace the ports are set to public again +gh codespace ports visibility 9142:public -c $CODESPACE_NAME +gh codespace ports visibility 9143:public -c $CODESPACE_NAME diff --git a/.devcontainer/splash.md b/.devcontainer/splash.md new file mode 100644 index 000000000..8ff746a9b --- /dev/null +++ b/.devcontainer/splash.md @@ -0,0 +1,35 @@ +# Green Metrics Tool Codespaces Quickstart + +Thank you for trying out the Green Metrics Tool :-) + +Please run the following command in the terminal to set up everything! 🚀 + +```sh +bash .devcontainer/codespace-setup.sh +``` + +It will take about 3 minutes. + +Afterwards, load the python environment: + +```sh +source venv/bin/activate +``` + +Do your first measurement run like this: + +```sh +python3 runner.py --name "Simple Test" --uri "/workspaces/green-metrics-tool/example-applications/" --filename "stress/usage_scenario.yml" --skip-system-checks --dev-no-optimizations --dev-cache-build +``` + +Then, if you want to see a more representative repository, try running our Bakery Demo repository we did together with the Wagtail Community: + +```sh +python3 runner.py --uri https://github.com/green-coding-solutions/bakerydemo --branch gmt --skip-system-checks --dev-no-optimization --dev-cache-build --skip-unsafe --name "Bakery Demo Test" +``` + +To see the Metrics front end, go to your ports tab and follow the forwarding address for port 9143. + +Make sure the `api` port (9142) is public. If it's private, the metrics frontend will not be able to access the API due to CORS issues. + +If you are experiencing problems, see the file [.devcontainer/troubleshooting.md](./troubleshooting.md) for some common troubleshooting tips. diff --git a/.devcontainer/troubleshooting.md b/.devcontainer/troubleshooting.md new file mode 100644 index 000000000..0ffb3f62a --- /dev/null +++ b/.devcontainer/troubleshooting.md @@ -0,0 +1,27 @@ +# Troubleshooting + +## Frontend can't be open + +Make sure the ports 9142 (`api`) und 9143 (`metrics page`) are public. If they are private, the metrics frontend will not be able to access the API due to CORS issues. After a restart of the codespace the ports are set to private, so you have to change the visibility manually. + +You can use the following commands in the terminal to make the ports public: + +```sh +gh codespace ports visibility 9142:public -c $CODESPACE_NAME +gh codespace ports visibility 9143:public -c $CODESPACE_NAME +``` + +## Connection to server failed + +If you entcounter an error like + +```log +error connecting in 'pool-1': connection failed: connection to server at "127.0.0.1", port 9573 failed: Connection refused + Is the server running on that host and accepting TCP/IP connections? +``` + +then ensure that the Docker containers of GMT are running. + +```sh +docker compose -f docker/compose.yml up -d +``` diff --git a/.github/workflows/build-codespace-container.yml b/.github/workflows/build-codespace-container.yml index c108e6912..e8ee990df 100644 --- a/.github/workflows/build-codespace-container.yml +++ b/.github/workflows/build-codespace-container.yml @@ -20,6 +20,9 @@ jobs: gmt-api-token: ${{ secrets.GMT_API_TOKEN }} electricitymaps-api-token: ${{ secrets.ELECTRICITYMAPS_TOKEN }} + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -31,13 +34,21 @@ jobs: with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ inputs.github-token }} + password: ${{ secrets.GITHUB_TOKEN }} + + # - name: Build and push gunicorn container + # uses: docker/build-push-action@v5 + # with: + # context: ./docker + # file: ./docker/Dockerfile-gunicorn + # push: true + # tags: ghcr.io/green-coding-berlin/green-coding-gunicorn-container:latest - name: Build and push Docker container uses: docker/build-push-action@v6 with: context: . - file: /.devcontainer/containerized/Dockerfile # Path to the Dockerfile + file: ./.devcontainer/containerized/Dockerfile push: true tags: ghcr.io/green-coding-solutions/codespace-container:latest diff --git a/.gitignore b/.gitignore index 3102d8269..286f02e18 100644 --- a/.gitignore +++ b/.gitignore @@ -14,13 +14,13 @@ metric-provider-binary .vscode static-binary .pytest_cache -/docker/test-compose.yml -/tests/structure.sql -/tools/sgx_enable -/venv/ /lib/hardware_info_root.py /tools/cluster/cleanup.sh /node_modules/ /lib/c/parse_int.o /tools/backup -/manager-config.yml \ No newline at end of file +/manager-config.yml +/venv/ +Makefile.bak +/docker/test-compose.yml +/tests/structure.sql diff --git a/README.md b/README.md index 0ca9ec8ea..665b0f7a6 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [](https://metrics.green-coding.io/ci.html?repo=green-coding-solutions/green-metrics-tool&branch=main&workflow=45267393) (This is the energy cost of running our CI-Pipelines on Github. [Find out more about Eco-CI](https://www.green-coding.io/projects/eco-ci/)) +[](https://codespaces.new/green-coding-berlin/green-metrics-tool) + # Introduction The Green Metrics Tool is a developer tool indented for measuring the energy and CO2 consumption of software through a software life cycle analysis (SLCA). diff --git a/api/api_helpers.py b/api/api_helpers.py index dba531850..16ff15741 100644 --- a/api/api_helpers.py +++ b/api/api_helpers.py @@ -1,5 +1,6 @@ +import sys import faulthandler -faulthandler.enable() # will catch segfaults and write to STDERR +faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr from urllib.parse import urlparse @@ -64,7 +65,7 @@ def rescale_energy_value(value, unit): # We only expect values to be uJ for energy in the future. Changing values now temporarily. # TODO: Refactor this once all data in the DB is uJ if unit != 'uJ' and not unit.startswith('ugCO2e/'): - raise RuntimeError('Unexpected unit occured for energy rescaling: ', unit) + raise ValueError('Unexpected unit occured for energy rescaling: ', unit) unit_type = unit[1:] @@ -218,6 +219,7 @@ def get_timeline_query(uri, filename, machine_id, branch, metrics, phase, start_ AND r.filename = %s AND r.branch = %s AND r.end_measurement IS NOT NULL + AND r.failed != TRUE AND r.machine_id = %s AND p.phase LIKE %s {metrics_condition} diff --git a/api/main.py b/api/main.py index 4b6687248..12206e59e 100644 --- a/api/main.py +++ b/api/main.py @@ -1,8 +1,8 @@ -import faulthandler - # It seems like FastAPI already enables faulthandler as it shows stacktrace on SEGFAULT # Is the redundant call problematic? -faulthandler.enable() # will catch segfaults and write to STDERR +import sys +import faulthandler +faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr import orjson from xml.sax.saxutils import escape as xml_escape @@ -632,7 +632,7 @@ async def software_add(software: Software, user: User = Depends(authenticate)): if not user.can_schedule_job(software.schedule_mode): raise RequestValidationError('Your user does not have the permissions to use that schedule mode.') - utils.check_repo(software.url) # if it exists through the git api + utils.check_repo(software.url, software.branch) # if it exists through the git api if software.schedule_mode in ['daily', 'weekly', 'commit', 'commit-variance', 'tag', 'tag-variance']: diff --git a/config.yml.example b/config.yml.example index 620b95616..02e8c3d2f 100644 --- a/config.yml.example +++ b/config.yml.example @@ -78,7 +78,7 @@ machine: measurement: system_check_threshold: 3 # Can be 1=INFO, 2=WARN or 3=ERROR pre-test-sleep: 5 - idle-duration: 5 + idle-duration: 10 baseline-duration: 5 post-test-sleep: 5 phase-transition-time: 1 @@ -181,6 +181,9 @@ measurement: ######### The value for memory must be in GB not in GiB # HW_MemAmountGB: 16 # Hardware_Availability_Year: 2011 +######### vhost_ratio is the virtualization degree of the machine. For Bare Metal this is 1. For 1 out of 4 VMs this would be 0.25 etc. +# VHost_Ratio: 1 + #--- END diff --git a/cron/client.py b/cron/client.py index d8478d2d5..732548ff9 100644 --- a/cron/client.py +++ b/cron/client.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import sys import faulthandler -faulthandler.enable() # will catch segfaults and write to stderr +faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr import os -import sys import time import subprocess import json diff --git a/cron/jobs.py b/cron/jobs.py index 094aeabf3..f2d4fa4a2 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # pylint: disable=cyclic-import +import sys import faulthandler -faulthandler.enable() # will catch segfaults and write to stderr +faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr -import sys import os from datetime import datetime import argparse diff --git a/cron/timeline_projects.py b/cron/timeline_projects.py index ffcfbfbf9..6ebe2e4f7 100644 --- a/cron/timeline_projects.py +++ b/cron/timeline_projects.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import sys import faulthandler -faulthandler.enable() # will catch segfaults and write to stderr +faulthandler.enable(file=sys.__stderr__) # will catch segfaults and write to stderr import os import pprint diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 08eba4724..e0ec8641b 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -12,6 +12,7 @@ events { http { + server_names_hash_bucket_size 128; include /etc/nginx/mime.types; default_type application/octet-stream; @@ -34,4 +35,4 @@ http { gzip_disable "MSIE [1-6]\."; include /etc/nginx/conf.d/*.conf; -} \ No newline at end of file +} diff --git a/docker/requirements.txt b/docker/requirements.txt index 51236bec6..5e0f0bd18 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,6 +1,6 @@ gunicorn==23.0.0 psycopg[binary]==3.2.3 -psycopg_pool==3.2.3 +psycopg_pool==3.2.4 fastapi[standard]==0.115.5 starlette>=0.35 uvicorn[standard]==0.32.0 diff --git a/frontend/carbondb.html b/frontend/carbondb.html deleted file mode 120000 index 10fdaf52d..000000000 --- a/frontend/carbondb.html +++ /dev/null @@ -1 +0,0 @@ -../ee/frontend/carbondb.html \ No newline at end of file diff --git a/frontend/js/helpers/config.js.example b/frontend/js/helpers/config.js.example index 173985c74..397d90130 100644 --- a/frontend/js/helpers/config.js.example +++ b/frontend/js/helpers/config.js.example @@ -38,12 +38,24 @@ METRIC_MAPPINGS = { 'explanation': 'Container energy estimated via CPU-% share', }, + 'psu_energy_cgroup_slice': { + 'clean_name': 'Container Energy (+Idle)', + 'source': 'estimation', + 'explanation': 'Container energy estimated via CPU-% share (incl. idle)', + }, + 'psu_power_cgroup_container': { 'clean_name': 'Container Power', 'source': 'estimation', 'explanation': 'Container power estimated via CPU-% share', }, + 'psu_power_cgroup_slice': { + 'clean_name': 'Container Power (+Idle)', + 'source': 'estimation', + 'explanation': 'Container power estimated via CPU-% share incl. Idle', + }, + 'psu_carbon_dc_rapl_msr_machine': { 'clean_name': 'Machine CO₂', 'source': 'RAPL', diff --git a/frontend/js/helpers/metric-boxes.js b/frontend/js/helpers/metric-boxes.js index 1b3825310..24691ad0d 100644 --- a/frontend/js/helpers/metric-boxes.js +++ b/frontend/js/helpers/metric-boxes.js @@ -149,7 +149,7 @@ class PhaseMetrics extends HTMLElement {