Merge branch 'main' into fix-sudoers-venv-bug

* main: (110 commits) Adding report url instead of just ID (#531) Makes error optional except if there is data we need (#530) Bump uvicorn[standard] from 0.23.2 to 0.24.0.post1 (#526) Adding current clocksource in linux according to timing problems outlined here: https://www.brendangregg.com/blog/2021-09-26/the-speed-of-time.html (#524) Bump orjson from 3.9.9 to 3.9.10 (#519) Bump pandas from 2.1.1 to 2.1.2 (#520) Update Dockerfile - Removed playwright version pinning (#523) Bump fastapi from 0.104.0 to 0.104.1 (#522) Casts everything to bigint in hog api Qol Updates (#521) build containers workflow: removed conditional from job scope and moved to step added options for build containers manual run to specify which container to build fix file permissions for build-containers script Bump playwright/python in /docker/auxiliary-containers/gcb_playwright (#513) Bump pytest from 7.4.2 to 7.4.3 (#515) Added lshw to hardware_info_root (#514) Tag dockerhub containers (#518) Update XGBoost model Update README.md Bump pylint from 3.0.1 to 3.0.2 (#512) ...
green-coding-solutions · Nov 10, 2023 · 2bfb3df · 2bfb3df
2 parents 131e210 + 520b5a1
commit 2bfb3df
Show file tree

Hide file tree

Showing 191 changed files with 5,449 additions and 2,464 deletions.
diff --git a/.github/actions/gmt-pytest/action.yml b/.github/actions/gmt-pytest/action.yml
@@ -9,10 +9,6 @@ inputs:
     description: 'The root directory of the gmt repository'
     required: false
     default: '.'
-  tests-directory:
-    description: 'The directory where to run the tests from'
-    required: false
-    default: './test'
   tests-command:
     description: 'The command to run the tests'
     required: false
@@ -24,26 +20,33 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - name: setup python
+    - name: setup_python
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
-        cache: 'pip'
-
-    - name: pip install
-      working-directory: ${{ inputs.gmt-directory }}
+
+    - id: python_cache
+      uses: actions/cache@v3
+      with:
+        path: venv
+        key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt') }}
+
+    - name: install script and packages
       shell: bash
+      working-directory: ${{ inputs.gmt-directory }}
       run: |
-        pip install -r requirements-dev.txt
-        pip install -r metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt
-
-    - name: Run Install / Setup scripts
+        ./install_linux.sh -p testpw -a http://api.green-coding.internal:9142 -m http://metrics.green-coding.internal:9142 -n -t
+        source venv/bin/activate
+        python3 -m pip install -r requirements-dev.txt
+        python3 -m pip install -r metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt
+      
+    - name: disable unneeded metric providers and run test setup script
       shell: bash
       working-directory: ${{ inputs.gmt-directory }}
       run: |
-        ./install_linux.sh -p testpw -a http://api.green-coding.internal:9142 -m http://metrics.green-coding.internal:9142 -n -t -w
+        source venv/bin/activate
         python3 disable_metric_providers.py ${{ inputs.metrics-to-turn-off }}
-        cd test && python3 setup-test-env.py --no-docker-build
+        cd tests && python3 setup-test-env.py --no-docker-build
 
     - name: Set up Docker Buildx
       id: buildx
@@ -63,30 +66,33 @@ runs:
 
     - name: Start Test container
       shell: bash
-      working-directory: ${{ inputs.gmt-directory }}/test
+      working-directory: ${{ inputs.gmt-directory }}/tests
       run: |
-        ./start-test-containers.sh -d
+        source ../venv/bin/activate && ./start-test-containers.sh -d
    
     - name: Sleep for 10 seconds
       run: sleep 10s
       shell: bash
-
+
+    # - name: Setup upterm session
+    #   uses: lhotari/action-upterm@v1
+
     - name: Run Tests
       shell: bash
-      working-directory: ${{ inputs.tests-directory }}
+      working-directory: ${{ inputs.gmt-directory }}/tests
       run: |
-        ${{ inputs.tests-command }} -rA | tee /tmp/test-results.txt
+        source ../venv/bin/activate
+        python3 -m ${{ inputs.tests-command }} -rA | tee /tmp/test-results.txt
 
     - name: Display Results
       shell: bash
       if: always()
-      working-directory: ${{ inputs.tests-directory }}
       run: |
         cat /tmp/test-results.txt | grep -oPz '(=*) short test summary(.*\n)*' >> $GITHUB_STEP_SUMMARY
 
     - name: Stop Containers
       shell: bash
       if: always()
-      working-directory: ${{ inputs.gmt-directory }}/test
+      working-directory: ${{ inputs.gmt-directory }}/tests
       run: |
         ./stop-test-containers.sh
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -11,7 +11,12 @@ updates:
     schedule:
       interval: "daily"
   - package-ecosystem: "docker"
-    directory: "/"
+    directory: "/docker/"
+    target-branch: "main"
+    schedule:
+      interval: "weekly"  
+  - package-ecosystem: "docker"
+    directory: "/docker/auxiliary-containers/gcb_playwright/"
     target-branch: "main"
     schedule:
       interval: "weekly"

diff --git a/.github/workflows/build-and-push-containers.yml b/.github/workflows/build-and-push-containers.yml
@@ -0,0 +1,55 @@
+name: Build and Push Containers
+on:
+  pull_request:
+    types:
+      - closed
+    paths:
+      - 'docker/auxiliary-containers/**/Dockerfile'
+
+  workflow_dispatch:
+    inputs:
+      container:
+        type: choice
+        description: The tag to update
+        required: true
+        options:
+          - gcb_playwright
+
+jobs:
+  build-and-push-containers:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+        ## This is needed for multi-architecture builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - if: github.event_name == 'pull_request' && github.event.pull_request.merged == true
+        name: Build and Push auxiliary-containers (PR)
+        shell: bash
+        run: |
+          PR_FILES=$(curl -s "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" | jq -r '.[] | .filename')
+          CHANGED_SUBFOLDERS=$(echo "$PR_FILES" | grep -E '^docker/auxiliary-containers/[^/]+/' | sed -r 's|^docker/auxiliary-containers/([^/]+)/.*|\1|' | sort -u)
+
+          echo $CHANGED_SUBFOLDERS
+          ./docker/auxiliary-containers/build-containers.sh $CHANGED_SUBFOLDERS
+
+      - if: github.event_name == 'workflow_dispatch'
+        name: Build and Push auxiliary-containers (manual)
+        shell: bash
+        run: |
+          ./docker/auxiliary-containers/build-containers.sh ${{ github.event.inputs.container }}
diff --git a/.github/workflows/tests-bare-metal-main.yml b/.github/workflows/tests-bare-metal-main.yml
@@ -22,7 +22,7 @@ jobs:
 
       # - if: ${{ github.event_name == 'workflow_dispatch' || steps.check-date.outputs.should_run == 'true'}}
       - name: 'Checkout repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'main'
           submodules: 'true'

diff --git a/.github/workflows/tests-eco-ci-energy-estimation.yaml b/.github/workflows/tests-eco-ci-energy-estimation.yaml
@@ -11,7 +11,7 @@ jobs:
       contents: read
     steps:
       - name: 'Checkout repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'main'
           submodules: 'true'
@@ -30,7 +30,7 @@ jobs:
         name: 'Setup, Run, and Teardown Tests'
         uses: ./.github/actions/gmt-pytest
         with:
-         metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS --providers CpuFrequencySysfsCoreProvider'
+         metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS'
          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Eco CI Energy Estimation - Get Measurement

diff --git a/.github/workflows/tests-vm-main.yml b/.github/workflows/tests-vm-main.yml
@@ -20,7 +20,7 @@ jobs:
 
       - if: ${{ github.event_name == 'workflow_dispatch' || steps.check-date.outputs.should_run == 'true'}}
         name: 'Checkout repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'main'
           submodules: 'true'
@@ -34,7 +34,7 @@ jobs:
         name: 'Setup, Run, and Teardown Tests'
         uses: ./.github/actions/gmt-pytest
         with:
-         metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS --providers CpuFrequencySysfsCoreProvider'
+         metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS'
          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Eco CI Energy Estimation - Get Measurement

diff --git a/.github/workflows/tests-vm-pr.yml b/.github/workflows/tests-vm-pr.yml
@@ -12,7 +12,7 @@ jobs:
       pull-requests: write
     steps:
       - name: 'Checkout repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.ref }}
           submodules: 'true'
@@ -25,7 +25,7 @@ jobs:
       - name: 'Setup, Run, and Teardown Tests'
         uses: ./.github/actions/gmt-pytest
         with:
-         metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS --providers CpuFrequencySysfsCoreProvider'
+         metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS'
          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Eco CI Energy Estimation - Get Measurement

diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,6 @@ static-binary
 .pytest_cache
 test-compose.yml
 test-config.yml
-test/structure.sql
+tests/structure.sql
 tools/sgx_enable
 venv/
diff --git a/.pylintrc b/.pylintrc
@@ -25,9 +25,15 @@ disable=missing-function-docstring,
     too-many-branches,
     too-many-statements,
     too-many-arguments,
+    too-many-return-statements,
+    too-many-instance-attributes,
+    invalid-name,
+    wrong-import-position,
+    wrong-import-order,
+    ungrouped-imports,
+    fixme
 
 
-#    import-error
 
 [MASTER]
 ignore=env

diff --git a/README.md b/README.md
@@ -5,7 +5,18 @@
 
 # Introduction
 
-The Green Metrics Tool is a developer tool is indented for measuring the energy consumption of software and doing life-cycle-analysis.
+The Green Metrics Tool is a developer tool indented for measuring the energy and CO2 consumption of software through a software life cycle analysis (SLCA).
+
+Key features are:
+- Reproducible measurements through configuration/setup-as-code
+- [POSIX style metric providers](https://docs.green-coding.berlin/docs/measuring/metric-providers/metric-providers-overview/) for many sensors (RAPL, IPMI, PSU, Docker, Temperature, CPU ...)
+- [Low overhead](https://docs.green-coding.berlin/docs/measuring/metric-providers/overhead-of-measurement-providers/)
+- Statististical frontend with charts - [DEMO](https://metrics.green-coding.berlin/stats.html?id=7169e39e-6938-4636-907b-68aa421994b2)
+- API - [DEMO](https://api.green-coding.berlin)
+- [Cluster setup](https://docs.green-coding.berlin/docs/installation/installation-cluster/)
+- [Free Hosted service for more precise measurements](https://docs.green-coding.berlin/docs/measuring/measurement-cluster/)
+- Timeline-View: Monitor software projects over time - [DEMO for Wagtail](https://metrics.green-coding.berlin/timeline.html?uri=https://github.com/green-coding-berlin/bakerydemo-gold-benchmark&filename=usage_scenario_warm.yml&branch=&machine_id=7) / [DEMO Overview](https://metrics.green-coding.berlin/energy-timeline.html)
+- [Energy-ID Score-Cards](https://www.green-coding.berlin/projects/energy-id/) for software (Also see below)
 
 It is designed to re-use existing infrastructure and testing files as much as possible to be easily integrateable into every software repository and create transparency around software energy consumption.
 
@@ -14,28 +25,32 @@ It can orchestrate Docker containers according to a given specificaion in a `usa
 These containers will be setup on the host system and the testing specification in the `usage_scenario.yml` will be
 run by sending the commands to the containers accordingly.
 
-During this process the performance metrics of the containers are read through different metric providers like:
-- CPU / DRAM energy (RAPL)
-- System energy (IMPI / PowerSpy2 / Machine-Learning-Model / SDIA Model)
-- container CPU utilization
-- container memory utilization
-- etc.
-
 This repository contains the command line tools to schedule and run the measurement report
 as well as a web interface to view the measured metrics in some nice charts.
 
 # Frontend
 To see the frontend in action and get an idea of what kind of metrics the tool can collect and display go to out [Green Metrics Frontend](https://metrics.green-coding.berlin)
 
-
 # Documentation
 
 To see the the documentation and how to install and use the tool please go to [Green Metrics Tool Documentation](https://docs.green-coding.berlin)
 
-# Screenshots
+# Screenshots of Single Run View
 
-![Web Flow Demo with CPU measurement provider](https://www.green-coding.berlin/img/projects/gmt-screenshot-1.webp "Web Charts demo with docker stats provider instead of energy")
-> Web Flow Demo with CPU measurement provider
+![](https://www.green-coding.berlin/img/projects/gmt-screenshot-1.webp)
+![](https://www.green-coding.berlin/img/projects/gmt-screenshot-2.webp)
+![](https://www.green-coding.berlin/img/projects/gmt-screenshot-3.webp)
+![](https://www.green-coding.berlin/img/projects/gmt-screenshot-4.webp)
 
-![Web Flow Demo with energy measurement provider](https://www.green-coding.berlin/img/projects/gmt-screenshot-2.webp "Web Charts demo with docker stats provider instead of energy")
-> Web Flow Demo with energy measurement provider
+
+# Screenshots of Comparison View
+![](https://www.green-coding.berlin/img/projects/gmt-screenshot-5.webp)
+![](https://www.green-coding.berlin/img/projects/gmt-screenshot-6.webp)
+
+# Energy-ID Scorecards
+<img width="1034" alt="Screenshot 2023-10-24 at 10 43 28 AM" src="https://github.com/green-coding-berlin/green-metrics-tool/assets/250671/7e3e3faa-5452-4722-af70-a65114f930ac">
+
+Details: [Energy-ID project page](https://www.green-coding.berlin/projects/energy-id/
+)
+
+