diff --git a/.github/actions/gmt-pytest/action.yml b/.github/actions/gmt-pytest/action.yml index da151b8ca..102f1d1f6 100644 --- a/.github/actions/gmt-pytest/action.yml +++ b/.github/actions/gmt-pytest/action.yml @@ -9,10 +9,6 @@ inputs: description: 'The root directory of the gmt repository' required: false default: '.' - tests-directory: - description: 'The directory where to run the tests from' - required: false - default: './test' tests-command: description: 'The command to run the tests' required: false @@ -24,26 +20,33 @@ inputs: runs: using: 'composite' steps: - - name: setup python + - name: setup_python uses: actions/setup-python@v4 with: python-version: '3.10' - cache: 'pip' - - - name: pip install - working-directory: ${{ inputs.gmt-directory }} + + - id: python_cache + uses: actions/cache@v3 + with: + path: venv + key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt') }} + + - name: install script and packages shell: bash + working-directory: ${{ inputs.gmt-directory }} run: | - pip install -r requirements-dev.txt - pip install -r metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt - - - name: Run Install / Setup scripts + ./install_linux.sh -p testpw -a http://api.green-coding.internal:9142 -m http://metrics.green-coding.internal:9142 -n -t + source venv/bin/activate + python3 -m pip install -r requirements-dev.txt + python3 -m pip install -r metric_providers/psu/energy/ac/xgboost/machine/model/requirements.txt + + - name: disable unneeded metric providers and run test setup script shell: bash working-directory: ${{ inputs.gmt-directory }} run: | - ./install_linux.sh -p testpw -a http://api.green-coding.internal:9142 -m http://metrics.green-coding.internal:9142 -n -t -w + source venv/bin/activate python3 disable_metric_providers.py ${{ inputs.metrics-to-turn-off }} - cd test && python3 setup-test-env.py --no-docker-build + cd tests && python3 setup-test-env.py --no-docker-build - name: Set up Docker Buildx id: buildx @@ -63,30 +66,33 @@ runs: - name: Start Test container shell: bash - working-directory: ${{ inputs.gmt-directory }}/test + working-directory: ${{ inputs.gmt-directory }}/tests run: | - ./start-test-containers.sh -d + source ../venv/bin/activate && ./start-test-containers.sh -d - name: Sleep for 10 seconds run: sleep 10s shell: bash - + + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 + - name: Run Tests shell: bash - working-directory: ${{ inputs.tests-directory }} + working-directory: ${{ inputs.gmt-directory }}/tests run: | - ${{ inputs.tests-command }} -rA | tee /tmp/test-results.txt + source ../venv/bin/activate + python3 -m ${{ inputs.tests-command }} -rA | tee /tmp/test-results.txt - name: Display Results shell: bash if: always() - working-directory: ${{ inputs.tests-directory }} run: | cat /tmp/test-results.txt | grep -oPz '(=*) short test summary(.*\n)*' >> $GITHUB_STEP_SUMMARY - name: Stop Containers shell: bash if: always() - working-directory: ${{ inputs.gmt-directory }}/test + working-directory: ${{ inputs.gmt-directory }}/tests run: | ./stop-test-containers.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b25c4c4a3..b07b40b7c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,7 +11,12 @@ updates: schedule: interval: "daily" - package-ecosystem: "docker" - directory: "/" + directory: "/docker/" + target-branch: "main" + schedule: + interval: "weekly" + - package-ecosystem: "docker" + directory: "/docker/auxiliary-containers/gcb_playwright/" target-branch: "main" schedule: interval: "weekly" diff --git a/.github/workflows/build-and-push-containers.yml b/.github/workflows/build-and-push-containers.yml new file mode 100644 index 000000000..4911859ff --- /dev/null +++ b/.github/workflows/build-and-push-containers.yml @@ -0,0 +1,36 @@ +name: Build and Push Containers +on: + pull_request: + types: + - closed + paths: + - 'docker/auxiliary-containers/**/Dockerfile' + + workflow_dispatch: + +jobs: + build-and-push-containers: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + ## This is needed for multi-architecture builds + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push Containers + run: bash ./docker/auxiliary-containers/build-containers.sh \ No newline at end of file diff --git a/.github/workflows/tests-bare-metal-main.yml b/.github/workflows/tests-bare-metal-main.yml index 7a72cfb2c..0fbaa43a7 100644 --- a/.github/workflows/tests-bare-metal-main.yml +++ b/.github/workflows/tests-bare-metal-main.yml @@ -22,7 +22,7 @@ jobs: # - if: ${{ github.event_name == 'workflow_dispatch' || steps.check-date.outputs.should_run == 'true'}} - name: 'Checkout repository' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: 'main' submodules: 'true' diff --git a/.github/workflows/tests-eco-ci-energy-estimation.yaml b/.github/workflows/tests-eco-ci-energy-estimation.yaml index e1a184d1c..fecf400f8 100644 --- a/.github/workflows/tests-eco-ci-energy-estimation.yaml +++ b/.github/workflows/tests-eco-ci-energy-estimation.yaml @@ -11,7 +11,7 @@ jobs: contents: read steps: - name: 'Checkout repository' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: 'main' submodules: 'true' @@ -30,7 +30,7 @@ jobs: name: 'Setup, Run, and Teardown Tests' uses: ./.github/actions/gmt-pytest with: - metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS --providers CpuFrequencySysfsCoreProvider' + metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS' github-token: ${{ secrets.GITHUB_TOKEN }} - name: Eco CI Energy Estimation - Get Measurement diff --git a/.github/workflows/tests-vm-main.yml b/.github/workflows/tests-vm-main.yml index bc46f217f..b1698912e 100644 --- a/.github/workflows/tests-vm-main.yml +++ b/.github/workflows/tests-vm-main.yml @@ -20,13 +20,13 @@ jobs: - if: ${{ github.event_name == 'workflow_dispatch' || steps.check-date.outputs.should_run == 'true'}} name: 'Checkout repository' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: 'main' submodules: 'true' - name: Eco CI Energy Estimation - Initialize - uses: green-coding-berlin/eco-ci-energy-estimation@701b5f2f4ba601be587823cd0786f07cb6ae2ee6 + uses: green-coding-berlin/eco-ci-energy-estimation@v2 with: task: start-measurement @@ -34,17 +34,17 @@ jobs: name: 'Setup, Run, and Teardown Tests' uses: ./.github/actions/gmt-pytest with: - metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS --providers CpuFrequencySysfsCoreProvider' + metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS' github-token: ${{ secrets.GITHUB_TOKEN }} - name: Eco CI Energy Estimation - Get Measurement - uses: green-coding-berlin/eco-ci-energy-estimation@701b5f2f4ba601be587823cd0786f07cb6ae2ee6 + uses: green-coding-berlin/eco-ci-energy-estimation@v2 with: task: get-measurement branch: main - name: Eco CI Energy Estimation - End Measurement - uses: green-coding-berlin/eco-ci-energy-estimation@701b5f2f4ba601be587823cd0786f07cb6ae2ee6 + uses: green-coding-berlin/eco-ci-energy-estimation@v2 with: task: display-results branch: main diff --git a/.github/workflows/tests-vm-pr.yml b/.github/workflows/tests-vm-pr.yml index f16f01261..ab79032d7 100644 --- a/.github/workflows/tests-vm-pr.yml +++ b/.github/workflows/tests-vm-pr.yml @@ -12,7 +12,7 @@ jobs: pull-requests: write steps: - name: 'Checkout repository' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} submodules: 'true' @@ -25,7 +25,7 @@ jobs: - name: 'Setup, Run, and Teardown Tests' uses: ./.github/actions/gmt-pytest with: - metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS --providers CpuFrequencySysfsCoreProvider' + metrics-to-turn-off: '--categories RAPL Machine Sensors Debug CGroupV2 MacOS' github-token: ${{ secrets.GITHUB_TOKEN }} - name: Eco CI Energy Estimation - Get Measurement diff --git a/.gitignore b/.gitignore index c43ed7604..86615b0fe 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,6 @@ static-binary .pytest_cache test-compose.yml test-config.yml -test/structure.sql +tests/structure.sql tools/sgx_enable venv/ diff --git a/.pylintrc b/.pylintrc index 686de9f34..772c57041 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,9 +25,15 @@ disable=missing-function-docstring, too-many-branches, too-many-statements, too-many-arguments, + too-many-return-statements, + too-many-instance-attributes, + invalid-name, + wrong-import-position, + wrong-import-order, + ungrouped-imports, + fixme -# import-error [MASTER] ignore=env diff --git a/README.md b/README.md index 811f18779..f2e018e61 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ The Green Metrics Tool is a developer tool is indented for measuring the energy consumption of software and doing life-cycle-analysis. +One of it's key features is the detailed statistical dashboard and the measurement cluster for making reproducible measurements. + It is designed to re-use existing infrastructure and testing files as much as possible to be easily integrateable into every software repository and create transparency around software energy consumption. It can orchestrate Docker containers according to a given specificaion in a `usage_scenario.yml` file. @@ -27,15 +29,18 @@ as well as a web interface to view the measured metrics in some nice charts. # Frontend To see the frontend in action and get an idea of what kind of metrics the tool can collect and display go to out [Green Metrics Frontend](https://metrics.green-coding.berlin) - # Documentation To see the the documentation and how to install and use the tool please go to [Green Metrics Tool Documentation](https://docs.green-coding.berlin) -# Screenshots +# Screenshots of Single Run View -![Web Flow Demo with CPU measurement provider](https://www.green-coding.berlin/img/projects/gmt-screenshot-1.webp "Web Charts demo with docker stats provider instead of energy") -> Web Flow Demo with CPU measurement provider +![](https://www.green-coding.berlin/img/projects/gmt-screenshot-1.webp) +![](https://www.green-coding.berlin/img/projects/gmt-screenshot-2.webp) +![](https://www.green-coding.berlin/img/projects/gmt-screenshot-3.webp) +![](https://www.green-coding.berlin/img/projects/gmt-screenshot-4.webp) -![Web Flow Demo with energy measurement provider](https://www.green-coding.berlin/img/projects/gmt-screenshot-2.webp "Web Charts demo with docker stats provider instead of energy") -> Web Flow Demo with energy measurement provider + +# Screenshots of Comparison View +![](https://www.green-coding.berlin/img/projects/gmt-screenshot-5.webp) +![](https://www.green-coding.berlin/img/projects/gmt-screenshot-6.webp) diff --git a/api/api.py b/api/api.py deleted file mode 100644 index 105c03b2f..000000000 --- a/api/api.py +++ /dev/null @@ -1,596 +0,0 @@ - -# pylint: disable=import-error -# pylint: disable=no-name-in-module -# pylint: disable=wrong-import-position - -import faulthandler -import sys -import os - -from xml.sax.saxutils import escape as xml_escape -from fastapi import FastAPI, Request, Response, status -from fastapi.responses import ORJSONResponse -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import RequestValidationError -from fastapi.middleware.cors import CORSMiddleware - -from starlette.responses import RedirectResponse -from pydantic import BaseModel - -sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../lib') -sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../tools') - -from global_config import GlobalConfig -from db import DB -import jobs -import email_helpers -import error_helpers -import anybadge -from api_helpers import (add_phase_stats_statistics, determine_comparison_case, - html_escape_multi, get_phase_stats, get_phase_stats_object, - is_valid_uuid, rescale_energy_value, get_timeline_query, - get_project_info, get_machine_list) - -# It seems like FastAPI already enables faulthandler as it shows stacktrace on SEGFAULT -# Is the redundant call problematic -faulthandler.enable() # will catch segfaults and write to STDERR - -app = FastAPI() - -async def log_exception(request: Request, body, exc): - error_message = f""" - Error in API call - - URL: {request.url} - - Query-Params: {request.query_params} - - Client: {request.client} - - Headers: {str(request.headers)} - - Body: {body} - - Exception: {exc} - """ - error_helpers.log_error(error_message) - email_helpers.send_error_email( - GlobalConfig().config['admin']['email'], - error_helpers.format_error(error_message), - project_id=None, - ) - -@app.exception_handler(RequestValidationError) -async def validation_exception_handler(request: Request, exc: RequestValidationError): - await log_exception(request, exc.body, exc) - return ORJSONResponse( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}), - ) - -async def catch_exceptions_middleware(request: Request, call_next): - #pylint: disable=broad-except - try: - return await call_next(request) - except Exception as exc: - # body = await request.body() # This blocks the application. Unclear atm how to handle it properly - # seems like a bug: https://github.com/tiangolo/fastapi/issues/394 - # Although the issue is closed the "solution" still behaves with same failure - await log_exception(request, None, exc) - return ORJSONResponse( - content={ - 'success': False, - 'err': 'Technical error with getting data from the API - Please contact us: info@green-coding.berlin', - }, - status_code=500, - ) - - -# Binding the Exception middleware must confusingly come BEFORE the CORS middleware. -# Otherwise CORS will not be sent in response -app.middleware('http')(catch_exceptions_middleware) - -origins = [ - GlobalConfig().config['cluster']['metrics_url'], - GlobalConfig().config['cluster']['api_url'], -] - -app.add_middleware( - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=['*'], - allow_headers=['*'], -) - - -@app.get('/') -async def home(): - return RedirectResponse(url='/docs') - - -# A route to return all of the available entries in our catalog. -@app.get('/v1/notes/{project_id}') -async def get_notes(project_id): - if project_id is None or not is_valid_uuid(project_id): - return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400) - - query = """ - SELECT project_id, detail_name, note, time - FROM notes - WHERE project_id = %s - ORDER BY created_at DESC -- important to order here, the charting library in JS cannot do that automatically! - """ - data = DB().fetch_all(query, (project_id,)) - if data is None or data == []: - return Response(status_code=204) # No-Content - - escaped_data = [html_escape_multi(note) for note in data] - return ORJSONResponse({'success': True, 'data': escaped_data}) - -# return a list of all possible registered machines -@app.get('/v1/machines/') -async def get_machines(): - - data = get_machine_list() - if data is None or data == []: - return Response(status_code=204) # No-Content - - return ORJSONResponse({'success': True, 'data': data}) - - -# A route to return all of the available entries in our catalog. -@app.get('/v1/projects') -async def get_projects(repo: str, filename: str): - query = """ - SELECT a.id, a.name, a.uri, COALESCE(a.branch, 'main / master'), a.end_measurement, a.last_run, a.invalid_project, a.filename, b.description, a.commit_hash - FROM projects as a - LEFT JOIN machines as b on a.machine_id = b.id - WHERE 1=1 - """ - params = [] - - filename = filename.strip() - if filename not in ('', 'null'): - query = f"{query} AND a.filename LIKE %s \n" - params.append(f"%{filename}%") - - repo = repo.strip() - if repo not in ('', 'null'): - query = f"{query} AND a.uri LIKE %s \n" - params.append(f"%{repo}%") - - query = f"{query} ORDER BY a.created_at DESC -- important to order here, the charting library in JS cannot do that automatically!" - - data = DB().fetch_all(query, params=tuple(params)) - if data is None or data == []: - return Response(status_code=204) # No-Content - - escaped_data = [html_escape_multi(project) for project in data] - - return ORJSONResponse({'success': True, 'data': escaped_data}) - - -# Just copy and paste if we want to deprecate URLs -# @app.get('/v1/measurements/uri', deprecated=True) # Here you can see, that URL is nevertheless accessible as variable -# later if supplied. Also deprecation shall be used once we move to v2 for all v1 routesthrough - -@app.get('/v1/compare') -async def compare_in_repo(ids: str): - if ids is None or not ids.strip(): - return ORJSONResponse({'success': False, 'err': 'Project_id is empty'}, status_code=400) - ids = ids.split(',') - if not all(is_valid_uuid(id) for id in ids): - return ORJSONResponse({'success': False, 'err': 'One of Project IDs is not a valid UUID or empty'}, status_code=400) - - try: - case = determine_comparison_case(ids) - except RuntimeError as err: - return ORJSONResponse({'success': False, 'err': str(err)}, status_code=400) - try: - phase_stats = get_phase_stats(ids) - except RuntimeError: - return Response(status_code=204) # No-Content - try: - phase_stats_object = get_phase_stats_object(phase_stats, case) - phase_stats_object = add_phase_stats_statistics(phase_stats_object) - phase_stats_object['common_info'] = {} - - project_info = get_project_info(ids[0]) - - machine_list = get_machine_list() - machines = {machine[0]: machine[1] for machine in machine_list} - - machine = machines[project_info['machine_id']] - uri = project_info['uri'] - usage_scenario = project_info['usage_scenario']['name'] - branch = project_info['branch'] if project_info['branch'] is not None else 'main / master' - commit = project_info['commit_hash'] - filename = project_info['filename'] - - match case: - case 'Repeated Run': - # same repo, same usage scenarios, same machines, same branches, same commit hashes - phase_stats_object['common_info']['Repository'] = uri - phase_stats_object['common_info']['Filename'] = filename - phase_stats_object['common_info']['Usage Scenario'] = usage_scenario - phase_stats_object['common_info']['Machine'] = machine - phase_stats_object['common_info']['Branch'] = branch - phase_stats_object['common_info']['Commit'] = commit - case 'Usage Scenario': - # same repo, diff usage scenarios, same machines, same branches, same commit hashes - phase_stats_object['common_info']['Repository'] = uri - phase_stats_object['common_info']['Machine'] = machine - phase_stats_object['common_info']['Branch'] = branch - phase_stats_object['common_info']['Commit'] = commit - case 'Machine': - # same repo, same usage scenarios, diff machines, same branches, same commit hashes - phase_stats_object['common_info']['Repository'] = uri - phase_stats_object['common_info']['Filename'] = filename - phase_stats_object['common_info']['Usage Scenario'] = usage_scenario - phase_stats_object['common_info']['Branch'] = branch - phase_stats_object['common_info']['Commit'] = commit - case 'Commit': - # same repo, same usage scenarios, same machines, diff commit hashes - phase_stats_object['common_info']['Repository'] = uri - phase_stats_object['common_info']['Filename'] = filename - phase_stats_object['common_info']['Usage Scenario'] = usage_scenario - phase_stats_object['common_info']['Machine'] = machine - case 'Repository': - # diff repo, diff usage scenarios, same machine, same branches, diff/same commits_hashes - phase_stats_object['common_info']['Machine'] = machine - phase_stats_object['common_info']['Branch'] = branch - case 'Branch': - # same repo, same usage scenarios, same machines, diff branch - phase_stats_object['common_info']['Repository'] = uri - phase_stats_object['common_info']['Filename'] = filename - phase_stats_object['common_info']['Usage Scenario'] = usage_scenario - phase_stats_object['common_info']['Machine'] = machine - - except RuntimeError as err: - return ORJSONResponse({'success': False, 'err': str(err)}, status_code=500) - - return ORJSONResponse({'success': True, 'data': phase_stats_object}) - - -@app.get('/v1/phase_stats/single/{project_id}') -async def get_phase_stats_single(project_id: str): - if project_id is None or not is_valid_uuid(project_id): - return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400) - - try: - phase_stats = get_phase_stats([project_id]) - phase_stats_object = get_phase_stats_object(phase_stats, None) - phase_stats_object = add_phase_stats_statistics(phase_stats_object) - - except RuntimeError: - return Response(status_code=204) # No-Content - - return ORJSONResponse({'success': True, 'data': phase_stats_object}) - - -# This route gets the measurements to be displayed in a timeline chart -@app.get('/v1/measurements/single/{project_id}') -async def get_measurements_single(project_id: str): - if project_id is None or not is_valid_uuid(project_id): - return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400) - - query = """ - SELECT measurements.detail_name, measurements.time, measurements.metric, - measurements.value, measurements.unit - FROM measurements - WHERE measurements.project_id = %s - """ - - # extremely important to order here, cause the charting library in JS cannot do that automatically! - - query = f" {query} ORDER BY measurements.metric ASC, measurements.detail_name ASC, measurements.time ASC" - - params = params = (project_id, ) - - data = DB().fetch_all(query, params=params) - - if data is None or data == []: - return Response(status_code=204) # No-Content - - return ORJSONResponse({'success': True, 'data': data}) - -@app.get('/v1/timeline') -async def get_timeline_stats(uri: str, machine_id: int, branch: str | None = None, filename: str | None = None, start_date: str | None = None, end_date: str | None = None, metrics: str | None = None, phase: str | None = None, sorting: str | None = None,): - if uri is None or uri.strip() == '': - return ORJSONResponse({'success': False, 'err': 'URI is empty'}, status_code=400) - - query, params = get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_date=start_date, end_date=end_date, sorting=sorting) - - data = DB().fetch_all(query, params=params) - - if data is None or data == []: - return Response(status_code=204) # No-Content - - return ORJSONResponse({'success': True, 'data': data}) - -@app.get('/v1/badge/timeline') -async def get_timeline_badge(detail_name: str, uri: str, machine_id: int, branch: str | None = None, filename: str | None = None, metrics: str | None = None, phase: str | None = None): - if uri is None or uri.strip() == '': - return ORJSONResponse({'success': False, 'err': 'URI is empty'}, status_code=400) - - if detail_name is None or detail_name.strip() == '': - return ORJSONResponse({'success': False, 'err': 'Detail Name is mandatory'}, status_code=400) - - query, params = get_timeline_query(uri,filename,machine_id, branch, metrics, phase, detail_name=detail_name, limit_365=True) - - query = f""" - WITH trend_data AS ( - {query} - ) SELECT - MAX(row_num::float), - regr_slope(value, row_num::float) AS trend_slope, - regr_intercept(value, row_num::float) AS trend_intercept, - MAX(unit) - FROM trend_data; - """ - - data = DB().fetch_one(query, params=params) - - if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result - return Response(status_code=204) # No-Content - - cost = data[1]/data[0] - cost = f"+{round(float(cost), 2)}" if abs(cost) == cost else f"{round(float(cost), 2)}" - - badge = anybadge.Badge( - label=xml_escape('Project Trend'), - value=xml_escape(f"{cost} {data[3]} per day"), - num_value_padding_chars=1, - default_color='orange') - return Response(content=str(badge), media_type="image/svg+xml") - - -# A route to return all of the available entries in our catalog. -@app.get('/v1/badge/single/{project_id}') -async def get_badge_single(project_id: str, metric: str = 'ml-estimated'): - - if project_id is None or not is_valid_uuid(project_id): - return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400) - - query = ''' - SELECT - SUM(value), MAX(unit) - FROM - phase_stats - WHERE - project_id = %s - AND metric LIKE %s - AND phase LIKE '%%_[RUNTIME]' - ''' - - value = None - label = 'Energy Cost' - via = '' - if metric == 'ml-estimated': - value = 'psu_energy_ac_xgboost_machine' - via = 'via XGBoost ML' - elif metric == 'RAPL': - value = '%_energy_rapl_%' - via = 'via RAPL' - elif metric == 'AC': - value = 'psu_energy_ac_%' - via = 'via PSU (AC)' - elif metric == 'SCI': - label = 'SCI' - value = 'software_carbon_intensity_global' - else: - return ORJSONResponse({'success': False, 'err': f"Unknown metric '{metric}' submitted"}, status_code=400) - - params = (project_id, value) - data = DB().fetch_one(query, params=params) - - if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result - badge_value = 'No energy data yet' - else: - [energy_value, energy_unit] = rescale_energy_value(data[0], data[1]) - badge_value= f"{energy_value:.2f} {energy_unit} {via}" - - badge = anybadge.Badge( - label=xml_escape(label), - value=xml_escape(badge_value), - num_value_padding_chars=1, - default_color='cornflowerblue') - return Response(content=str(badge), media_type="image/svg+xml") - - -class Project(BaseModel): - name: str - url: str - email: str - filename: str - branch: str - machine_id: int - -@app.post('/v1/project/add') -async def post_project_add(project: Project): - if project.url is None or project.url.strip() == '': - return ORJSONResponse({'success': False, 'err': 'URL is empty'}, status_code=400) - - if project.name is None or project.name.strip() == '': - return ORJSONResponse({'success': False, 'err': 'Name is empty'}, status_code=400) - - if project.email is None or project.email.strip() == '': - return ORJSONResponse({'success': False, 'err': 'E-mail is empty'}, status_code=400) - - if project.branch.strip() == '': - project.branch = None - - if project.filename.strip() == '': - project.filename = 'usage_scenario.yml' - - if project.machine_id == 0: - project.machine_id = None - project = html_escape_multi(project) - - # Note that we use uri here as the general identifier, however when adding through web interface we only allow urls - query = """ - INSERT INTO projects (uri,name,email,branch,filename) - VALUES (%s, %s, %s, %s, %s) - RETURNING id - """ - params = (project.url, project.name, project.email, project.branch, project.filename) - project_id = DB().fetch_one(query, params=params)[0] - - # This order as selected on purpose. If the admin mail fails, we currently do - # not want the job to be queued, as we want to monitor every project execution manually - config = GlobalConfig().config - if (config['admin']['notify_admin_for_own_project_add'] or config['admin']['email'] != project.email): - email_helpers.send_admin_email( - f"New project added from Web Interface: {project.name}", project - ) # notify admin of new project - - jobs.insert_job('project', project_id, project.machine_id) - - return ORJSONResponse({'success': True}, status_code=202) - - -@app.get('/v1/project/{project_id}') -async def get_project(project_id: str): - if project_id is None or not is_valid_uuid(project_id): - return ORJSONResponse({'success': False, 'err': 'Project ID is not a valid UUID or empty'}, status_code=400) - - data = get_project_info(project_id) - - if data is None or data == []: - return Response(status_code=204) # No-Content - - data = html_escape_multi(data) - - return ORJSONResponse({'success': True, 'data': data}) - -@app.get('/robots.txt') -async def robots_txt(): - data = "User-agent: *\n" - data += "Disallow: /" - - return Response(content=data, media_type='text/plain') - -# pylint: disable=invalid-name -class CI_Measurement(BaseModel): - energy_value: int - energy_unit: str - repo: str - branch: str - cpu: str - cpu_util_avg: float - commit_hash: str - workflow: str - run_id: str - project_id: str - source: str - label: str - duration: int - -@app.post('/v1/ci/measurement/add') -async def post_ci_measurement_add(measurement: CI_Measurement): - for key, value in measurement.model_dump().items(): - match key: - case 'project_id': - if value is None or value.strip() == '': - measurement.project_id = None - continue - if not is_valid_uuid(value.strip()): - return ORJSONResponse({'success': False, 'err': f"project_id '{value}' is not a valid uuid"}, status_code=400) - continue - - case 'unit': - if value is None or value.strip() == '': - return ORJSONResponse({'success': False, 'err': f"{key} is empty"}, status_code=400) - if value != 'mJ': - return ORJSONResponse({'success': False, 'err': "Unit is unsupported - only mJ currently accepted"}, status_code=400) - continue - - case 'label': # Optional fields - continue - - case _: - if value is None: - return ORJSONResponse({'success': False, 'err': f"{key} is empty"}, status_code=400) - if isinstance(value, str): - if value.strip() == '': - return ORJSONResponse({'success': False, 'err': f"{key} is empty"}, status_code=400) - - measurement = html_escape_multi(measurement) - - query = """ - INSERT INTO - ci_measurements (energy_value, energy_unit, repo, branch, workflow, run_id, project_id, label, source, cpu, commit_hash, duration, cpu_util_avg) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) - """ - params = (measurement.energy_value, measurement.energy_unit, measurement.repo, measurement.branch, - measurement.workflow, measurement.run_id, measurement.project_id, - measurement.label, measurement.source, measurement.cpu, measurement.commit_hash, - measurement.duration, measurement.cpu_util_avg) - - DB().query(query=query, params=params) - return ORJSONResponse({'success': True}, status_code=201) - -@app.get('/v1/ci/measurements') -async def get_ci_measurements(repo: str, branch: str, workflow: str): - query = """ - SELECT energy_value, energy_unit, run_id, created_at, label, cpu, commit_hash, duration, source, cpu_util_avg - FROM ci_measurements - WHERE repo = %s AND branch = %s AND workflow = %s - ORDER BY run_id ASC, created_at ASC - """ - params = (repo, branch, workflow) - data = DB().fetch_all(query, params=params) - if data is None or data == []: - return Response(status_code=204) # No-Content - - return ORJSONResponse({'success': True, 'data': data}) - -@app.get('/v1/ci/projects') -async def get_ci_projects(): - query = """ - SELECT repo, branch, workflow, source, MAX(created_at) - FROM ci_measurements - GROUP BY repo, branch, workflow, source - ORDER BY repo ASC - """ - - data = DB().fetch_all(query) - if data is None or data == []: - return Response(status_code=204) # No-Content - - return ORJSONResponse({'success': True, 'data': data}) - -@app.get('/v1/ci/badge/get') -async def get_ci_badge_get(repo: str, branch: str, workflow:str): - query = """ - SELECT SUM(energy_value), MAX(energy_unit), MAX(run_id) - FROM ci_measurements - WHERE repo = %s AND branch = %s AND workflow = %s - GROUP BY run_id - ORDER BY MAX(created_at) DESC - LIMIT 1 - """ - - params = (repo, branch, workflow) - data = DB().fetch_one(query, params=params) - - if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result - return Response(status_code=204) # No-Content - - energy_value = data[0] - energy_unit = data[1] - - [energy_value, energy_unit] = rescale_energy_value(energy_value, energy_unit) - badge_value= f"{energy_value:.2f} {energy_unit}" - - badge = anybadge.Badge( - label='Energy Used', - value=xml_escape(badge_value), - num_value_padding_chars=1, - default_color='green') - return Response(content=str(badge), media_type="image/svg+xml") - - -if __name__ == '__main__': - app.run() diff --git a/api/api_helpers.py b/api/api_helpers.py index 4284e921f..4a64d4ce4 100644 --- a/api/api_helpers.py +++ b/api/api_helpers.py @@ -1,23 +1,17 @@ -#pylint: disable=fixme, import-error, wrong-import-position - -import sys -import os import uuid import faulthandler from functools import cache from html import escape as html_escape -import psycopg import numpy as np import scipy.stats -# pylint: disable=no-name-in-module + +from psycopg.rows import dict_row as psycopg_rows_dict_row + from pydantic import BaseModel faulthandler.enable() # will catch segfaults and write to STDERR -sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../lib') -sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../tools') - -from db import DB +from lib.db import DB def rescale_energy_value(value, unit): # We only expect values to be mJ for energy! @@ -30,7 +24,6 @@ def rescale_energy_value(value, unit): value = value / (10**3) unit = f"m{unit_type}" - # pylint: disable=multiple-statements if value > 1_000_000_000: return [value/(10**12), f"G{unit_type}"] if value > 1_000_000_000: return [value/(10**9), f"M{unit_type}"] if value > 1_000_000: return [value/(10**6), f"k{unit_type}"] @@ -94,20 +87,20 @@ def get_machine_list(): """ return DB().fetch_all(query) -def get_project_info(project_id): +def get_run_info(run_id): query = """ SELECT id, name, uri, branch, commit_hash, - (SELECT STRING_AGG(t.name, ', ' ) FROM unnest(projects.categories) as elements + (SELECT STRING_AGG(t.name, ', ' ) FROM unnest(runs.categories) as elements LEFT JOIN categories as t on t.id = elements) as categories, filename, start_measurement, end_measurement, measurement_config, machine_specs, machine_id, usage_scenario, - last_run, created_at, invalid_project, phases, logs - FROM projects + created_at, invalid_run, phases, logs + FROM runs WHERE id = %s """ - params = (project_id,) - return DB().fetch_one(query, params=params, row_factory=psycopg.rows.dict_row) + params = (run_id,) + return DB().fetch_one(query, params=params, row_factory=psycopg_rows_dict_row) def get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_date=None, end_date=None, detail_name=None, limit_365=False, sorting='run'): @@ -115,84 +108,76 @@ def get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_da if filename is None or filename.strip() == '': filename = 'usage_scenario.yml' - params = [uri, filename, machine_id] + params = [uri, filename, machine_id, f"%{phase}"] - branch_condition = '' + branch_condition = 'AND r.branch IS NULL' if branch is not None and branch.strip() != '': - branch_condition = 'AND projects.branch = %s' + branch_condition = 'AND r.branch = %s' params.append(branch) metrics_condition = '' if metrics is None or metrics.strip() == '' or metrics.strip() == 'key': - metrics_condition = "AND (metric LIKE '%%_energy_%%' OR metric = 'software_carbon_intensity_global')" + metrics_condition = "AND (p.metric LIKE '%%_energy_%%' OR metric = 'software_carbon_intensity_global')" elif metrics.strip() != 'all': - metrics_condition = "AND metric = %s" + metrics_condition = "AND p.metric = %s" params.append(metrics) - phase_condition = '' - if phase is not None and phase.strip() != '': - phase_condition = "AND (phase LIKE %s)" - params.append(f"%{phase}") - start_date_condition = '' if start_date is not None and start_date.strip() != '': - start_date_condition = "AND DATE(projects.last_run) >= TO_DATE(%s, 'YYYY-MM-DD')" + start_date_condition = "AND DATE(r.created_at) >= TO_DATE(%s, 'YYYY-MM-DD')" params.append(start_date) end_date_condition = '' if end_date is not None and end_date.strip() != '': - end_date_condition = "AND DATE(projects.last_run) <= TO_DATE(%s, 'YYYY-MM-DD')" + end_date_condition = "AND DATE(r.created_at) <= TO_DATE(%s, 'YYYY-MM-DD')" params.append(end_date) detail_name_condition = '' if detail_name is not None and detail_name.strip() != '': - detail_name_condition = "AND phase_stats.detail_name = %s" + detail_name_condition = "AND p.detail_name = %s" params.append(detail_name) limit_365_condition = '' if limit_365: - limit_365_condition = "AND projects.last_run >= CURRENT_DATE - INTERVAL '365 days'" + limit_365_condition = "AND r.created_at >= CURRENT_DATE - INTERVAL '365 days'" - sorting_condition = 'projects.commit_timestamp ASC, projects.last_run ASC' + sorting_condition = 'r.commit_timestamp ASC, r.created_at ASC' if sorting is not None and sorting.strip() == 'run': - sorting_condition = 'projects.last_run ASC, projects.commit_timestamp ASC' - + sorting_condition = 'r.created_at ASC, r.commit_timestamp ASC' query = f""" SELECT - projects.id, projects.name, projects.last_run, phase_stats.metric, phase_stats.detail_name, phase_stats.phase, - phase_stats.value, phase_stats.unit, projects.commit_hash, projects.commit_timestamp, + r.id, r.name, r.created_at, p.metric, p.detail_name, p.phase, + p.value, p.unit, r.commit_hash, r.commit_timestamp, row_number() OVER () AS row_num - FROM projects - LEFT JOIN phase_stats ON - projects.id = phase_stats.project_id + FROM runs as r + LEFT JOIN phase_stats as p ON + r.id = p.run_id WHERE - projects.uri = %s - AND projects.filename = %s - AND projects.end_measurement IS NOT NULL - AND projects.last_run IS NOT NULL - AND machine_id = %s - {metrics_condition} + r.uri = %s + AND r.filename = %s + AND r.end_measurement IS NOT NULL + AND r.machine_id = %s + AND p.phase LIKE %s {branch_condition} - {phase_condition} + {metrics_condition} {start_date_condition} {end_date_condition} {detail_name_condition} {limit_365_condition} - AND projects.commit_timestamp IS NOT NULL + AND r.commit_timestamp IS NOT NULL ORDER BY - phase_stats.metric ASC, phase_stats.detail_name ASC, - phase_stats.phase ASC, {sorting_condition} + p.metric ASC, p.detail_name ASC, + p.phase ASC, {sorting_condition} """ - print(query) return (query, params) def determine_comparison_case(ids): query = ''' WITH uniques as ( - SELECT uri, filename, machine_id, commit_hash, COALESCE(branch, 'main / master') as branch FROM projects + SELECT uri, filename, machine_id, commit_hash, COALESCE(branch, 'main / master') as branch FROM runs WHERE id = ANY(%s::uuid[]) GROUP BY uri, filename, machine_id, commit_hash, branch ) @@ -209,7 +194,7 @@ def determine_comparison_case(ids): [repos, usage_scenarios, machine_ids, commit_hashes, branches] = data # If we have one or more measurement in a phase_stat it will currently just be averaged - # however, when we allow comparing projects we will get same phase_stats but with different repo etc. + # however, when we allow comparing runs we will get same phase_stats but with different repo etc. # these cannot be just averaged. But they have to be split and then compared via t-test # For the moment I think it makes sense to restrict to two repositories. Comparing three is too much to handle I believe if we do not want to drill down to one specific metric @@ -282,7 +267,6 @@ def determine_comparison_case(ids): raise RuntimeError('Less than 1 or more than 2 Usage scenarios per repo not supported.') else: - # TODO: Metric drilldown has to be implemented at some point ... # The functionality I imagine here is, because comparing more than two repos is very complex with # multiple t-tests / ANOVA etc. and hard to grasp, only a focus on one metric shall be provided. raise RuntimeError('Less than 1 or more than 2 repos not supported for overview. Please apply metric filter.') @@ -295,11 +279,11 @@ def get_phase_stats(ids): a.phase, a.metric, a.detail_name, a.value, a.type, a.max_value, a.min_value, a.unit, b.uri, c.description, b.filename, b.commit_hash, COALESCE(b.branch, 'main / master') as branch FROM phase_stats as a - LEFT JOIN projects as b on b.id = a.project_id + LEFT JOIN runs as b on b.id = a.run_id LEFT JOIN machines as c on c.id = b.machine_id WHERE - a.project_id = ANY(%s::uuid[]) + a.run_id = ANY(%s::uuid[]) ORDER BY a.phase ASC, a.metric ASC, @@ -316,7 +300,6 @@ def get_phase_stats(ids): raise RuntimeError('Data is empty') return data -# TODO: This method needs proper database caching # Would be interesting to know if in an application server like gunicor @cache # Will also work for subsequent requests ...? ''' Object structure @@ -347,8 +330,8 @@ def get_phase_stats(ids): // although we can have 2 commits on 2 repos, we do not keep // track of the multiple commits here as key - // currently the system is limited to compare only two projects until we have - // figured out how big our StdDev is and how many projects we can run per day + // currently the system is limited to compare only two runs until we have + // figured out how big our StdDev is and how many runs we can run per day // at all (and how many repetitions are needed and possbile time-wise) repo/usage_scenarios/machine/commit/: mean: // mean per commit/repo etc. @@ -361,10 +344,10 @@ def get_phase_stats(ids): data: dict -> key: repo/usage_scenarios/machine/commit/branch - project_1: dict - project_2: dict + run_1: dict + run_2: dict ... - project_x : dict -> key: phase_name + run_x : dict -> key: phase_name [BASELINE]: dict [INSTALLATION]: dict .... diff --git a/api/main.py b/api/main.py new file mode 100644 index 000000000..02a3f0b27 --- /dev/null +++ b/api/main.py @@ -0,0 +1,1144 @@ +import faulthandler + +# It seems like FastAPI already enables faulthandler as it shows stacktrace on SEGFAULT +# Is the redundant call problematic +faulthandler.enable() # will catch segfaults and write to STDERR + +import zlib +import base64 +import json +from typing import List +from xml.sax.saxutils import escape as xml_escape +import math +from fastapi import FastAPI, Request, Response +from fastapi.responses import ORJSONResponse +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware + +from starlette.responses import RedirectResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from pydantic import BaseModel + +import anybadge + +from api.object_specifications import Measurement +from api.api_helpers import (add_phase_stats_statistics, determine_comparison_case, + html_escape_multi, get_phase_stats, get_phase_stats_object, + is_valid_uuid, rescale_energy_value, get_timeline_query, + get_run_info, get_machine_list) + +from lib.global_config import GlobalConfig +from lib.db import DB +from lib import email_helpers +from lib import error_helpers +from tools.jobs import Job +from tools.timeline_projects import TimelineProject + + +app = FastAPI() + +async def log_exception(request: Request, exc, body=None, details=None): + error_message = f""" + Error in API call + + URL: {request.url} + + Query-Params: {request.query_params} + + Client: {request.client} + + Headers: {str(request.headers)} + + Body: {body} + + Optional details: {details} + + Exception: {exc} + """ + error_helpers.log_error(error_message) + + # This saves us from crawler requests to the IP directly, or to our DNS reverse PTR etc. + # which would create email noise + request_url = str(request.url).replace('https://', '').replace('http://', '') + api_url = GlobalConfig().config['cluster']['api_url'].replace('https://', '').replace('http://', '') + + if not request_url.startswith(api_url): + return + + if GlobalConfig().config['admin']['no_emails'] is False: + email_helpers.send_error_email( + GlobalConfig().config['admin']['email'], + error_helpers.format_error(error_message), + run_id=None, + ) + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + await log_exception(request, exc, body=exc.body, details=exc.errors()) + return ORJSONResponse( + status_code=422, # HTTP_422_UNPROCESSABLE_ENTITY + content=jsonable_encoder({'success': False, 'err': exc.errors(), 'body': exc.body}), + ) + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request, exc): + await log_exception(request, exc, body='StarletteHTTPException handler cannot read body atm. Waiting for FastAPI upgrade.', details=exc.detail) + return ORJSONResponse( + status_code=exc.status_code, + content=jsonable_encoder({'success': False, 'err': exc.detail}), + ) + +async def catch_exceptions_middleware(request: Request, call_next): + #pylint: disable=broad-except + try: + return await call_next(request) + except Exception as exc: + # body = await request.body() # This blocks the application. Unclear atm how to handle it properly + # seems like a bug: https://github.com/tiangolo/fastapi/issues/394 + # Although the issue is closed the "solution" still behaves with same failure + # Actually Starlette, the underlying library to FastAPI has already introduced this functionality: + # https://github.com/encode/starlette/pull/1692 + # However FastAPI does not support the new Starlette 0.31.1 + # The PR relevant here is: https://github.com/tiangolo/fastapi/pull/9939 + await log_exception(request, exc, body='Middleware cannot read body atm. Waiting for FastAPI upgrade') + return ORJSONResponse( + content={ + 'success': False, + 'err': 'Technical error with getting data from the API - Please contact us: info@green-coding.berlin', + }, + status_code=500, + ) + + +# Binding the Exception middleware must confusingly come BEFORE the CORS middleware. +# Otherwise CORS will not be sent in response +app.middleware('http')(catch_exceptions_middleware) + +origins = [ + GlobalConfig().config['cluster']['metrics_url'], + GlobalConfig().config['cluster']['api_url'], +] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=['*'], + allow_headers=['*'], +) + + +@app.get('/') +async def home(): + return RedirectResponse(url='/docs') + + +# A route to return all of the available entries in our catalog. +@app.get('/v1/notes/{run_id}') +async def get_notes(run_id): + if run_id is None or not is_valid_uuid(run_id): + raise RequestValidationError('Run ID is not a valid UUID or empty') + + query = """ + SELECT run_id, detail_name, note, time + FROM notes + WHERE run_id = %s + ORDER BY created_at DESC -- important to order here, the charting library in JS cannot do that automatically! + """ + data = DB().fetch_all(query, (run_id,)) + if data is None or data == []: + return Response(status_code=204) # No-Content + + escaped_data = [html_escape_multi(note) for note in data] + return ORJSONResponse({'success': True, 'data': escaped_data}) + +@app.get('/v1/network/{run_id}') +async def get_network(run_id): + if run_id is None or not is_valid_uuid(run_id): + raise RequestValidationError('Run ID is not a valid UUID or empty') + + query = """ + SELECT * + FROM network_intercepts + WHERE run_id = %s + ORDER BY time + """ + data = DB().fetch_all(query, (run_id,)) + + escaped_data = html_escape_multi(data) + return ORJSONResponse({'success': True, 'data': escaped_data}) + + +# return a list of all possible registered machines +@app.get('/v1/machines/') +async def get_machines(): + + data = get_machine_list() + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + +@app.get('/v1/repositories') +async def get_repositories(uri: str | None = None, branch: str | None = None, machine_id: int | None = None, machine: str | None = None, filename: str | None = None, ): + query = """ + SELECT DISTINCT(r.uri) + FROM runs as r + LEFT JOIN machines as m on r.machine_id = m.id + WHERE 1=1 + """ + params = [] + + if uri: + query = f"{query} AND r.uri LIKE %s \n" + params.append(f"%{uri}%") + + if branch: + query = f"{query} AND r.branch LIKE %s \n" + params.append(f"%{branch}%") + + if filename: + query = f"{query} AND r.filename LIKE %s \n" + params.append(f"%{filename}%") + + if machine_id: + query = f"{query} AND m.id = %s \n" + params.append(machine_id) + + if machine: + query = f"{query} AND m.description LIKE %s \n" + params.append(f"%{machine}%") + + + query = f"{query} ORDER BY r.uri ASC" + + data = DB().fetch_all(query, params=tuple(params)) + if data is None or data == []: + return Response(status_code=204) # No-Content + + escaped_data = [html_escape_multi(run) for run in data] + + return ORJSONResponse({'success': True, 'data': escaped_data}) + +# A route to return all of the available entries in our catalog. +@app.get('/v1/runs') +async def get_runs(uri: str | None = None, branch: str | None = None, machine_id: int | None = None, machine: str | None = None, filename: str | None = None, limit: int | None = None): + + query = """ + SELECT r.id, r.name, r.uri, COALESCE(r.branch, 'main / master'), r.created_at, r.invalid_run, r.filename, m.description, r.commit_hash, r.end_measurement + FROM runs as r + LEFT JOIN machines as m on r.machine_id = m.id + WHERE 1=1 + """ + params = [] + + if uri: + query = f"{query} AND r.uri LIKE %s \n" + params.append(f"%{uri}%") + + if branch: + query = f"{query} AND r.branch LIKE %s \n" + params.append(f"%{branch}%") + + if filename: + query = f"{query} AND r.filename LIKE %s \n" + params.append(f"%{filename}%") + + if machine_id: + query = f"{query} AND m.id = %s \n" + params.append(machine_id) + + if machine: + query = f"{query} AND m.description LIKE %s \n" + params.append(f"%{machine}%") + + query = f"{query} ORDER BY r.created_at DESC" + + if limit: + query = f"{query} LIMIT %s" + params.append(limit) + + + data = DB().fetch_all(query, params=tuple(params)) + if data is None or data == []: + return Response(status_code=204) # No-Content + + escaped_data = [html_escape_multi(run) for run in data] + + return ORJSONResponse({'success': True, 'data': escaped_data}) + + +# Just copy and paste if we want to deprecate URLs +# @app.get('/v1/measurements/uri', deprecated=True) # Here you can see, that URL is nevertheless accessible as variable +# later if supplied. Also deprecation shall be used once we move to v2 for all v1 routesthrough + +@app.get('/v1/compare') +async def compare_in_repo(ids: str): + if ids is None or not ids.strip(): + raise RequestValidationError('run_id is empty') + ids = ids.split(',') + if not all(is_valid_uuid(id) for id in ids): + raise RequestValidationError('One of Run IDs is not a valid UUID or empty') + + try: + case = determine_comparison_case(ids) + except RuntimeError as err: + raise RequestValidationError(str(err)) from err + try: + phase_stats = get_phase_stats(ids) + except RuntimeError: + return Response(status_code=204) # No-Content + try: + phase_stats_object = get_phase_stats_object(phase_stats, case) + phase_stats_object = add_phase_stats_statistics(phase_stats_object) + phase_stats_object['common_info'] = {} + + run_info = get_run_info(ids[0]) + + machine_list = get_machine_list() + machines = {machine[0]: machine[1] for machine in machine_list} + + machine = machines[run_info['machine_id']] + uri = run_info['uri'] + usage_scenario = run_info['usage_scenario']['name'] + branch = run_info['branch'] if run_info['branch'] is not None else 'main / master' + commit = run_info['commit_hash'] + filename = run_info['filename'] + + match case: + case 'Repeated Run': + # same repo, same usage scenarios, same machines, same branches, same commit hashes + phase_stats_object['common_info']['Repository'] = uri + phase_stats_object['common_info']['Filename'] = filename + phase_stats_object['common_info']['Usage Scenario'] = usage_scenario + phase_stats_object['common_info']['Machine'] = machine + phase_stats_object['common_info']['Branch'] = branch + phase_stats_object['common_info']['Commit'] = commit + case 'Usage Scenario': + # same repo, diff usage scenarios, same machines, same branches, same commit hashes + phase_stats_object['common_info']['Repository'] = uri + phase_stats_object['common_info']['Machine'] = machine + phase_stats_object['common_info']['Branch'] = branch + phase_stats_object['common_info']['Commit'] = commit + case 'Machine': + # same repo, same usage scenarios, diff machines, same branches, same commit hashes + phase_stats_object['common_info']['Repository'] = uri + phase_stats_object['common_info']['Filename'] = filename + phase_stats_object['common_info']['Usage Scenario'] = usage_scenario + phase_stats_object['common_info']['Branch'] = branch + phase_stats_object['common_info']['Commit'] = commit + case 'Commit': + # same repo, same usage scenarios, same machines, diff commit hashes + phase_stats_object['common_info']['Repository'] = uri + phase_stats_object['common_info']['Filename'] = filename + phase_stats_object['common_info']['Usage Scenario'] = usage_scenario + phase_stats_object['common_info']['Machine'] = machine + case 'Repository': + # diff repo, diff usage scenarios, same machine, same branches, diff/same commits_hashes + phase_stats_object['common_info']['Machine'] = machine + phase_stats_object['common_info']['Branch'] = branch + case 'Branch': + # same repo, same usage scenarios, same machines, diff branch + phase_stats_object['common_info']['Repository'] = uri + phase_stats_object['common_info']['Filename'] = filename + phase_stats_object['common_info']['Usage Scenario'] = usage_scenario + phase_stats_object['common_info']['Machine'] = machine + + except RuntimeError as err: + raise RequestValidationError(str(err)) from err + + return ORJSONResponse({'success': True, 'data': phase_stats_object}) + + +@app.get('/v1/phase_stats/single/{run_id}') +async def get_phase_stats_single(run_id: str): + if run_id is None or not is_valid_uuid(run_id): + raise RequestValidationError('Run ID is not a valid UUID or empty') + + try: + phase_stats = get_phase_stats([run_id]) + phase_stats_object = get_phase_stats_object(phase_stats, None) + phase_stats_object = add_phase_stats_statistics(phase_stats_object) + + except RuntimeError: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': phase_stats_object}) + + +# This route gets the measurements to be displayed in a timeline chart +@app.get('/v1/measurements/single/{run_id}') +async def get_measurements_single(run_id: str): + if run_id is None or not is_valid_uuid(run_id): + raise RequestValidationError('Run ID is not a valid UUID or empty') + + query = """ + SELECT measurements.detail_name, measurements.time, measurements.metric, + measurements.value, measurements.unit + FROM measurements + WHERE measurements.run_id = %s + """ + + # extremely important to order here, cause the charting library in JS cannot do that automatically! + + query = f" {query} ORDER BY measurements.metric ASC, measurements.detail_name ASC, measurements.time ASC" + + params = params = (run_id, ) + + data = DB().fetch_all(query, params=params) + + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + +@app.get('/v1/timeline') +async def get_timeline_stats(uri: str, machine_id: int, branch: str | None = None, filename: str | None = None, start_date: str | None = None, end_date: str | None = None, metrics: str | None = None, phase: str | None = None, sorting: str | None = None,): + if uri is None or uri.strip() == '': + raise RequestValidationError('URI is empty') + + if phase is None or phase.strip() == '': + raise RequestValidationError('Phase is empty') + + query, params = get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_date=start_date, end_date=end_date, sorting=sorting) + + data = DB().fetch_all(query, params=params) + + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + +@app.get('/v1/badge/timeline') +async def get_timeline_badge(detail_name: str, uri: str, machine_id: int, branch: str | None = None, filename: str | None = None, metrics: str | None = None): + if uri is None or uri.strip() == '': + raise RequestValidationError('URI is empty') + + if detail_name is None or detail_name.strip() == '': + raise RequestValidationError('Detail Name is mandatory') + + query, params = get_timeline_query(uri,filename,machine_id, branch, metrics, '[RUNTIME]', detail_name=detail_name, limit_365=True) + + query = f""" + WITH trend_data AS ( + {query} + ) SELECT + MAX(row_num::float), + regr_slope(value, row_num::float) AS trend_slope, + regr_intercept(value, row_num::float) AS trend_intercept, + MAX(unit) + FROM trend_data; + """ + + data = DB().fetch_one(query, params=params) + + if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result + return Response(status_code=204) # No-Content + + cost = data[1]/data[0] + cost = f"+{round(float(cost), 2)}" if abs(cost) == cost else f"{round(float(cost), 2)}" + + badge = anybadge.Badge( + label=xml_escape('Run Trend'), + value=xml_escape(f"{cost} {data[3]} per day"), + num_value_padding_chars=1, + default_color='orange') + return Response(content=str(badge), media_type="image/svg+xml") + + +# A route to return all of the available entries in our catalog. +@app.get('/v1/badge/single/{run_id}') +async def get_badge_single(run_id: str, metric: str = 'ml-estimated'): + + if run_id is None or not is_valid_uuid(run_id): + raise RequestValidationError('Run ID is not a valid UUID or empty') + + query = ''' + SELECT + SUM(value), MAX(unit) + FROM + phase_stats + WHERE + run_id = %s + AND metric LIKE %s + AND phase LIKE '%%_[RUNTIME]' + ''' + + value = None + label = 'Energy Cost' + via = '' + if metric == 'ml-estimated': + value = 'psu_energy_ac_xgboost_machine' + via = 'via XGBoost ML' + elif metric == 'RAPL': + value = '%_energy_rapl_%' + via = 'via RAPL' + elif metric == 'AC': + value = 'psu_energy_ac_%' + via = 'via PSU (AC)' + elif metric == 'SCI': + label = 'SCI' + value = 'software_carbon_intensity_global' + else: + raise RequestValidationError(f"Unknown metric '{metric}' submitted") + + params = (run_id, value) + data = DB().fetch_one(query, params=params) + + if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result + badge_value = 'No energy data yet' + else: + [energy_value, energy_unit] = rescale_energy_value(data[0], data[1]) + badge_value= f"{energy_value:.2f} {energy_unit} {via}" + + badge = anybadge.Badge( + label=xml_escape(label), + value=xml_escape(badge_value), + num_value_padding_chars=1, + default_color='cornflowerblue') + return Response(content=str(badge), media_type="image/svg+xml") + + +@app.get('/v1/timeline-projects') +async def get_timeline_projects(): + # Do not get the email jobs as they do not need to be display in the frontend atm + # Also do not get the email field for privacy + query = """ + SELECT + p.id, p.name, p.url, + ( + SELECT STRING_AGG(t.name, ', ' ) + FROM unnest(p.categories) as elements + LEFT JOIN categories as t on t.id = elements + ) as categories, + p.branch, p.filename, p.machine_id, m.description, p.schedule_mode, p.last_scheduled, p.created_at, p.updated_at, + ( + SELECT created_at + FROM runs as r + WHERE + p.url = r.uri + AND COALESCE(p.branch, 'main / master') = COALESCE(r.branch, 'main / master') + AND COALESCE(p.filename, 'usage_scenario.yml') = COALESCE(r.filename, 'usage_scenario.yml') + AND p.machine_id = r.machine_id + ORDER BY r.created_at DESC + LIMIT 1 + ) as "last_run" + FROM timeline_projects as p + LEFT JOIN machines as m ON m.id = p.machine_id + ORDER BY p.url ASC; + """ + data = DB().fetch_all(query) + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + + +@app.get('/v1/jobs') +async def get_jobs(): + # Do not get the email jobs as they do not need to be display in the frontend atm + query = """ + SELECT j.id, j.name, j.url, j.filename, j.branch, m.description, j.state, j.updated_at, j.created_at + FROM jobs as j + LEFT JOIN machines as m on m.id = j.machine_id + ORDER BY j.updated_at DESC, j.created_at ASC + """ + data = DB().fetch_all(query) + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + +#### + +class HogMeasurement(BaseModel): + time: int + data: str + settings: str + machine_uuid: str + +def replace_nan_with_zero(obj): + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, (dict, list)): + replace_nan_with_zero(v) + elif isinstance(v, float) and math.isnan(v): + obj[k] = 0 + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, (dict, list)): + replace_nan_with_zero(item) + elif isinstance(item, float) and math.isnan(item): + obj[i] = 0 + return obj + + + +@app.post('/v1/hog/add') +async def hog_add(measurements: List[HogMeasurement]): + + for measurement in measurements: + decoded_data = base64.b64decode(measurement.data) + decompressed_data = zlib.decompress(decoded_data) + measurement_data = json.loads(decompressed_data.decode()) + + # For some reason we sometimes get NaN in the data. + measurement_data = replace_nan_with_zero(measurement_data) + + #Check if the data is valid, if not this will throw an exception and converted into a request by the middleware + try: + _ = Measurement(**measurement_data) + except RequestValidationError as exc: + print(f"Caught Exception {exc}") + print(f"Errors are: {exc.errors()}") + raise exc + + coalitions = [] + for coalition in measurement_data['coalitions']: + if coalition['name'] == 'com.googlecode.iterm2' or \ + coalition['name'] == 'com.apple.Terminal' or \ + coalition['name'] == 'com.vix.cron' or \ + coalition['name'].strip() == '': + tmp = coalition['tasks'] + for tmp_el in tmp: + tmp_el['tasks'] = [] + coalitions.extend(tmp) + else: + coalitions.append(coalition) + + # We remove the coalitions as we don't want to save all the data in hog_measurements + del measurement_data['coalitions'] + del measurement.data + + cpu_energy_data = {} + energy_impact = round(measurement_data['all_tasks'].get('energy_impact_per_s') * measurement_data['elapsed_ns'] / 1_000_000_000) + if 'ane_energy' in measurement_data['processor']: + cpu_energy_data = { + 'combined_energy': round(measurement_data['processor'].get('combined_power', 0) * measurement_data['elapsed_ns'] / 1_000_000_000.0), + 'cpu_energy': round(measurement_data['processor'].get('cpu_energy', 0)), + 'gpu_energy': round(measurement_data['processor'].get('gpu_energy', 0)), + 'ane_energy': round(measurement_data['processor'].get('ane_energy', 0)), + 'energy_impact': energy_impact, + } + elif 'package_joules' in measurement_data['processor']: + # Intel processors report in joules/ watts and not mJ + cpu_energy_data = { + 'combined_energy': round(measurement_data['processor'].get('package_joules', 0) * 1_000), + 'cpu_energy': round(measurement_data['processor'].get('cpu_joules', 0) * 1_000), + 'gpu_energy': round(measurement_data['processor'].get('igpu_watts', 0) * measurement_data['elapsed_ns'] / 1_000_000_000.0 * 1_000), + 'ane_energy': 0, + 'energy_impact': energy_impact, + } + else: + raise RequestValidationError("input not valid") + + query = """ + INSERT INTO + hog_measurements ( + time, + machine_uuid, + elapsed_ns, + combined_energy, + cpu_energy, + gpu_energy, + ane_energy, + energy_impact, + thermal_pressure, + settings, + data) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """ + params = ( + measurement.time, + measurement.machine_uuid, + measurement_data['elapsed_ns'], + cpu_energy_data['combined_energy'], + cpu_energy_data['cpu_energy'], + cpu_energy_data['gpu_energy'], + cpu_energy_data['ane_energy'], + cpu_energy_data['energy_impact'], + measurement_data['thermal_pressure'], + measurement.settings, + json.dumps(measurement_data), + ) + + measurement_db_id = DB().fetch_one(query=query, params=params)[0] + + + # Save hog_measurements + for coalition in coalitions: + + if coalition['energy_impact'] < 1.0: + # If the energy_impact is too small we just skip the coalition. + continue + + c_tasks = coalition['tasks'].copy() + del coalition['tasks'] + + c_energy_impact = round((coalition['energy_impact_per_s'] / 1_000_000_000) * measurement_data['elapsed_ns']) + c_cputime_ns = ((coalition['cputime_ms_per_s'] * 1_000_000) / 1_000_000_000) * measurement_data['elapsed_ns'] + + query = """ + INSERT INTO + hog_coalitions ( + measurement, + name, + cputime_ns, + cputime_per, + energy_impact, + diskio_bytesread, + diskio_byteswritten, + intr_wakeups, + idle_wakeups, + data) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """ + params = ( + measurement_db_id, + coalition['name'], + c_cputime_ns, + int(c_cputime_ns / measurement_data['elapsed_ns'] * 100), + c_energy_impact, + coalition['diskio_bytesread'], + coalition['diskio_byteswritten'], + coalition['intr_wakeups'], + coalition['idle_wakeups'], + json.dumps(coalition) + ) + + coaltion_db_id = DB().fetch_one(query=query, params=params)[0] + + for task in c_tasks: + t_energy_impact = round((task['energy_impact_per_s'] / 1_000_000_000) * measurement_data['elapsed_ns']) + t_cputime_ns = ((task['cputime_ms_per_s'] * 1_000_000) / 1_000_000_000) * measurement_data['elapsed_ns'] + + query = """ + INSERT INTO + hog_tasks ( + coalition, + name, + cputime_ns, + cputime_per, + energy_impact, + bytes_received, + bytes_sent, + diskio_bytesread, + diskio_byteswritten, + intr_wakeups, + idle_wakeups, + data) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """ + params = ( + coaltion_db_id, + task['name'], + t_cputime_ns, + int(t_cputime_ns / measurement_data['elapsed_ns'] * 100), + t_energy_impact, + task.get('bytes_received', 0), + task.get('bytes_sent', 0), + task.get('diskio_bytesread', 0), + task.get('diskio_byteswritten', 0), + task.get('intr_wakeups', 0), + task.get('idle_wakeups', 0), + json.dumps(task) + + ) + DB().fetch_one(query=query, params=params) + + return Response(status_code=204) # No-Content + + +@app.get('/v1/hog/top_processes') +async def hog_get_top_processes(): + query = """ + SELECT + name, + (SUM(energy_impact)::bigint) AS total_energy_impact + FROM + hog_coalitions + GROUP BY + name + ORDER BY + total_energy_impact DESC + LIMIT 100; + """ + data = DB().fetch_all(query) + + if data is None: + data = [] + + query = """ + SELECT COUNT(DISTINCT machine_uuid) FROM hog_measurements; + """ + + machine_count = DB().fetch_one(query)[0] + + return ORJSONResponse({'success': True, 'process_data': data, 'machine_count': machine_count}) + + +@app.get('/v1/hog/machine_details/{machine_uuid}') +async def hog_get_machine_details(machine_uuid: str): + + if machine_uuid is None or not is_valid_uuid(machine_uuid): + return ORJSONResponse({'success': False, 'err': 'machine_uuid is empty or malformed'}, status_code=400) + + query = """ + SELECT + time, + combined_energy, + cpu_energy, + gpu_energy, + ane_energy, + energy_impact::bigint, + id + FROM + hog_measurements + WHERE + machine_uuid = %s + ORDER BY + time + """ + + data = DB().fetch_all(query, (machine_uuid,)) + + return ORJSONResponse({'success': True, 'data': data}) + + +@app.get('/v1/hog/coalitions_tasks/{machine_uuid}/{measurements_id_start}/{measurements_id_end}') +async def hog_get_coalitions_tasks(machine_uuid: str, measurements_id_start: int, measurements_id_end: int): + + if machine_uuid is None or not is_valid_uuid(machine_uuid): + return ORJSONResponse({'success': False, 'err': 'machine_uuid is empty'}, status_code=400) + + if measurements_id_start is None: + return ORJSONResponse({'success': False, 'err': 'measurements_id_start is empty'}, status_code=400) + + if measurements_id_end is None: + return ORJSONResponse({'success': False, 'err': 'measurements_id_end is empty'}, status_code=400) + + + coalitions_query = """ + SELECT + name, + (SUM(hc.energy_impact)::bigint) AS total_energy_impact, + (SUM(hc.diskio_bytesread)::bigint) AS total_diskio_bytesread, + (SUM(hc.diskio_byteswritten)::bigint) AS total_diskio_byteswritten, + (SUM(hc.intr_wakeups)::bigint) AS total_intr_wakeups, + (SUM(hc.idle_wakeups)::bigint) AS total_idle_wakeups, + (AVG(hc.cputime_per)::integer) AS avg_cpu_per + FROM + hog_coalitions AS hc + JOIN + hog_measurements AS hm ON hc.measurement = hm.id + WHERE + hc.measurement BETWEEN %s AND %s + AND hm.machine_uuid = %s + GROUP BY + name + ORDER BY + total_energy_impact DESC + LIMIT 100; + """ + + measurements_query = """ + SELECT + (SUM(combined_energy)::bigint) AS total_combined_energy, + (SUM(cpu_energy)::bigint) AS total_cpu_energy, + (SUM(gpu_energy)::bigint) AS total_gpu_energy, + (SUM(ane_energy)::bigint) AS total_ane_energy, + (SUM(energy_impact)::bigint) AS total_energy_impact + FROM + hog_measurements + WHERE + id BETWEEN %s AND %s + AND machine_uuid = %s + + """ + + coalitions_data = DB().fetch_all(coalitions_query, (measurements_id_start, measurements_id_end, machine_uuid)) + + energy_data = DB().fetch_one(measurements_query, (measurements_id_start, measurements_id_end, machine_uuid)) + + return ORJSONResponse({'success': True, 'data': coalitions_data, 'energy_data': energy_data}) + +@app.get('/v1/hog/tasks_details/{machine_uuid}/{measurements_id_start}/{measurements_id_end}/{coalition_name}') +async def hog_get_task_details(machine_uuid: str, measurements_id_start: int, measurements_id_end: int, coalition_name: str): + + if machine_uuid is None or not is_valid_uuid(machine_uuid): + return ORJSONResponse({'success': False, 'err': 'machine_uuid is empty'}, status_code=400) + + if measurements_id_start is None: + return ORJSONResponse({'success': False, 'err': 'measurements_id_start is empty'}, status_code=400) + + if measurements_id_end is None: + return ORJSONResponse({'success': False, 'err': 'measurements_id_end is empty'}, status_code=400) + + if coalition_name is None or not coalition_name.strip(): + return ORJSONResponse({'success': False, 'err': 'coalition_name is empty'}, status_code=400) + + tasks_query = """ + SELECT + t.name, + COUNT(t.id) AS number_of_tasks, + (SUM(t.energy_impact)::bigint) AS total_energy_impact, + SUM(t.cputime_ns) AS total_cputime_ns, + SUM(t.bytes_received) AS total_bytes_received, + SUM(t.bytes_sent) AS total_bytes_sent, + SUM(t.diskio_bytesread) AS total_diskio_bytesread, + SUM(t.diskio_byteswritten) AS total_diskio_byteswritten, + SUM(t.intr_wakeups) AS total_intr_wakeups, + SUM(t.idle_wakeups) AS total_idle_wakeups + FROM + hog_tasks t + JOIN hog_coalitions c ON t.coalition = c.id + JOIN hog_measurements m ON c.measurement = m.id + WHERE + c.name = %s + AND c.measurement BETWEEN %s AND %s + AND m.machine_uuid = %s + GROUP BY + t.name + ORDER BY + total_energy_impact DESC; + """ + + coalitions_query = """ + SELECT + c.name, + (SUM(c.energy_impact)::bigint) AS total_energy_impact, + (SUM(c.diskio_bytesread)::bigint) AS total_diskio_bytesread, + (SUM(c.diskio_byteswritten)::bigint) AS total_diskio_byteswritten, + (SUM(c.intr_wakeups)::bigint) AS total_intr_wakeups, + (SUM(c.idle_wakeups)::bigint) AS total_idle_wakeups + FROM + hog_coalitions c + JOIN hog_measurements m ON c.measurement = m.id + WHERE + c.name = %s + AND c.measurement BETWEEN %s AND %s + AND m.machine_uuid = %s + GROUP BY + c.name + ORDER BY + total_energy_impact DESC + LIMIT 100; + """ + + tasks_data = DB().fetch_all(tasks_query, (coalition_name, measurements_id_start,measurements_id_end, machine_uuid)) + coalitions_data = DB().fetch_one(coalitions_query, (coalition_name, measurements_id_start, measurements_id_end, machine_uuid)) + + return ORJSONResponse({'success': True, 'tasks_data': tasks_data, 'coalitions_data': coalitions_data}) + + + +#### + +class Software(BaseModel): + name: str + url: str + email: str + filename: str + branch: str + machine_id: int + schedule_mode: str + +@app.post('/v1/software/add') +async def software_add(software: Software): + + software = html_escape_multi(software) + + if software.name is None or software.name.strip() == '': + raise RequestValidationError('Name is empty') + + # Note that we use uri as the general identifier, however when adding through web interface we only allow urls + if software.url is None or software.url.strip() == '': + raise RequestValidationError('URL is empty') + + if software.name is None or software.name.strip() == '': + raise RequestValidationError('Name is empty') + + if software.email is None or software.email.strip() == '': + raise RequestValidationError('E-mail is empty') + + if not DB().fetch_one('SELECT id FROM machines WHERE id=%s AND available=TRUE', params=(software.machine_id,)): + raise RequestValidationError('Machine does not exist') + + + if software.branch.strip() == '': + software.branch = None + + if software.filename.strip() == '': + software.filename = 'usage_scenario.yml' + + if software.schedule_mode not in ['one-off', 'time', 'commit', 'variance']: + raise RequestValidationError(f"Please select a valid measurement interval. ({software.schedule_mode}) is unknown.") + + # notify admin of new add + if GlobalConfig().config['admin']['no_emails'] is False: + email_helpers.send_admin_email(f"New run added from Web Interface: {software.name}", software) + + if software.schedule_mode == 'one-off': + Job.insert(software.name, software.url, software.email, software.branch, software.filename, software.machine_id) + elif software.schedule_mode == 'variance': + for _ in range(0,3): + Job.insert(software.name, software.url, software.email, software.branch, software.filename, software.machine_id) + else: + TimelineProject.insert(software.name, software.url, software.branch, software.filename, software.machine_id, software.schedule_mode) + + return ORJSONResponse({'success': True}, status_code=202) + + +@app.get('/v1/run/{run_id}') +async def get_run(run_id: str): + if run_id is None or not is_valid_uuid(run_id): + raise RequestValidationError('Run ID is not a valid UUID or empty') + + data = get_run_info(run_id) + + if data is None or data == []: + return Response(status_code=204) # No-Content + + data = html_escape_multi(data) + + return ORJSONResponse({'success': True, 'data': data}) + +@app.get('/robots.txt') +async def robots_txt(): + data = "User-agent: *\n" + data += "Disallow: /" + + return Response(content=data, media_type='text/plain') + +# pylint: disable=invalid-name +class CI_Measurement(BaseModel): + energy_value: int + energy_unit: str + repo: str + branch: str + cpu: str + cpu_util_avg: float + commit_hash: str + workflow: str # workflow_id, change when we make API change of workflow_name being mandatory + run_id: str + source: str + label: str + duration: int + workflow_name: str = None + +@app.post('/v1/ci/measurement/add') +async def post_ci_measurement_add(measurement: CI_Measurement): + for key, value in measurement.model_dump().items(): + match key: + case 'unit': + if value is None or value.strip() == '': + raise RequestValidationError(f"{key} is empty") + if value != 'mJ': + raise RequestValidationError("Unit is unsupported - only mJ currently accepted") + continue + + case 'label' | 'workflow_name': # Optional fields + continue + + case _: + if value is None: + raise RequestValidationError(f"{key} is empty") + if isinstance(value, str): + if value.strip() == '': + raise RequestValidationError(f"{key} is empty") + + measurement = html_escape_multi(measurement) + + query = """ + INSERT INTO + ci_measurements (energy_value, energy_unit, repo, branch, workflow_id, run_id, label, source, cpu, commit_hash, duration, cpu_util_avg, workflow_name) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """ + params = (measurement.energy_value, measurement.energy_unit, measurement.repo, measurement.branch, + measurement.workflow, measurement.run_id, measurement.label, measurement.source, measurement.cpu, + measurement.commit_hash, measurement.duration, measurement.cpu_util_avg, measurement.workflow_name) + + DB().query(query=query, params=params) + return ORJSONResponse({'success': True}, status_code=201) + +@app.get('/v1/ci/measurements') +async def get_ci_measurements(repo: str, branch: str, workflow: str): + query = """ + SELECT energy_value, energy_unit, run_id, created_at, label, cpu, commit_hash, duration, source, cpu_util_avg, + (SELECT workflow_name FROM ci_measurements AS latest_workflow + WHERE latest_workflow.repo = ci_measurements.repo + AND latest_workflow.branch = ci_measurements.branch + AND latest_workflow.workflow_id = ci_measurements.workflow_id + ORDER BY latest_workflow.created_at DESC + LIMIT 1) AS workflow_name + FROM ci_measurements + WHERE repo = %s AND branch = %s AND workflow_id = %s + ORDER BY run_id ASC, created_at ASC + """ + params = (repo, branch, workflow) + data = DB().fetch_all(query, params=params) + + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + +@app.get('/v1/ci/projects') +async def get_ci_projects(): + query = """ + SELECT repo, branch, workflow_id, source, MAX(created_at), + (SELECT workflow_name FROM ci_measurements AS latest_workflow + WHERE latest_workflow.repo = ci_measurements.repo + AND latest_workflow.branch = ci_measurements.branch + AND latest_workflow.workflow_id = ci_measurements.workflow_id + ORDER BY latest_workflow.created_at DESC + LIMIT 1) AS workflow_name + FROM ci_measurements + GROUP BY repo, branch, workflow_id, source + ORDER BY repo ASC + """ + + data = DB().fetch_all(query) + if data is None or data == []: + return Response(status_code=204) # No-Content + + return ORJSONResponse({'success': True, 'data': data}) + +@app.get('/v1/ci/badge/get') +async def get_ci_badge_get(repo: str, branch: str, workflow:str): + query = """ + SELECT SUM(energy_value), MAX(energy_unit), MAX(run_id) + FROM ci_measurements + WHERE repo = %s AND branch = %s AND workflow_id = %s + GROUP BY run_id + ORDER BY MAX(created_at) DESC + LIMIT 1 + """ + + params = (repo, branch, workflow) + data = DB().fetch_one(query, params=params) + + if data is None or data == [] or data[1] is None: # special check for data[1] as this is aggregate query which always returns result + return Response(status_code=204) # No-Content + + energy_value = data[0] + energy_unit = data[1] + + [energy_value, energy_unit] = rescale_energy_value(energy_value, energy_unit) + badge_value= f"{energy_value:.2f} {energy_unit}" + + badge = anybadge.Badge( + label='Energy Used', + value=xml_escape(badge_value), + num_value_padding_chars=1, + default_color='green') + return Response(content=str(badge), media_type="image/svg+xml") + + +if __name__ == '__main__': + app.run() diff --git a/api/object_specifications.py b/api/object_specifications.py new file mode 100644 index 000000000..4f57826fd --- /dev/null +++ b/api/object_specifications.py @@ -0,0 +1,56 @@ +from typing import List, Dict, Optional +from pydantic import BaseModel + +class Task(BaseModel): + # We need to set the optional to a value as otherwise the key is required in the input + # https://docs.pydantic.dev/latest/migration/#required-optional-and-nullable-fields + name: str + cputime_ns: int + timer_wakeups: List + diskio_bytesread: Optional[int] = 0 + diskio_byteswritten: Optional[int] = 0 + packets_received: int + packets_sent: int + bytes_received: int + bytes_sent: int + energy_impact: float + +class Coalition(BaseModel): + name: str + cputime_ns: int + diskio_bytesread: int = 0 + diskio_byteswritten: int = 0 + energy_impact: float + tasks: List[Task] + +class Processor(BaseModel): + # https://docs.pydantic.dev/latest/migration/#required-optional-and-nullable-fields + clusters: Optional[List] = None + cpu_power_zones_engaged: Optional[float] = None + cpu_energy: Optional[int] = None + cpu_power: Optional[float] = None + gpu_energy: Optional[int] = None + gpu_power: Optional[float] = None + ane_energy: Optional[int] = None + ane_power: Optional[float] = None + combined_power: Optional[float] = None + package_joules: Optional[float] = None + cpu_joules: Optional[float] = None + igpu_watts: Optional[float] = None + +class GPU(BaseModel): + gpu_energy: Optional[int] = None + +class Measurement(BaseModel): + is_delta: bool + elapsed_ns: int + timestamp: int + coalitions: List[Coalition] + all_tasks: Dict + network: Optional[Dict] = None # network is optional when system is in flight mode / network turned off + disk: Dict + interrupts: List + processor: Processor + thermal_pressure: str + sfi: Dict + gpu: Optional[GPU] = None diff --git a/api_test.py b/api_test.py index 31a3530ed..5c73fbfd3 100644 --- a/api_test.py +++ b/api_test.py @@ -1,17 +1,10 @@ -import sys, os import json -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/lib") -from db import DB -from global_config import GlobalConfig - -from api.api_helpers import * +from api import api_helpers if __name__ == '__main__': import argparse - from pathlib import Path parser = argparse.ArgumentParser() @@ -21,9 +14,9 @@ ids = args.ids.split(',') - case = determine_comparison_case(ids) - phase_stats = get_phase_stats(ids) - phase_stats_object = get_phase_stats_object(phase_stats, case) - phase_stats_object = add_phase_stats_statistics(phase_stats_object) + case = api_helpers.determine_comparison_case(ids) + phase_stats = api_helpers.get_phase_stats(ids) + phase_stats_object = api_helpers.get_phase_stats_object(phase_stats, case) + phase_stats_object = api_helpers.add_phase_stats_statistics(phase_stats_object) print(json.dumps(phase_stats_object, indent=4)) diff --git a/config.yml.example b/config.yml.example index 5e61bd773..38ae881f0 100644 --- a/config.yml.example +++ b/config.yml.example @@ -16,34 +16,33 @@ smtp: user: SMTP_AUTH_USER admin: - # This address will get an email, when a new project was added through the frontend + # This address will get an email, for any error or new project added etc. email: myemail@dev.local + # This email will always get a copy of every email sent, even for user-only mails like the "Your report is ready" mail. Put an empty string if you do not want that: "" + bcc_email: "" # no_emails: True will suppress all emails. Helpful in development servers no_emails: True - # notifiy_admin_for_own_project_*: False will suppress an email if a project is added / ready with the - # same email address as the admin. - # If no_emails is set to True, this will have no effect - notify_admin_for_own_project_add: False - notify_admin_for_own_project_ready: False - cluster: api_url: __API_URL__ metrics_url: __METRICS_URL__ + client: - sleep_time: 300 + sleep_time_no_job: 300 + sleep_time_after_job: 300 machine: id: 1 description: "Development machine for testing" # Takes a file path to log all the errors to it. This is disabled if False error_log_file: False + jobs_processing: random measurement: idle-time-start: 10 idle-time-end: 5 - flow-process-runtime: 1800 + flow-process-runtime: 3800 phase-transition-time: 1 metric-providers: @@ -58,8 +57,6 @@ measurement: #--- Always-On - We recommend these providers to be always enabled cpu.utilization.procfs.system.provider.CpuUtilizationProcfsSystemProvider: resolution: 100 - cpu.frequency.sysfs.core.provider.CpuFrequencySysfsCoreProvider: - resolution: 100 #--- CGroupV2 - Turn these on if you have CGroupsV2 working on your machine cpu.utilization.cgroup.container.provider.CpuUtilizationCgroupContainerProvider: resolution: 100 @@ -81,7 +78,7 @@ measurement: # resolution: 100 # psu.energy.ac.ipmi.machine.provider.PsuEnergyAcIpmiMachineProvider: # resolution: 100 - #--- Sensors + #--- Sensors - these providers need the lm-sensors package installed # lm_sensors.temperature.component.provider.LmSensorsTemperatureComponentProvider: # resolution: 100 # Please change these values according to the names in '$ sensors' @@ -92,7 +89,9 @@ measurement: # Please change these values according to the names in '$ sensors' # chips: ['thinkpad-isa-0000'] # features: ['fan1', 'fan2'] - #--- Debug - These providers are just for development of the tool itself + #--- Debug - These providers should only be needed for debugging and introspection purposes +# cpu.frequency.sysfs.core.provider.CpuFrequencySysfsCoreProvider: +# resolution: 100 # cpu.time.cgroup.container.provider.CpuTimeCgroupContainerProvider: # resolution: 100 # cpu.time.cgroup.system.provider.CpuTimeCgroupSystemProvider: @@ -101,12 +100,14 @@ measurement: # resolution: 100 #--- Architecture - MacOS macos: - #--- MacOS: On Mac you only need this provider. Please delete all others! + #--- MacOS: On Mac you only need this provider. Please remove all others! powermetrics.provider.PowermetricsProvider: resolution: 100 #--- Architecture - Common common: - #--- Model based +# network.connections.proxy.container.provider.NetworkConnectionsProxyContainerProvider: +## host_ip: 192.168.1.2 # This only needs to be enabled if automatic detection fails + #--- Model based - These providers estimate rather than measure. Helpful where measuring is not possible, like in VMs # psu.energy.ac.sdia.machine.provider.PsuEnergyAcSdiaMachineProvider: # resolution: 100 #-- This is a default configuration. Please change this to your system! diff --git a/docker/Dockerfile-gunicorn b/docker/Dockerfile-gunicorn index 2f77559c4..94a85f6cd 100644 --- a/docker/Dockerfile-gunicorn +++ b/docker/Dockerfile-gunicorn @@ -1,8 +1,15 @@ # syntax=docker/dockerfile:1 -FROM python:3.11.4-slim-bookworm +FROM python:3.11.5-slim-bookworm ENV DEBIAN_FRONTEND=noninteractive +WORKDIR /var/www/startup/ COPY requirements.txt requirements.txt -RUN pip3 install -r requirements.txt +RUN python -m venv venv +RUN venv/bin/pip install --upgrade pip +RUN venv/bin/pip install -r requirements.txt +RUN find venv -type d -name "site-packages" -exec sh -c 'echo /var/www/green-metrics-tool > "$0/gmt-lib.pth"' {} \; -ENTRYPOINT ["/usr/local/bin/gunicorn", "--workers=2", "--access-logfile=-", "--error-logfile=-", "--worker-tmp-dir=/dev/shm", "--threads=4", "--worker-class=gthread", "--bind", "unix:/tmp/green-coding-api.sock", "-m", "007", "--user", "www-data", "--chdir", "/var/www/green-metrics-tool/api", "-k", "uvicorn.workers.UvicornWorker", "api:app"] \ No newline at end of file + +COPY startup_gunicorn.sh /var/www/startup/startup_gunicorn.sh + +ENTRYPOINT ["/bin/bash", "/var/www/startup/startup_gunicorn.sh"] diff --git a/docker/auxiliary-containers/build-containers.sh b/docker/auxiliary-containers/build-containers.sh new file mode 100644 index 000000000..d819e9ef8 --- /dev/null +++ b/docker/auxiliary-containers/build-containers.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -euo pipefail + +# The names of the folders within "auxiliary-containers" must match the repository name in dockerhub! + +# Get the list of subdirectories within "auxiliary-containers" directory containing a Dockerfile +subdirs=($(find ./docker/auxiliary-containers -type f -name 'Dockerfile' -exec dirname {} \;)) + +# Loop through each subdirectory, build and push the Docker image +for subdir in "${subdirs[@]}"; do + folder=$(basename "${subdir}") + docker buildx build \ + --push \ + --tag "greencoding/${folder}:latest" \ + --platform linux/amd64,linux/arm64 \ + "${subdir}" +done \ No newline at end of file diff --git a/docker/auxiliary-containers/gcb_playwright/Dockerfile b/docker/auxiliary-containers/gcb_playwright/Dockerfile new file mode 100644 index 000000000..475da385e --- /dev/null +++ b/docker/auxiliary-containers/gcb_playwright/Dockerfile @@ -0,0 +1,13 @@ +FROM mcr.microsoft.com/playwright/python:v1.35.0-jammy + +# Install dependencies +RUN apt-get update && apt-get install -y curl wget gnupg && rm -rf /var/lib/apt/lists/* + +# Install Playwright +RUN pip install playwright==1.35.0 + +# Set up Playwright dependencies for Chromium, Firefox and Webkit +RUN playwright install +RUN playwright install-deps + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/compose.yml.example b/docker/compose.yml.example index e76e90f04..9f65d6da5 100644 --- a/docker/compose.yml.example +++ b/docker/compose.yml.example @@ -1,5 +1,8 @@ services: green-coding-postgres: + # No need to fix version anymore than major version here + # for measurement accuracy the db container is not relevant as it + # should not run on the measurement node anyway image: postgres:15 shm_size: 256MB container_name: green-coding-postgres-container @@ -24,6 +27,9 @@ services: -p 9573 # This option can potentially speed up big queries: https://www.twilio.com/blog/sqlite-postgresql-complicated green-coding-nginx: + # No need to fix the version here, as we just waant to use latest, never have experienced + # incompatibilities and for measurement accuracy the web container is not relevant as it + # should not run on the measurement node anyway image: nginx container_name: green-coding-nginx-container depends_on: diff --git a/docker/requirements.txt b/docker/requirements.txt index a81d7769c..f6815fecd 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,10 +1,10 @@ gunicorn==21.2.0 -psycopg[binary]==3.1.10 -fastapi==0.101.1 +psycopg[binary]==3.1.12 +fastapi==0.104.0 uvicorn[standard]==0.23.2 -pandas==2.0.3 +pandas==2.1.1 PyYAML==6.0.1 anybadge==1.14.0 -scipy==1.11.1 -orjson==3.9.4 +orjson==3.9.9 +scipy==1.11.3 schema==0.7.5 diff --git a/docker/startup_gunicorn.sh b/docker/startup_gunicorn.sh new file mode 100644 index 000000000..99dd3ab8b --- /dev/null +++ b/docker/startup_gunicorn.sh @@ -0,0 +1,16 @@ +#!/bin/sh +source /var/www/startup/venv/bin/activate + +/var/www/startup/venv/bin/gunicorn \ +--workers=2 \ +--access-logfile=- \ +--error-logfile=- \ +--worker-tmp-dir=/dev/shm \ +--threads=4 \ +--worker-class=gthread \ +--bind unix:/tmp/green-coding-api.sock \ +-m 007 \ +--user www-data \ +--chdir /var/www/green-metrics-tool/api \ +-k uvicorn.workers.UvicornWorker \ +main:app \ No newline at end of file diff --git a/docker/structure.sql b/docker/structure.sql index 504874fcf..483d1a588 100644 --- a/docker/structure.sql +++ b/docker/structure.sql @@ -2,9 +2,41 @@ CREATE DATABASE "green-coding"; \c green-coding; CREATE EXTENSION "uuid-ossp"; +CREATE EXTENSION "moddatetime"; -CREATE TABLE projects ( +CREATE TABLE machines ( + id SERIAL PRIMARY KEY, + description text, + available boolean DEFAULT false, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER machines_moddatetime + BEFORE UPDATE ON machines + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TABLE jobs ( + id SERIAL PRIMARY KEY, + state text, + name text, + email text, + url text, + branch text, + filename text, + categories int[], + machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER jobs_moddatetime + BEFORE UPDATE ON jobs + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TABLE runs ( id uuid DEFAULT uuid_generate_v4() PRIMARY KEY, + job_id integer REFERENCES jobs(id) ON DELETE SET NULL ON UPDATE CASCADE UNIQUE, name text, uri text, branch text, @@ -15,85 +47,109 @@ CREATE TABLE projects ( usage_scenario json, filename text, machine_specs jsonb, - machine_id int DEFAULT 1, + runner_arguments json, + machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE, gmt_hash text, measurement_config jsonb, start_measurement bigint, end_measurement bigint, - phases JSON DEFAULT null, - logs text DEFAULT null, - invalid_project text, - last_run timestamp with time zone, - created_at timestamp with time zone DEFAULT now() + phases JSON, + logs text, + invalid_run text, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone ); +CREATE TRIGGER runs_moddatetime + BEFORE UPDATE ON runs + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + CREATE TABLE measurements ( id SERIAL PRIMARY KEY, - project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE , + run_id uuid NOT NULL REFERENCES runs(id) ON DELETE CASCADE ON UPDATE CASCADE , detail_name text NOT NULL, metric text NOT NULL, value bigint NOT NULL, unit text NOT NULL, time bigint NOT NULL, - created_at timestamp with time zone DEFAULT now() + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone ); - -CREATE UNIQUE INDEX measurements_get ON measurements(project_id ,metric ,detail_name ,time ); -CREATE INDEX measurements_build_and_store_phase_stats ON measurements(project_id, metric, unit, detail_name); +CREATE UNIQUE INDEX measurements_get ON measurements(run_id ,metric ,detail_name ,time ); +CREATE INDEX measurements_build_and_store_phase_stats ON measurements(run_id, metric, unit, detail_name); CREATE INDEX measurements_build_phases ON measurements(metric, unit, detail_name); +CREATE TRIGGER measurements_moddatetime + BEFORE UPDATE ON measurements + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); -CREATE TABLE categories ( +CREATE TABLE network_intercepts ( id SERIAL PRIMARY KEY, - name text, - parent_id int REFERENCES categories(id) ON DELETE CASCADE ON UPDATE CASCADE, - created_at timestamp with time zone DEFAULT now() + run_id uuid NOT NULL REFERENCES runs(id) ON DELETE CASCADE ON UPDATE CASCADE , + time bigint NOT NULL, + connection_type text NOT NULL, + protocol text NOT NULL, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone ); +CREATE TRIGGER network_intercepts_moddatetime + BEFORE UPDATE ON network_intercepts + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); -CREATE TABLE machines ( + +CREATE TABLE categories ( id SERIAL PRIMARY KEY, - description text, - available boolean DEFAULT false, + name text, + parent_id int REFERENCES categories(id) ON DELETE CASCADE ON UPDATE CASCADE, created_at timestamp with time zone DEFAULT now(), - updated_at timestamp with time zone DEFAULT NULL + updated_at timestamp with time zone ); +CREATE TRIGGER categories_moddatetime + BEFORE UPDATE ON categories + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + CREATE TABLE phase_stats ( id SERIAL PRIMARY KEY, - project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE, + run_id uuid NOT NULL REFERENCES runs(id) ON DELETE CASCADE ON UPDATE CASCADE, metric text NOT NULL, detail_name text NOT NULL, phase text NOT NULL, value bigint NOT NULL, type text NOT NULL, - max_value bigint DEFAULT NULL, - min_value bigint DEFAULT NULL, + max_value bigint, + min_value bigint, unit text NOT NULL, - created_at timestamp with time zone DEFAULT now() + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone ); -CREATE INDEX "phase_stats_project_id" ON "phase_stats" USING HASH ("project_id"); +CREATE INDEX "phase_stats_run_id" ON "phase_stats" USING HASH ("run_id"); +CREATE TRIGGER phase_stats_moddatetime + BEFORE UPDATE ON phase_stats + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + -CREATE TABLE jobs ( - id SERIAL PRIMARY KEY, - project_id uuid REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE DEFAULT null, - type text, - machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE DEFAULT null, - failed boolean DEFAULT false, - running boolean DEFAULT false, - last_run timestamp with time zone, - created_at timestamp with time zone DEFAULT now() -); CREATE TABLE notes ( id SERIAL PRIMARY KEY, - project_id uuid REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE, + run_id uuid REFERENCES runs(id) ON DELETE CASCADE ON UPDATE CASCADE, detail_name text, note text, time bigint, - created_at timestamp with time zone DEFAULT now() + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone ); -CREATE INDEX "notes_project_id" ON "notes" USING HASH ("project_id"); +CREATE INDEX "notes_run_id" ON "notes" USING HASH ("run_id"); +CREATE TRIGGER notes_moddatetime + BEFORE UPDATE ON notes + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); CREATE TABLE ci_measurements ( @@ -102,24 +158,126 @@ CREATE TABLE ci_measurements ( energy_unit text, repo text, branch text, - workflow text, + workflow_id text, + workflow_name text, run_id text, - cpu text DEFAULT NULL, + cpu text, cpu_util_avg int, - commit_hash text DEFAULT NULL, + commit_hash text, label text, duration bigint, source text, - project_id uuid REFERENCES projects(id) ON DELETE SET NULL ON UPDATE CASCADE DEFAULT null, - created_at timestamp with time zone DEFAULT now() + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone ); -CREATE INDEX "ci_measurements_get" ON ci_measurements(repo, branch, workflow, run_id, created_at); +CREATE INDEX "ci_measurements_get" ON ci_measurements(repo, branch, workflow_id, run_id, created_at); +CREATE TRIGGER ci_measurements_moddatetime + BEFORE UPDATE ON ci_measurements + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + CREATE TABLE client_status ( id SERIAL PRIMARY KEY, status_code TEXT NOT NULL, - machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE DEFAULT null, + machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE, "data" TEXT, - project_id uuid REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE, - created_at timestamp with time zone DEFAULT now() -); \ No newline at end of file + run_id uuid REFERENCES runs(id) ON DELETE CASCADE ON UPDATE CASCADE, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER client_status_moddatetime + BEFORE UPDATE ON client_status + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TABLE timeline_projects ( + id SERIAL PRIMARY KEY, + name text, + url text, + categories integer[], + branch text DEFAULT 'NULL'::text, + filename text, + machine_id integer REFERENCES machines(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL, + schedule_mode text NOT NULL, + last_scheduled timestamp with time zone, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER timeline_projects_moddatetime + BEFORE UPDATE ON timeline_projects + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TABLE hog_measurements ( + id SERIAL PRIMARY KEY, + time bigint NOT NULL, + machine_uuid uuid NOT NULL, + elapsed_ns bigint NOT NULL, + combined_energy int, + cpu_energy int, + gpu_energy int, + ane_energy int, + energy_impact int, + thermal_pressure text, + settings jsonb, + data jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER hog_measurements_moddatetime + BEFORE UPDATE ON hog_measurements + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE INDEX idx_hog_measurements_machine_uuid ON hog_measurements USING hash (machine_uuid); +CREATE INDEX idx_hog_measurements_time ON hog_measurements (time); + + +CREATE TABLE hog_coalitions ( + id SERIAL PRIMARY KEY, + measurement integer REFERENCES hog_measurements(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL, + name text NOT NULL, + cputime_ns bigint, + cputime_per int, + energy_impact int, + diskio_bytesread bigint, + diskio_byteswritten bigint, + intr_wakeups bigint, + idle_wakeups bigint, + data jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER hog_coalitions_moddatetime + BEFORE UPDATE ON hog_coalitions + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE INDEX idx_coalition_energy_impact ON hog_coalitions(energy_impact); +CREATE INDEX idx_coalition_name ON hog_coalitions(name); + +CREATE TABLE hog_tasks ( + id SERIAL PRIMARY KEY, + coalition integer REFERENCES hog_coalitions(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL, + name text NOT NULL, + cputime_ns bigint, + cputime_per int, + energy_impact int, + bytes_received bigint, + bytes_sent bigint, + diskio_bytesread bigint, + diskio_byteswritten bigint, + intr_wakeups bigint, + idle_wakeups bigint, + + data jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER hog_tasks_moddatetime + BEFORE UPDATE ON hog_tasks + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE INDEX idx_task_coalition ON hog_tasks(coalition); diff --git a/frontend/ci-index.html b/frontend/ci-index.html index 6ed613ea1..90cfd0cdf 100644 --- a/frontend/ci-index.html +++ b/frontend/ci-index.html @@ -40,11 +40,10 @@

--> - +
- - + diff --git a/frontend/ci.html b/frontend/ci.html index de275557a..3041c960f 100644 --- a/frontend/ci.html +++ b/frontend/ci.html @@ -18,6 +18,7 @@ + @@ -104,7 +105,7 @@

Run Stats

- + @@ -137,7 +138,7 @@

Avg. CPU Util.

diff --git a/frontend/compare.html b/frontend/compare.html index d887a17a8..0fa00d9a4 100644 --- a/frontend/compare.html +++ b/frontend/compare.html @@ -38,10 +38,10 @@

-

Project Data

+

Run Data

-

Repositories ( / )Branch Repository ( / )
Label Energy TimeAvg. CPU Util.
Avg. CPU Util.
Total
Duration Commit Hash
+
diff --git a/frontend/css/green-coding.css b/frontend/css/green-coding.css index abe2ba76f..164802fae 100644 --- a/frontend/css/green-coding.css +++ b/frontend/css/green-coding.css @@ -18,11 +18,6 @@ Thanks to https://css-tricks.com/transitions-only-after-page-load/ */ margin-bottom: 26px !important; } -#project-uri { - word-break: break-all; - overflow-wrap: break-all; -} - #chart-container { margin-top: 14px; } @@ -60,11 +55,11 @@ Thanks to https://css-tricks.com/transitions-only-after-page-load/ */ } /* request.html: */ -#new-project-description>div { +#new-software-description>div { margin-bottom: 10px; } -#new-project-name-div { +#new-software-name-div { margin-top: 10px; } @@ -265,3 +260,6 @@ a, float: right; } +.wide.card { + width: 400px !important; +} \ No newline at end of file diff --git a/frontend/data-analysis.html b/frontend/data-analysis.html index 4f3b48b91..4db514d18 100644 --- a/frontend/data-analysis.html +++ b/frontend/data-analysis.html @@ -24,7 +24,7 @@

Data Analysis

-
+
Data Analysis of our Open Data API
diff --git a/frontend/energy-timeline.html b/frontend/energy-timeline.html new file mode 100644 index 000000000..ea9222f48 --- /dev/null +++ b/frontend/energy-timeline.html @@ -0,0 +1,48 @@ + + + + + + + + + + + + + Green Metrics Tool + + + + + + + + + + + + + + + +
+

+ + Green Metrics Tool - Energy Timeline +

+
+
+
Energy Timeline projects overview
+
+

This page shows you all projects that are monitored recurringly.

+

The Green Metrics Tool calls this Energy Timeline.

+

In this chart view you can see how the energy cost for a specific scenario in a project has developed over time.

+

From this view you can go the detail statistics page for every run and also conveniently see the badges for the corresponding projects in one view.

+
+
+
+
+
+ + \ No newline at end of file diff --git a/frontend/hog-details.html b/frontend/hog-details.html new file mode 100644 index 000000000..4bababea2 --- /dev/null +++ b/frontend/hog-details.html @@ -0,0 +1,88 @@ + + + + + + + + + + + + + Green Metrics Tool + + + + + + + + + + + + + + + + + + + + + +
+

+ + Green Metrics Tool - Power HOG +

+
+
+
Your machine stats
+
+

You can click on a bar that you want more details about. Please note that we don't show this one bar + as the surrounding processes might also influence the results. Please use the zoom to look at one specific bar. +

+

+
+
+
+
+ +
Loading and processing data
+ +
+
+ Overall energy statistics +
+

+
+
+ + +
+
+ Per process energy statistics +
+

+ + + + + + \ No newline at end of file diff --git a/frontend/hog.html b/frontend/hog.html new file mode 100644 index 000000000..7f41baa60 --- /dev/null +++ b/frontend/hog.html @@ -0,0 +1,51 @@ + + + + + + + + + + + + + Green Metrics Tool + + + + + + + + + + + + + + + + + +
+

+ + Green Metrics Tool - Power HOG +

+
+
+
Status overview
+
+

This list shows you all processes we have monitored on the machines that run the power hog.

+

If you are looking for the data from your install please click on "Details" in the power hog desktop app.

+

Number of Machines this data is based on: 0

+

+
+
+
+

Processes

+
+
+ + \ No newline at end of file diff --git a/frontend/index.html b/frontend/index.html index 61f7c9500..f9ad27305 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -14,13 +14,14 @@ + + - @@ -31,28 +32,21 @@

- Green Metrics Tool Dashboard + Green Metrics Tool - Home

-
- +
-

- On the overview page you can go into the details for a specific measurement, or compare them. -

-

Comparing is possible for different repos, different branches and different runs as long as they are on the same machine. Also different machines are supported if you compare the same repo. +

Welcome to the home page
+
+

The home page of the Green Metrics Tool shows you the last 50 runs +

+

From here you can go into the details for a specific measurement, or compare them.

+

Comparing is possible for different repos, different branches and different runs as long as they are on the same machine. Also different machines are supported if you compare the same repo.

Currently we do not support comparing different machines AND different repos at the same time. The tool will notifiy you if you attempt to do so :)

+
- - \ No newline at end of file diff --git a/frontend/js/ci-index.js b/frontend/js/ci-index.js index 1951d67af..89b032a0b 100644 --- a/frontend/js/ci-index.js +++ b/frontend/js/ci-index.js @@ -1,78 +1,97 @@ (async () => { try { - var api_data = await makeAPICall('/v1/ci/projects') + var api_data = await makeAPICall('/v1/ci/projects'); } catch (err) { - showNotification('Could not get data from API', err); - return; + showNotification('Could not get data from API', err); + return; } - api_data.data.forEach(el => { - - const repo = el[0] - const branch = el[1] - const workflow = el[2] - const source = el[3] - const last_run = el[4] - let uri_display = repo; - let uri = repo; + const projectsTableBody = document.querySelector('#projects-table tbody'); + let currentRepoRow = null; // Track the current repository row - if (source == 'github'){ - uri_display = `${repo}`; - uri = `https://www.github.com/${repo}`; - } else if (source == 'gitlab'){ - uri_display = `${repo}`; - uri = `https://www.gitlab.com/${repo}`; - } else if (source == 'bitbucket'){ - uri_display = `${repo}`; - uri = `https://bitbucket.com/${repo}`; - } - - uri_link = `${uri_display} `; + api_data.data.forEach(el => { + const repo = el[0]; + const branch = el[1]; + const workflow_id = el[2]; + const source = el[3]; + const last_run = el[4]; + let workflow_name = el[5]; + if (workflow_name == '' || workflow_name == null) { + workflow_name = workflow_id; + } - // insert new accordion row if repository not known - let td_node = document.querySelector(`td[data-uri='${uri}_${branch}']`) - if (td_node == null || td_node == undefined) { - let row = document.querySelector('#projects-table tbody').insertRow() - row.innerHTML = ` - + const repo_esc = escapeString(repo); + // Check if it's a new repository + if (currentRepoRow === null || currentRepoRow.repo !== repo_esc) { + // Create a row for the repository with an accordion + currentRepoRow = projectsTableBody.insertRow(); + currentRepoRow.repo = repo_esc; + currentRepoRow.innerHTML = ` +
-
- ${uri_link} -
-
-
+
+ + ${getRepoLink(repo_esc, source)} +
+
+ + + + + + + + + + + +
WorkflowBranchLast RunWorkflow IDSource
+
- ${branch}`; - let content = document.querySelector(`#projects-table td[data-uri='${uri}_${branch}'] div.content`); - content.innerHTML = ` - - - - - - - - - - - -
WorkflowBranchLast RunSource
`; + `; } + const content = currentRepoRow.querySelector('.content table tbody'); - let inner_row = document.querySelector(`#projects-table td[data-uri='${uri}_${branch}'] div.content table tbody`).insertRow(); - - inner_row.innerHTML = ` - ${workflow} - ${branch} - ${dateToYMD(new Date(last_run))} - ${source}`; - + // Add branch as a row within the accordion content + const branchRow = content.insertRow(); + branchRow.innerHTML = ` + ${escapeString(workflow_name)} + ${escapeString(branch)} + ${dateToYMD(new Date(last_run))} + ${escapeString(workflow_id)} + ${escapeString(source)} + `; }); + // Initialize the accordion $('.ui.accordion').accordion(); - $('#projects-table table').tablesort(); - })(); + +// Function to generate the repository link +function getRepoLink(repo, source) { + let iconClass = ''; + if (source.startsWith('github')) { + iconClass = 'github'; + } else if (source.startsWith('gitlab')) { + iconClass = 'gitlab'; + } else if (source.startsWith('bitbucket')) { + iconClass = 'bitbucket'; + } + + // Assumes the repo var is sanitized before being sent to this function + return `${repo} `; +} + +// Function to generate the repository URI +function getRepoUri(repo, source) { + if (source.startsWith('github')) { + return `https://www.github.com/${repo}`; + } else if (source.startsWith('gitlab')) { + return `https://www.gitlab.com/${repo}`; + } else if (source.startsWith('bitbucket')) { + return `https://bitbucket.com/${repo}`; + } +} diff --git a/frontend/js/ci.js b/frontend/js/ci.js index 0df45b0f4..e78b0f5ba 100644 --- a/frontend/js/ci.js +++ b/frontend/js/ci.js @@ -1,100 +1,123 @@ -const convertValue = (value, unit) => { - switch (unit) { - case 'mJ': - return [value / 1000, 'Joules']; - break; - default: - return [value, unit]; // no conversion in default calse - } - -} - -const calculateStats = (measurements) => { - let energyMeasurements = measurements.map(measurement => measurement[0]); - let energySum = energyMeasurements.reduce((a, b) => a + b, 0); - let timeMeasurements = measurements.map(measurement => measurement[7]); - let timeSum = timeMeasurements.reduce((a, b) => a + b, 0); - let cpuUtilMeasurments = measurements.map(measurement => measurement[9]); +const numberFormatter = new Intl.NumberFormat('en-US', { + style: 'decimal', + maximumFractionDigits: 2, +}); - let energyAverage = math.mean(energyMeasurements); - let timeAverage = math.mean(timeMeasurements); - let cpuUtilAverage = math.mean(cpuUtilMeasurments); +const calculateStats = (energy_measurements, time_measurements, cpu_util_measurements) => { + let energyAverage = '--' + let energyStdDeviation = '--' + let energyStdDevPercent = '--' + let energySum = '--'; + + let timeAverage = '--' + let timeStdDeviation = '--' + let timeStdDevPercent = '--' + let timeSum = '--'; + + let cpuUtilStdDeviation = '--' + let cpuUtilAverage = '--' + let cpuUtilStdDevPercent = '--' + + if (energy_measurements.length > 0) { + energyStdDeviation = Math.round(math.std(energy_measurements, normalization="uncorrected")); + energyAverage = Math.round(math.mean(energy_measurements)); + energyStdDevPercent = Math.round((energyStdDeviation / energyAverage) * 100); + energySum = Math.round(math.sum(energy_measurements)); + } - let energyStdDeviation = math.std(energyMeasurements); - let timeStdDeviation = math.std(timeMeasurements); - let cpuUtilStdDeviation = math.std(cpuUtilMeasurments); + if (time_measurements.length > 0) { + timeStdDeviation = Math.round(math.std(time_measurements, normalization="uncorrected")); + timeAverage = Math.round(math.mean(time_measurements)); + timeStdDevPercent = Math.round((timeStdDeviation / timeAverage) * 100); + timeSum = Math.round(math.sum(time_measurements)); + } - let energyStdDevPercent = (energyStdDeviation / energyAverage) * 100; - let timeStdDevPercent = (timeStdDeviation / timeAverage) * 100; - let cpuUtilStdDevPercent = (cpuUtilStdDeviation / cpuUtilAverage) * 100; + if (cpu_util_measurements.length > 0) { + cpuUtilStdDeviation = Math.round(math.std(cpu_util_measurements, normalization="uncorrected")); + cpuUtilAverage = Math.round(math.mean(cpu_util_measurements)); + cpuUtilStdDevPercent = Math.round((cpuUtilStdDeviation / cpuUtilAverage) * 100); + } return { energy: { - average: Math.round(energyAverage), - stdDeviation: Math.round(energyStdDeviation), - stdDevPercent: Math.round(energyStdDevPercent), - total: Math.round(energySum) + average: energyAverage, + stdDeviation: energyStdDeviation, + stdDevPercent: energyStdDevPercent, + total: energySum }, time: { - average: Math.round(timeAverage), - stdDeviation: Math.round(timeStdDeviation), - stdDevPercent: Math.round(timeStdDevPercent), - total: Math.round(timeSum) + average: timeAverage, + stdDeviation: timeStdDeviation, + stdDevPercent: timeStdDevPercent, + total: timeSum }, cpu_util: { - average: Math.round(cpuUtilAverage), - stdDeviation: Math.round(cpuUtilStdDeviation), - stdDevPercent: Math.round(cpuUtilStdDevPercent) + average: cpuUtilAverage, + stdDeviation: cpuUtilStdDeviation, + stdDevPercent: cpuUtilStdDevPercent }, - count: measurements.length }; }; -const getStatsofLabel = (measurements, label) => { - let filteredMeasurements = measurements.filter(measurement => measurement[4] === label); - - if (filteredMeasurements.length === 0) { - return { average: NaN, stdDeviation: NaN }; - } - - return calculateStats(filteredMeasurements); -}; - -const getFullRunStats = (measurements) => { - let combinedMeasurements = []; - - let sumByRunId = {}; +const createStatsArrays = (measurements) => { // iterates 2n times (1 full, 1 by run ID) + const measurementsByRun = {} + const measurementsByLabel = {} measurements.forEach(measurement => { - const runId = measurement[2]; - - if (!sumByRunId[runId]) { - sumByRunId[runId] = { - energySum: 0, - timeSum: 0, - cpuUtilSum: 0, + const run_id = measurement[2] + const energy = measurement[0] + const time = measurement[7] + const cpuUtil = measurement[9] + const label = measurement[4] + + if (!measurementsByLabel[label]) { + measurementsByLabel[label] = { + energy: [], + time: [], + cpu_util: [], count: 0 }; } + if (!measurementsByRun[run_id]) { + measurementsByRun[run_id] = { + energy: 0, + time: 0, + cpu_util: [] + }; + } - sumByRunId[runId].energySum += measurement[0]; - sumByRunId[runId].timeSum += measurement[7]; - sumByRunId[runId].cpuUtilSum += measurement[9]; - sumByRunId[runId].count++; + if (energy != null) { + measurementsByLabel[label].energy.push(energy); + measurementsByRun[run_id].energy += energy; + } + if (time != null) { + measurementsByLabel[label].time.push(time); + measurementsByRun[run_id].time += time; + } + if (cpuUtil != null) { + measurementsByLabel[label].cpu_util.push(cpuUtil); + measurementsByRun[run_id].cpu_util.push(cpuUtil); + } + measurementsByLabel[label].count += 1; }); - for (const runId in sumByRunId) { - const avgCpuUtil = sumByRunId[runId].cpuUtilSum / sumByRunId[runId].count; // Calculate the average - combinedMeasurements.push({ - 0: sumByRunId[runId].energySum, - 7: sumByRunId[runId].timeSum, - 9: avgCpuUtil, // Use the calculated average - 2: runId - }); + const measurementsForFullRun = { + energy: [], + time: [], + cpu_util: [], + count: 0 + }; + + for (const run_id in measurementsByRun) { + if (measurementsByRun[run_id].energy) measurementsForFullRun.energy.push(measurementsByRun[run_id].energy); + if (measurementsByRun[run_id].time) measurementsForFullRun.time.push(measurementsByRun[run_id].time); + if (measurementsByRun[run_id].cpu_util.length > 0) measurementsForFullRun.cpu_util.push(math.mean(measurementsByRun[run_id].cpu_util)); + measurementsForFullRun.count += 1; } - return calculateStats(combinedMeasurements); -}; + return [measurementsForFullRun, measurementsByLabel]; + +} const createChartContainer = (container, el) => { const chart_node = document.createElement("div") @@ -140,34 +163,34 @@ const getEChartsOptions = () => { }; } -const filterMeasurements = (measurements, start_date, end_date, selectedLegends) => { - let filteredMeasurements = []; - let discard_measurements = []; +const filterMeasurements = (measurements, start_date, end_date) => { + const filteredMeasurements = []; + const discardMeasurements = []; measurements.forEach(measurement => { - let run_id = measurement[2]; - let timestamp = new Date(measurement[3]); + const run_id = measurement[2]; + const timestamp = new Date(measurement[3]); - if (timestamp >= start_date && timestamp <= end_date && selectedLegends[measurement[5]]) { + if (timestamp >= start_date && timestamp <= end_date){ filteredMeasurements.push(measurement); } else { - discard_measurements.push(run_id); + discardMeasurements.push(run_id); } }); - displayStatsTable(filteredMeasurements); // Update stats table return filteredMeasurements; } const getChartOptions = (measurements, chart_element) => { - let options = getEChartsOptions(); + const options = getEChartsOptions(); options.title.text = `Workflow energy cost per run [mJ]`; - let legend = new Set() - let labels = [] + const legend = new Set() + const labels = [] measurements.forEach(measurement => { // iterate over all measurements, which are in row order let [value, unit, run_id, timestamp, label, cpu, commit_hash, duration, source, cpu_util] = measurement; + cpu_util = cpu_util ? cpu_util : '--'; options.series.push({ type: 'bar', smooth: true, @@ -185,16 +208,22 @@ const getChartOptions = (measurements, chart_element) => { }); options.legend.data = Array.from(legend) + // set options.legend.selected to true for all cpus + options.legend.selected = {} + options.legend.data.forEach(cpu => { + options.legend.selected[cpu] = true + }) + options.tooltip = { trigger: 'item', formatter: function (params, ticket, callback) { - return `${labels[params.componentIndex].labels[params.dataIndex]}
- run_id: ${labels[params.componentIndex].run_id}
+ return `${escapeString(labels[params.componentIndex].labels[params.dataIndex])}
+ run_id: ${escapeString(labels[params.componentIndex].run_id)}
timestamp: ${labels[params.componentIndex].timestamp}
- commit_hash: ${labels[params.componentIndex].commit_hash}
- value: ${labels[params.componentIndex].value} ${labels[params.componentIndex].unit}
- duration: ${labels[params.componentIndex].duration} seconds
- avg. cpu. utilization: ${labels[params.componentIndex].cpu_util}%
+ commit_hash: ${escapeString(labels[params.componentIndex].commit_hash)}
+ value: ${escapeString(labels[params.componentIndex].value)} ${escapeString(labels[params.componentIndex].unit)}
+ duration: ${escapeString(labels[params.componentIndex].duration)} seconds
+ avg. cpu. utilization: ${escapeString(labels[params.componentIndex].cpu_util)}%
`; } }; @@ -205,7 +234,7 @@ const getChartOptions = (measurements, chart_element) => { const displayGraph = (measurements) => { const element = createChartContainer("#chart-container", "run-energy"); - const options = getChartOptions(measurements, element); + const options = getChartOptions(measurements, element); // iterates const chart_instance = echarts.init(element); chart_instance.setOption(options); @@ -219,10 +248,12 @@ const displayGraph = (measurements) => { // are >= startValue <= endValue // either copying element or reducing it by checking if int or not + + chart_instance.on('dataZoom', function (evt) { let sum = 0; if (!('startValue' in evt.batch[0])) return - for (var i = evt.batch[0].startValue; i <= evt.batch[0].endValue; i++) { + for (let i = evt.batch[0].startValue; i <= evt.batch[0].endValue; i++) { sum = sum + options.dataset.source[i].slice(1).reduce((partialSum, a) => partialSum + a, 0); } }) @@ -230,67 +261,61 @@ const displayGraph = (measurements) => { chart_instance.resize(); } - chart_instance.on('legendselectchanged', function (params) { - const selectedLegends = params.selected; - const filteredMeasurements = measurements.filter(measurement => selectedLegends[measurement[5]]); - - displayStatsTable(filteredMeasurements); - }); - return chart_instance; } const displayStatsTable = (measurements) => { - let labels = new Set() - measurements.forEach(measurement => { - labels.add(measurement[4]) - }); + const [fullRunArray, labelsArray] = createStatsArrays(measurements); // iterates 2n times const tableBody = document.querySelector("#label-stats-table"); tableBody.innerHTML = ""; - const label_full_stats_node = document.createElement("tr") - full_stats = getFullRunStats(measurements) - label_full_stats_node.innerHTML += ` - Full Run - ${full_stats.energy.average} mJ - ${full_stats.energy.stdDeviation} mJ - ${full_stats.energy.stdDevPercent}% - ${full_stats.time.average}s - ${full_stats.time.stdDeviation}s - ${full_stats.time.stdDevPercent}% - ${full_stats.cpu_util.average}% - ${full_stats.energy.total} mJ - ${full_stats.time.total}s - ${full_stats.count} + const full_run_stats_node = document.createElement("tr") + full_run_stats = calculateStats(fullRunArray.energy, fullRunArray.time, fullRunArray.cpu_util) + + full_run_stats_node.innerHTML += ` + Full Run + ${numberFormatter.format(full_run_stats.energy.average)} mJ + ${numberFormatter.format(full_run_stats.energy.stdDeviation)} mJ + ${full_run_stats.energy.stdDevPercent}% + ${numberFormatter.format(full_run_stats.time.average)}s + ${numberFormatter.format(full_run_stats.time.stdDeviation)}s + ${full_run_stats.time.stdDevPercent}% + ${numberFormatter.format(full_run_stats.cpu_util.average)}% + ${numberFormatter.format(full_run_stats.energy.total)} mJ + ${numberFormatter.format(full_run_stats.time.total)}s + ${numberFormatter.format(fullRunArray.count)} ` - tableBody.appendChild(label_full_stats_node); + tableBody.appendChild(full_run_stats_node); - labels.forEach(label => { + for (const label in labelsArray) { + const label_stats = calculateStats(labelsArray[label].energy, labelsArray[label].time, labelsArray[label].cpu_util) const label_stats_node = document.createElement("tr") - let stats = getStatsofLabel(measurements, label); label_stats_node.innerHTML += ` - ${label} - ${stats.energy.average} mJ - ${stats.energy.stdDeviation} mJ - ${stats.energy.stdDevPercent}% - ${stats.time.average}s - ${stats.time.stdDeviation}s - ${stats.time.stdDevPercent}% - ${stats.cpu_util.average}% - ${stats.energy.total} mJ - ${stats.time.total}s - ${stats.count} + ${label} + ${numberFormatter.format(label_stats.energy.average)} mJ + ${numberFormatter.format(label_stats.energy.stdDeviation)} mJ + ${label_stats.energy.stdDevPercent}% + ${numberFormatter.format(label_stats.time.average)}s + ${numberFormatter.format(label_stats.time.stdDeviation)}s + ${label_stats.time.stdDevPercent}% + ${numberFormatter.format(label_stats.cpu_util.average)}% + ${numberFormatter.format(label_stats.energy.total)} mJ + ${numberFormatter.format(label_stats.time.total)}s + ${numberFormatter.format(labelsArray[label].count)} ` - document.querySelector("#label-stats-table").appendChild(label_stats_node); - }); + document.querySelector("#label-stats-table").appendChild(label_stats_node); + }; } const displayCITable = (measurements, url_params) => { + + const repo_esc = escapeString(url_params.get('repo')) + measurements.forEach(el => { const li_node = document.createElement("tr"); - [energy_value, energy_unit] = convertValue(el[0], el[1]) + const [energy_value, energy_unit] = convertValue(el[0], el[1]) const value = `${energy_value} ${energy_unit}`; const run_id = el[2]; @@ -299,17 +324,20 @@ const displayCITable = (measurements, url_params) => { const short_hash = commit_hash.substring(0, 7); const tooltip = `title="${commit_hash}"`; const source = el[8]; - const cpu_avg = el[9]; + const cpu_avg = el[9] ? el[9] : '--'; + + let run_link = ''; + + const run_id_esc = escapeString(run_id) - var run_link = '' if(source == 'github') { - run_link = `https://github.com/${escapeString(url_params.get('repo'))}/actions/runs/${escapeString(run_id)}`; + run_link = `https://github.com/${repo_esc}/actions/runs/${run_id_esc}`; } else if (source == 'gitlab') { - run_link = `https://gitlab.com/${escapeString(url_params.get('repo'))}/-/pipelines/${escapeString(run_id)}` + run_link = `https://gitlab.com/${repo_esc}/-/pipelines/${run_id_esc}` } - const run_link_node = `${escapeString(run_id)}` + const run_link_node = `${run_id_esc}` const created_at = el[3] @@ -320,10 +348,10 @@ const displayCITable = (measurements, url_params) => { ${run_link_node}\ ${escapeString(label)}\ ${dateToYMD(new Date(created_at))}\ - ${escapeString(value)}\ + ${escapeString(numberFormatter.format(value))}\ ${escapeString(cpu)}\ - ${cpu_avg}% - ${duration} seconds + ${escapeString(cpu_avg)}% + ${escapeString(duration)} seconds ${escapeString(short_hash)}\ `; document.querySelector("#ci-table").appendChild(li_node); @@ -365,6 +393,7 @@ $(document).ready((e) => { const link_node = document.createElement("a") const img_node = document.createElement("img") img_node.src = `${API_URL}/v1/ci/badge/get?repo=${url_params.get('repo')}&branch=${url_params.get('branch')}&workflow=${url_params.get('workflow')}` + link_node.href = window.location.href link_node.appendChild(img_node) document.querySelector("span.energy-badge-container").appendChild(link_node) document.querySelector(".copy-badge").addEventListener('click', copyToClipboard) @@ -373,7 +402,7 @@ $(document).ready((e) => { } try { - api_string=`/v1/ci/measurements?repo=${url_params.get('repo')}&branch=${url_params.get('branch')}&workflow=${url_params.get('workflow')}`; + const api_string=`/v1/ci/measurements?repo=${url_params.get('repo')}&branch=${url_params.get('branch')}&workflow=${url_params.get('workflow')}`; var measurements = await makeAPICall(api_string); } catch (err) { showNotification('Could not get data from API', err); @@ -381,7 +410,13 @@ $(document).ready((e) => { } let repo_link = '' - let source = measurements.data[0][8] + const source = measurements.data[0][8] + const workflow_id = escapeString(url_params.get('workflow')) + let workflow_name = measurements.data[0][10] + + if (workflow_name == '' || workflow_name == null) { + workflow_name = workflow_id ; + } if(source == 'github') { repo_link = `https://github.com/${escapeString(url_params.get('repo'))}`; @@ -389,26 +424,58 @@ $(document).ready((e) => { else if(source == 'gitlab') { repo_link = `https://gitlab.com/${escapeString(url_params.get('repo'))}`; } - //${repo_link} + const repo_link_node = `${escapeString(url_params.get('repo'))}` - document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `Repository:${repo_link_node}`) - document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `Branch:${escapeString(url_params.get('branch'))}`) - document.querySelector('#ci-data').insertAdjacentHTML('afterbegin', `Workflow:${escapeString(url_params.get('workflow'))}`) + const ci_data_node = document.querySelector('#ci-data') + ci_data_node.insertAdjacentHTML('afterbegin', `Repository:${repo_link_node}`) + ci_data_node.insertAdjacentHTML('afterbegin', `Branch:${escapeString(url_params.get('branch'))}`) + ci_data_node.insertAdjacentHTML('afterbegin', `Workflow ID:${escapeString(workflow_id)}`) + ci_data_node.insertAdjacentHTML('afterbegin', `Workflow:${escapeString(workflow_name)}`) - displayCITable(measurements.data, url_params); + displayCITable(measurements.data, url_params); // Iterates I (total: 1) - chart_instance = displayGraph(measurements.data) - displayStatsTable(measurements.data) + chart_instance = displayGraph(measurements.data) // iterates I (total: 2) + + displayStatsTable(measurements.data) // iterates II (total: 4) dateTimePicker(); - $('#submit').on('click', function () { - var startDate = new Date($('#rangestart input').val()); - var endDate = new Date($('#rangeend input').val()); + // On legend change, recalculate stats table + chart_instance.on('legendselectchanged', function (params) { + // get list of all legends that are on + const selectedLegends = params.selected; + const filteredMeasurements = measurements.data.filter(measurement => selectedLegends[measurement[5]]); + + displayStatsTable(filteredMeasurements); + }); + // When the user selects a subset of the measurement data via the date-picker + $('#submit').on('click', function () { + const startDate = new Date($('#rangestart input').val()); + const endDate = new Date($('#rangeend input').val()); + + const filteredMeasurements = filterMeasurements(measurements.data, startDate, endDate); // iterates I + displayStatsTable(filteredMeasurements); //iterates II + const options = getChartOptions(filteredMeasurements); //iterates I + + /* The following functionality is to "remember" a user's legend settings as they date switch + * it is turned off because if the user selects a subset that doesn't contain a cpu + * that cpu is treated as "off" even if the user didn't select it off themselves + * and therefore it is "misremembered" from a user point of view + * + * So for now, changing the date resets the legends to all true + * + // get the selected legends of the old chart instance, to remember what the user toggled on/off const selectedLegends = chart_instance.getOption().legend[0].selected; - const filteredMeasurements = filterMeasurements(measurements.data, startDate, endDate, selectedLegends); + + // go through options and turn off all legends that are not selected + for(const legend in options.legend.selected) { + if (!selectedLegends[legend]) { + options.legend.selected[legend] = false; + } + } + */ - options = getChartOptions(filteredMeasurements); + // set new chart instance options chart_instance.clear(); chart_instance.setOption(options); }); diff --git a/frontend/js/compare.js b/frontend/js/compare.js index 8709c4607..b82cabae3 100644 --- a/frontend/js/compare.js +++ b/frontend/js/compare.js @@ -31,11 +31,11 @@ $(document).ready( (e) => { let comparison_details = phase_stats_data.comparison_details.map((el) => replaceRepoIcon(el)); comparison_details = comparison_details.join(' vs. ') - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `Comparison Type${phase_stats_data.comparison_case}`) - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `Number of runs compared${runs}`) - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `${phase_stats_data.comparison_case}${comparison_details}`) + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `Comparison Type${phase_stats_data.comparison_case}`) + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `Number of runs compared${runs}`) + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `${phase_stats_data.comparison_case}${comparison_details}`) Object.keys(phase_stats_data['common_info']).forEach(function(key) { - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `${key}${phase_stats_data['common_info'][key]}`) + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `${key}${phase_stats_data['common_info'][key]}`) }); displayComparisonMetrics(phase_stats_data) diff --git a/frontend/js/energy-timeline.js b/frontend/js/energy-timeline.js new file mode 100644 index 000000000..ba2d7e8f2 --- /dev/null +++ b/frontend/js/energy-timeline.js @@ -0,0 +1,135 @@ +$(document).ready(function () { + + (async () => { + try { + var measurements = await makeAPICall('/v1/timeline-projects'); + } catch (err) { + showNotification('Could not get data from API', err); + return; + } + measurements.data.forEach(measurement => { + let [id, name, url, categories, branch, filename, machine_id, machine_description, schedule_mode, last_scheduled, created_at, updated_at, last_run, metrics] = measurement + filename = filename == null ? '': filename + branch = branch == null ? '': branch + + const chart_node = document.createElement("div") + chart_node.classList.add("card"); + chart_node.classList.add('ui') + chart_node.classList.add('wide') + + const url_link = `${replaceRepoIcon(url)} `; + chart_node.innerHTML = ` +
+
${name}
+
+ ${url_link} +
+
+
+

Monitoring since: ${dateToYMD(new Date(created_at), true)}

+

Branch: ${branch == '' ? '-': branch}

+

Filename: ${filename == '' ? '-': filename}

+

Machine: ${machine_description}

+

Schedule Mode: ${schedule_mode}

+

Last Run: ${last_run == '' ? '-' : dateToYMD(new Date(last_run))}

+ ` + + DEFAULT_ENERGY_TIMELINE_BADGE_METRICS.forEach(metric => { + const [metric_name, detail_name] = metric + chart_node.innerHTML = `${chart_node.innerHTML} +
+
+
+ ${METRIC_MAPPINGS[metric_name]['clean_name']} via + ${METRIC_MAPPINGS[metric_name]['source']} + - ${detail_name} + + + +
+ Image Failed to Load + +
+
+

` + }) + + chart_node.innerHTML = `${chart_node.innerHTML} +
+
+ + Show Timeline + +
+ + Show All Measurements + ` + + document.querySelector('#timeline-cards').appendChild(chart_node) + }); + document.querySelectorAll(".copy-badge").forEach(el => { + el.addEventListener('click', copyToClipboard) + }) + + })(); +}); + + +/* + $('#timeline-projects-table').DataTable({ + ajax: `${API_URL}/v1/timeline-projects`, + columns: [ + { data: 0, title: 'ID'}, + { data: 1, title: 'Url'}, +// { data: 2, title: 'Categories'}, + { data: 3, title: 'Branch'}, + { data: 4, title: 'Filename'}, + { data: 6, title: 'Machine'}, + { data: 7, title: 'Schedule Mode'}, + { data: 8, title: 'Last Scheduled', render: (data) => data == null ? '-' : dateToYMD(new Date(data)) }, + { data: 9, title: 'Created At', render: (data) => data == null ? '-' : dateToYMD(new Date(data)) }, + { data: 10, title: 'Updated At', render: (data) => data == null ? '-' : dateToYMD(new Date(data)) }, + { + data: 0, + title: 'Timeline Link', + render: function(name, type, row) { + return `Show Timeline` + }, + }, + { + data: 0, + title: 'Show all measurements', + render: function(name, type, row) { + return `Show all measurements` + }, + }, + { + data: 0, + title: 'Badges', + render: function(name, type, row) { + // iterate over the key metrics that shalle be displayed as badge + return `
+
+ ${METRIC_MAPPINGS['cores_energy_powermetrics_component']['clean_name']} via + ${METRIC_MAPPINGS['cores_energy_powermetrics_component']['source']} + - ${`[COMPONENT]`} + + + +
+ + +
+

` + + }, + }, + + + ], + deferRender: true, + order: [] // API determines order + }); + +*/ + diff --git a/frontend/js/helpers/charts.js b/frontend/js/helpers/charts.js index 6e17174b8..bca3a3492 100644 --- a/frontend/js/helpers/charts.js +++ b/frontend/js/helpers/charts.js @@ -98,7 +98,7 @@ const calculateMA = (series, factor) => { continue; } var sum = 0; - for (var j = 0; j < factor; j++) { + for (let j = 0; j < factor; j++) { sum += series[i - j].value; } result.push(sum / factor); @@ -481,7 +481,7 @@ const displayTotalChart = (legend, labels, data) => { let myChart = echarts.init(chartDom); let series = []; - for (key in data) { + for (const key in data) { series.push({ name: key, type: 'bar', diff --git a/frontend/js/helpers/config.js.example b/frontend/js/helpers/config.js.example index 500e95520..63609cc0e 100644 --- a/frontend/js/helpers/config.js.example +++ b/frontend/js/helpers/config.js.example @@ -235,6 +235,19 @@ METRIC_MAPPINGS = { }, } +// Here you can statically define the badges that shall be shown in the timeline view +// although this could also be done dynamically it would be a slightly more heavy lifting for the database and +// only reflect the latest run. +// This gives you a more fixed picture of what you want to show for the user and does not always change if you try out +// some configurations in your machine setups +const DEFAULT_ENERGY_TIMELINE_BADGE_METRICS = [ + // ['cpu_energy_rapl_msr_component','Package_0'], // uncomment if you want RAPL CPU energy in timeline overview + // ['memory_energy_rapl_msr_component','Package_0'], // uncomment if you want RAPL DRAM energy in timeline overview + // ['network_energy_formula_global','[FORMULA]'], // uncomment if you want network in timeline overview + ['psu_energy_ac_mcp_machine','[machine]'], + // ['software_carbon_intensity_global','[SYSTEM]'] // uncomment if you want SCI as badge in timeline overview +] + // title and filter function for the top left most chart in the Detailed Metrics / Compare view const TOTAL_CHART_BOTTOM_TITLE = 'Total energy consumption'; const TOTAL_CHART_BOTTOM_LABEL = 'Machine Energy'; diff --git a/frontend/js/helpers/converters.js b/frontend/js/helpers/converters.js index dda32d67d..e1bfa3c06 100644 --- a/frontend/js/helpers/converters.js +++ b/frontend/js/helpers/converters.js @@ -44,4 +44,5 @@ const rescaleCO2Value = (value,unit) => { else if(value > 1_000_000) return [(value/(10**6)).toFixed(2), 'g']; else if(value > 1_000) return [(value/(10**3)).toFixed(2), 'mg']; return [value.toFixed(2) , unit]; -} \ No newline at end of file +} + diff --git a/frontend/js/helpers/main.js b/frontend/js/helpers/main.js index a623e7396..a651e390c 100644 --- a/frontend/js/helpers/main.js +++ b/frontend/js/helpers/main.js @@ -15,6 +15,12 @@ class GMTMenu extends HTMLElement { Home + + Repositories + + + Energy Timeline + Measure software @@ -24,6 +30,12 @@ class GMTMenu extends HTMLElement { Eco-CI + + Status + + + Power Hog + Settings @@ -75,8 +87,10 @@ const showNotification = (message_title, message_text, type='warning') => { } const copyToClipboard = (e) => { + e.preventDefault(); if (navigator && navigator.clipboard && navigator.clipboard.writeText) - return navigator.clipboard.writeText(e.currentTarget.closest('.field').querySelector('span').innerHTML) + navigator.clipboard.writeText(e.currentTarget.previousElementSibling.innerHTML) + return false alert('Copying badge on local is not working due to browser security models') return Promise.reject('The Clipboard API is not available.'); @@ -140,34 +154,33 @@ async function makeAPICall(path, values=null) { return json_response; }; -(() => { - /* Menu toggling */ - let openMenu = function(e){ - $(this).removeClass('closed').addClass('opened'); - $(this).find('i').removeClass('right').addClass('left'); - $('#menu').removeClass('closed').addClass('opened'); - $('#main').removeClass('closed').addClass('opened'); - setTimeout(function(){window.dispatchEvent(new Event('resize'))}, 500) // needed for the graphs to resize - } - $(document).on('click','#menu-toggle.closed', openMenu); +/* Menu toggling */ +let openMenu = function(e){ + $(this).removeClass('closed').addClass('opened'); + $(this).find('i').removeClass('right').addClass('left'); + $('#menu').removeClass('closed').addClass('opened'); + $('#main').removeClass('closed').addClass('opened'); + setTimeout(function(){window.dispatchEvent(new Event('resize'))}, 500) // needed for the graphs to resize +} - let closeMenu = function(e){ - $(this).removeClass('opened').addClass('closed'); - $(this).find('i').removeClass('left').addClass('right'); - $('#menu').removeClass('opened').addClass('closed'); - $('#main').removeClass('opened').addClass('closed'); - setTimeout(function(){window.dispatchEvent(new Event('resize'))}, 500) // needed for the graphs to resize - } +let closeMenu = function(e){ + $(this).removeClass('opened').addClass('closed'); + $(this).find('i').removeClass('left').addClass('right'); + $('#menu').removeClass('opened').addClass('closed'); + $('#main').removeClass('opened').addClass('closed'); + setTimeout(function(){window.dispatchEvent(new Event('resize'))}, 500) // needed for the graphs to resize +} +$(document).ready(function () { + $(document).on('click','#menu-toggle.closed', openMenu); $(document).on('click','#menu-toggle.opened', closeMenu); - $(document).ready(function () { - if ($(window).width() < 960) { - $('#menu-toggle').removeClass('opened').addClass('closed'); - } - }); - $(window).on('load', function() { - $("body").removeClass("preload"); // activate tranisition CSS properties again - }); + if ($(window).width() < 960) { + $('#menu-toggle').removeClass('opened').addClass('closed'); + } +}); + +$(window).on('load', function() { + $("body").removeClass("preload"); // activate tranisition CSS properties again +}); -})(); diff --git a/frontend/js/helpers/phase-stats.js b/frontend/js/helpers/phase-stats.js index 3ad39bf36..ff8af1c10 100644 --- a/frontend/js/helpers/phase-stats.js +++ b/frontend/js/helpers/phase-stats.js @@ -105,7 +105,7 @@ const displayComparisonMetrics = (phase_stats_object) => { let total_chart_bottom_legend = {}; let total_chart_bottom_labels = []; - for (phase in phase_stats_object['data']) { + for (const phase in phase_stats_object['data']) { createPhaseTab(phase); // will not create already existing phase tabs createTableHeader( phase, @@ -131,11 +131,11 @@ const displayComparisonMetrics = (phase_stats_object) => { let found_bottom_chart_metric = false; const bottom_chart_present_keys = Object.fromEntries(phase_stats_object.comparison_details.map(e => [e, false])) - for (metric_name in phase_data) { + for (const metric_name in phase_data) { let metric_data = phase_data[metric_name] let found_radar_chart_item = false; - for (detail_name in metric_data['data']) { + for (const detail_name in metric_data['data']) { let detail_data = metric_data['data'][detail_name] /* @@ -230,7 +230,7 @@ const displayComparisonMetrics = (phase_stats_object) => { // a phase had no bottom chart metric and must be null-filled // this can for instance happen if a phase is too short and no metric was reported in the timespan - for (key in bottom_chart_present_keys) { + for (const key in bottom_chart_present_keys) { if(bottom_chart_present_keys[key] == false) { if(total_chart_bottom_data?.[`${TOTAL_CHART_BOTTOM_LABEL} - ${key}`] == null) { total_chart_bottom_data[`${TOTAL_CHART_BOTTOM_LABEL} - ${key}`] = [] diff --git a/frontend/js/helpers/runs.js b/frontend/js/helpers/runs.js new file mode 100644 index 000000000..66ed8e824 --- /dev/null +++ b/frontend/js/helpers/runs.js @@ -0,0 +1,165 @@ +const compareButton = () => { + let checkedBoxes = document.querySelectorAll('input[type=checkbox]:checked'); + + let link = '/compare.html?ids='; + checkedBoxes.forEach(checkbox => { + link = `${link}${checkbox.value},`; + }); + window.open(link.substr(0,link.length-1), '_blank'); +} +const updateCompareCount = () => { + const countButton = document.getElementById('compare-button'); + const checkedCount = document.querySelectorAll('input[type=checkbox]:checked').length; + countButton.textContent = `Compare: ${checkedCount} Run(s)`; +} + + +const allow_group_select_checkboxes = (checkbox_wrapper_id) => { + let lastChecked = null; + let checkboxes = document.querySelectorAll(checkbox_wrapper_id); + + for (let i=0;ij) { + [i, j] = [j, i] + } + + for (let c=0; c { + const urlSearchParams = new URLSearchParams(window.location.search); + urlSearchParams.delete(paramName); + const newUrl = `${window.location.pathname}?${urlSearchParams.toString()}`; + window.location.href = newUrl; +} + +const showActiveFilters = (key, value) => { + document.querySelector(`.ui.warning.message`).classList.remove('hidden'); + const newListItem = document.createElement("span"); + newListItem.innerHTML = `
${escapeString(key)}: ${escapeString(value)}
`; + document.querySelector(`.ui.warning.message ul`).appendChild(newListItem); + +} + +const getFilterQueryStringFromURI = () => { + const url_params = (new URLSearchParams(window.location.search)) + let query_string = ''; + if (url_params.get('uri') != null && url_params.get('uri').trim() != '') { + const uri = url_params.get('uri').trim() + query_string = `${query_string}&uri=${uri}` + showActiveFilters('uri', uri) + } + if (url_params.get('filename') != null && url_params.get('filename').trim() != '') { + const filename = url_params.get('filename').trim() + query_string = `${query_string}&filename=${filename}` + showActiveFilters('filename', filename) + } + if (url_params.get('branch') != null && url_params.get('branch').trim() != '') { + const branch = url_params.get('branch').trim() + query_string = `${query_string}&branch=${branch}` + showActiveFilters('branch', branch) + } + if (url_params.get('machine_id') != null && url_params.get('machine_id').trim() != '') { + const machine_id = url_params.get('machine_id').trim() + query_string = `${query_string}&machine_id=${machine_id}` + showActiveFilters('machine_id', machine_id) + } + if (url_params.get('machine') != null && url_params.get('machine').trim() != '') { + const machine = url_params.get('machine').trim() + query_string = `${query_string}&machine=${machine}` + showActiveFilters('machine', machine) + } + + + return query_string +} + +const getRunsTable = (el, url, include_uri=true, include_button=true, searching=false) => { + + const columns = [ + { + data: 1, + title: 'Name', + render: function(name, type, row) { + + if(row[9] == null) name = `${name} (in progress 🔥)`; + if(row[5] != null) name = `${name} invalidated`; + return `${name}` + }, + }, + ] + + if(include_uri) { + columns.push({ + data: 2, + title: '( / / etc.) Repo', + render: function(uri, type, row) { + let uri_link = replaceRepoIcon(uri); + + if (uri.startsWith("http")) { + uri_link = `${uri_link} `; + } + return uri_link + }, + }) + } + + columns.push({ data: 3, title: 'Branch'}); + + columns.push({ + data: 8, + title: 'Commit', + render: function(commit, type, row) { + // Modify the content of the "Name" column here + return commit == null ? null : `${commit.substr(0,3)}...${commit.substr(-3,3)}` + }, + }); + + columns.push({ data: 6, title: 'Filename', }); + columns.push({ data: 7, title: 'Machine' }); + columns.push({ data: 4, title: 'Last run', render: (data) => data == null ? '-' : dateToYMD(new Date(data)) }); + + const button_title = include_button ? '' : ''; + + columns.push({ + data: 0, + title: button_title, + render: function(id, type, row) { + // Modify the content of the "Name" column here + return ` ` + } + }); + + el.DataTable({ + // searchPanes: { + // initCollapsed: true, + // }, + searching: searching, + ajax: url, + columns: columns, + deferRender: true, + drawCallback: function(settings) { + document.querySelectorAll('input[type="checkbox"]').forEach((e) =>{ + e.addEventListener('change', updateCompareCount); + }) + allow_group_select_checkboxes('input[type="checkbox"]'); + updateCompareCount(); + }, + order: [[columns.length-2, 'desc']] // API also orders, but we need to indicate order for the user + }); +} \ No newline at end of file diff --git a/frontend/js/hog-details.js b/frontend/js/hog-details.js new file mode 100644 index 000000000..f1b188b50 --- /dev/null +++ b/frontend/js/hog-details.js @@ -0,0 +1,385 @@ +$(document).ready(function () { + function getURLParameter(name) { + return new URLSearchParams(window.location.search).get(name); + } + + (async () => { + let mData + const machine_uuid = getURLParameter('machine_uuid') + try { + + var measurements = await makeAPICall(`/v1/hog/machine_details/${machine_uuid}`); + } catch (err) { + showNotification('Could not get data from API', err); + return; + } + if (measurements.data.length == 0){ + showNotification('No data', 'We could not find any data. Did you follow a correct URL?') + return + } + mData = measurements.data.map(item => { + item[0] = new Date(item[0]); + return item; + }); + mData.unshift(['time', 'combined_energy', 'cpu_energy', 'gpu_energy','ane_energy','energy_impact', 'id']) + + const myChart = echarts.init(document.getElementById('chart-container')); + + options = { + legend: { + orient: 'horizontal', + top: 'top', + data: ['combined_energy', 'cpu_energy', 'gpu_energy', 'ane_energy','energy_impact'], + selected: { + 'combined_energy': false, + 'cpu_energy': false, + 'gpu_energy': false, + 'ane_energy': false, + 'energy_impact': true + } + + }, + tooltip: { + trigger: 'axis', + axisPointer: { + type: 'shadow', + label: { + show: true + } + } + }, + dataset: { + source: mData + }, + grid: { + top: '12%', + left: '1%', + right: '10%', + containLabel: true + }, + + xAxis: { + type: 'category', + name: 'Time' + }, + + yAxis: [ + { + type: 'value', + name: 'mJ', + position: 'left', + }, + { + type: 'value', + name: 'energy_impact', + position: 'right', + }, + ], + series: [ + { type: 'bar', yAxisIndex: 0 }, + { type: 'bar', yAxisIndex: 0 }, + { type: 'bar', yAxisIndex: 0 }, + { type: 'bar', yAxisIndex: 0 }, + { type: 'bar', yAxisIndex: 1 }], + calculable: true, + dataZoom: [ + { + show: true, + start: 0, + end: 100 + }, + { + type: 'inside', + start: 0, + end: 100 + }, + { + show: true, + yAxisIndex: 0, + filterMode: 'empty', + width: 30, + height: '80%', + showDataShadow: false, + left: '93%' + } + ], + toolbox: { + show: true, + feature: { + mark: { show: true }, + magicType: { show: true, type: ['line', 'bar', 'stack'] }, + restore: { show: true }, + saveAsImage: { show: true }, + dataZoom: { yAxisIndex: false}, + } + }, + + }; + + + function handleZoomEvent(){ + let zoomTimeout; + $('#table-loader').addClass('active'); + + clearTimeout(zoomTimeout); + + zoomTimeout = setTimeout(async function() { + const dataZoomOption = myChart.getOption().dataZoom[0]; + const startPercent = dataZoomOption.start; + const endPercent = dataZoomOption.end; + const totalDataPoints = mData.length; + const startIndex = Math.floor(startPercent / 100 * totalDataPoints); + const endIndex = Math.ceil(endPercent / 100 * totalDataPoints) - 1; + let firstValue = mData[startIndex]; + let lastValue = mData[endIndex]; + if (firstValue[6] == 'id'){ + firstValue = mData[1]; + } + if(typeof lastValue === "undefined"){ + lastValue = mData[mData.length]; + } + try { + + var coalitions = await makeAPICall(`/v1/hog/coalitions_tasks/${machine_uuid}/${firstValue[6]}/${lastValue[6]}`); + energy_html = ` +
+
+
+
Combined System Energy
+ ${coalitions.energy_data[0].toLocaleString()} mJ +
+
+
+
+
Cpu Energy
+ ${coalitions.energy_data[1].toLocaleString()} mJ +
+
+
+
+
Gpu Energy
+ ${coalitions.energy_data[2].toLocaleString()} mJ +
+
+
+
+
Ane Energy
+ ${coalitions.energy_data[3].toLocaleString()} mJ +
+
+
+
+
Energy Impact
+ ${coalitions.energy_data[4].toLocaleString()} +
+
+
+ ` + $("#energy_segment").html(energy_html) + $('#process-table').DataTable({ + autoWidth: false, + destroy: true, + data: coalitions.data, + columns: [ + { data: 0, title: 'Name'}, + { + data: 1, + title: 'Energy Impact', + className: "dt-body-right", + render: function(data, type, row) { + if (type === 'display' || type === 'filter') { + return (data.toLocaleString()) + } + return data; + } + }, + { + data: 2, + title: 'Mb Read', + className: "dt-body-right", + render: function(data, type, row) { + if (type === 'display' || type === 'filter') { + return Math.trunc(data / 1048576).toLocaleString(); + } + return data; + } + }, + { + data: 3, + title: 'Mb Written', + className: "dt-body-right", + render: function(data, type, row) { + if (type === 'display' || type === 'filter') { + return Math.trunc(data / 1048576).toLocaleString(); + } + return data; + } + }, + { data: 4, title: 'Intr Wakeups',className: "dt-body-right"}, + { data: 5, title: 'Idle Wakeups', className: "dt-body-right"}, + { data: 6, title: 'Avg cpu time %', className: "dt-body-right"}, + + { + data: null, + title: '', + render: function(data, type, row) { + return ``; + }, + orderable: false, + searchable: false + } + ], + deferRender: true, + order: [], + }); + $('#table-loader').removeClass('active'); + + $('.js-task-info').click(async function() { + + $("#coaliton-segment").addClass("loading") + $("#task-segment").addClass("loading") + + $('#task-details').modal('show'); + + var tasks = await makeAPICall(`/v1/hog/tasks_details/${machine_uuid}/${$(this).data('start')}/${$(this).data('end')}/${$(this).data('name')}`); + + coalition_string=` +

${tasks.coalitions_data[0]}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AttributeValue
total_energy_impact${tasks.coalitions_data[1]}
total_diskio_bytesread${tasks.coalitions_data[2]}
total_diskio_byteswritten${tasks.coalitions_data[3]}
total_intr_wakeups${tasks.coalitions_data[4]}
total_idle_wakeups${tasks.coalitions_data[5]}
` + const tasks_string = tasks.tasks_data.map(subArr => ` +

${subArr[0]}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AttributeValue
Name${subArr[1]}
Occurrence${subArr[2]}
total_energy_impact${subArr[3]}
cputime_ns${subArr[4]}
bytes_received${subArr[5]}
bytes_sent${subArr[6]}
diskio_bytesread${subArr[7]}
diskio_byteswritten${subArr[8]}
intr_wakeups${subArr[9]}
idle_wakeups${subArr[10]}
+ `).join(' '); + $("#coaliton-segment").html(coalition_string) + $("#coaliton-segment").removeClass("loading") + + $("#task-segment").html(tasks_string) + $("#task-segment").removeClass("loading") + + + }); + + } catch (err) { + showNotification('Could not get data from API', err); + return; + } + }, 1000); + + } + + function focusOnBar(dataIndex) { + const zoomFactor = 8; + const dataLength = mData.length -1 ; + const startPercent = (dataIndex - zoomFactor / 2) / dataLength * 100; + const endPercent = (dataIndex + zoomFactor / 2) / dataLength * 100; + + myChart.setOption({ + dataZoom: [{ + start: Math.max(0, startPercent), + end: Math.min(100, endPercent) + }] + }); + } + myChart.setOption(options); + handleZoomEvent(); + + myChart.on('click', function(params) { + if (params.componentType === 'series' && params.seriesType === 'bar') { + focusOnBar(params.dataIndex); + handleZoomEvent(); + } + }); + + myChart.on('datazoom', function() { + handleZoomEvent(); + }); + + myChart.on('restore', function() { + handleZoomEvent(); + }); + + window.addEventListener('resize', function() { + myChart.resize(); + }); + + + })(); +}); diff --git a/frontend/js/hog.js b/frontend/js/hog.js new file mode 100644 index 000000000..52c47f955 --- /dev/null +++ b/frontend/js/hog.js @@ -0,0 +1,32 @@ +$(document).ready(function () { + + (async () => { + try { + var measurements = await makeAPICall('/v1/hog/top_processes'); + } catch (err) { + showNotification('Could not get data from API', err); + return; + } + $('#process-table').DataTable({ + data: measurements.process_data, + autoWidth: false, + columns: [ + { data: 0, title: 'Name'}, + { + data: 1, + title: 'Energy Impact', + className: "dt-body-right", + render: function(data, type, row) { + if (type === 'display' || type === 'filter') { + return (data.toLocaleString()) + } + return data; + } + }, + ], + deferRender: true, + order: [] // API determines order + }); + $('#machine_count').text(measurements.machine_count); + })(); +}); diff --git a/frontend/js/index.js b/frontend/js/index.js index 002bb9c72..803690d44 100644 --- a/frontend/js/index.js +++ b/frontend/js/index.js @@ -1,184 +1,5 @@ -const compareButton = () => { - let checkedBoxes = document.querySelectorAll('input[name=chbx-proj]:checked'); - - let link = '/compare.html?ids='; - checkedBoxes.forEach(checkbox => { - link = `${link}${checkbox.value},`; - }); - window.open(link.substr(0,link.length-1), '_blank'); -} -const updateCompareCount = () => { - const countButton = document.getElementById('compare-button'); - const checkedCount = document.querySelectorAll('input[name=chbx-proj]:checked').length; - countButton.textContent = `Compare: ${checkedCount} Run(s)`; -} - -const removeFilter = (paramName) => { - const urlSearchParams = new URLSearchParams(window.location.search); - urlSearchParams.delete(paramName); - const newUrl = `${window.location.pathname}?${urlSearchParams.toString()}`; - window.location.href = newUrl; -} - -const showActiveFilters = (key, value) => { - document.querySelector(`.ui.warning.message`).classList.remove('hidden'); - const newListItem = document.createElement("span"); - newListItem.innerHTML = `
${escapeString(key)}: ${escapeString(value)}
`; - document.querySelector(`.ui.warning.message ul`).appendChild(newListItem); - -} - - -const allow_group_select_checkboxes = (checkbox_wrapper_id) => { - let lastChecked = null; - let checkboxes = document.querySelectorAll(checkbox_wrapper_id); - - for (let i=0;ij) { - [i, j] = [j, i] - } - - for (let c=0; c { - try { - const url_params = (new URLSearchParams(window.location.search)) - let repo_filter = ''; - if (url_params.get('repo') != null && url_params.get('repo').trim() != '') { - repo_filter = url_params.get('repo').trim() - showActiveFilters('repo', repo_filter) - } - let filename_filter = ''; - if (url_params.get('filename') != null && url_params.get('filename').trim() != '') { - filename_filter = url_params.get('filename').trim() - showActiveFilters('filename', filename_filter) - } - var api_data = await makeAPICall(`/v1/projects?repo=${repo_filter}&filename=${filename_filter}`) - } catch (err) { - showNotification('Could not get data from API', err); - return; - } - - api_data.data.forEach(el => { - - const id = el[0] - let name = el[1] - const uri = el[2] - let branch = el[3] - const end_measurement = el[4] - const last_run = el[5] - const invalid_project = el[6] - const filename = el[7] - const machine = el[8] - const commit_hash = el[9] - const commit_hash_short = commit_hash == null ? null : `${commit_hash.substr(0,3)}...${commit_hash.substr(-3,3)}` - - - - let uri_link = replaceRepoIcon(uri); - - if (uri.startsWith("http")) { - uri_link = `${uri_link} `; - } - - - // insert new accordion row if repository not known - let td_node = document.querySelector(`td[data-uri='${uri}']`) - if (td_node == null || td_node == undefined) { - let row = document.querySelector('#projects-table tbody').insertRow() - row.innerHTML = ` - -
-
- ${uri_link} -
-
-
-
- `; - let content = document.querySelector(`#projects-table td[data-uri='${uri}'] div.content`); - content.innerHTML = ` - - - - - - - - - - - - - -
NameFilenameMachineBranchCommitLast run
`; - } - - if(end_measurement == null) name = `${name} (no data yet 🔥)`; - if(invalid_project != null) name = `${name} invalidated`; - - let inner_row = document.querySelector(`#projects-table td[data-uri='${uri}'] div.content table tbody`).insertRow(); - - inner_row.innerHTML = ` - ${name} - ${filename} - ${machine} - ${branch} - ${commit_hash_short} - ${dateToYMD(new Date(last_run))} -  `; - }); - - - - $('.ui.accordion').accordion(); - setTimeout(function() { - $('#projects-table table').DataTable({ -// searchPanes: { -// initCollapsed: true, -// }, - "order": [[5, 'desc']], // sort by last_run by default - }); - - }, 1000); // Delay of 2000 milliseconds (2 seconds) - - /* - This code would be most efficient. But it has bad UI experience due to lag - $('.ui.accordion').accordion({ - onOpen: function(value, text) { - table = this.querySelector('table') - if(!$.fn.DataTable.isDataTable(table)) { - $(table).DataTable({ - // searchPanes: { - // initCollapsed: true, - // }, - "order": [[5, 'desc']], // sort by last_run by default - }); - } - }}); - */ - - - document.querySelectorAll('input[name=chbx-proj]').forEach((e) =>{ - e.addEventListener('change', updateCompareCount); - }) - - allow_group_select_checkboxes('#projects-table input[type="checkbox"]'); - -})(); +$(document).ready(function () { + (async () => { + getRunsTable($('#runs-table'), `${API_URL}/v1/runs?${getFilterQueryStringFromURI()}&limit=50`) + })(); +}); \ No newline at end of file diff --git a/frontend/js/repositories.js b/frontend/js/repositories.js new file mode 100644 index 000000000..4a4d91887 --- /dev/null +++ b/frontend/js/repositories.js @@ -0,0 +1,44 @@ +(async () => { + + try { + + var api_data = await makeAPICall(`/v1/repositories?${getFilterQueryStringFromURI()}`) + } catch (err) { + showNotification('Could not get data from API', err); + return + } + + + api_data.data.forEach(el => { + + const uri = el[0] + let uri_link = replaceRepoIcon(uri); + + if (uri.startsWith("http")) { + uri_link = `${uri_link} `; + } + + + let row = document.querySelector('#repositories-table tbody').insertRow() + row.innerHTML = ` + +
+
+ ${uri_link} +
+
+
+
+
+ `; + }); + $('.ui.accordion').accordion({ + onOpen: function(value, text) { + const table = this.querySelector('table'); + + if(!$.fn.DataTable.isDataTable(table)) { + const uri = this.getAttribute('data-uri'); + getRunsTable($(table), `${API_URL}/v1/runs?uri=${uri}`, false, false, true) + } + }}); +})(); diff --git a/frontend/js/request.js b/frontend/js/request.js index 1ed7058c8..2abd6d484 100644 --- a/frontend/js/request.js +++ b/frontend/js/request.js @@ -21,7 +21,7 @@ const values = Object.fromEntries(data.entries()); try { - await makeAPICall('/v1/project/add', values); + await makeAPICall('/v1/software/add', values); form.reset() showNotification('Success', 'Save successful. Check your mail in 10-15 minutes', 'success'); } catch (err) { diff --git a/frontend/js/stats.js b/frontend/js/stats.js index 95074c191..7f4da5217 100644 --- a/frontend/js/stats.js +++ b/frontend/js/stats.js @@ -72,67 +72,67 @@ class CO2Tangible extends HTMLElement { customElements.define('co2-tangible', CO2Tangible); -const fillProjectData = (project_data, key = null) => { +const fillRunData = (run_data, key = null) => { - for (item in project_data) { + for (const item in run_data) { if (item == 'machine_specs') { - fillProjectTab('#machine-specs', project_data[item]); // recurse + fillRunTab('#machine-specs', run_data[item]); // recurse } else if(item == 'usage_scenario') { - document.querySelector("#usage-scenario").insertAdjacentHTML('beforeend', `
${json2yaml(project_data?.[item])}
`) + document.querySelector("#usage-scenario").insertAdjacentHTML('beforeend', `
${json2yaml(run_data?.[item])}
`) } else if(item == 'logs') { - document.querySelector("#logs").insertAdjacentHTML('beforeend', `
${project_data?.[item]}
`) + document.querySelector("#logs").insertAdjacentHTML('beforeend', `
${run_data?.[item]}
`) } else if(item == 'measurement_config') { - fillProjectTab('#measurement-config', project_data[item]); // recurse + fillRunTab('#measurement-config', run_data[item]); // recurse } else if(item == 'phases' || item == 'id') { // skip } else if(item == 'commit_hash') { - if (project_data?.[item] == null) continue; // some old projects did not save it - let commit_link = buildCommitLink(project_data); - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `${item}${project_data?.[item]}`) + if (run_data?.[item] == null) continue; // some old runs did not save it + let commit_link = buildCommitLink(run_data); + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `${item}${run_data?.[item]}`) } else if(item == 'name' || item == 'filename') { - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `${item}${project_data?.[item]}`) + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `${item}${run_data?.[item]}`) } else if(item == 'uri') { - let entry = project_data?.[item]; - if(project_data?.[item].indexOf('http') === 0) entry = `${project_data?.[item]}`; - document.querySelector('#project-data-top').insertAdjacentHTML('beforeend', `${item}${entry}`); + let entry = run_data?.[item]; + if(run_data?.[item].indexOf('http') === 0) entry = `${run_data?.[item]}`; + document.querySelector('#run-data-top').insertAdjacentHTML('beforeend', `${item}${entry}`); } else { - document.querySelector('#project-data-accordion').insertAdjacentHTML('beforeend', `${item}${project_data?.[item]}`) + document.querySelector('#run-data-accordion').insertAdjacentHTML('beforeend', `${item}${run_data?.[item]}`) } } // create new custom field // timestamp is in microseconds, therefore divide by 10**6 - const measurement_duration_in_s = (project_data.end_measurement - project_data.start_measurement) / 1000000 - document.querySelector('#project-data-accordion').insertAdjacentHTML('beforeend', `duration${measurement_duration_in_s} s`) + const measurement_duration_in_s = (run_data.end_measurement - run_data.start_measurement) / 1000000 + document.querySelector('#run-data-accordion').insertAdjacentHTML('beforeend', `duration${measurement_duration_in_s} s`) - $('.ui.secondary.menu .item').tab({childrenOnly: true, context: '.project-data-container'}); // activate tabs for project data + $('.ui.secondary.menu .item').tab({childrenOnly: true, context: '.run-data-container'}); // activate tabs for run data $('.ui.accordion').accordion(); - if (project_data.invalid_project) { - showNotification('Project measurement has been marked as invalid', project_data.invalid_project); + if (run_data.invalid_run) { + showNotification('Run measurement has been marked as invalid', run_data.invalid_run); document.body.classList.add("invalidated-measurement") } } -const buildCommitLink = (project_data) => { +const buildCommitLink = (run_data) => { let commit_link; - commit_link = project_data['uri'].endsWith('.git') ? project_data['uri'].slice(0, -4) : project_data['uri'] - if (project_data['uri'].includes('github')) { - commit_link = commit_link + '/tree/' + project_data['commit_hash'] + commit_link = run_data['uri'].endsWith('.git') ? run_data['uri'].slice(0, -4) : run_data['uri'] + if (run_data['uri'].includes('github')) { + commit_link = commit_link + '/tree/' + run_data['commit_hash'] } - else if (project_data['uri'].includes('gitlab')) { - commit_link = commit_link + '/-/tree/' + project_data ['commit_hash'] + else if (run_data['uri'].includes('gitlab')) { + commit_link = commit_link + '/-/tree/' + run_data ['commit_hash'] } return commit_link; } -const fillProjectTab = (selector, data, parent = '') => { +const fillRunTab = (selector, data, parent = '') => { for (item in data) { if(typeof data[item] == 'object') - fillProjectTab(selector, data[item], `${item}.`) + fillRunTab(selector, data[item], `${item}.`) else document.querySelector(selector).insertAdjacentHTML('beforeend', `${parent}${item}${data?.[item]}`) @@ -220,14 +220,14 @@ const displayTimelineCharts = (metrics, notes) => { const chart_instances = []; const t0 = performance.now(); - for ( metric_name in metrics) { + for (const metric_name in metrics) { const element = createChartContainer("#chart-container", `${METRIC_MAPPINGS[metric_name]['clean_name']} via ${METRIC_MAPPINGS[metric_name]['source']} `); let legend = []; let series = []; - for (detail_name in metrics[metric_name].series) { + for (const detail_name in metrics[metric_name].series) { legend.push(detail_name) series.push({ name: detail_name, @@ -285,9 +285,9 @@ const displayTimelineCharts = (metrics, notes) => { async function makeAPICalls(url_params) { try { - var project_data = await makeAPICall('/v1/project/' + url_params.get('id')) + var run_data = await makeAPICall('/v1/run/' + url_params.get('id')) } catch (err) { - showNotification('Could not get project data from API', err); + showNotification('Could not get run data from API', err); } try { @@ -295,18 +295,26 @@ async function makeAPICalls(url_params) { } catch (err) { showNotification('Could not get stats data from API', err); } + try { var notes_data = await makeAPICall('/v1/notes/' + url_params.get('id')) } catch (err) { showNotification('Could not get notes data from API', err); } + + try { + var network_data = await makeAPICall('/v1/network/' + url_params.get('id')) + } catch (err) { + showNotification('Could not get network intercepts data from API', err); + } + try { var phase_stats_data = await makeAPICall('/v1/phase_stats/single/' + url_params.get('id')) } catch (err) { showNotification('Could not get phase_stats data from API', err); } - return [project_data?.data, measurement_data?.data, notes_data?.data, phase_stats_data?.data]; + return [run_data?.data, measurement_data?.data, notes_data?.data, phase_stats_data?.data, network_data?.data]; } const renderBadges = (url_params) => { @@ -324,12 +332,24 @@ const renderBadges = (url_params) => { }) } +const displayNetworkIntercepts = (network_data) => { + if (network_data.length === 0) { + document.querySelector("#network-divider").insertAdjacentHTML('afterEnd', '

No external network connections were detected.

') + } else { + for (const item of network_data) { + date = new Date(Number(item[2])); + date = date.toLocaleString(); + document.querySelector("#network-intercepts").insertAdjacentHTML('beforeend', `${date}${item[3]}${item[4]}`) + } + } +} + const getURLParams = () => { const query_string = window.location.search; const url_params = (new URLSearchParams(query_string)) if(url_params.get('id') == null || url_params.get('id') == '' || url_params.get('id') == 'null') { - showNotification('No project id', 'ID parameter in URL is empty or not present. Did you follow a correct URL?'); + showNotification('No run id', 'ID parameter in URL is empty or not present. Did you follow a correct URL?'); throw "Error"; } return url_params; @@ -342,17 +362,17 @@ $(document).ready( (e) => { let url_params = getURLParams(); if(url_params.get('id') == null || url_params.get('id') == '' || url_params.get('id') == 'null') { - showNotification('No project id', 'ID parameter in URL is empty or not present. Did you follow a correct URL?'); + showNotification('No run id', 'ID parameter in URL is empty or not present. Did you follow a correct URL?'); return; } - let [project_data, measurements_data, notes_data, phase_stats_data] = await makeAPICalls(url_params); + let [run_data, measurements_data, notes_data, phase_stats_data, network_data] = await makeAPICalls(url_params); - if (project_data == undefined) return; + if (run_data == undefined) return; renderBadges(url_params); - fillProjectData(project_data); + fillRunData(run_data); if(phase_stats_data != null) { displayComparisonMetrics(phase_stats_data) @@ -364,6 +384,8 @@ $(document).ready( (e) => { if (notes_data == undefined) return; displayTimelineCharts(metrics, notes_data); + displayNetworkIntercepts(network_data); + // after all charts instances have been placed // the flexboxes might have rearranged. We need to trigger resize setTimeout(function(){console.log("Resize"); window.dispatchEvent(new Event('resize'))}, 500); diff --git a/frontend/js/status.js b/frontend/js/status.js new file mode 100644 index 000000000..56faf70b4 --- /dev/null +++ b/frontend/js/status.js @@ -0,0 +1,20 @@ +$(document).ready(function () { + (async () => { + $('#jobs-table').DataTable({ + ajax: `${API_URL}/v1/jobs`, + columns: [ + { data: 0, title: 'ID'}, + { data: 1, title: 'Name'}, + { data: 2, title: 'Url'}, + { data: 3, title: 'Filename'}, + { data: 4, title: 'Branch'}, + { data: 5, title: 'Machine'}, + { data: 6, title: 'State'}, + { data: 7, title: 'Last Update', render: (data) => data == null ? '-' : dateToYMD(new Date(data)) }, + { data: 8, title: 'Created at', render: (data) => data == null ? '-' : dateToYMD(new Date(data)) }, + ], + deferRender: true, + order: [[7, 'desc']] // API also orders, but we need to indicate order for the user + }); + })(); +}); diff --git a/frontend/js/timeline.js b/frontend/js/timeline.js index 93493a4d3..fc0222655 100644 --- a/frontend/js/timeline.js +++ b/frontend/js/timeline.js @@ -156,10 +156,10 @@ const loadCharts = async () => { let legends = {}; let series = {}; - let pproject_id = null + let prun_id = null phase_stats_data.forEach( (data) => { - let [project_id, project_name, last_run, metric_name, detail_name, phase, value, unit, commit_hash, commit_timestamp] = data + let [run_id, run_name, created_at, metric_name, detail_name, phase, value, unit, commit_hash, commit_timestamp] = data if (series[`${metric_name} - ${detail_name}`] == undefined) { @@ -169,19 +169,19 @@ const loadCharts = async () => { series[`${metric_name} - ${detail_name}`].labels.push(commit_timestamp) series[`${metric_name} - ${detail_name}`].values.push({value: value, commit_hash: commit_hash}) series[`${metric_name} - ${detail_name}`].notes.push({ - project_name: project_name, - last_run: last_run, + run_name: run_name, + created_at: created_at, commit_timestamp: commit_timestamp, commit_hash: commit_hash, phase: phase, - project_id: project_id, - pproject_id: pproject_id, + run_id: run_id, + prun_id: prun_id, }) - pproject_id = project_id + prun_id = run_id }) - for(my_series in series) { + for(const my_series in series) { let badge = `
@@ -192,7 +192,7 @@ const loadCharts = async () => {
- +

` @@ -224,8 +224,8 @@ const loadCharts = async () => { trigger: 'item', formatter: function (params, ticket, callback) { if(params.componentType != 'series') return; // no notes for the MovingAverage - return `${series[params.seriesName].notes[params.dataIndex].project_name}
- date: ${series[params.seriesName].notes[params.dataIndex].last_run}
+ return `${series[params.seriesName].notes[params.dataIndex].run_name}
+ date: ${series[params.seriesName].notes[params.dataIndex].created_at}
metric_name: ${params.seriesName}
phase: ${series[params.seriesName].notes[params.dataIndex].phase}
value: ${numberFormatter.format(series[params.seriesName].values[params.dataIndex].value)}
@@ -239,7 +239,7 @@ const loadCharts = async () => { chart_instance.on('click', function (params) { if(params.componentType != 'series') return; // no notes for the MovingAverage - window.open(`/compare.html?ids=${series[params.seriesName].notes[params.dataIndex].project_id},${series[params.seriesName].notes[params.dataIndex].pproject_id}`, '_blank'); + window.open(`/compare.html?ids=${series[params.seriesName].notes[params.dataIndex].run_id},${series[params.seriesName].notes[params.dataIndex].prun_id}`, '_blank'); }); @@ -258,7 +258,7 @@ const loadCharts = async () => { $(document).ready( (e) => { (async () => { - $('.ui.secondary.menu .item').tab({childrenOnly: true, context: '.project-data-container'}); // activate tabs for project data + $('.ui.secondary.menu .item').tab({childrenOnly: true, context: '.run-data-container'}); // activate tabs for run data $('#rangestart').calendar({ type: 'date', endCalendar: $('#rangeend'), diff --git a/frontend/repositories.html b/frontend/repositories.html new file mode 100644 index 000000000..427392daa --- /dev/null +++ b/frontend/repositories.html @@ -0,0 +1,73 @@ + + + + + + + + + + + + + Green Metrics Tool + + + + + + + + + + + + + + + + + + +
+

+ + Green Metrics Tool - All Repositories +

+
+
+
Repository overview
+
+

The repository page of the Green Metrics Tool shows you all known repositories +

+

By clicking on a repository name a drawer will open with all the runs known for that repository.

+

From there you can go into the details for a specific measurement, or compare them with other runs or even repositories.

+

Comparing is possible for different repos, different branches and different runs as long as they are on the same machine. Also different machines are supported if you compare the same repo. +

+

Currently we do not support comparing different machines AND different repos at the same time. The tool will notifiy you if you attempt to do so :)

+
+
+
+ + + + + + + + + + +
+ Repositories ( / / etc.) +
+
+
+ + + \ No newline at end of file diff --git a/frontend/request.html b/frontend/request.html index e77af87ee..afc518f12 100644 --- a/frontend/request.html +++ b/frontend/request.html @@ -31,60 +31,70 @@

-
INTRODUCTION
+
Measuring software introduction
-

Green Metric can measure your software according to different schemas.

-

In this demo, only the Green-Coding-Berlin schema is implemented. In the future - it is - planned to support Blue - Angel (in Germany known as Blauer Engel) and also the SCI - from Green Software Foundation -

-

In the box below please put your Github-Repo URL and an E-mail where we send the +

The Green Metrics Tool can measure your software either in a one-off run or in a continuous monitoring.

+

In the box below please put your Github/Gitlab/Bitbucket-Repo URL and an E-mail where we send the link to the statistics.

-

Your certification request will be put in a queue and processing takes about 5-15 - Minutes (+ the runtime of your usage scenario)

-

Note: Only open-source software can and will be - certified. It does not have to free software though. This means we only process - public github repositories. -

-

The certifier expects a schema-correct +

Your run will be put in a queue and processed in the next free slot.

+

The Green Metrics Tool expects a schema-correct usage_scenario.yml file in your repository root. Please check the Demo Software Repository for example and documentation.
Override the usage scenario file name or provide relative path if it is not in the repository root with the optional filename field.

-

If you have any - questions - or issues please email us at info@green-coding.berlin

+
+ +
+
+ Caveats of the free version +
+
    +
  • Only public repositories are supported
    • +
  • Only one run per day per repository is supported
    • +
  • For a premium account with no restrictions please contact us at info@green-coding.berlin
    • +
+
+
-
ADD NEW PROJECT
+
Submit software for measurement
-
-
+
+
- + +
+
+ +
+
+
- +
- +
- +
-
- +
diff --git a/frontend/stats.html b/frontend/stats.html index c1b168087..89ef8b065 100644 --- a/frontend/stats.html +++ b/frontend/stats.html @@ -41,9 +41,9 @@

-

Project Data

+

Run Data

-
+
General Measurement @@ -51,15 +51,16 @@

Project Data

Machine Usage Scenario Logs + Network
-
+
Click here for more data ...
-
+
@@ -70,34 +71,34 @@

Project Data

+
XGBoost estimated AC energy (Runtime)
-
+
RAPL component energy (Runtime)
-
+
Measured AC energy (Runtime)
-
+
SCI (Runtime)
-
@@ -110,6 +111,11 @@

Project Data

+
+ This is a list of all external network connections the system requested. +
+
+
diff --git a/frontend/status.html b/frontend/status.html new file mode 100644 index 000000000..7e90d54e6 --- /dev/null +++ b/frontend/status.html @@ -0,0 +1,51 @@ + + + + + + + + + + + + + Green Metrics Tool + + + + + + + + + + + + + + + + + +
+

+ + Green Metrics Tool - Status +

+
+
+
Status overview
+
+

This page shows the database status. In the current version only the jobs in the database are shown.

+

+

With it you can estimate the length of the jobs queue. Be aware of the the processing scheme configured (Random or FIFO).

+

In a future version this will also provide statistics such as jobs run per day, failed jobs, most measured software etc.

+
+
+
+

Jobs queue

+
+
+ + \ No newline at end of file diff --git a/frontend/timeline.html b/frontend/timeline.html index bbbbef040..4f37eca9d 100644 --- a/frontend/timeline.html +++ b/frontend/timeline.html @@ -45,7 +45,7 @@

Timeline View

-
+

Timeline View Details

diff --git a/install_linux.sh b/install_linux.sh index 6e68cb581..c5ad2fafa 100755 --- a/install_linux.sh +++ b/install_linux.sh @@ -10,6 +10,12 @@ function print_message { echo "$1" } +function generate_random_password() { + local length=$1 + LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c "$length" + echo +} + db_pw='' api_url='' metrics_url='' @@ -17,6 +23,8 @@ no_build=false no_hosts=false ask_tmpfs=true +reboot_echo_flag=false + while getopts "p:a:m:nht" o; do case "$o" in p) @@ -51,12 +59,19 @@ if [[ -z $metrics_url ]] ; then metrics_url=${metrics_url:-"http://metrics.green-coding.internal:9142"} fi +if [[ -f config.yml ]]; then + password_from_file=$(awk '/postgresql:/ {flag=1; next} flag && /password:/ {print $2; exit}' config.yml) +fi + +default_password=${password_from_file:-$(generate_random_password 12)} + if [[ -z "$db_pw" ]] ; then - read -sp "Please enter the new password to be set for the PostgreSQL DB: " db_pw - echo "" # force a newline, because print -sp will consume it + read -sp "Please enter the new password to be set for the PostgreSQL DB (default: $default_password): " db_pw + echo "" # force a newline, because read -sp will consume it + db_pw=${db_pw:-"$default_password"} fi -if [[ $ask_tmpfs == true ]] ; then +if ! mount | grep -E '\s/tmp\s' | grep -Eq '\stmpfs\s' && [[ $ask_tmpfs == true ]]; then read -p "We strongly recommend mounting /tmp on a tmpfs. Do you want to do that? (y/N)" tmpfs if [[ "$tmpfs" == "Y" || "$tmpfs" == "y" ]] ; then if lsb_release -is | grep -q "Fedora"; then @@ -64,6 +79,7 @@ if [[ $ask_tmpfs == true ]] ; then else sudo systemctl enable /usr/share/systemd/tmp.mount fi + reboot_echo_flag=true fi fi @@ -98,13 +114,12 @@ sed -i -e "s|__METRICS_URL__|$metrics_url|" frontend/js/helpers/config.js print_message "Checking out further git submodules ..." git submodule update --init -sudo apt-get update - print_message "Installing needed binaries for building ..." if lsb_release -is | grep -q "Fedora"; then - sudo dnf -y install lm_sensors lm_sensors-devel glib2 glib2-devel + sudo dnf -y install lm_sensors lm_sensors-devel glib2 glib2-devel tinyproxy else - sudo apt-get install -y lm-sensors libsensors-dev libglib2.0-0 libglib2.0-dev + sudo apt-get update + sudo apt-get install -y lm-sensors libsensors-dev libglib2.0-0 libglib2.0-dev tinyproxy fi print_message "Building binaries ..." @@ -121,6 +136,13 @@ while IFS= read -r subdir; do fi done +print_message "Setting up python venv" +python3 -m venv venv +source venv/bin/activate + +print_message "Setting GMT in include path for python via .pth file" +find venv -type d -name "site-packages" -exec sh -c 'echo $PWD > "$0/gmt-lib.pth"' {} \; + print_message "Building sgx binaries" make -C lib/sgx-software-enable mv lib/sgx-software-enable/sgx_enable tools/ @@ -132,7 +154,12 @@ PWD=$(pwd) echo "ALL ALL=(ALL) NOPASSWD:$PYTHON_PATH $PWD/lib/hardware_info_root.py" | sudo tee /etc/sudoers.d/green_coding_hardware_info print_message "Installing IPMI tools" -sudo apt-get install -y freeipmi-tools ipmitool +if lsb_release -is | grep -q "Fedora"; then + sudo dnf -y install ipmitool +else + sudo apt-get install -y freeipmi-tools ipmitool +fi + print_message "Adding IPMI to sudoers file" echo "ALL ALL=(ALL) NOPASSWD:/usr/sbin/ipmi-dcmi --get-system-power-statistics" | sudo tee /etc/sudoers.d/ipmi_get_machine_energy_stat @@ -163,13 +190,25 @@ fi if [[ $no_build != true ]] ; then print_message "Building / Updating docker containers" - docker compose -f docker/compose.yml down - docker compose -f docker/compose.yml build + if docker info 2>/dev/null | grep rootless; then + print_message "Docker is running in rootless mode. Using non-sudo call ..." + docker compose -f docker/compose.yml down + docker compose -f docker/compose.yml build + else + print_message "Docker is running in default root mode. Using sudo call ..." + sudo docker compose -f docker/compose.yml down + sudo docker compose -f docker/compose.yml build + fi print_message "Updating python requirements" + python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt fi echo "" echo -e "${GREEN}Successfully installed Green Metrics Tool!${NC}" -echo -e "${GREEN}If you have newly requested to mount /tmp as tmpfs please reboot your system now.${NC}" +echo -e "Please remember to always activate your venv when using the GMT with 'source venv/bin/activate'" + +if $reboot_echo_flag; then + echo -e "${GREEN}If you have newly requested to mount /tmp as tmpfs please reboot your system now.${NC}" +fi diff --git a/install_mac.sh b/install_mac.sh index f9f547d6e..e579e8757 100755 --- a/install_mac.sh +++ b/install_mac.sh @@ -9,6 +9,12 @@ function print_message { echo "$1" } +function generate_random_password() { + local length=$1 + LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c "$length" + echo +} + db_pw='' api_url='' metrics_url='' @@ -37,8 +43,16 @@ if [[ -z $metrics_url ]] ; then metrics_url=${metrics_url:-"http://metrics.green-coding.internal:9142"} fi +if [[ -f config.yml ]]; then + password_from_file=$(awk '/postgresql:/ {flag=1; next} flag && /password:/ {print $2; exit}' config.yml) +fi + +default_password=${password_from_file:-$(generate_random_password 12)} + if [[ -z "$db_pw" ]] ; then - read -sp "Please enter the new password to be set for the PostgreSQL DB: " db_pw + read -sp "Please enter the new password to be set for the PostgreSQL DB (default: $default_password): " db_pw + echo "" # force a newline, because read -sp will consume it + db_pw=${db_pw:-"$default_password"} fi print_message "Clearing old api.conf and frontend.conf files" @@ -72,7 +86,14 @@ sed -i '' -e "s|__METRICS_URL__|$metrics_url|" frontend/js/helpers/config.js print_message "Checking out further git submodules ..." git submodule update --init -print_message "Adding hardware_info_root.py to sudoers file" +print_message "Setting up python venv" +python3 -m venv venv +source venv/bin/activate +# This will set the include path for the project +find venv -type d -name "site-packages" -exec sh -c 'echo $PWD > "$0/gmt-lib.pth"' {} \; + + +print_message "Adding powermetrics to sudoers file" echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/powermetrics" | sudo tee /etc/sudoers.d/green_coding_powermetrics echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/killall powermetrics" | sudo tee /etc/sudoers.d/green_coding_kill_powermetrics echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/killall -9 powermetrics" | sudo tee /etc/sudoers.d/green_coding_kill_powermetrics_sigkill @@ -99,12 +120,19 @@ if [[ ${host_metrics_url} == *".green-coding.internal"* ]];then fi fi +if ! command -v stdbuf &> /dev/null; then + print_message "Trying to install 'coreutils' via homebew. If this fails (because you do not have brew or use another package manager), please install it manually ..." + brew install coreutils +fi + print_message "Building / Updating docker containers" docker compose -f docker/compose.yml down docker compose -f docker/compose.yml build print_message "Updating python requirements" +python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt echo "" echo -e "${GREEN}Successfully installed Green Metrics Tool!${NC}" +echo -e "Please remember to always activate your venv when using the GMT with 'source venv/bin/activate'" diff --git a/lib/db.py b/lib/db.py index d9075821f..56a5b12a5 100644 --- a/lib/db.py +++ b/lib/db.py @@ -1,5 +1,6 @@ import psycopg -from global_config import GlobalConfig + +from lib.global_config import GlobalConfig class DB: @@ -81,7 +82,7 @@ def copy_from(self, file, table, columns, sep=','): if __name__ == '__main__': DB() DB() - print(DB().fetch_all('SELECT * FROM projects')) - # DB().query('SELECT * FROM projects') - # DB().query('SELECT * FROM projects') - # DB().query('SELECT * FROM projects') + print(DB().fetch_all('SELECT * FROM runs')) + # DB().query('SELECT * FROM runs') + # DB().query('SELECT * FROM runs') + # DB().query('SELECT * FROM runs') diff --git a/lib/debug_helper.py b/lib/debug_helper.py index a6516cf83..82f977143 100644 --- a/lib/debug_helper.py +++ b/lib/debug_helper.py @@ -1,5 +1,6 @@ import sys -from terminal_colors import TerminalColors + +from lib.terminal_colors import TerminalColors class DebugHelper: diff --git a/lib/email_helpers.py b/lib/email_helpers.py index 28036ae33..f612c8826 100644 --- a/lib/email_helpers.py +++ b/lib/email_helpers.py @@ -1,19 +1,10 @@ -import sys -import os import smtplib import ssl -from global_config import GlobalConfig - -sys.path.append(os.path.dirname(os.path.abspath(__file__))+'/../lib') - +from lib.global_config import GlobalConfig def send_email(message, receiver_email): config = GlobalConfig().config - - if config['admin']['no_emails'] is True: - return - context = ssl.create_default_context() with smtplib.SMTP_SSL(config['smtp']['server'], config['smtp']['port'], context=context) as server: # No need to set server.auth manually. server.login will iterater over all available methods @@ -21,9 +12,7 @@ def send_email(message, receiver_email): server.login(config['smtp']['user'], config['smtp']['password']) server.sendmail(config['smtp']['sender'], receiver_email, message.encode('utf-8')) - def send_admin_email(subject, body): - config = GlobalConfig().config message = """\ From: {smtp_sender} To: {receiver_email} @@ -34,53 +23,56 @@ def send_admin_email(subject, body): -- {url}""" + config = GlobalConfig().config message = message.format( subject=subject, body=body, url=config['cluster']['metrics_url'], receiver_email=config['admin']['email'], smtp_sender=config['smtp']['sender']) - send_email(message, config['admin']['email']) + send_email(message, [config['admin']['email'], config['admin']['bcc_email']]) -def send_error_email(receiver_email, error, project_id=None, name=None, machine=None): - config = GlobalConfig().config +def send_error_email(receiver_email, error, run_id=None, name=None, machine=None): message = """\ From: {smtp_sender} To: {receiver_email} +Bcc: {bcc_email} Subject: Your Green Metrics analysis has encountered problems Unfortunately, your Green Metrics analysis has run into some issues and could not be completed. Name: {name} -Project id: {project_id} +Run Id: {run_id} Machine: {machine} -Link: {url}/stats.html?id={project_id} +Link: {url}/stats.html?id={run_id} {errors} -- {url}""" + config = GlobalConfig().config message = message.format( receiver_email=receiver_email, errors=error, name=name, machine=machine, + bcc_email=config['admin']['bcc_email'], url=config['cluster']['metrics_url'], - project_id=project_id, + run_id=run_id, smtp_sender=config['smtp']['sender']) - send_email(message, receiver_email) + send_email(message, [receiver_email, config['admin']['bcc_email']]) def send_report_email(receiver_email, report_id, name, machine=None): - config = GlobalConfig().config message = """\ From: {smtp_sender} To: {receiver_email} +Bcc: {bcc_email} Subject: Your Green Metric report is ready -Project name: {name} +Run Name: {name} Machine: {machine} Your report is now accessible under the URL: {url}/stats.html?id={report_id} @@ -88,14 +80,16 @@ def send_report_email(receiver_email, report_id, name, machine=None): -- {url}""" + config = GlobalConfig().config message = message.format( receiver_email=receiver_email, report_id=report_id, machine=machine, name=name, + bcc_email=config['admin']['bcc_email'], url=config['cluster']['metrics_url'], smtp_sender=config['smtp']['sender']) - send_email(message, receiver_email) + send_email(message, [receiver_email, config['admin']['bcc_email']]) if __name__ == '__main__': diff --git a/lib/error_helpers.py b/lib/error_helpers.py index ce0857710..40bb252cd 100644 --- a/lib/error_helpers.py +++ b/lib/error_helpers.py @@ -1,8 +1,8 @@ import sys -import os import traceback -from terminal_colors import TerminalColors -from global_config import GlobalConfig + +from lib.terminal_colors import TerminalColors +from lib.global_config import GlobalConfig def end_error(*errors): @@ -32,7 +32,7 @@ def log_error(*errors): if error_log_file: try: - with open(error_log_file, 'a') as file: + with open(error_log_file, 'a', encoding='utf-8') as file: print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=file) print('Error: ', *errors, file=file) print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=file) @@ -41,10 +41,11 @@ def log_error(*errors): except (IOError ,FileNotFoundError, PermissionError): print(TerminalColors.FAIL, "\nError: Cannot create file in the specified location because file is not found or not writable", TerminalColors.ENDC, file=sys.stderr) + # For terminal logging we invert the order. It is better readable if the error is at the bottom print(TerminalColors.FAIL, '\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=sys.stderr) - print('Error: ', *errors, file=sys.stderr) - print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) - print('\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', + print('\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', file=sys.stderr) + print('Error: ', *errors, file=sys.stderr) + print('\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 0_o >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n', TerminalColors.ENDC, file=sys.stderr) diff --git a/lib/global_config.py b/lib/global_config.py index f2069a2f6..3ccc9ef20 100644 --- a/lib/global_config.py +++ b/lib/global_config.py @@ -3,6 +3,7 @@ class GlobalConfig: + # for unknown reasons pylint needs this argument set, although we don't use it. Only in __init__ # pylint: disable=unused-argument def __new__(cls, config_name='config.yml'): if not hasattr(cls, 'instance'): diff --git a/lib/hardware_info.py b/lib/hardware_info.py index 2d99596a9..2dfd5229f 100755 --- a/lib/hardware_info.py +++ b/lib/hardware_info.py @@ -4,9 +4,10 @@ ''' import re import os -import subprocess import platform import pprint +import subprocess +import sys REGEX_PARAMS = re.MULTILINE | re.IGNORECASE @@ -77,6 +78,9 @@ def read_directory_recursive(directory): [rpwr, 'Hardware Model', '/usr/bin/hostnamectl', r'Hardware Model:\s*(?P.*)'], [rpwr, 'Docker Info', 'docker info', r'(?P.*)', re.IGNORECASE | re.DOTALL], [rpwr, 'Docker Version', 'docker version', r'(?P.*)', re.IGNORECASE | re.DOTALL], + [rpwr, 'Docker Containers', 'docker ps -a', r'(?P.*)'], + [rpwr, 'Installed System Packages', 'if [ -f /etc/lsb-release ]; then dpkg -l ; elif [ -f /etc/redhat-release ]; then dnf list installed ; fi', r'(?P.*)', re.IGNORECASE | re.DOTALL], + [rpwr, 'Installed Python Packages', f"{sys.executable} -m pip freeze", r'(?P.*)', re.IGNORECASE | re.DOTALL], [rpwr, 'Processes', '/usr/bin/ps -aux', r'(?P.*)', re.IGNORECASE | re.DOTALL], [ rpwrs, @@ -106,6 +110,7 @@ def read_directory_recursive(directory): [rpwr, 'Uname', 'uname -a', r'(?P.*)'], [rpwr, 'Docker Info', 'docker info', r'(?P.*)', re.IGNORECASE | re.DOTALL], [rpwr, 'Docker Version', 'docker version', r'(?P.*)', re.IGNORECASE | re.DOTALL], + [rpwr, 'Docker Containers', 'docker ps -a', r'(?P.*)'], [rpwr, 'Processes', '/bin/ps -ax', r'(?P.*)', re.IGNORECASE | re.DOTALL], [rpwr, 'Network Interfaces', 'ifconfig | grep -E "flags|ether"', r'(?P.*)', re.IGNORECASE | re.DOTALL], diff --git a/lib/hardware_info_root.py b/lib/hardware_info_root.py index ddc57cbd0..8b7f7d0e9 100755 --- a/lib/hardware_info_root.py +++ b/lib/hardware_info_root.py @@ -6,7 +6,7 @@ import json import platform -from hardware_info import rdr, get_values +from lib.hardware_info import rdr, get_values root_info_list = [ [rdr, 'CPU scheduling', '/sys/kernel/debug/sched'], diff --git a/lib/notes.py b/lib/notes.py index d4c83e84a..aa2b6d77b 100644 --- a/lib/notes.py +++ b/lib/notes.py @@ -1,9 +1,7 @@ -#pylint: disable=import-error,wrong-import-position - from html import escape from re import fullmatch -from db import DB +from lib.db import DB class Notes(): @@ -14,16 +12,16 @@ def get_notes(self): return self.__notes - def save_to_db(self, project_id): + def save_to_db(self, run_id): for note in self.__notes: DB().query(""" INSERT INTO notes - ("project_id", "detail_name", "note", "time", "created_at") + ("run_id", "detail_name", "note", "time", "created_at") VALUES (%s, %s, %s, %s, NOW()) """, - params=(project_id, escape(note['detail_name']), escape(note['note']), int(note['timestamp'])) + params=(run_id, escape(note['detail_name']), escape(note['note']), int(note['timestamp'])) ) def parse_note(self, line): @@ -39,7 +37,7 @@ def add_note(self, note): import time parser = argparse.ArgumentParser() - parser.add_argument('project_id', help='Please supply a project_id to attribute the measurements to') + parser.add_argument('run_id', help='Please supply a run_id to attribute the measurements to') args = parser.parse_args() # script will exit if arguments not present @@ -47,4 +45,4 @@ def add_note(self, note): notes.add_note({'note': 'This is my note', 'timestamp': int(time.time_ns() / 1000), 'detail_name': 'Arnes_ Container'}) - notes.save_to_db(args.project_id) + notes.save_to_db(args.run_id) diff --git a/lib/process_helpers.py b/lib/process_helpers.py index a9317bf12..49302f7fa 100644 --- a/lib/process_helpers.py +++ b/lib/process_helpers.py @@ -49,7 +49,6 @@ def timeout(process, cmd: str, duration: int): except subprocess.TimeoutExpired as exc2: print("Process could not terminate in 5s time. Killing ...") process.kill() - #pylint: disable=raise-missing-from raise RuntimeError(f"Process could not terminate in 5s time and was killed: {cmd}") from exc2 raise RuntimeError(f"Process exceeded runtime of {duration}s: {cmd}") from exc diff --git a/lib/schema_checker.py b/lib/schema_checker.py index 29e1449f4..8eff447ee 100644 --- a/lib/schema_checker.py +++ b/lib/schema_checker.py @@ -2,7 +2,7 @@ import string import re from schema import Schema, SchemaError, Optional, Or, Use -# https://docs.green-coding.berlin/docs/measuring/usage-scenario/ +# # networks documentation is different than what i see in the wild! # name: str # also isn't networks optional? @@ -67,6 +67,19 @@ def valid_service_types(self, value): raise SchemaError(f"{value} is not 'container'") return value + def validate_networks_no_invalid_chars(self, networks): + if isinstance(networks, list): + for item in networks: + if item is not None: + self.contains_no_invalid_chars(item) + elif isinstance(networks, dict): + for key, value in networks.items(): + self.contains_no_invalid_chars(key) + if value is not None: + self.contains_no_invalid_chars(value) + else: + raise SchemaError("'networks' should be a list or a dictionary") + def check_usage_scenario(self, usage_scenario): # Anything with Optional() is not needed, but if it exists must conform to the definition specified @@ -75,14 +88,13 @@ def check_usage_scenario(self, usage_scenario): "author": str, "description": str, - Optional("networks"): { - Use(self.contains_no_invalid_chars): None - }, + Optional("networks"): Or(list, dict), Optional("services"): { Use(self.contains_no_invalid_chars): { Optional("type"): Use(self.valid_service_types), Optional("image"): str, + Optional("build"): Or(Or({str:str},list),str), Optional("networks"): self.single_or_list(Use(self.contains_no_invalid_chars)), Optional("environment"): self.single_or_list(Or(dict,str)), Optional("ports"): self.single_or_list(Or(str, int)), @@ -102,17 +114,26 @@ def check_usage_scenario(self, usage_scenario): Optional("detach"): bool, Optional("note"): str, Optional("read-notes-stdout"): bool, - Optional("ignore-errors"): bool + Optional("ignore-errors"): bool, + Optional("shell"): str, + Optional("log-stdout"): bool, + Optional("log-stderr"): bool, }], }], - Optional("builds"): { - str:str - }, - Optional("compose-file"): Use(self.validate_compose_include) }, ignore_extra_keys=True) + # This check is necessary to do in a seperate pass. If tried to bake into the schema object above, + # it will not know how to handle the value passed when it could be either a dict or list + if 'networks' in usage_scenario: + self.validate_networks_no_invalid_chars(usage_scenario['networks']) + + for service_name in usage_scenario.get('services'): + service = usage_scenario['services'][service_name] + if 'image' not in service and 'build' not in service: + raise SchemaError("The 'image' key under services is required when 'build' key is not present.") + usage_scenario_schema.validate(usage_scenario) @@ -120,7 +141,6 @@ def check_usage_scenario(self, usage_scenario): # import yaml # with open("test-file.yml", encoding='utf8') as f: -# # with open("test-file-2.yaml", encoding='utf8') as f: # usage_scenario = yaml.safe_load(f) # SchemaChecker = SchemaChecker(validate_compose_flag=True) diff --git a/lib/system_checks.py b/lib/system_checks.py new file mode 100644 index 000000000..1e5fa014d --- /dev/null +++ b/lib/system_checks.py @@ -0,0 +1,116 @@ +# This file handles the checking of the system +# There is a list of checks that is made up of tuples structured the following way: +# - the function to call to check. This will return True or None for success and False for failure +# - What severity the False return value has. If the Status is Error we raise and exit the GMT +# - A string what is being checked +# - A string to output on WARN or INFO +# It is possible for one of the checkers or metric providers to raise an exception if something should fail specifically +# otherwise you can just return False and set the Status to ERROR for the program to abort. + +import sys +import os +from enum import Enum +import subprocess +import psutil + +from psycopg import OperationalError as psycopg_OperationalError + +from lib import utils +from lib import error_helpers +from lib.db import DB +from lib.global_config import GlobalConfig +from lib.terminal_colors import TerminalColors + +Status = Enum('Status', ['ERROR', 'INFO', 'WARN']) + +GMT_Resources = { + 'free_disk': 1024 ** 3, # 1GB in bytes + 'free_memory': 1024 ** 3, # 1GB in bytes +} +class ConfigurationCheckError(Exception): + pass + +######## CHECK FUNCTIONS ######## +def check_db(): + try: + DB() + except psycopg_OperationalError: + error_helpers.log_error('DB is not available. Did you start the docker containers?') + os._exit(1) + return True + +def check_one_psu_provider(): + metric_providers = utils.get_metric_providers(GlobalConfig().config).keys() + energy_machine_providers = [provider for provider in metric_providers if ".energy" in provider and ".machine" in provider] + return len(energy_machine_providers) <= 1 + +def check_tmpfs_mount(): + return not any(partition.mountpoint == '/tmp' and partition.fstype != 'tmpfs' for partition in psutil.disk_partitions()) + +def check_free_disk(): + free_space_bytes = psutil.disk_usage(os.path.dirname(os.path.abspath(__file__))).free + return free_space_bytes >= GMT_Resources['free_disk'] + +def check_free_memory(): + return psutil.virtual_memory().available >= GMT_Resources['free_memory'] + +def check_containers_running(): + result = subprocess.run(['docker', 'ps', '--format', '{{.Names}}'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, encoding='UTF-8') + return not bool(result.stdout.strip()) + +def check_docker_daemon(): + result = subprocess.run(['docker', 'version'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, encoding='UTF-8') + return result.returncode == 0 + + +######## END CHECK FUNCTIONS ######## + +start_checks = [ + (check_db, Status.ERROR, 'db online', 'This text will never be triggered, please look in the function itself'), + (check_one_psu_provider, Status.ERROR, 'single PSU provider', 'Please only select one PSU provider'), + (check_tmpfs_mount, Status.INFO, 'tmpfs mount', 'We recommend to mount tmp on tmpfs'), + (check_free_disk, Status.ERROR, '1GB free hdd space', 'We recommend to free up some disk space'), + (check_free_memory, Status.ERROR, 'free memory', 'No free memory! Please kill some programs'), + (check_docker_daemon, Status.ERROR, 'docker daemon', 'The docker daemon could not be reached. Are you running in rootless mode or have added yourself to the docker group? See installation: [See https://docs.green-coding.berlin/docs/installation/]'), + (check_containers_running, Status.WARN, 'Running containers', 'You have other containers running on the system. This is usually what you want in local development, but for undisturbed measurements consider going for a measurement cluster [See https://docs.green-coding.berlin/docs/installation/installation-cluster/].'), + +] + + +def check_start(): + print(TerminalColors.HEADER, '\nRunning System Checks', TerminalColors.ENDC) + max_key_length = max(len(key[2]) for key in start_checks) + + for check in start_checks: + retval = None + try: + retval = check[0]() + except ConfigurationCheckError as exp: + raise exp + finally: + formatted_key = check[2].ljust(max_key_length) + if retval or retval is None: + output = f"{TerminalColors.OKGREEN}OK{TerminalColors.ENDC}" + else: + if check[1] == Status.WARN: + output = f"{TerminalColors.WARNING}WARN{TerminalColors.ENDC} ({check[3]})" + elif check[1] == Status.INFO: + output = f"{TerminalColors.OKCYAN}INFO{TerminalColors.ENDC} ({check[3]})" + else: + output = f"{TerminalColors.FAIL}ERROR{TerminalColors.ENDC}" + + exc_type, _, _ = sys.exc_info() + if exc_type is not None: + output = f"{TerminalColors.FAIL}EXCEPTION{TerminalColors.ENDC}" + + print(f"Checking {formatted_key} : {output}") + + if retval is False and check[1] == Status.ERROR: + # Error needs to raise + raise ConfigurationCheckError(check[3]) diff --git a/lib/utils.py b/lib/utils.py index 252b3167c..c98230928 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,25 +1,23 @@ -#pylint: disable=no-member -#pylint: disable=invalid-name import random import string import subprocess import psycopg -from db import DB +from lib.db import DB def randomword(length): letters = string.ascii_lowercase return ''.join(random.choice(letters) for i in range(length)) -def get_project_data(project_name): +def get_run_data(run_name): query = """ SELECT * FROM - projects + runs WHERE name = %s """ - data = DB().fetch_one(query, (project_name, ), row_factory=psycopg.rows.dict_row) + data = DB().fetch_one(query, (run_name, ), row_factory=psycopg.rows.dict_row) if data is None or data == []: return None return data diff --git a/lib/venv_checker.py b/lib/venv_checker.py new file mode 100644 index 000000000..cea7b7f80 --- /dev/null +++ b/lib/venv_checker.py @@ -0,0 +1,12 @@ +import os +import sys +from lib.terminal_colors import TerminalColors + +def check_venv(): + CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + venv_path = os.path.realpath(os.path.join(CURRENT_DIR, '..', 'venv')) + if sys.prefix != venv_path: + print(TerminalColors.FAIL) + print(f"Error:\n\nYou are not using a venv, or venv is not in expected directory {venv_path}\nCurrent venv is in {sys.prefix}\n\nThe Green Metrics Tool needs a venv to correctly find installed packages and also necessary include paths.\nPlease check the installation instructions on https://docs.green-coding.berlin/docs/installation/\n\nMaybe you just forgot to activate your venv? Try:\n$ source venv/bin/activate") + print(TerminalColors.ENDC) + sys.exit(-1) diff --git a/metric_providers/base.py b/metric_providers/base.py index 2999e3265..d90f912f2 100644 --- a/metric_providers/base.py +++ b/metric_providers/base.py @@ -1,5 +1,3 @@ -# pylint: disable=no-member,consider-using-with,subprocess-popen-preexec-fn,import-error,too-many-instance-attributes,too-many-arguments - import os from pathlib import Path import subprocess @@ -8,6 +6,10 @@ from io import StringIO import pandas +from lib.system_checks import ConfigurationCheckError + +class MetricProviderConfigurationError(ConfigurationCheckError): + pass class BaseMetricProvider: @@ -20,6 +22,7 @@ def __init__( current_dir, metric_provider_executable='metric-provider-binary', sudo=False, + disable_buffer=True ): self._metric_name = metric_name self._metrics = metrics @@ -29,6 +32,8 @@ def __init__( self._metric_provider_executable = metric_provider_executable self._sudo = sudo self._has_started = False + self._disable_buffer = disable_buffer + self._rootless = None self._tmp_folder = '/tmp/green-metrics-tool' self._ps = None @@ -38,6 +43,12 @@ def __init__( self._filename = f"{self._tmp_folder}/{self._metric_name}.log" + self.check_system() + + # this is the default function that will be overridden in the children + def check_system(self): + pass + # implemented as getter function and not direct access, so it can be overloaded # some child classes might not actually have _ps attribute set def get_stderr(self): @@ -46,7 +57,7 @@ def get_stderr(self): def has_started(self): return self._has_started - def read_metrics(self, project_id, containers): + def read_metrics(self, run_id, containers=None): with open(self._filename, 'r', encoding='utf-8') as file: csv_data = file.read() @@ -79,13 +90,17 @@ def read_metrics(self, project_id, containers): df['unit'] = self._unit df['metric'] = self._metric_name - df['project_id'] = project_id + df['run_id'] = run_id return df def start_profiling(self, containers=None): - call_string = f"{self._metric_provider_executable} -i {self._resolution}" + if self._resolution is None: + call_string = self._metric_provider_executable + else: + call_string = f"{self._metric_provider_executable} -i {self._resolution}" + if self._metric_provider_executable[0] != '/': call_string = f"{self._current_dir}/{call_string}" @@ -101,10 +116,18 @@ def start_profiling(self, containers=None): if self._metrics.get('container_id') is not None: call_string += ' -s ' call_string += ','.join(containers.keys()) + + if self._rootless is True: + call_string += ' --rootless ' + call_string += f" > {self._filename}" + if self._disable_buffer: + call_string = f"stdbuf -o0 {call_string}" + print(call_string) + #pylint: disable=consider-using-with,subprocess-popen-preexec-fn self._ps = subprocess.Popen( [call_string], shell=True, diff --git a/metric_providers/cpu/energy/RAPL/MSR/component/provider.py b/metric_providers/cpu/energy/RAPL/MSR/component/provider.py index cc95211f3..9d6b420ff 100644 --- a/metric_providers/cpu/energy/RAPL/MSR/component/provider.py +++ b/metric_providers/cpu/energy/RAPL/MSR/component/provider.py @@ -1,14 +1,13 @@ import os -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class CpuEnergyRaplMsrComponentProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="cpu_energy_rapl_msr_component", - metrics={"time": int, "value": int, "package_id": str}, + metric_name='cpu_energy_rapl_msr_component', + metrics={'time': int, 'value': int, 'package_id': str}, resolution=resolution, - unit="mJ", + unit='mJ', current_dir=os.path.dirname(os.path.abspath(__file__)), ) diff --git a/metric_providers/cpu/frequency/sysfs/core/README.md b/metric_providers/cpu/frequency/sysfs/core/README.md index fbd5e1924..d8b16ae58 100644 --- a/metric_providers/cpu/frequency/sysfs/core/README.md +++ b/metric_providers/cpu/frequency/sysfs/core/README.md @@ -1,7 +1,3 @@ -# Information +# Documentation -This provider uses [IPMI](https://www.intel.com/content/www/us/en/products/docs/servers/ipmi/ipmi-home.html) to get -the current system power statistics. - -It requires specific hardware to run and we have also identified a delay in the readings, close to one second. -This provider is currently a proof of concept, and is uncertain if it will be developed further. +Please see https://docs.green-coding.berlin/docs/measuring/metric-providers/cpu-frequency-sysfs-core/ for details \ No newline at end of file diff --git a/metric_providers/cpu/frequency/sysfs/core/provider.py b/metric_providers/cpu/frequency/sysfs/core/provider.py index 10a174595..1de47ba0a 100644 --- a/metric_providers/cpu/frequency/sysfs/core/provider.py +++ b/metric_providers/cpu/frequency/sysfs/core/provider.py @@ -1,15 +1,24 @@ import os -#pylint: disable=import-error -from metric_providers.base import BaseMetricProvider +from metric_providers.base import MetricProviderConfigurationError, BaseMetricProvider class CpuFrequencySysfsCoreProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="cpu_frequency_sysfs_core", - metrics={"time": int, "value": int, "core_id": int}, + metric_name='cpu_frequency_sysfs_core', + metrics={'time': int, 'value': int, 'core_id': int}, resolution=0.001*resolution, - unit="Hz", + unit='Hz', current_dir=os.path.dirname(os.path.abspath(__file__)), - metric_provider_executable="get-scaling-cur-freq.sh", + metric_provider_executable='get-scaling-cur-freq.sh', ) + + def check_system(self): + file_path = "/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" + if os.path.exists(file_path): + try: + with open(file_path, 'r', encoding='utf-8') as file: + file.read() + except PermissionError as exc: + raise MetricProviderConfigurationError(f"{self._metric_name} provider could not be started.\nCannot read the path for the CPU frequency in sysfs.\n\nAre you running in a VM / cloud / shared hosting?\nIf so please disable the {self._metric_name} provider in the config.yml") from exc + raise MetricProviderConfigurationError(f"{self._metric_name} provider could not be started.\nCould not find the path for the CPU frequency in sysfs.\n\nAre you running in a VM / cloud / shared hosting? \nIf so please disable the {self._metric_name} provider in the config.yml") diff --git a/metric_providers/cpu/time/cgroup/container/provider.py b/metric_providers/cpu/time/cgroup/container/provider.py index cf17dba83..5c354b81b 100644 --- a/metric_providers/cpu/time/cgroup/container/provider.py +++ b/metric_providers/cpu/time/cgroup/container/provider.py @@ -1,14 +1,14 @@ import os -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class CpuTimeCgroupContainerProvider(BaseMetricProvider): - def __init__(self, resolution): + def __init__(self, resolution, rootless=False): super().__init__( - metric_name="cpu_time_cgroup_container", - metrics={"time": int, "value": int, "container_id": str}, + metric_name='cpu_time_cgroup_container', + metrics={'time': int, 'value': int, 'container_id': str}, resolution=resolution, - unit="us", + unit='us', current_dir=os.path.dirname(os.path.abspath(__file__)), ) + self._rootless = rootless diff --git a/metric_providers/cpu/time/cgroup/container/source.c b/metric_providers/cpu/time/cgroup/container/source.c index 52f9ac896..ffa0e665b 100644 --- a/metric_providers/cpu/time/cgroup/container/source.c +++ b/metric_providers/cpu/time/cgroup/container/source.c @@ -5,6 +5,7 @@ #include #include #include // for strtok +#include typedef struct container_t { // struct is a specification and this static makes no sense here char path[BUFSIZ]; @@ -14,17 +15,19 @@ typedef struct container_t { // struct is a specification and this static makes // All variables are made static, because we believe that this will // keep them local in scope to the file and not make them persist in state // between Threads. -// TODO: If this code ever gets multi-threaded please review this assumption to -// not pollute another threads state +// in any case, none of these variables should change between threads static int user_id = 0; static long int user_hz; static unsigned int msleep_time=1000; -static container_t *containers = NULL; static long int read_cpu_cgroup(char* filename) { long int cpu_usage = -1; FILE* fd = NULL; - fd = fopen(filename, "r"); // check for general readability only once + fd = fopen(filename, "r"); + if ( fd == NULL) { + fprintf(stderr, "Error - Could not open path for reading: %s. Maybe the container is not running anymore? Are you using --rootless mode? Errno: %d\n", filename, errno); + exit(1); + } fscanf(fd, "usage_usec %ld", &cpu_usage); fclose(fd); return cpu_usage; @@ -42,18 +45,60 @@ static int output_stats(container_t *containers, int length) { return 1; } -// TODO: better arguement parsing, atm it assumes first argument is msleep_time, -// and rest are container ids with no real error checking +static int parse_containers(container_t** containers, char* containers_string, int rootless_mode) { + if(containers_string == NULL) { + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); + exit(1); + } + + *containers = malloc(sizeof(container_t)); + char *id = strtok(containers_string,","); + int length = 0; + + for (; id != NULL; id = strtok(NULL, ",")) { + //printf("Token: %s\n", id); + length++; + *containers = realloc(*containers, length * sizeof(container_t)); + (*containers)[length-1].id = id; + if(rootless_mode) { + sprintf((*containers)[length-1].path, + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cpu.stat", + user_id, user_id, id); + } else { + sprintf((*containers)[length-1].path, + "/sys/fs/cgroup/system.slice/docker-%s.scope/cpu.stat", + id); + } + } + + if(length == 0) { + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); + exit(1); + } + return length; +} + int main(int argc, char **argv) { int c; - int length = 0; + int rootless_mode = 0; // docker root is default + char *containers_string = NULL; // Dynamic buffer to store optarg + container_t *containers = NULL; setvbuf(stdout, NULL, _IONBF, 0); user_hz = sysconf(_SC_CLK_TCK); user_id = getuid(); - while ((c = getopt (argc, argv, "i:s:h")) != -1) { + static struct option long_options[] = + { + {"rootless", no_argument, NULL, 'r'}, + {"help", no_argument, NULL, 'h'}, + {"interval", no_argument, NULL, 'i'}, + {"containers", no_argument, NULL, 's'}, + {NULL, 0, NULL, 0} + }; + + while ((c = getopt_long(argc, argv, "ri:s:h", long_options, NULL)) != -1) { switch (c) { case 'h': printf("Usage: %s [-i msleep_time] [-h]\n\n",argv[0]); @@ -74,27 +119,14 @@ int main(int argc, char **argv) { case 'i': msleep_time = atoi(optarg); break; - case 's': - containers = malloc(sizeof(container_t)); - char *id = strtok(optarg,","); - for (; id != NULL; id = strtok(NULL, ",")) { - //printf("Token: %s\n", id); - length++; - containers = realloc(containers, length * sizeof(container_t)); - containers[length-1].id = id; - sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cpu.stat", - user_id, user_id, id); - - FILE* fd = NULL; - fd = fopen(containers[length-1].path, "r"); // check for general readability only once - if ( fd == NULL) { - fprintf(stderr, "Error - file %s failed to open: errno: %d\n", containers[length-1].path, errno); - exit(1); - } - fclose(fd); - } + case 'r': + rootless_mode = 1; + break; + + case 's': + containers_string = (char *)malloc(strlen(optarg) + 1); // Allocate memory + strncpy(containers_string, optarg, strlen(optarg)); break; default: fprintf(stderr,"Unknown option %c\n",c); @@ -102,10 +134,7 @@ int main(int argc, char **argv) { } } - if(containers == NULL) { - fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); - exit(1); - } + int length = parse_containers(&containers, containers_string, rootless_mode); while(1) { output_stats(containers, length); diff --git a/metric_providers/cpu/time/cgroup/system/provider.py b/metric_providers/cpu/time/cgroup/system/provider.py index de7228c9f..6e5c7d051 100644 --- a/metric_providers/cpu/time/cgroup/system/provider.py +++ b/metric_providers/cpu/time/cgroup/system/provider.py @@ -1,14 +1,13 @@ import os -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class CpuTimeCgroupSystemProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="cpu_time_cgroup_system", - metrics={"time": int, "value": int}, + metric_name='cpu_time_cgroup_system', + metrics={'time': int, 'value': int}, resolution=resolution, - unit="us", + unit='us', current_dir=os.path.dirname(os.path.abspath(__file__)), ) diff --git a/metric_providers/cpu/time/procfs/system/provider.py b/metric_providers/cpu/time/procfs/system/provider.py index 489a6cc45..db7bc5198 100644 --- a/metric_providers/cpu/time/procfs/system/provider.py +++ b/metric_providers/cpu/time/procfs/system/provider.py @@ -1,14 +1,13 @@ import os -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class CpuTimeProcfsSystemProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="cpu_time_procfs_system", - metrics={"time": int, "value": int}, + metric_name='cpu_time_procfs_system', + metrics={'time': int, 'value': int}, resolution=resolution, - unit="us", + unit='us', current_dir=os.path.dirname(os.path.abspath(__file__)), ) diff --git a/metric_providers/cpu/utilization/cgroup/container/provider.py b/metric_providers/cpu/utilization/cgroup/container/provider.py index ff8632cef..986d36701 100644 --- a/metric_providers/cpu/utilization/cgroup/container/provider.py +++ b/metric_providers/cpu/utilization/cgroup/container/provider.py @@ -1,14 +1,14 @@ import os -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class CpuUtilizationCgroupContainerProvider(BaseMetricProvider): - def __init__(self, resolution): + def __init__(self, resolution, rootless=False): super().__init__( - metric_name="cpu_utilization_cgroup_container", + metric_name='cpu_utilization_cgroup_container', metrics={'time': int, 'value': int, 'container_id': str}, resolution=resolution, - unit="Ratio", + unit='Ratio', current_dir=os.path.dirname(os.path.abspath(__file__)), ) + self._rootless = rootless diff --git a/metric_providers/cpu/utilization/cgroup/container/source.c b/metric_providers/cpu/utilization/cgroup/container/source.c index 05e5fe3ef..f83c67a57 100644 --- a/metric_providers/cpu/utilization/cgroup/container/source.c +++ b/metric_providers/cpu/utilization/cgroup/container/source.c @@ -5,6 +5,7 @@ #include #include #include // for strtok +#include typedef struct container_t { // struct is a specification and this static makes no sense here char path[BUFSIZ]; @@ -14,12 +15,10 @@ typedef struct container_t { // struct is a specification and this static makes // All variables are made static, because we believe that this will // keep them local in scope to the file and not make them persist in state // between Threads. -// TODO: If this code ever gets multi-threaded please review this assumption to -// not pollute another threads state +// in any case, none of these variables should change between threads static int user_id = 0; static long int user_hz; static unsigned int msleep_time=1000; -static container_t *containers = NULL; static long int read_cpu_proc(FILE *fd) { long int user_time, nice_time, system_time, idle_time, iowait_time, irq_time, softirq_time, steal_time; @@ -46,9 +45,10 @@ static long int get_cpu_stat(char* filename, int mode) { long int result=-1; fd = fopen(filename, "r"); + if ( fd == NULL) { - fprintf(stderr, "Error - file %s failed to open: errno: %d\n", filename, errno); - exit(1); + fprintf(stderr, "Error - Could not open path for reading: %s. Maybe the container is not running anymore? Are you using --rootless mode? Errno: %d\n", filename, errno); + exit(1); } if(mode == 1) { result = read_cpu_cgroup(fd); @@ -62,7 +62,7 @@ static long int get_cpu_stat(char* filename, int mode) { } -static int output_stats(container_t *containers, int length) { +static int output_stats(container_t* containers, int length) { long int main_cpu_reading_before, main_cpu_reading_after, main_cpu_reading; long int cpu_readings_before[length]; @@ -76,6 +76,7 @@ static int output_stats(container_t *containers, int length) { // Get Energy Readings, set timestamp mark gettimeofday(&now, NULL); for(i=0; i #include #include // for strtok +#include typedef struct container_t { // struct is a specification and this static makes no sense here char path[BUFSIZ]; @@ -13,19 +14,17 @@ typedef struct container_t { // struct is a specification and this static makes // All variables are made static, because we believe that this will // keep them local in scope to the file and not make them persist in state // between Threads. -// TODO: If this code ever gets multi-threaded please review this assumption to -// not pollute another threads state +// in any case, none of these variables should change between threads static int user_id = 0; static unsigned int msleep_time=1000; -static container_t *containers = NULL; static long int get_memory_cgroup(char* filename) { long int memory = -1; FILE * fd = fopen(filename, "r"); if ( fd == NULL) { - fprintf(stderr, "Error - file %s failed to open: errno: %d\n", filename, errno); - exit(1); + fprintf(stderr, "Error - Could not open path for reading: %s. Maybe the container is not running anymore? Are you using --rootless mode? Errno: %d\n", filename, errno); + exit(1); } fscanf(fd, "%ld", &memory); @@ -34,7 +33,7 @@ static long int get_memory_cgroup(char* filename) { return memory; } else { - fprintf(stderr, "Error - memory.current could not be read"); + fprintf(stderr, "Error - memory.current could not be read or was < 0."); fclose(fd); exit(1); } @@ -55,15 +54,59 @@ static int output_stats(container_t *containers, int length) { return 1; } +static int parse_containers(container_t** containers, char* containers_string, int rootless_mode) { + if(containers_string == NULL) { + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); + exit(1); + } + + *containers = malloc(sizeof(container_t)); + char *id = strtok(containers_string,","); + int length = 0; + + for (; id != NULL; id = strtok(NULL, ",")) { + //printf("Token: %s\n", id); + length++; + *containers = realloc(*containers, length * sizeof(container_t)); + (*containers)[length-1].id = id; + if(rootless_mode) { + sprintf((*containers)[length-1].path, + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/memory.current", + user_id, user_id, id); + } else { + sprintf((*containers)[length-1].path, + "/sys/fs/cgroup/system.slice/docker-%s.scope/memory.current", + id); + } + } + + if(length == 0) { + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); + exit(1); + } + return length; +} + int main(int argc, char **argv) { int c; - int length = 0; + int rootless_mode = 0; // docker root is default + char *containers_string = NULL; // Dynamic buffer to store optarg + container_t *containers = NULL; setvbuf(stdout, NULL, _IONBF, 0); user_id = getuid(); - while ((c = getopt (argc, argv, "i:s:h")) != -1) { + static struct option long_options[] = + { + {"rootless", no_argument, NULL, 'r'}, + {"help", no_argument, NULL, 'h'}, + {"interval", no_argument, NULL, 'i'}, + {"containers", no_argument, NULL, 's'}, + {NULL, 0, NULL, 0} + }; + + while ((c = getopt_long(argc, argv, "ri:s:h", long_options, NULL)) != -1) { switch (c) { case 'h': printf("Usage: %s [-i msleep_time] [-h]\n\n",argv[0]); @@ -74,27 +117,12 @@ int main(int argc, char **argv) { case 'i': msleep_time = atoi(optarg); break; + case 'r': + rootless_mode = 1; + break; case 's': - containers = malloc(sizeof(container_t)); - char *id = strtok(optarg,","); - for (; id != NULL; id = strtok(NULL, ",")) { - //printf("Token: %s\n", id); - length++; - containers = realloc(containers, length * sizeof(container_t)); - containers[length-1].id = id; - sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/memory.current", - user_id, user_id, id); - - FILE* fd = NULL; - fd = fopen(containers[length-1].path, "r"); // check for general readability only once - if ( fd == NULL) { - fprintf(stderr, "Error - file %s failed to open: errno: %d\n", containers[length-1].path, errno); - exit(1); - } - fclose(fd); - } - + containers_string = (char *)malloc(strlen(optarg) + 1); // Allocate memory + strncpy(containers_string, optarg, strlen(optarg)); break; default: fprintf(stderr,"Unknown option %c\n",c); @@ -102,10 +130,7 @@ int main(int argc, char **argv) { } } - if(containers == NULL) { - fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); - exit(1); - } + int length = parse_containers(&containers, containers_string, rootless_mode); while(1) { output_stats(containers, length); diff --git a/metric_providers/network/connections/proxy/container/provider.py b/metric_providers/network/connections/proxy/container/provider.py new file mode 100644 index 000000000..354e58b22 --- /dev/null +++ b/metric_providers/network/connections/proxy/container/provider.py @@ -0,0 +1,95 @@ +# This code handles the setup of the proxy we use to monitor the network connections in the docker containers. +# Structurally it is a copy of the BaseMetricProvider but because we need to do things slightly different it is a copy. +# In the future this might be implemented as a proper provider. + +import os +import re +from datetime import datetime, timezone +import platform +import subprocess +from packaging.version import parse + +from lib.db import DB +from metric_providers.base import MetricProviderConfigurationError, BaseMetricProvider +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + +class NetworkConnectionsProxyContainerProvider(BaseMetricProvider): + def __init__(self, *, host_ip=None): + super().__init__( + metric_name='network_connections_proxy_container_dockerproxy', + metrics={}, + resolution=None, + unit=None, + current_dir=os.path.dirname(os.path.abspath(__file__)), + ) + + self._conf_file = f"{CURRENT_DIR}/proxy_conf.conf" + self._filename = f"{self._tmp_folder}/proxy.log" + self._host_ip = host_ip + + tinyproxy_path = subprocess.getoutput('which tinyproxy') + self._metric_provider_executable = f"{tinyproxy_path} -d -c {self._conf_file} > {self._filename}" + + + def check_system(self): + + output = subprocess.check_output(['tinyproxy', '-v'], stderr=subprocess.STDOUT, text=True) + version_string = output.strip().split()[1].split('-')[0] + if parse(version_string) >= parse('1.11'): + return True + + raise MetricProviderConfigurationError('Tinyproxy needs to be version 1.11 or greater.') + + + def get_docker_params(self, no_proxy_list=''): + + proxy_addr = '' + if self._host_ip: + proxy_addr = self._host_ip + elif platform.system() == 'Linux': + # Under Linux there is no way to directly link to the host + cmd = "ip addr show dev $(ip route | grep default | awk '{print $5}') | grep 'inet '| awk '{print $2}'| cut -f1 -d'/'" + ps = subprocess.run(cmd, shell=True, check=True, text=True, capture_output=True) + proxy_addr = ps.stdout.strip() + else: + proxy_addr = 'host.docker.internal' + + # See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ for a discussion on the env vars + # To be sure we include all variants + return ['--env', f"HTTP_PROXY=http://{proxy_addr}:8889", + '--env', f"HTTPS_PROXY=http://{proxy_addr}:8889", + '--env', f"http_proxy=http://{proxy_addr}:8889", + '--env', f"https_proxy=http://{proxy_addr}:8889", + '--env', f"NO_PROXY={no_proxy_list}", + '--env', f"no_proxy={no_proxy_list}"] + + + def read_metrics(self, run_id, containers=None): + records_added = 0 + with open(self._filename, 'r', encoding='utf-8') as file: + lines = file.readlines() + + pattern = re.compile(r"CONNECT\s+([A-Za-z]{3} \d{2} \d{2}:\d{2}:\d{2}(?:\.\d{3})?) \[\d+\]: Request \(file descriptor \d+\): (.+) (.+)") + + for line in lines: + match = pattern.search(line) + if match: + date_str, connection_type, protocol = match.groups() + # parse the date and time + try: + date = datetime.strptime(date_str, '%b %d %H:%M:%S.%f').replace(year=datetime.now().year) + except ValueError: + date = datetime.strptime(date_str, '%b %d %H:%M:%S').replace(year=datetime.now().year) + + time = int(date.replace(tzinfo=timezone.utc).timestamp() * 1000) + + query = ''' + INSERT INTO network_intercepts (run_id, time, connection_type, protocol) + VALUES (%s, %s, %s, %s) + RETURNING id + ''' + params = (run_id, time, connection_type, protocol) + DB().fetch_one(query, params=params) + records_added += 1 + + return records_added diff --git a/metric_providers/network/connections/proxy/container/proxy_conf.conf b/metric_providers/network/connections/proxy/container/proxy_conf.conf new file mode 100644 index 000000000..bc5e6f3a0 --- /dev/null +++ b/metric_providers/network/connections/proxy/container/proxy_conf.conf @@ -0,0 +1,6 @@ +User nobody +Group nogroup + +Port 8889 +Timeout 600 +LogLevel Connect diff --git a/metric_providers/network/io/cgroup/container/provider.py b/metric_providers/network/io/cgroup/container/provider.py index 2800e07f5..af1d599de 100644 --- a/metric_providers/network/io/cgroup/container/provider.py +++ b/metric_providers/network/io/cgroup/container/provider.py @@ -1,14 +1,14 @@ import os -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class NetworkIoCgroupContainerProvider(BaseMetricProvider): - def __init__(self, resolution): + def __init__(self, resolution, rootless=False): super().__init__( - metric_name="network_io_cgroup_container", - metrics={"time": int, "value": int, "container_id": str}, + metric_name='network_io_cgroup_container', + metrics={'time': int, 'value': int, 'container_id': str}, resolution=resolution, - unit="Bytes", + unit='Bytes', current_dir=os.path.dirname(os.path.abspath(__file__)), ) + self._rootless = rootless diff --git a/metric_providers/network/io/cgroup/container/source.c b/metric_providers/network/io/cgroup/container/source.c index fe8833717..8867c223e 100644 --- a/metric_providers/network/io/cgroup/container/source.c +++ b/metric_providers/network/io/cgroup/container/source.c @@ -8,6 +8,7 @@ #include #include #include +#include typedef struct container_t { // struct is a specification and this static makes no sense here char path[BUFSIZ]; @@ -18,11 +19,9 @@ typedef struct container_t { // struct is a specification and this static makes // All variables are made static, because we believe that this will // keep them local in scope to the file and not make them persist in state // between Threads. -// TODO: If this code ever gets multi-threaded please review this assumption to -// not pollute another threads state +// in any case, none of these variables should change between threads static int user_id = 0; static unsigned int msleep_time=1000; -static container_t *containers = NULL; static char *trimwhitespace(char *str) { char *end; @@ -53,14 +52,14 @@ static unsigned long int get_network_cgroup(unsigned int pid) { int fd_ns = open(ns_path, O_RDONLY); /* Get descriptor for namespace */ if (fd_ns == -1) { - fprintf(stderr, "open failed"); + fprintf(stderr, "open namespace failed for pid %u", pid); exit(1); } // printf("Entering namespace /proc/%u/ns/net \n", pid); if (setns(fd_ns, 0) == -1) { // argument 0 means that any type of NS (IPC, Network, UTS) is allowed - fprintf(stderr, "setns failed"); + fprintf(stderr, "setns failed for pid %u", pid); exit(1); } @@ -69,8 +68,8 @@ static unsigned long int get_network_cgroup(unsigned int pid) { // by testing on our machine though ip link also returned significantly smaller values (~50% less) FILE * fd = fopen("/proc/net/dev", "r"); if ( fd == NULL) { - fprintf(stderr, "Error - file %s failed to open: errno: %d\n", "/proc/net/dev", errno); - exit(1); + fprintf(stderr, "Error - file %s failed to open. Is the container still running? Errno: %d\n", "/proc/net/dev", errno); + exit(1); } // skip first two lines @@ -111,15 +110,68 @@ static int output_stats(container_t *containers, int length) { return 1; } +static int parse_containers(container_t** containers, char* containers_string, int rootless_mode) { + if(containers_string == NULL) { + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); + exit(1); + } + + *containers = malloc(sizeof(container_t)); + char *id = strtok(containers_string,","); + int length = 0; + + for (; id != NULL; id = strtok(NULL, ",")) { + //printf("Token: %s\n", id); + length++; + *containers = realloc(*containers, length * sizeof(container_t)); + (*containers)[length-1].id = id; + if(rootless_mode) { + sprintf((*containers)[length-1].path, + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cgroup.procs", + user_id, user_id, id); + } else { + sprintf((*containers)[length-1].path, + "/sys/fs/cgroup/system.slice/docker-%s.scope/cgroup.procs", + id); + } + FILE* fd = NULL; + fd = fopen((*containers)[length-1].path, "r"); // check for general readability only once + if ( fd == NULL) { + fprintf(stderr, "Error - cgroup.procs file %s failed to open: errno: %d\n", (*containers)[length-1].path, errno); + exit(1); + } + fscanf(fd, "%u", &(*containers)[length-1].pid); + fclose(fd); + } + + if(length == 0) { + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); + exit(1); + } + + return length; +} + int main(int argc, char **argv) { int c; - int length = 0; + int rootless_mode = 0; // docker root is default + char *containers_string = NULL; // Dynamic buffer to store optarg + container_t *containers = NULL; setvbuf(stdout, NULL, _IONBF, 0); user_id = getuid(); // because the file is run without sudo but has the suid bit set we only need getuid and not geteuid - while ((c = getopt (argc, argv, "i:s:h")) != -1) { + static struct option long_options[] = + { + {"rootless", no_argument, NULL, 'r'}, + {"help", no_argument, NULL, 'h'}, + {"interval", no_argument, NULL, 'i'}, + {"containers", no_argument, NULL, 's'}, + {NULL, 0, NULL, 0} + }; + + while ((c = getopt_long(argc, argv, "ri:s:h", long_options, NULL)) != -1) { switch (c) { case 'h': printf("Usage: %s [-i msleep_time] [-h]\n\n",argv[0]); @@ -130,28 +182,12 @@ int main(int argc, char **argv) { case 'i': msleep_time = atoi(optarg); break; + case 'r': + rootless_mode = 1; + break; case 's': - containers = malloc(sizeof(container_t)); - char *id = strtok(optarg,","); - for (; id != NULL; id = strtok(NULL, ",")) { - //printf("Token: %s\n", id); - length++; - containers = realloc(containers, length * sizeof(container_t)); - containers[length-1].id = id; - sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cgroup.procs", - user_id, user_id, id); - - FILE* fd = NULL; - fd = fopen(containers[length-1].path, "r"); // check for general readability only once - if ( fd == NULL) { - fprintf(stderr, "Error - file %s failed to open: errno: %d\n", containers[length-1].path, errno); - exit(1); - } - fscanf(fd, "%u", &containers[length-1].pid); - fclose(fd); - } - + containers_string = (char *)malloc(strlen(optarg) + 1); // Allocate memory + strncpy(containers_string, optarg, strlen(optarg)); break; default: fprintf(stderr,"Unknown option %c\n",c); @@ -159,10 +195,7 @@ int main(int argc, char **argv) { } } - if(containers == NULL) { - fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); - exit(1); - } + int length = parse_containers(&containers, containers_string, rootless_mode); while(1) { output_stats(containers, length); diff --git a/metric_providers/network/io/docker/stats/container/provider.py b/metric_providers/network/io/docker/stats/container/provider.py index 758f684c4..b3821a3b0 100644 --- a/metric_providers/network/io/docker/stats/container/provider.py +++ b/metric_providers/network/io/docker/stats/container/provider.py @@ -1,17 +1,15 @@ import os import subprocess -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider - class NetworkIoDockerStatsContainerProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="network_io_docker_stats_container", - metrics={"time": int, "value": int, "container_id": str}, + metric_name='network_io_docker_stats_container', + metrics={'time': int, 'value': int, 'container_id': str}, resolution=resolution, - unit="Bytes", + unit='Bytes', current_dir=os.path.dirname(os.path.abspath(__file__)), ) @@ -28,7 +26,6 @@ def start_profiling(self, containers=None): preexec_fn=os.setsid, encoding='UTF-8') - #pylint: disable=unused-argument - def read_metrics(self, project_id, containers): + def read_metrics(self, run_id, containers=None): print('Read Metrics is overloaded for docker_stats, since values are not time-keyed. \ Reporter is only for manual falsification. Never use in production!') diff --git a/metric_providers/powermetrics/provider.py b/metric_providers/powermetrics/provider.py index 569987ec3..37cd35e80 100644 --- a/metric_providers/powermetrics/provider.py +++ b/metric_providers/powermetrics/provider.py @@ -6,18 +6,16 @@ import xml import pandas -#pylint: disable=import-error -from db import DB -from metric_providers.base import BaseMetricProvider - +from lib.db import DB +from metric_providers.base import MetricProviderConfigurationError, BaseMetricProvider class PowermetricsProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="powermetrics", + metric_name='powermetrics', metrics={'time': int, 'value': int}, resolution=resolution, - unit="mJ", + unit='mJ', current_dir=os.path.dirname(os.path.abspath(__file__)), metric_provider_executable='/usr/bin/powermetrics', sudo=True, @@ -25,23 +23,25 @@ def __init__(self, resolution): # We can't use --show-all here as this sometimes triggers output on stderr self._extra_switches = [ - "--show-process-io", - "--show-process-gpu", - "--show-process-netstats", - "--show-process-energy", - "--show-process-coalition", + '--show-process-io', + '--show-process-gpu', + '--show-process-netstats', + '--show-process-energy', + '--show-process-coalition', '-f', 'plist', '-o', self._filename] - def is_powermetrics_running(self): - try: - output = subprocess.check_output('pgrep -x powermetrics', shell=True) - return bool(output.strip()) # If the output is not empty, the process is running. + def check_system(self): + if self.is_powermetrics_running(): + raise MetricProviderConfigurationError('Another instance of powermetrics is already running on the system!\nPlease close it before running the Green Metrics Tool.') - except subprocess.CalledProcessError: # If the process is not running, 'pgrep' returns non-zero exit code. + def is_powermetrics_running(self): + ps = subprocess.run(['pgrep', '-qx', 'powermetrics'], check=False) + if ps.returncode == 1: return False + return True def stop_profiling(self): @@ -69,8 +69,7 @@ def stop_profiling(self): self._ps = None - # pylint: disable=too-many-locals - def read_metrics(self, project_id, containers=None): + def read_metrics(self, run_id, containers=None): with open(self._filename, 'rb') as metrics_file: datas = metrics_file.read() @@ -170,11 +169,11 @@ def read_metrics(self, project_id, containers=None): df = pandas.DataFrame.from_records(dfs, columns=['time', 'value', 'metric', 'detail_name', 'unit']) - df['project_id'] = project_id + df['run_id'] = run_id - # Set the invalid project string to indicate, that it was mac and we can't rely on the data + # Set the invalid run string to indicate, that it was mac and we can't rely on the data invalid_message = 'Measurements are not reliable as they are done on a Mac. See our blog for details.' - DB().query('UPDATE projects SET invalid_project=%s WHERE id = %s', params=(invalid_message, project_id)) + DB().query('UPDATE runs SET invalid_run=%s WHERE id = %s', params=(invalid_message, run_id)) return df diff --git a/metric_providers/psu/energy/ac/gude/machine/provider.py b/metric_providers/psu/energy/ac/gude/machine/provider.py index 96f68dc5e..b0070b232 100644 --- a/metric_providers/psu/energy/ac/gude/machine/provider.py +++ b/metric_providers/psu/energy/ac/gude/machine/provider.py @@ -1,21 +1,19 @@ import os import subprocess -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class PsuEnergyAcGudeMachineProvider(BaseMetricProvider): def __init__(self, resolution): super().__init__( - metric_name="psu_energy_ac_gude_machine", - metrics={"time": int, "value": int}, + metric_name='psu_energy_ac_gude_machine', + metrics={'time': int, 'value': int}, resolution=resolution, - unit="mJ", + unit='mJ', current_dir=os.path.dirname(os.path.abspath(__file__)), ) - #pylint: disable=unused-argument def start_profiling(self, containers=None): call_string = f"{self._current_dir}/check_gude_modified.py -i {self._resolution}" @@ -23,7 +21,7 @@ def start_profiling(self, containers=None): print(call_string) - #pylint:disable=subprocess-popen-preexec-fn,consider-using-with,attribute-defined-outside-init + #pylint:disable=subprocess-popen-preexec-fn,consider-using-with self._ps = subprocess.Popen( [call_string], shell=True, diff --git a/metric_providers/psu/energy/ac/ipmi/machine/provider.py b/metric_providers/psu/energy/ac/ipmi/machine/provider.py index 5020205bf..26e5462d7 100644 --- a/metric_providers/psu/energy/ac/ipmi/machine/provider.py +++ b/metric_providers/psu/energy/ac/ipmi/machine/provider.py @@ -1,6 +1,5 @@ import os -#pylint: disable=import-error, invalid-name from metric_providers.base import BaseMetricProvider class PsuEnergyAcIpmiMachineProvider(BaseMetricProvider): @@ -14,10 +13,26 @@ def __init__(self, resolution): metric_provider_executable='ipmi-get-machine-energy-stat.sh', ) - def read_metrics(self, project_id, containers): - df = super().read_metrics(project_id, containers) - # Conversion to Joules + def read_metrics(self, run_id, containers=None): + df = super().read_metrics(run_id, containers) + + ''' + Conversion to Joules + + If ever in need to convert the database from Joules back to a power format: + + WITH times as ( + SELECT id, value, detail_name, time, (time - LAG(time) OVER (ORDER BY detail_name ASC, time ASC)) AS diff, unit + FROM measurements + WHERE run_id = RUN_ID AND metric = 'psu_energy_ac_ipmi_machine' + + ORDER BY detail_name ASC, time ASC) + SELECT *, value / (diff / 1000) as power FROM times; + + One can see that the value only changes once per second + ''' + intervals = df['time'].diff() intervals[0] = intervals.mean() # approximate first interval df['interval'] = intervals # in microseconds diff --git a/metric_providers/psu/energy/ac/powerspy2/machine/metric-provider.py b/metric_providers/psu/energy/ac/powerspy2/machine/metric-provider.py index fb449171f..7a4866ac6 100755 --- a/metric_providers/psu/energy/ac/powerspy2/machine/metric-provider.py +++ b/metric_providers/psu/energy/ac/powerspy2/machine/metric-provider.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -#pylint: disable=invalid-name -from powerspy2 import PowerSpy2 +from metric_providers.psu.energy.ac.powerspy2.machine.powerspy2 import PowerSpy2 if __name__ == '__main__': import argparse diff --git a/metric_providers/psu/energy/ac/powerspy2/machine/provider.py b/metric_providers/psu/energy/ac/powerspy2/machine/provider.py index 6bab4f45c..69d186866 100755 --- a/metric_providers/psu/energy/ac/powerspy2/machine/provider.py +++ b/metric_providers/psu/energy/ac/powerspy2/machine/provider.py @@ -1,6 +1,4 @@ import os - -#pylint: disable=import-error from metric_providers.base import BaseMetricProvider class PsuEnergyAcPowerspy2MachineProvider(BaseMetricProvider): diff --git a/metric_providers/psu/energy/ac/sdia/machine/provider.py b/metric_providers/psu/energy/ac/sdia/machine/provider.py index 02643a06f..f5b331a8a 100644 --- a/metric_providers/psu/energy/ac/sdia/machine/provider.py +++ b/metric_providers/psu/energy/ac/sdia/machine/provider.py @@ -1,19 +1,11 @@ -import sys import os from io import StringIO import pandas -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../../../../../lib") -sys.path.append(CURRENT_DIR) - - -#pylint: disable=import-error, wrong-import-position -from global_config import GlobalConfig from metric_providers.base import BaseMetricProvider class PsuEnergyAcSdiaMachineProvider(BaseMetricProvider): - def __init__(self, resolution): + def __init__(self, *, resolution, CPUChips, TDP): super().__init__( metric_name='psu_energy_ac_sdia_machine', metrics={'time': int, 'value': int}, @@ -21,6 +13,9 @@ def __init__(self, resolution): unit='mJ', current_dir=os.path.dirname(os.path.abspath(__file__)), ) + self.cpu_chips = CPUChips + self.tpd = TDP + # Since no process is ever started we just return None def get_stderr(self): @@ -30,7 +25,7 @@ def get_stderr(self): def start_profiling(self, containers=None): self._has_started = True - def read_metrics(self, project_id, containers): + def read_metrics(self, run_id, containers=None): if not os.path.isfile('/tmp/green-metrics-tool/cpu_utilization_procfs_system.log'): raise RuntimeError('could not find the /tmp/green-metrics-tool/cpu_utilization_procfs_system.log file.\ @@ -51,24 +46,21 @@ def read_metrics(self, project_id, containers): df['detail_name'] = '[DEFAULT]' # standard container name when no further granularity was measured df['metric'] = self._metric_name - df['project_id'] = project_id + df['run_id'] = run_id #Z = df.loc[:, ['value']] - provider_config = GlobalConfig( - ).config['measurement']['metric-providers']['common']\ - ['psu.energy.ac.sdia.machine.provider.PsuEnergyAcSdiaMachineProvider'] - if 'CPUChips' not in provider_config: + if not self.cpu_chips: raise RuntimeError( 'Please set the CPUChips config option for PsuEnergyAcSdiaMachineProvider in the config.yml') - if 'TDP' not in provider_config: + if not self.tpd: raise RuntimeError('Please set the TDP config option for PsuEnergyAcSdiaMachineProvider in the config.yml') # since the CPU-Utilization is a ratio, we technically have to divide by 10,000 to get a 0...1 range. # And then again at the end multiply with 1000 to get mW. We take the # shortcut and just mutiply the 0.65 ratio from the SDIA by 10 -> 6.5 - df.value = ((df.value * provider_config['TDP']) / 6.5) * provider_config['CPUChips'] # will result in mW + df.value = ((df.value * self.tpd) / 6.5) * self.cpu_chips # will result in mW df.value = (df.value * df.time.diff()) / 1_000_000 # mW * us / 1_000_000 will result in mJ df['unit'] = self._unit diff --git a/metric_providers/psu/energy/ac/xgboost/machine/provider.py b/metric_providers/psu/energy/ac/xgboost/machine/provider.py index 37297a160..1df7341e2 100644 --- a/metric_providers/psu/energy/ac/xgboost/machine/provider.py +++ b/metric_providers/psu/energy/ac/xgboost/machine/provider.py @@ -1,19 +1,17 @@ -import sys import os +import sys from io import StringIO import pandas CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../../../../../../lib") sys.path.append(CURRENT_DIR) -#pylint: disable=import-error, wrong-import-position import model.xgb as mlmodel -from global_config import GlobalConfig from metric_providers.base import BaseMetricProvider class PsuEnergyAcXgboostMachineProvider(BaseMetricProvider): - def __init__(self, resolution): + def __init__(self, *, resolution, HW_CPUFreq, CPUChips, CPUThreads, TDP, + HW_MemAmountGB, CPUCores=None, Hardware_Availability_Year=None): super().__init__( metric_name="psu_energy_ac_xgboost_machine", metrics={"time": int, "value": int}, @@ -21,6 +19,13 @@ def __init__(self, resolution): unit="mJ", current_dir=os.path.dirname(os.path.abspath(__file__)), ) + self.HW_CPUFreq = HW_CPUFreq + self.CPUChips = CPUChips + self.CPUThreads = CPUThreads + self.TDP = TDP + self.HW_MemAmountGB = HW_MemAmountGB + self.CPUCores = CPUCores + self.Hardware_Availability_Year=Hardware_Availability_Year # Since no process is ever started we just return None def get_stderr(self): @@ -30,7 +35,7 @@ def get_stderr(self): def start_profiling(self, containers=None): self._has_started = True - def read_metrics(self, project_id, containers): + def read_metrics(self, run_id, containers=None): if not os.path.isfile('/tmp/green-metrics-tool/cpu_utilization_procfs_system.log'): raise RuntimeError('could not find the /tmp/green-metrics-tool/cpu_utilization_procfs_system.log file. \ @@ -51,45 +56,26 @@ def read_metrics(self, project_id, containers): df['detail_name'] = '[DEFAULT]' # standard container name when no further granularity was measured df['metric'] = self._metric_name - df['project_id'] = project_id + df['run_id'] = run_id Z = df.loc[:, ['value']] - provider_config = GlobalConfig().config['measurement']['metric-providers']['common']\ - ['psu.energy.ac.xgboost.machine.provider.PsuEnergyAcXgboostMachineProvider'] - - if 'HW_CPUFreq' not in provider_config: - raise RuntimeError( - 'Please set the HW_CPUFreq config option for PsuEnergyAcXgboostMachineProvider in the config.yml') - if 'CPUChips' not in provider_config: - raise RuntimeError( - 'Please set the CPUChips config option for PsuEnergyAcXgboostMachineProvider in the config.yml') - if 'CPUThreads' not in provider_config: - raise RuntimeError( - 'Please set the CPUThreads config option for PsuEnergyAcXgboostMachineProvider in the config.yml') - if 'TDP' not in provider_config: - raise RuntimeError('Please set the TDP config option for \ - PsuEnergyAcXgboostMachineProvider in the config.yml') - if 'HW_MemAmountGB' not in provider_config: - raise RuntimeError( - 'Please set the HW_MemAmountGB config option for PsuEnergyAcXgboostMachineProvider in the config.yml') - - Z['HW_CPUFreq'] = provider_config['HW_CPUFreq'] - Z['CPUThreads'] = provider_config['CPUThreads'] - Z['TDP'] = provider_config['TDP'] - Z['HW_MemAmountGB'] = provider_config['HW_MemAmountGB'] + Z['HW_CPUFreq'] = self.HW_CPUFreq + Z['CPUThreads'] = self.CPUThreads + Z['TDP'] = self.TDP + Z['HW_MemAmountGB'] = self.HW_MemAmountGB # now we process the optional parameters - if 'CPUCores' in provider_config: - Z['CPUCores'] = provider_config['CPUCores'] + if self.CPUCores: + Z['CPUCores'] = self.CPUCores - if 'Hardware_Availability_Year' in provider_config: - Z['Hardware_Availability_Year'] = provider_config['Hardware_Availability_Year'] + if self.Hardware_Availability_Year: + Z['Hardware_Availability_Year'] = self.Hardware_Availability_Year Z = Z.rename(columns={'value': 'utilization'}) Z.utilization = Z.utilization / 100 - model = mlmodel.train_model(provider_config['CPUChips'], Z) + model = mlmodel.train_model(self.CPUChips, Z) inferred_predictions = mlmodel.infer_predictions(model, Z) interpolated_predictions = mlmodel.interpolate_predictions(inferred_predictions) diff --git a/migrations/2023_06_21_adds_proxy.sql b/migrations/2023_06_21_adds_proxy.sql new file mode 100644 index 000000000..78a910fda --- /dev/null +++ b/migrations/2023_06_21_adds_proxy.sql @@ -0,0 +1,8 @@ +CREATE TABLE network_intercepts ( + id SERIAL PRIMARY KEY, + project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE , + time bigint NOT NULL, + connection_type text NOT NULL, + protocol text NOT NULL, + created_at timestamp with time zone DEFAULT now() +); diff --git a/migrations/2023_09_01_network_intercepts.sql b/migrations/2023_09_01_network_intercepts.sql new file mode 100644 index 000000000..f0ce9de96 --- /dev/null +++ b/migrations/2023_09_01_network_intercepts.sql @@ -0,0 +1,8 @@ +CREATE TABLE network_intercepts ( + id SERIAL PRIMARY KEY, + project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE , + time bigint NOT NULL, + connection_type text NOT NULL, + protocol text NOT NULL, + created_at timestamp with time zone DEFAULT now() +); \ No newline at end of file diff --git a/migrations/2023_09_02_projects_to_runs.sql b/migrations/2023_09_02_projects_to_runs.sql new file mode 100644 index 000000000..bbe6c5d83 --- /dev/null +++ b/migrations/2023_09_02_projects_to_runs.sql @@ -0,0 +1,117 @@ +ALTER TABLE projects RENAME TO "runs"; +ALTER TABLE runs RENAME COLUMN "invalid_project" TO "invalid_run"; +ALTER TABLE runs ALTER COLUMN "machine_id" DROP DEFAULT; +ALTER TABLE runs ADD CONSTRAINT "machines_fk" FOREIGN KEY ("machine_id") REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE; +UPDATE runs SET created_at = last_run WHERE last_run IS NOT NULL; +ALTER TABLE runs DROP COLUMN "last_run"; +ALTER TABLE runs + ADD COLUMN "job_id" int, + ADD CONSTRAINT "job_fk" FOREIGN KEY ("job_id") REFERENCES jobs(id) ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE runs ADD UNIQUE (job_id); + + +ALTER TABLE measurements RENAME COLUMN "project_id" TO "run_id"; +ALTER TABLE phase_stats RENAME COLUMN "project_id" TO "run_id"; +ALTER TABLE client_status RENAME COLUMN "project_id" TO "run_id"; +ALTER TABLE notes RENAME COLUMN "project_id" TO "run_id"; +ALTER TABLE network_intercepts RENAME COLUMN "project_id" TO "run_id"; + +ALTER TABLE ci_measurements DROP COLUMN "project_id"; + +-- create the timeline_projects table +CREATE TABLE timeline_projects ( + id SERIAL PRIMARY KEY, + name text, + url text, + categories integer[], + branch text DEFAULT 'NULL'::text, + filename text, + machine_id integer REFERENCES machines(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL, + schedule_mode text NOT NULL, + last_scheduled timestamp with time zone, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +-- Alter the jobs table to the new view + +ALTER TABLE jobs RENAME COLUMN "project_id" TO "run_id"; +ALTER TABLE jobs RENAME COLUMN "type" TO "state"; +ALTER TABLE jobs ADD COLUMN "name" text; +ALTER TABLE jobs ADD COLUMN "email" text; +ALTER TABLE jobs ADD COLUMN "url" text; +ALTER TABLE jobs ADD COLUMN "filename" text; +ALTER TABLE jobs ADD COLUMN "branch" text; +ALTER TABLE jobs ADD CONSTRAINT "machines_fk" FOREIGN KEY ("machine_id") REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE jobs ADD COLUMN "updated_at" timestamp with time zone DEFAULT now(); +ALTER TABLE jobs DROP COLUMN "failed", DROP COLUMN "running", DROP COLUMN "last_run"; + + +-- now we create updated_at columns for every table +CREATE EXTENSION "moddatetime"; + +ALTER TABLE measurements ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE phase_stats ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE client_status ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE notes ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE network_intercepts ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE ci_measurements ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE runs ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE categories ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE machines ADD COLUMN "updated_at" timestamp with time zone ; +ALTER TABLE ci_measurements ADD COLUMN "updated_at" timestamp with time zone ; + + +CREATE TRIGGER measurements_moddatetime + BEFORE UPDATE ON measurements + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER phase_stats_moddatetime + BEFORE UPDATE ON phase_stats + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER client_status_moddatetime + BEFORE UPDATE ON client_status + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER notes_moddatetime + BEFORE UPDATE ON notes + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER network_intercepts_moddatetime + BEFORE UPDATE ON network_intercepts + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER ci_measurements_moddatetime + BEFORE UPDATE ON ci_measurements + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER runs_moddatetime + BEFORE UPDATE ON runs + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER categories_moddatetime + BEFORE UPDATE ON categories + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER machines_moddatetime + BEFORE UPDATE ON machines + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER timeline_projects_moddatetime + BEFORE UPDATE ON timeline_projects + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE TRIGGER jobs_moddatetime + BEFORE UPDATE ON jobs + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); diff --git a/migrations/2023_09_09_runner_arguments.sql b/migrations/2023_09_09_runner_arguments.sql new file mode 100644 index 000000000..086681258 --- /dev/null +++ b/migrations/2023_09_09_runner_arguments.sql @@ -0,0 +1 @@ +ALTER TABLE runs ADD COLUMN "runner_arguments" json; diff --git a/migrations/2023_09_18_workflow_name.sql b/migrations/2023_09_18_workflow_name.sql new file mode 100644 index 000000000..737097a0f --- /dev/null +++ b/migrations/2023_09_18_workflow_name.sql @@ -0,0 +1,2 @@ +ALTER TABLE "ci_measurements" ADD "workflow_name" text NULL; +ALTER TABLE "ci_measurements" RENAME "workflow" to "workflow_id"; \ No newline at end of file diff --git a/migrations/2023_09_28_hog.sql b/migrations/2023_09_28_hog.sql new file mode 100644 index 000000000..37586371e --- /dev/null +++ b/migrations/2023_09_28_hog.sql @@ -0,0 +1,72 @@ +CREATE TABLE hog_measurements ( + id SERIAL PRIMARY KEY, + time bigint NOT NULL, + machine_uuid uuid NOT NULL, + elapsed_ns bigint NOT NULL, + combined_energy int, + cpu_energy int, + gpu_energy int, + ane_energy int, + energy_impact int, + thermal_pressure text, + settings jsonb, + data jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER hog_measurements_moddatetime + BEFORE UPDATE ON hog_measurements + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE INDEX idx_hog_measurements_machine_uuid ON hog_measurements USING hash (machine_uuid); +CREATE INDEX idx_hog_measurements_time ON hog_measurements (time); + + +CREATE TABLE hog_coalitions ( + id SERIAL PRIMARY KEY, + measurement integer REFERENCES hog_measurements(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL, + name text NOT NULL, + cputime_ns bigint, + cputime_per int, + energy_impact int, + diskio_bytesread bigint, + diskio_byteswritten bigint, + intr_wakeups bigint, + idle_wakeups bigint, + data jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER hog_coalitions_moddatetime + BEFORE UPDATE ON hog_coalitions + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE INDEX idx_coalition_energy_impact ON hog_coalitions(energy_impact); +CREATE INDEX idx_coalition_name ON hog_coalitions(name); + +CREATE TABLE hog_tasks ( + id SERIAL PRIMARY KEY, + coalition integer REFERENCES hog_coalitions(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL, + name text NOT NULL, + cputime_ns bigint, + cputime_per int, + energy_impact int, + bytes_received bigint, + bytes_sent bigint, + diskio_bytesread bigint, + diskio_byteswritten bigint, + intr_wakeups bigint, + idle_wakeups bigint, + + data jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone +); +CREATE TRIGGER hog_tasks_moddatetime + BEFORE UPDATE ON hog_tasks + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + +CREATE INDEX idx_task_coalition ON hog_tasks(coalition); \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index a40a61424..8bb2a9a35 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,11 @@ -r requirements.txt -pydantic==2.1.1 -pytest==7.4.0 +pydantic==2.4.2 +pytest==7.4.2 requests==2.31.0 -pylint==2.17.5 +pylint==3.0.1 +fastapi==0.104.0 +anybadge==1.14.0 + +# just to clear the pylint errors for the files in /api +scipy==1.11.3 +orjson==3.9.9 diff --git a/requirements.txt b/requirements.txt index 688caa320..9973e1b8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ PyYAML==6.0.1 -pandas==2.0.3 -psycopg[binary]==3.1.10 +pandas==2.1.1 +psycopg[binary]==3.1.12 pyserial==3.5 -schema==0.7.5 \ No newline at end of file +psutil==5.9.6 +schema==0.7.5 +anybadge==1.14.0 diff --git a/runner.py b/runner.py index d27a048db..1115fca3d 100755 --- a/runner.py +++ b/runner.py @@ -1,18 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# We disable naming convention to allow names like p,kv etc. Even if it is not 'allowed' it makes the code more readable -#pylint: disable=invalid-name - -# As pretty much everything is done in one big flow we trigger all the too-many-* checks. Which normally makes sense -# but in this case it would make the code a lot more complicated separating this out into loads of sub-functions -#pylint: disable=too-many-branches,too-many-statements,too-many-arguments,too-many-instance-attributes - -# Using a very broad exception makes sense in this case as we have excepted all the specific ones before -#pylint: disable=broad-except +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr -# I can't make these go away, but the imports all work fine on my system >.< -#pylint: disable=wrong-import-position, import-error +from lib.venv_checker import check_venv +check_venv() # this check must even run before __main__ as imports might not get resolved import subprocess import json @@ -22,7 +15,6 @@ from html import escape import sys import importlib -import faulthandler import re from io import StringIO from pathlib import Path @@ -30,24 +22,23 @@ import shutil import yaml -faulthandler.enable() # will catch segfaults and write to stderr CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/lib") -sys.path.append(f"{CURRENT_DIR}/tools") - -from debug_helper import DebugHelper -from terminal_colors import TerminalColors -from schema_checker import SchemaChecker -import process_helpers -import hardware_info -import hardware_info_root -import error_helpers -from db import DB -from global_config import GlobalConfig -import utils -from notes import Notes +from lib import utils +from lib import process_helpers +from lib import hardware_info +from lib import hardware_info_root +from lib import error_helpers +from lib.debug_helper import DebugHelper +from lib.terminal_colors import TerminalColors +from lib.schema_checker import SchemaChecker +from lib.db import DB +from lib.global_config import GlobalConfig +from lib.notes import Notes +from lib import system_checks + +from tools.machine import Machine def arrows(text): return f"\n\n>>>> {text} <<<<\n\n" @@ -94,20 +85,21 @@ def join_paths(path, path2, mode=None): class Runner: def __init__(self, - uri, uri_type, pid, filename='usage_scenario.yml', branch=None, - debug_mode=False, allow_unsafe=False, no_file_cleanup=False, skip_config_check=False, + name, uri, uri_type, filename='usage_scenario.yml', branch=None, + debug_mode=False, allow_unsafe=False, no_file_cleanup=False, skip_system_checks=False, skip_unsafe=False, verbose_provider_boot=False, full_docker_prune=False, - dry_run=False, dev_repeat_run=False, docker_prune=False): + dry_run=False, dev_repeat_run=False, docker_prune=False, job_id=None): if skip_unsafe is True and allow_unsafe is True: raise RuntimeError('Cannot specify both --skip-unsafe and --allow-unsafe') # variables that should not change if you call run multiple times + self._name = name self._debugger = DebugHelper(debug_mode) self._allow_unsafe = allow_unsafe self._no_file_cleanup = no_file_cleanup self._skip_unsafe = skip_unsafe - self._skip_config_check = skip_config_check + self._skip_system_checks = skip_system_checks self._verbose_provider_boot = verbose_provider_boot self._full_docker_prune = full_docker_prune self._docker_prune = docker_prune @@ -115,13 +107,15 @@ def __init__(self, self._dev_repeat_run = dev_repeat_run self._uri = uri self._uri_type = uri_type - self._project_id = pid self._original_filename = filename self._branch = branch self._tmp_folder = '/tmp/green-metrics-tool' self._usage_scenario = {} self._architecture = utils.get_architecture() self._sci = {'R_d': None, 'R': 0} + self._job_id = job_id + self._arguments = locals() + del self._arguments['self'] # self is not needed and also cannot be serialzed. We remove it # transient variables that are created by the runner itself @@ -139,52 +133,45 @@ def __init__(self, self.__end_measurement = None self.__services_to_pause_phase = {} self.__join_default_network = False + self.__docker_params = [] self.__folder = f"{self._tmp_folder}/repo" # default if not changed in checkout_repository + self.__run_id = None # we currently do not use this variable # self.__filename = self._original_filename # this can be changed later if working directory changes - def custom_sleep(self, sleep_time): if not self._dry_run: print(TerminalColors.HEADER, '\nSleeping for : ', sleep_time, TerminalColors.ENDC) time.sleep(sleep_time) + def initialize_run(self): + # We issue a fetch_one() instead of a query() here, cause we want to get the RUN_ID + self.__run_id = DB().fetch_one(""" + INSERT INTO runs (job_id, name, uri, email, branch, runner_arguments, created_at) + VALUES (%s, %s, %s, 'manual', %s, %s, NOW()) + RETURNING id + """, params=(self._job_id, self._name, self._uri, self._branch, json.dumps(self._arguments)))[0] + return self.__run_id + def initialize_folder(self, path): shutil.rmtree(path, ignore_errors=True) Path(path).mkdir(parents=True, exist_ok=True) def save_notes_runner(self): print(TerminalColors.HEADER, '\nSaving notes: ', TerminalColors.ENDC, self.__notes_helper.get_notes()) - self.__notes_helper.save_to_db(self._project_id) + self.__notes_helper.save_to_db(self.__run_id) - def check_configuration(self): - print(TerminalColors.HEADER, '\nStarting configuration check', TerminalColors.ENDC) - - if self._skip_config_check: - print("Configuration check skipped") + def check_system(self, mode='start'): + if self._skip_system_checks: + print("System check skipped") return - errors = [] - config = GlobalConfig().config - metric_providers = list(utils.get_metric_providers(config).keys()) - - psu_energy_providers = sum(True for provider in metric_providers if ".energy" in provider and ".machine" in provider) - - if psu_energy_providers > 1: - errors.append("Multiple PSU Energy providers enabled!") - - if not errors: - print("Configuration check passed") - return - - printable_errors = '\n'.join(errors) + '\n' + if mode =='start': + system_checks.check_start() + else: + raise RuntimeError('Unknown mode for system check:', mode) - raise ValueError( - "Configuration check failed - not running measurement\n" - f"Configuration errors:\n{printable_errors}\n" - "If however that is what you want to do (for debug purposes), please set the --skip-config-check switch" - ) def checkout_repository(self): print(TerminalColors.HEADER, '\nChecking out repository', TerminalColors.ENDC) @@ -255,12 +242,12 @@ def checkout_repository(self): parsed_timestamp = datetime.strptime(commit_timestamp, "%Y-%m-%d %H:%M:%S %z") DB().query(""" - UPDATE projects + UPDATE runs SET commit_hash=%s, commit_timestamp=%s WHERE id = %s - """, params=(commit_hash, parsed_timestamp, self._project_id)) + """, params=(commit_hash, parsed_timestamp, self.__run_id)) # This method loads the yml file and takes care that the includes work and are secure. # It uses the tagging infrastructure provided by https://pyyaml.org/wiki/PyYAMLDocumentation @@ -381,7 +368,7 @@ def check_running_containers(self): stderr=subprocess.PIPE, check=True, encoding='UTF-8') for line in result.stdout.splitlines(): - for running_container in line.split(','): + for running_container in line.split(','): # if docker container has multiple tags, they will be split by comma, so we only want to for service_name in self._usage_scenario.get('services', []): if 'container_name' in self._usage_scenario['services'][service_name]: container_name = self._usage_scenario['services'][service_name]['container_name'] @@ -425,7 +412,7 @@ def remove_docker_images(self): ''' A machine will always register in the database on run. This means that it will write its machine_id and machine_descroption to the machines table - and then link itself in the projects table accordingly. + and then link itself in the runs table accordingly. ''' def register_machine_id(self): config = GlobalConfig().config @@ -435,18 +422,8 @@ def register_machine_id(self): or config['machine']['description'] == '': raise RuntimeError('You must set machine id and machine description') - DB().query(""" - INSERT INTO machines - ("id", "description", "available", "created_at") - VALUES - (%s, %s, TRUE, 'NOW()') - ON CONFLICT (id) DO - UPDATE SET description = %s -- no need to make where clause here for correct row - """, params=(config['machine']['id'], - config['machine']['description'], - config['machine']['description'] - ) - ) + machine = Machine(machine_id=config['machine'].get('id'), description=config['machine'].get('description')) + machine.register() def update_and_insert_specs(self): config = GlobalConfig().config @@ -463,7 +440,7 @@ def update_and_insert_specs(self): # There are two ways we get hardware info. First things we don't need to be root to do which we get through # a method call. And then things we need root privilege which we need to call as a subprocess with sudo. The - # install.sh script should have called the makefile which adds the script to the sudoes file. + # install.sh script should have added the script to the sudoes file. machine_specs = hardware_info.get_default_values() if len(hardware_info_root.get_root_list()) > 0: @@ -479,10 +456,10 @@ def update_and_insert_specs(self): # Insert auxilary info for the run. Not critical. DB().query(""" - UPDATE projects + UPDATE runs SET machine_id=%s, machine_specs=%s, measurement_config=%s, - usage_scenario = %s, filename=%s, gmt_hash=%s, last_run = NOW() + usage_scenario = %s, filename=%s, gmt_hash=%s WHERE id = %s """, params=( config['machine']['id'], @@ -491,7 +468,7 @@ def update_and_insert_specs(self): escape(json.dumps(self._usage_scenario), quote=False), self._original_filename, gmt_hash, - self._project_id) + self.__run_id) ) def import_metric_providers(self): @@ -505,19 +482,33 @@ def import_metric_providers(self): print(TerminalColors.WARNING, arrows('No metric providers were configured in config.yml. Was this intentional?'), TerminalColors.ENDC) return - # will iterate over keys - for metric_provider in metric_providers: + docker_ps = subprocess.run(["docker", "info"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, encoding='UTF-8', check=True) + rootless = False + if 'rootless' in docker_ps.stdout: + rootless = True + + for metric_provider in metric_providers: # will iterate over keys module_path, class_name = metric_provider.rsplit('.', 1) module_path = f"metric_providers.{module_path}" + conf = metric_providers[metric_provider] or {} + + if rootless and '.cgroup.' in module_path: + conf['rootless'] = True print(f"Importing {class_name} from {module_path}") - print(f"Configuration is {metric_providers[metric_provider]}") + print(f"Configuration is {conf}") + module = importlib.import_module(module_path) - # the additional () creates the instance - metric_provider_obj = getattr(module, class_name)(resolution=metric_providers[metric_provider]['resolution']) + + metric_provider_obj = getattr(module, class_name)(**conf) self.__metric_providers.append(metric_provider_obj) + if hasattr(metric_provider_obj, 'get_docker_params'): + services_list = ",".join(list(self._usage_scenario['services'].keys())) + self.__docker_params += metric_provider_obj.get_docker_params(no_proxy_list=services_list) + + self.__metric_providers.sort(key=lambda item: 'rapl' not in item.__class__.__name__.lower()) def download_dependencies(self): @@ -589,13 +580,16 @@ def build_docker_images(self): f"--tar-path=/output/{tmp_img_name}.tar", '--no-push'] + if self.__docker_params: + docker_build_command[2:2] = self.__docker_params + print(" ".join(docker_build_command)) ps = subprocess.run(docker_build_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='UTF-8', check=False) if ps.returncode != 0: print(f"Error: {ps.stderr} \n {ps.stdout}") - raise OSError("Docker build failed") + raise OSError(f"Docker build failed\nStderr: {ps.stderr}\nStdout: {ps.stdout}") # import the docker image locally image_import_command = ['docker', 'load', '-q', '-i', f"{temp_dir}/{tmp_img_name}.tar"] @@ -678,6 +672,10 @@ def setup_services(self): else: docker_run_string.append(f"{self.__folder}:/tmp/repo:ro") + if self.__docker_params: + docker_run_string[2:2] = self.__docker_params + + if 'volumes' in service: if self._allow_unsafe: # On old docker clients we experience some weird error, that we deem legacy @@ -1028,7 +1026,12 @@ def stop_metric_providers(self): metric_provider.stop_profiling() - df = metric_provider.read_metrics(self._project_id, self.__containers) + df = metric_provider.read_metrics(self.__run_id, self.__containers) + if isinstance(df, int): + print('Imported', TerminalColors.HEADER, df, TerminalColors.ENDC, 'metrics from ', metric_provider.__class__.__name__) + # If df returns an int the data has already been committed to the db + continue + print('Imported', TerminalColors.HEADER, df.shape[0], TerminalColors.ENDC, 'metrics from ', metric_provider.__class__.__name__) if df is None or df.shape[0] == 0: errors.append(f"No metrics were able to be imported from: {metric_provider.__class__.__name__}") @@ -1089,10 +1092,10 @@ def end_measurement(self): def update_start_and_end_times(self): print(TerminalColors.HEADER, '\nUpdating start and end measurement times', TerminalColors.ENDC) DB().query(""" - UPDATE projects + UPDATE runs SET start_measurement=%s, end_measurement=%s WHERE id = %s - """, params=(self.__start_measurement, self.__end_measurement, self._project_id)) + """, params=(self.__start_measurement, self.__end_measurement, self.__run_id)) def store_phases(self): print(TerminalColors.HEADER, '\nUpdating phases in DB', TerminalColors.ENDC) @@ -1101,10 +1104,10 @@ def store_phases(self): # We did not make this before, as we needed the duplicate checking of dicts self.__phases = list(self.__phases.values()) DB().query(""" - UPDATE projects + UPDATE runs SET phases=%s WHERE id = %s - """, params=(json.dumps(self.__phases), self._project_id)) + """, params=(json.dumps(self.__phases), self.__run_id)) def read_container_logs(self): print(TerminalColors.HEADER, '\nCapturing container logs', TerminalColors.ENDC) @@ -1141,10 +1144,10 @@ def save_stdout_logs(self): logs_as_str = logs_as_str.replace('\x00','') if logs_as_str: DB().query(""" - UPDATE projects + UPDATE runs SET logs=%s WHERE id = %s - """, params=(logs_as_str, self._project_id)) + """, params=(logs_as_str, self.__run_id)) def cleanup(self): @@ -1185,6 +1188,7 @@ def cleanup(self): self.__join_default_network = False #self.__filename = self._original_filename # # we currently do not use this variable self.__folder = f"{self._tmp_folder}/repo" + self.__run_id = None def run(self): ''' @@ -1197,19 +1201,21 @@ def run(self): Methods thus will behave differently given the runner was instantiated with different arguments. ''' + return_run_id = None try: config = GlobalConfig().config + self.check_system('start') + return_run_id = self.initialize_run() self.initialize_folder(self._tmp_folder) - self.check_configuration() self.checkout_repository() self.initial_parse() + self.import_metric_providers() self.populate_image_names() self.check_running_containers() self.remove_docker_images() self.download_dependencies() self.register_machine_id() self.update_and_insert_specs() - self.import_metric_providers() if self._debugger.active: self._debugger.pause('Initial load complete. Waiting to start metric providers') @@ -1274,6 +1280,7 @@ def run(self): self.custom_sleep(config['measurement']['idle-time-end']) self.store_phases() self.update_start_and_end_times() + except BaseException as exc: self.add_to_log(exc.__class__.__name__, str(exc)) raise exc @@ -1312,6 +1319,7 @@ def run(self): print(TerminalColors.OKGREEN, arrows('MEASUREMENT SUCCESSFULLY COMPLETED'), TerminalColors.ENDC) + return return_run_id # we cannot return self.__run_id as this is reset in cleanup() if __name__ == '__main__': import argparse @@ -1326,7 +1334,7 @@ def run(self): parser.add_argument('--debug', action='store_true', help='Activate steppable debug mode') parser.add_argument('--allow-unsafe', action='store_true', help='Activate unsafe volume bindings, ports and complex environment vars') parser.add_argument('--skip-unsafe', action='store_true', help='Skip unsafe volume bindings, ports and complex environment vars') - parser.add_argument('--skip-config-check', action='store_true', help='Skip checking the configuration') + parser.add_argument('--skip-system-checks', action='store_true', help='Skip checking the system if the GMT can run') parser.add_argument('--verbose-provider-boot', action='store_true', help='Boot metric providers gradually') parser.add_argument('--full-docker-prune', action='store_true', help='Stop and remove all containers, build caches, volumes and images on the system') parser.add_argument('--docker-prune', action='store_true', help='Prune all unassociated build caches, networks volumes and stopped containers on the system') @@ -1387,21 +1395,18 @@ def run(self): sys.exit(1) GlobalConfig(config_name=args.config_override) - # We issue a fetch_one() instead of a query() here, cause we want to get the project_id - project_id = DB().fetch_one(""" - INSERT INTO "projects" ("name","uri","email","last_run","created_at", "branch") - VALUES - (%s,%s,'manual',NULL,NOW(),%s) RETURNING id; - """, params=(args.name, args.uri, args.branch))[0] - - runner = Runner(uri=args.uri, uri_type=run_type, pid=project_id, filename=args.filename, + successful_run_id = None + runner = Runner(name=args.name, uri=args.uri, uri_type=run_type, filename=args.filename, branch=args.branch, debug_mode=args.debug, allow_unsafe=args.allow_unsafe, - no_file_cleanup=args.no_file_cleanup, skip_config_check =args.skip_config_check, + no_file_cleanup=args.no_file_cleanup, skip_system_checks=args.skip_system_checks, skip_unsafe=args.skip_unsafe,verbose_provider_boot=args.verbose_provider_boot, full_docker_prune=args.full_docker_prune, dry_run=args.dry_run, dev_repeat_run=args.dev_repeat_run, docker_prune=args.docker_prune) + + # Using a very broad exception makes sense in this case as we have excepted all the specific ones before + #pylint: disable=broad-except try: - runner.run() # Start main code + successful_run_id = runner.run() # Start main code # this code should live at a different position. # From a user perspective it makes perfect sense to run both jobs directly after each other @@ -1412,23 +1417,25 @@ def run(self): # get all the metrics from the measurements table grouped by metric # loop over them issueing separate queries to the DB - from phase_stats import build_and_store_phase_stats - build_and_store_phase_stats(project_id, runner._sci) + from tools.phase_stats import build_and_store_phase_stats + + print("Run id is", successful_run_id) + build_and_store_phase_stats(successful_run_id, runner._sci) print(TerminalColors.OKGREEN,'\n\n####################################################################################') - print(f"Please access your report with the ID: {project_id}") + print(f"Please access your report with the ID: {successful_run_id}") print('####################################################################################\n\n', TerminalColors.ENDC) except FileNotFoundError as e: - error_helpers.log_error('Docker command failed.', e, project_id) + error_helpers.log_error('Docker command failed.', e, successful_run_id) except subprocess.CalledProcessError as e: - error_helpers.log_error('Docker command failed', 'Stdout:', e.stdout, 'Stderr:', e.stderr, project_id) + error_helpers.log_error('Docker command failed', 'Stdout:', e.stdout, 'Stderr:', e.stderr, successful_run_id) except KeyError as e: - error_helpers.log_error('Was expecting a value inside the usage_scenario.yml file, but value was missing: ', e, project_id) + error_helpers.log_error('Was expecting a value inside the usage_scenario.yml file, but value was missing: ', e, successful_run_id) except RuntimeError as e: - error_helpers.log_error('RuntimeError occured in runner.py: ', e, project_id) + error_helpers.log_error('RuntimeError occured in runner.py: ', e, successful_run_id) except BaseException as e: - error_helpers.log_error('Base exception occured in runner.py: ', e, project_id) + error_helpers.log_error('Base exception occured in runner.py: ', e, successful_run_id) finally: if args.print_logs: print("Container logs:", runner.get_logs()) diff --git a/test/api/test_api.py b/test/api/test_api.py deleted file mode 100644 index d2e2d274c..000000000 --- a/test/api/test_api.py +++ /dev/null @@ -1,53 +0,0 @@ -#pylint: disable=no-name-in-module,wrong-import-position,import-error, redefined-outer-name, unused-argument -import os -import sys -import pytest -import requests - -current_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{current_dir}/../../api") -sys.path.append(f"{current_dir}/../../lib") - -from db import DB -import utils -from pydantic import BaseModel -from global_config import GlobalConfig -import test_functions as Tests - -class Project(BaseModel): - name: str - url: str - email: str - branch: str - filename: str - machine_id: int - - -config = GlobalConfig(config_name='test-config.yml').config -API_URL = config['cluster']['api_url'] -PROJECT_NAME = 'test_' + utils.randomword(12) -PROJECT = Project(name=PROJECT_NAME, url='testURL', email='testEmail', branch='', filename='', machine_id=0) - -@pytest.fixture() -def cleanup_projects(): - yield - DB().query('DELETE FROM projects') - -def test_post_project_add(cleanup_projects): - response = requests.post(f"{API_URL}/v1/project/add", json=PROJECT.model_dump(), timeout=15) - assert response.status_code == 202, Tests.assertion_info('success', response.text) - pid = utils.get_project_data(PROJECT_NAME)['id'] - assert pid is not None - - -def test_get_projects(cleanup_projects): - project_name = 'test_' + utils.randomword(12) - uri = os.path.abspath(os.path.join( - current_dir, 'stress-application/')) - pid = DB().fetch_one('INSERT INTO "projects" ("name","uri","email","last_run","created_at") \ - VALUES \ - (%s,%s,\'manual\',NULL,NOW()) RETURNING id;', params=(project_name, uri))[0] - response = requests.get(f"{API_URL}/v1/projects?repo=&filename=", timeout=15) - res_json = response.json() - assert response.status_code == 200 - assert res_json['data'][0][0] == str(pid) diff --git a/test/conftest.py b/test/conftest.py deleted file mode 100644 index 692dd5dd2..000000000 --- a/test/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -#pylint: disable=undefined-variable - -def pytest_collection_modifyitems(items): - for item in items: - if item.fspath.basename == 'test_functions.py': - item.add_marker(pytest.mark.skip(reason='Skipping this file')) diff --git a/test/lib/test_save_notes.py b/test/lib/test_save_notes.py deleted file mode 100644 index 6c60533e6..000000000 --- a/test/lib/test_save_notes.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import sys -from unittest.mock import MagicMock - -import pytest - -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../../lib") - -# pylint: disable=import-error,wrong-import-position -from notes import Notes -from db import DB -import test_functions as Tests - -invalid_test_data = [ - ("72e54687-ba3e-4ef6-a5a1-9f2d6af26239", "This is a note", "test", "string_instead_of_time"), -] -valid_test_data = [ - ("72e54687-ba3e-4ef6-a5a1-9f2d6af26239", "This is a note", "test", '1679393122123643'), - ("72e54687-ba3e-4ef6-a5a1-9f2d6af26239", "This is a note", "test", 1679393122123643), -] - - -@pytest.mark.parametrize("project_id,note,detail,timestamp", invalid_test_data) -def test_invalid_timestamp(project_id, note, detail, timestamp): - with pytest.raises(ValueError) as err: - notes = Notes() - notes.add_note({"note": note,"detail_name": detail,"timestamp": timestamp,}) - notes.save_to_db(project_id) - expected_exception = "invalid literal for int" - assert expected_exception in str(err.value), \ - Tests.assertion_info(f"Exception: {expected_exception}", str(err.value)) - -@pytest.mark.parametrize("project_id,note,detail,timestamp", valid_test_data) -def test_valid_timestamp(project_id, note, detail, timestamp): - mock_db = DB() - mock_db.query = MagicMock() - - notes = Notes() - notes.add_note({"note": note,"detail_name": detail,"timestamp": timestamp,}) - notes.save_to_db(project_id) - mock_db.query.assert_called_once() diff --git a/test/lib/test_schema_checker.py b/test/lib/test_schema_checker.py deleted file mode 100644 index 50b49d011..000000000 --- a/test/lib/test_schema_checker.py +++ /dev/null @@ -1,37 +0,0 @@ -#pylint: disable=import-error, wrong-import-position - -import os -import sys -import yaml -import pytest - -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../..") -sys.path.append(f"{CURRENT_DIR}/../../lib") - -from schema import SchemaError -from schema_checker import SchemaChecker -import test_functions as Tests - - -def test_schema_checker_valid(): - usage_scenario_name = 'schema_checker_valid.yml' - usage_scenario_path = os.path.join(CURRENT_DIR, '../data/usage_scenarios/', usage_scenario_name) - with open(usage_scenario_path, encoding='utf8') as file: - usage_scenario = yaml.safe_load(file) - schema_checker = SchemaChecker(validate_compose_flag=True) - schema_checker.check_usage_scenario(usage_scenario) - -def test_schema_checker_invalid(): - usage_scenario_name = 'schema_checker_invalid_1.yml' - usage_scenario_path = os.path.join(CURRENT_DIR, '../data/usage_scenarios/', usage_scenario_name) - with open(usage_scenario_path, encoding='utf8') as file: - usage_scenario = yaml.safe_load(file) - - schema_checker = SchemaChecker(validate_compose_flag=True) - with pytest.raises(SchemaError) as error: - schema_checker.check_usage_scenario(usage_scenario) - - expected_exception = "Missing key: 'description'" - assert expected_exception in str(error.value), \ - Tests.assertion_info(f"Exception: {expected_exception}", str(error.value)) diff --git a/test/README.MD b/tests/README.MD similarity index 100% rename from test/README.MD rename to tests/README.MD diff --git a/tests/api/test_api.py b/tests/api/test_api.py new file mode 100644 index 000000000..436d52d2e --- /dev/null +++ b/tests/api/test_api.py @@ -0,0 +1,86 @@ +import os +import pytest +import requests +import psycopg + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + +from lib.db import DB +from lib import utils +from lib.global_config import GlobalConfig +from tools.machine import Machine +from tests import test_functions as Tests + +config = GlobalConfig(config_name='test-config.yml').config +API_URL = config['cluster']['api_url'] + +from api.main import Software +from api.main import CI_Measurement + +@pytest.fixture(autouse=True, name="register_machine") +def register_machine_fixture(): + machine = Machine(machine_id=1, description='test-machine') + machine.register() + +def get_job_id(run_name): + query = """ + SELECT + * + FROM + jobs + WHERE name = %s + """ + data = DB().fetch_one(query, (run_name, ), row_factory=psycopg.rows.dict_row) + if data is None or data == []: + return None + return data['id'] + +def test_post_run_add(): + run_name = 'test_' + utils.randomword(12) + run = Software(name=run_name, url='testURL', email='testEmail', branch='', filename='', machine_id=1, schedule_mode='one-off') + response = requests.post(f"{API_URL}/v1/software/add", json=run.model_dump(), timeout=15) + assert response.status_code == 202, Tests.assertion_info('success', response.text) + + job_id = get_job_id(run_name) + assert job_id is not None + +def test_ci_measurement_add(): + measurement = CI_Measurement(energy_value=123, + energy_unit='mJ', + repo='testRepo', + branch='testBranch', + cpu='testCPU', + cpu_util_avg=50, + commit_hash='1234asdf', + workflow='testWorkflow', + run_id='testRunID', + source='testSource', + label='testLabel', + duration=20, + workflow_name='testWorkflowName') + response = requests.post(f"{API_URL}/v1/ci/measurement/add", json=measurement.model_dump(), timeout=15) + assert response.status_code == 201, Tests.assertion_info('success', response.text) + query = """ + SELECT * FROM ci_measurements WHERE run_id = %s + """ + data = DB().fetch_one(query, (measurement.run_id, ), row_factory=psycopg.rows.dict_row) + assert data is not None + for key in measurement.model_dump().keys(): + if key == 'workflow': + assert data['workflow_id'] == measurement.model_dump()[key], Tests.assertion_info(f"workflow_id: {data['workflow_id']}", measurement.model_dump()[key]) + else: + assert data[key] == measurement.model_dump()[key], Tests.assertion_info(f"{key}: {data[key]}", measurement.model_dump()[key]) + + +def todo_test_get_runs(): + run_name = 'test_' + utils.randomword(12) + uri = os.path.abspath(os.path.join( + CURRENT_DIR, 'stress-application/')) + pid = DB().fetch_one('INSERT INTO "runs" ("name","uri","email","last_run","created_at") \ + VALUES \ + (%s,%s,\'manual\',NULL,NOW()) RETURNING id;', params=(run_name, uri))[0] + + response = requests.get(f"{API_URL}/v1/runs?repo=&filename=", timeout=15) + res_json = response.json() + assert response.status_code == 200 + assert res_json['data'][0][0] == str(pid) diff --git a/test/api/test_api_helpers.py b/tests/api/test_api_helpers.py similarity index 70% rename from test/api/test_api_helpers.py rename to tests/api/test_api_helpers.py index 27d166eb8..6be2c1de4 100644 --- a/test/api/test_api_helpers.py +++ b/tests/api/test_api_helpers.py @@ -1,15 +1,8 @@ -#pylint: disable=wrong-import-position,import-error,invalid-name -import os -import sys - from pydantic import BaseModel -current_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{current_dir}/../../api") - -import api_helpers +from api import api_helpers -class Project(BaseModel): +class Run(BaseModel): name: str url: str email: str @@ -25,7 +18,6 @@ class CI_Measurement(BaseModel): commit_hash: str workflow: str run_id: str - project_id: str source: str label: str duration: int @@ -38,10 +30,10 @@ def test_escape_dict(): assert escaped['link'] == escaped_link -def test_escape_project(): - messy_project = Project(name="test", url='testURL', email='testEmail', branch='', machine_id=0) +def test_escape_run(): + messy_run = Run(name="test", url='testURL', email='testEmail', branch='', machine_id=0) escaped_name = 'test<?>' - escaped = api_helpers.html_escape_multi(messy_project.model_copy()) + escaped = api_helpers.html_escape_multi(messy_run.model_copy()) assert escaped.name == escaped_name @@ -55,7 +47,6 @@ def test_escape_measurement(): commit_hash='', workflow='', run_id='', - project_id='', source='', label='', duration=13, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..111c76f9d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,29 @@ +import pytest + +from lib.db import DB + +## VERY IMPORTANT to override the config file here +## otherwise it will automatically connect to non-test DB and delete all your real data +from lib.global_config import GlobalConfig +GlobalConfig().override_config(config_name='test-config.yml') + +def pytest_collection_modifyitems(items): + for item in items: + if item.fspath.basename == 'test_functions.py': + item.add_marker(pytest.mark.skip(reason='Skipping this file')) + +# should we hardcode test-db here? +@pytest.fixture(autouse=True) +def cleanup_after_test(): + yield + tables = DB().fetch_all("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'") + for table in tables: + table_name = table[0] + DB().query(f'TRUNCATE TABLE "{table_name}" RESTART IDENTITY CASCADE') + +### If you wish to turn off the above auto-cleanup per test, include the following in your +### test module: +# from conftest import cleanup_after_test +# @pytest.fixture(autouse=False) # Set autouse to False to override the fixture +# def cleanup_after_test(): +# pass diff --git a/test/data/config_files/two_psu_providers.yml b/tests/data/config_files/two_psu_providers.yml similarity index 100% rename from test/data/config_files/two_psu_providers.yml rename to tests/data/config_files/two_psu_providers.yml diff --git a/test/data/docker-compose-files/compose.yml b/tests/data/docker-compose-files/compose.yml similarity index 83% rename from test/data/docker-compose-files/compose.yml rename to tests/data/docker-compose-files/compose.yml index b266e5ea7..e9b39ca75 100644 --- a/test/data/docker-compose-files/compose.yml +++ b/tests/data/docker-compose-files/compose.yml @@ -4,4 +4,4 @@ services: build: . image: gcb_stress container_name: test-container - restart: always + restart: always \ No newline at end of file diff --git a/test/data/docker-compose-files/volume_load_etc_passwords.yml b/tests/data/docker-compose-files/volume_load_etc_passwords.yml similarity index 100% rename from test/data/docker-compose-files/volume_load_etc_passwords.yml rename to tests/data/docker-compose-files/volume_load_etc_passwords.yml diff --git a/test/data/docker-compose-files/volume_load_non_bind_mounts.yml b/tests/data/docker-compose-files/volume_load_non_bind_mounts.yml similarity index 100% rename from test/data/docker-compose-files/volume_load_non_bind_mounts.yml rename to tests/data/docker-compose-files/volume_load_non_bind_mounts.yml diff --git a/test/data/docker-compose-files/volume_load_references.yml b/tests/data/docker-compose-files/volume_load_references.yml similarity index 100% rename from test/data/docker-compose-files/volume_load_references.yml rename to tests/data/docker-compose-files/volume_load_references.yml diff --git a/test/data/docker-compose-files/volume_load_symlinks_negative.yml b/tests/data/docker-compose-files/volume_load_symlinks_negative.yml similarity index 100% rename from test/data/docker-compose-files/volume_load_symlinks_negative.yml rename to tests/data/docker-compose-files/volume_load_symlinks_negative.yml diff --git a/test/data/docker-compose-files/volume_load_within_proj.yml b/tests/data/docker-compose-files/volume_load_within_proj.yml similarity index 100% rename from test/data/docker-compose-files/volume_load_within_proj.yml rename to tests/data/docker-compose-files/volume_load_within_proj.yml diff --git a/test/data/test_cases/subdir_volume_loading/Dockerfile b/tests/data/test_cases/subdir_volume_loading/Dockerfile similarity index 100% rename from test/data/test_cases/subdir_volume_loading/Dockerfile rename to tests/data/test_cases/subdir_volume_loading/Dockerfile diff --git a/test/data/test_cases/subdir_volume_loading/README.md b/tests/data/test_cases/subdir_volume_loading/README.md similarity index 100% rename from test/data/test_cases/subdir_volume_loading/README.md rename to tests/data/test_cases/subdir_volume_loading/README.md diff --git a/test/data/test_cases/subdir_volume_loading/compose.yaml b/tests/data/test_cases/subdir_volume_loading/compose.yaml similarity index 100% rename from test/data/test_cases/subdir_volume_loading/compose.yaml rename to tests/data/test_cases/subdir_volume_loading/compose.yaml diff --git a/test/data/test_cases/subdir_volume_loading/subdir/Dockerfile b/tests/data/test_cases/subdir_volume_loading/subdir/Dockerfile similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/Dockerfile rename to tests/data/test_cases/subdir_volume_loading/subdir/Dockerfile diff --git a/test/data/test_cases/subdir_volume_loading/subdir/subdir2/Dockerfile2 b/tests/data/test_cases/subdir_volume_loading/subdir/subdir2/Dockerfile2 similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/subdir2/Dockerfile2 rename to tests/data/test_cases/subdir_volume_loading/subdir/subdir2/Dockerfile2 diff --git a/test/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/Dockerfile3 b/tests/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/Dockerfile3 similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/Dockerfile3 rename to tests/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/Dockerfile3 diff --git a/test/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/subdir4/testfile4 b/tests/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/subdir4/testfile4 similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/subdir4/testfile4 rename to tests/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/subdir4/testfile4 diff --git a/test/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/testfile3 b/tests/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/testfile3 similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/testfile3 rename to tests/data/test_cases/subdir_volume_loading/subdir/subdir2/subdir3/testfile3 diff --git a/test/data/test_cases/subdir_volume_loading/subdir/subdir2/testfile2 b/tests/data/test_cases/subdir_volume_loading/subdir/subdir2/testfile2 similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/subdir2/testfile2 rename to tests/data/test_cases/subdir_volume_loading/subdir/subdir2/testfile2 diff --git a/test/data/test_cases/subdir_volume_loading/subdir/subdir2/usage_scenario_subdir2.yml b/tests/data/test_cases/subdir_volume_loading/subdir/subdir2/usage_scenario_subdir2.yml similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/subdir2/usage_scenario_subdir2.yml rename to tests/data/test_cases/subdir_volume_loading/subdir/subdir2/usage_scenario_subdir2.yml diff --git a/test/data/test_cases/subdir_volume_loading/subdir/testfile b/tests/data/test_cases/subdir_volume_loading/subdir/testfile similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/testfile rename to tests/data/test_cases/subdir_volume_loading/subdir/testfile diff --git a/test/data/test_cases/subdir_volume_loading/subdir/usage_scenario_subdir.yml b/tests/data/test_cases/subdir_volume_loading/subdir/usage_scenario_subdir.yml similarity index 100% rename from test/data/test_cases/subdir_volume_loading/subdir/usage_scenario_subdir.yml rename to tests/data/test_cases/subdir_volume_loading/subdir/usage_scenario_subdir.yml diff --git a/test/data/test_cases/subdir_volume_loading/testfile-root b/tests/data/test_cases/subdir_volume_loading/testfile-root similarity index 100% rename from test/data/test_cases/subdir_volume_loading/testfile-root rename to tests/data/test_cases/subdir_volume_loading/testfile-root diff --git a/test/data/test_cases/subdir_volume_loading/usage_scenario.yml b/tests/data/test_cases/subdir_volume_loading/usage_scenario.yml similarity index 100% rename from test/data/test_cases/subdir_volume_loading/usage_scenario.yml rename to tests/data/test_cases/subdir_volume_loading/usage_scenario.yml diff --git a/test/data/usage_scenarios/basic_stress.yml b/tests/data/usage_scenarios/basic_stress.yml similarity index 100% rename from test/data/usage_scenarios/basic_stress.yml rename to tests/data/usage_scenarios/basic_stress.yml diff --git a/test/data/usage_scenarios/basic_stress_w_import.yml b/tests/data/usage_scenarios/basic_stress_w_import.yml similarity index 100% rename from test/data/usage_scenarios/basic_stress_w_import.yml rename to tests/data/usage_scenarios/basic_stress_w_import.yml diff --git a/test/data/usage_scenarios/cmd_stress.yml b/tests/data/usage_scenarios/cmd_stress.yml similarity index 100% rename from test/data/usage_scenarios/cmd_stress.yml rename to tests/data/usage_scenarios/cmd_stress.yml diff --git a/test/data/usage_scenarios/env_vars_stress.yml b/tests/data/usage_scenarios/env_vars_stress.yml similarity index 100% rename from test/data/usage_scenarios/env_vars_stress.yml rename to tests/data/usage_scenarios/env_vars_stress.yml diff --git a/test/data/usage_scenarios/env_vars_stress_unallowed.yml b/tests/data/usage_scenarios/env_vars_stress_unallowed.yml similarity index 100% rename from test/data/usage_scenarios/env_vars_stress_unallowed.yml rename to tests/data/usage_scenarios/env_vars_stress_unallowed.yml diff --git a/test/data/usage_scenarios/import_error.yml b/tests/data/usage_scenarios/import_error.yml similarity index 100% rename from test/data/usage_scenarios/import_error.yml rename to tests/data/usage_scenarios/import_error.yml diff --git a/test/data/usage_scenarios/import_one_flow.yml b/tests/data/usage_scenarios/import_one_flow.yml similarity index 100% rename from test/data/usage_scenarios/import_one_flow.yml rename to tests/data/usage_scenarios/import_one_flow.yml diff --git a/test/data/usage_scenarios/import_one_root.yml b/tests/data/usage_scenarios/import_one_root.yml similarity index 100% rename from test/data/usage_scenarios/import_one_root.yml rename to tests/data/usage_scenarios/import_one_root.yml diff --git a/test/data/usage_scenarios/import_one_services.yml b/tests/data/usage_scenarios/import_one_services.yml similarity index 100% rename from test/data/usage_scenarios/import_one_services.yml rename to tests/data/usage_scenarios/import_one_services.yml diff --git a/test/data/usage_scenarios/import_two_compose.yml b/tests/data/usage_scenarios/import_two_compose.yml similarity index 100% rename from test/data/usage_scenarios/import_two_compose.yml rename to tests/data/usage_scenarios/import_two_compose.yml diff --git a/test/data/usage_scenarios/import_two_root.yml b/tests/data/usage_scenarios/import_two_root.yml similarity index 100% rename from test/data/usage_scenarios/import_two_root.yml rename to tests/data/usage_scenarios/import_two_root.yml diff --git a/test/data/usage_scenarios/network_stress.yml b/tests/data/usage_scenarios/network_stress.yml similarity index 100% rename from test/data/usage_scenarios/network_stress.yml rename to tests/data/usage_scenarios/network_stress.yml diff --git a/test/data/usage_scenarios/port_bindings_stress.yml b/tests/data/usage_scenarios/port_bindings_stress.yml similarity index 100% rename from test/data/usage_scenarios/port_bindings_stress.yml rename to tests/data/usage_scenarios/port_bindings_stress.yml diff --git a/test/data/usage_scenarios/schema_checker_valid.yml b/tests/data/usage_scenarios/schema_checker/schema_checker_invalid_image_builds.yml similarity index 74% rename from test/data/usage_scenarios/schema_checker_valid.yml rename to tests/data/usage_scenarios/schema_checker/schema_checker_invalid_image_builds.yml index d67d60e79..077674b2a 100644 --- a/test/data/usage_scenarios/schema_checker_valid.yml +++ b/tests/data/usage_scenarios/schema_checker/schema_checker_invalid_image_builds.yml @@ -1,16 +1,14 @@ --- name: Test author: Dan Mateas -description: test +description: "test that image is required when build is not specified" networks: - network-name: + networkname: services: test-container: type: container - image: gcb_stress - build: . flow: - name: Stress diff --git a/test/data/usage_scenarios/schema_checker_invalid_1.yml b/tests/data/usage_scenarios/schema_checker/schema_checker_invalid_missing_description.yml similarity index 100% rename from test/data/usage_scenarios/schema_checker_invalid_1.yml rename to tests/data/usage_scenarios/schema_checker/schema_checker_invalid_missing_description.yml diff --git a/tests/data/usage_scenarios/schema_checker/schema_checker_invalid_wrong_type.yml b/tests/data/usage_scenarios/schema_checker/schema_checker_invalid_wrong_type.yml new file mode 100644 index 000000000..70e57c995 --- /dev/null +++ b/tests/data/usage_scenarios/schema_checker/schema_checker_invalid_wrong_type.yml @@ -0,0 +1,28 @@ +--- +name: Test +author: Dan Mateas +description: test + +networks: + network-name: + +networks: + - network-a + - network-b + +services: + test-container: + type: container + image: gcb_stress + build: . + +flow: + - name: Stress + container: test-container + commands: + - type: console + command: stress-ng -c 1 -t 1 -q + note: Starting Stress + shell: bash + log-stdout: true + log-stderr: "no" # should throw error, not a bool \ No newline at end of file diff --git a/tests/data/usage_scenarios/schema_checker/schema_checker_valid.yml b/tests/data/usage_scenarios/schema_checker/schema_checker_valid.yml new file mode 100644 index 000000000..637e09837 --- /dev/null +++ b/tests/data/usage_scenarios/schema_checker/schema_checker_valid.yml @@ -0,0 +1,28 @@ +--- +name: Test +author: Dan Mateas +description: test + +networks: + network-name: + network-name-2: + +services: + test-container: + type: container + image: gcb_stress + build: . + test-container-2: + type: container + image: fizzbump + +flow: + - name: Stress + container: test-container + commands: + - type: console + command: stress-ng -c 1 -t 1 -q + note: Starting Stress + shell: bash + log-stdout: true + log-stderr: false \ No newline at end of file diff --git a/tests/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_keys.yml b/tests/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_keys.yml new file mode 100644 index 000000000..e825275c7 --- /dev/null +++ b/tests/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_keys.yml @@ -0,0 +1,25 @@ +--- +name: Test +author: Dan Mateas +description: test + +networks: + network-name: + network-name-2: + +services: + test-container: + type: container + image: gcb_stress + build: . + +flow: + - name: Stress + container: test-container + commands: + - type: console + command: stress-ng -c 1 -t 1 -q + note: Starting Stress + shell: bash + log-stdout: true + log-stderr: false \ No newline at end of file diff --git a/tests/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_list.yml b/tests/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_list.yml new file mode 100644 index 000000000..306ffa12d --- /dev/null +++ b/tests/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_list.yml @@ -0,0 +1,25 @@ +--- +name: Test +author: Dan Mateas +description: test + +networks: + - network-a + - network-b + +services: + test-container: + type: container + image: gcb_stress + build: . + +flow: + - name: Stress + container: test-container + commands: + - type: console + command: stress-ng -c 1 -t 1 -q + note: Starting Stress + shell: bash + log-stdout: true + log-stderr: false \ No newline at end of file diff --git a/test/data/usage_scenarios/setup_commands_multiple_stress.yml b/tests/data/usage_scenarios/setup_commands_multiple_stress.yml similarity index 100% rename from test/data/usage_scenarios/setup_commands_multiple_stress.yml rename to tests/data/usage_scenarios/setup_commands_multiple_stress.yml diff --git a/test/data/usage_scenarios/setup_commands_stress.yml b/tests/data/usage_scenarios/setup_commands_stress.yml similarity index 100% rename from test/data/usage_scenarios/setup_commands_stress.yml rename to tests/data/usage_scenarios/setup_commands_stress.yml diff --git a/test/data/usage_scenarios/volume_bindings_stress.yml b/tests/data/usage_scenarios/volume_bindings_stress.yml similarity index 100% rename from test/data/usage_scenarios/volume_bindings_stress.yml rename to tests/data/usage_scenarios/volume_bindings_stress.yml diff --git a/test/edit-etc-hosts.sh b/tests/edit-etc-hosts.sh similarity index 80% rename from test/edit-etc-hosts.sh rename to tests/edit-etc-hosts.sh index 4cd735202..74780a18e 100755 --- a/test/edit-etc-hosts.sh +++ b/tests/edit-etc-hosts.sh @@ -4,7 +4,7 @@ set -euo pipefail etc_hosts_line_1="127.0.0.1 test-green-coding-postgres-container" echo "Writing to /etc/hosts file..." -if ! sudo grep -Fxq "$etc_hosts_line_1" /etc/hosts; then +if ! grep -Fxq "$etc_hosts_line_1" /etc/hosts; then echo $etc_hosts_line_1 | sudo tee -a /etc/hosts else echo "Entry was already present..." diff --git a/tests/lib/test_save_notes.py b/tests/lib/test_save_notes.py new file mode 100644 index 000000000..11f737e7d --- /dev/null +++ b/tests/lib/test_save_notes.py @@ -0,0 +1,35 @@ +from unittest.mock import patch +import pytest + +from lib.notes import Notes +from tests import test_functions as Tests + +invalid_test_data = [ + ("72e54687-ba3e-4ef6-a5a1-9f2d6af26239", "This is a note", "test", "string_instead_of_time"), +] +valid_test_data = [ + ("72e54687-ba3e-4ef6-a5a1-9f2d6af26239", "This is a note", "test", '1679393122123643'), + ("72e54687-ba3e-4ef6-a5a1-9f2d6af26239", "This is a note", "test", 1679393122123643), +] + + +@pytest.mark.parametrize("run_id,note,detail,timestamp", invalid_test_data) +def test_invalid_timestamp(run_id, note, detail, timestamp): + with pytest.raises(ValueError) as err: + notes = Notes() + notes.add_note({"note": note,"detail_name": detail,"timestamp": timestamp,}) + notes.save_to_db(run_id) + expected_exception = "invalid literal for int" + assert expected_exception in str(err.value), \ + Tests.assertion_info(f"Exception: {expected_exception}", str(err.value)) + +@pytest.mark.parametrize("run_id,note,detail,timestamp", valid_test_data) +@patch('lib.db.DB.query') +def test_valid_timestamp(mock_query, run_id, note, detail, timestamp): + mock_query.return_value = None # Replace with the desired return value + + notes = Notes() + notes.add_note({"note": note, "detail_name": detail, "timestamp": timestamp}) + notes.save_to_db(run_id) + + mock_query.assert_called_once() diff --git a/tests/lib/test_schema_checker.py b/tests/lib/test_schema_checker.py new file mode 100644 index 000000000..acec90062 --- /dev/null +++ b/tests/lib/test_schema_checker.py @@ -0,0 +1,81 @@ +import os +import yaml +import pytest + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + +from schema import SchemaError + +from lib.schema_checker import SchemaChecker +from tests import test_functions as Tests + + +def test_schema_checker_valid(): + usage_scenario_name = 'schema_checker_valid.yml' + usage_scenario_path = os.path.join(CURRENT_DIR, '../data/usage_scenarios/schema_checker/', usage_scenario_name) + with open(usage_scenario_path, encoding='utf8') as file: + usage_scenario = yaml.safe_load(file) + schema_checker = SchemaChecker(validate_compose_flag=True) + schema_checker.check_usage_scenario(usage_scenario) + +def test_schema_checker_both_network_types_valid(): + ## Check first that it works in case a, with the network listed as keys + usage_scenario_name_a = 'schema_checker_valid_network_as_keys.yml' + usage_scenario_path_a = os.path.join(CURRENT_DIR, '../data/usage_scenarios/schema_checker/', usage_scenario_name_a) + with open(usage_scenario_path_a, encoding='utf8') as file: + usage_scenario_a = yaml.safe_load(file) + schema_checker_a = SchemaChecker(validate_compose_flag=True) + schema_checker_a.check_usage_scenario(usage_scenario_a) + + ## Also check that it works in case b, with the networks as a list + usage_scenario_name_b = 'schema_checker_valid_network_as_list.yml' + usage_scenario_path_b = os.path.join(CURRENT_DIR, '../data/usage_scenarios/schema_checker/', usage_scenario_name_b) + with open(usage_scenario_path_b, encoding='utf8') as file: + usage_scenario_b = yaml.safe_load(file) + schema_checker_b = SchemaChecker(validate_compose_flag=True) + schema_checker_b.check_usage_scenario(usage_scenario_b) + + +def test_schema_checker_invalid_missing_description(): + usage_scenario_name = 'schema_checker_invalid_missing_description.yml' + usage_scenario_path = os.path.join(CURRENT_DIR, '../data/usage_scenarios/schema_checker/', usage_scenario_name) + with open(usage_scenario_path, encoding='utf8') as file: + usage_scenario = yaml.safe_load(file) + + schema_checker = SchemaChecker(validate_compose_flag=True) + with pytest.raises(SchemaError) as error: + schema_checker.check_usage_scenario(usage_scenario) + + expected_exception = "Missing key: 'description'" + assert expected_exception in str(error.value), \ + Tests.assertion_info(f"Exception: {expected_exception}", str(error.value)) + + +def test_schema_checker_invalid_image_req_when_no_build(): + usage_scenario_name = 'schema_checker_invalid_image_builds.yml' + usage_scenario_path = os.path.join(CURRENT_DIR, '../data/usage_scenarios/schema_checker/', usage_scenario_name) + with open(usage_scenario_path, encoding='utf8') as file: + usage_scenario = yaml.safe_load(file) + + schema_checker = SchemaChecker(validate_compose_flag=True) + with pytest.raises(SchemaError) as error: + schema_checker.check_usage_scenario(usage_scenario) + + expected_exception = "The 'image' key under services is required when 'build' key is not present." + assert expected_exception in str(error.value), \ + Tests.assertion_info(f"Exception: {expected_exception}", str(error.value)) + +def test_schema_checker_invalid_wrong_type(): + usage_scenario_name = 'schema_checker_invalid_wrong_type.yml' + usage_scenario_path = os.path.join(CURRENT_DIR, '../data/usage_scenarios/schema_checker/', usage_scenario_name) + with open(usage_scenario_path, encoding='utf8') as file: + usage_scenario = yaml.safe_load(file) + + schema_checker = SchemaChecker(validate_compose_flag=True) + with pytest.raises(SchemaError) as error: + schema_checker.check_usage_scenario(usage_scenario) + + expected_exception = "Key 'log-stderr' error:\n'no' should be instance of 'bool'" + print(error.value) + assert expected_exception in str(error.value), \ + Tests.assertion_info(f"Exception: {expected_exception}", str(error.value)) diff --git a/test/run-tests.sh b/tests/run-tests.sh similarity index 100% rename from test/run-tests.sh rename to tests/run-tests.sh diff --git a/test/setup-test-env.py b/tests/setup-test-env.py similarity index 96% rename from test/setup-test-env.py rename to tests/setup-test-env.py index 1a412b842..13439ea37 100644 --- a/test/setup-test-env.py +++ b/tests/setup-test-env.py @@ -1,15 +1,11 @@ -#pylint: disable=invalid-name,wrong-import-position,import-error - -import sys import os from copy import deepcopy import subprocess import yaml CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../lib/") -import utils +from lib import utils BASE_CONFIG_NAME = 'config.yml' BASE_COMPOSE_NAME = 'compose.yml.example' @@ -93,7 +89,7 @@ def edit_compose_file(): volume = volume.replace(k, f'test-{k}') volume = volume.replace('PATH_TO_GREEN_METRICS_TOOL_REPO', f'{current_dir}/../') - volume = volume.replace('./structure.sql', '../test/structure.sql') + volume = volume.replace('./structure.sql', '../tests/structure.sql') new_vol_list.append(volume) # Change the depends on: in services as well diff --git a/test/smoke_test.py b/tests/smoke_test.py similarity index 65% rename from test/smoke_test.py rename to tests/smoke_test.py index 2cd7d32af..7172e553d 100644 --- a/test/smoke_test.py +++ b/tests/smoke_test.py @@ -1,28 +1,36 @@ -#pylint: disable=fixme,import-error,wrong-import-position, global-statement, unused-argument, invalid-name -# unused-argument because its not happy with 'module', which is unfortunately necessary for pytest -# also disabled invalid-name because its not happy with single word for d in data , for example - import io import os -import sys import subprocess import re CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/..") -sys.path.append(f"{CURRENT_DIR}/../lib") from contextlib import redirect_stdout, redirect_stderr -from db import DB -import utils -from global_config import GlobalConfig +import pytest + +from lib.db import DB +from lib import utils +from lib.global_config import GlobalConfig from runner import Runner run_stderr = None run_stdout = None -project_name = 'test_' + utils.randomword(12) +RUN_NAME = 'test_' + utils.randomword(12) + +# override per test cleanup, as the module setup requires writing to DB +@pytest.fixture(autouse=False) +def cleanup_after_test(): + pass + +#pylint: disable=unused-argument # unused arguement off for now - because there are no running tests in this file +def cleanup_after_module(autouse=True, scope="module"): + yield + tables = DB().fetch_all("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'") + for table in tables: + table_name = table[0] + DB().query(f'TRUNCATE TABLE "{table_name}" RESTART IDENTITY CASCADE') # Runs once per file before any test( #pylint: disable=expression-not-assigned @@ -35,14 +43,11 @@ def setup_module(module): CURRENT_DIR, 'stress-application/')) subprocess.run(['docker', 'compose', '-f', uri+'/compose.yml', 'build'], check=True) - pid = DB().fetch_one('INSERT INTO "projects" ("name","uri","email","last_run","created_at") \ - VALUES \ - (%s,%s,\'manual\',NULL,NOW()) RETURNING id;', params=(project_name, uri))[0] - # Run the application - runner = Runner(uri=uri, uri_type='folder', pid=pid, dev_repeat_run=True, skip_config_check=True) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', dev_repeat_run=True, skip_system_checks=True) runner.run() + #pylint: disable=global-statement global run_stderr, run_stdout run_stderr = err.getvalue() run_stdout = out.getvalue() @@ -61,23 +66,27 @@ def test_db_rows_are_written_and_presented(): # also check (in the same test, to save on a DB call) that the output to STD.OUT # "Imported XXX metrics from {metric_provider}" displays the same count as in the DB - project_id = utils.get_project_data(project_name)['id'] - assert(project_id is not None or project_id != '') + run_id = utils.get_run_data(RUN_NAME)['id'] + assert(run_id is not None and run_id != '') query = """ SELECT metric, COUNT(*) as count FROM measurements - WHERE project_id = %s + WHERE run_id = %s GROUP BY metric """ - data = DB().fetch_all(query, (project_id,)) - assert(data is not None or data != []) + data = DB().fetch_all(query, (run_id,)) + assert(data is not None and data != []) config = GlobalConfig(config_name='test-config.yml').config metric_providers = utils.get_metric_providers_names(config) + # The network connection proxy provider writes to a different DB so we need to remove it here + if 'NetworkConnectionsProxyContainerProvider' in metric_providers: + metric_providers.remove('NetworkConnectionsProxyContainerProvider') + for d in data: d_provider = utils.get_pascal_case(d[0]) + 'Provider' d_count = d[1] diff --git a/test/start-test-containers.sh b/tests/start-test-containers.sh similarity index 100% rename from test/start-test-containers.sh rename to tests/start-test-containers.sh diff --git a/test/stop-test-containers.sh b/tests/stop-test-containers.sh similarity index 100% rename from test/stop-test-containers.sh rename to tests/stop-test-containers.sh diff --git a/test/stress-application/.gitignore b/tests/stress-application/.gitignore similarity index 100% rename from test/stress-application/.gitignore rename to tests/stress-application/.gitignore diff --git a/test/stress-application/Dockerfile b/tests/stress-application/Dockerfile similarity index 100% rename from test/stress-application/Dockerfile rename to tests/stress-application/Dockerfile diff --git a/test/stress-application/compose.yml b/tests/stress-application/compose.yml similarity index 100% rename from test/stress-application/compose.yml rename to tests/stress-application/compose.yml diff --git a/test/stress-application/usage_scenario.yml b/tests/stress-application/usage_scenario.yml similarity index 100% rename from test/stress-application/usage_scenario.yml rename to tests/stress-application/usage_scenario.yml diff --git a/test/test_config_opts.py b/tests/test_config_opts.py similarity index 73% rename from test/test_config_opts.py rename to tests/test_config_opts.py index 1a26226ce..85e46185e 100644 --- a/test/test_config_opts.py +++ b/tests/test_config_opts.py @@ -1,25 +1,21 @@ -#pylint: disable=redefined-outer-name, import-error, wrong-import-position, unused-argument - import os -import sys import subprocess import pytest CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/..") -sys.path.append(f"{CURRENT_DIR}/../lib") -from db import DB -import utils -import test_functions as Tests -from global_config import GlobalConfig +from lib.db import DB +from lib import utils +from lib.global_config import GlobalConfig +from tests import test_functions as Tests from runner import Runner -PROJECT_NAME = 'test_' + utils.randomword(12) + config = GlobalConfig(config_name='test-config.yml').config -@pytest.fixture -def reset_config(): +#pylint: disable=unused-argument # unused arguement off for now - because there are no running tests in this file +@pytest.fixture(name="reset_config") +def reset_config_fixture(): idle_start_time = config['measurement']['idle-time-start'] idle_time_end = config['measurement']['idle-time-end'] flow_process_runtime = config['measurement']['flow-process-runtime'] @@ -28,8 +24,8 @@ def reset_config(): config['measurement']['idle-time-end'] = idle_time_end config['measurement']['flow-process-runtime'] = flow_process_runtime -@pytest.fixture(autouse=True, scope="module") -def build_image(): +@pytest.fixture(autouse=True, scope="module", name="build_image") +def build_image_fixture(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) subprocess.run(['docker', 'compose', '-f', uri+'/compose.yml', 'build'], check=True) @@ -38,31 +34,28 @@ def build_image(): def run_runner(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) - pid = DB().fetch_one('INSERT INTO "projects" ("name","uri","email","last_run","created_at") \ - VALUES \ - (%s,%s,\'manual\',NULL,NOW()) RETURNING id;', params=(PROJECT_NAME, uri))[0] # Run the application - runner = Runner(uri=uri, uri_type='folder', pid=pid, verbose_provider_boot=True, dev_repeat_run=True, skip_config_check=True) - runner.run() - return pid + RUN_NAME = 'test_' + utils.randomword(12) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', verbose_provider_boot=True, dev_repeat_run=True, skip_system_checks=True) + return runner.run() # Rethink how to do this test entirely def wip_test_idle_start_time(reset_config): config['measurement']['idle-time-start'] = 2 - pid = run_runner() + run_id = run_runner() query = """ SELECT time, note FROM notes WHERE - project_id = %s + run_id = %s ORDER BY time """ - notes = DB().fetch_all(query, (pid,)) + notes = DB().fetch_all(query, (run_id,)) timestamp_preidle = [note for note in notes if "Booting" in note[1]][0][0] timestamp_start = [note for note in notes if note[1] == 'Start of measurement'][0][0] @@ -75,19 +68,19 @@ def wip_test_idle_start_time(reset_config): # Rethink how to do this test entirely def wip_test_idle_end_time(reset_config): config['measurement']['idle-time-end'] = 2 - pid = run_runner() + run_id = run_runner() query = """ SELECT time, note FROM notes WHERE - project_id = %s + run_id = %s ORDER BY time """ - notes = DB().fetch_all(query, (pid,)) + notes = DB().fetch_all(query, (run_id,)) timestamp_postidle = [note for note in notes if note[1] == 'End of post-measurement idle'][0][0] timestamp_end = [note for note in notes if note[1] == 'End of measurement'][0][0] diff --git a/test/test_functions.py b/tests/test_functions.py similarity index 87% rename from test/test_functions.py rename to tests/test_functions.py index df0775cde..f474d8ee6 100644 --- a/test/test_functions.py +++ b/tests/test_functions.py @@ -1,19 +1,13 @@ -#pylint: disable=wrong-import-position, import-error, no-name-in-module import os import re import shutil -import sys CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../.") -sys.path.append(f"{CURRENT_DIR}/../lib") from pathlib import Path -from global_config import GlobalConfig -from db import DB -import utils -#pylint:disable=import-error +from lib.global_config import GlobalConfig +from lib import utils from runner import Runner #create test/tmp directory with specified usage_scenario to be passed as uri to runner @@ -31,14 +25,6 @@ def make_proj_dir(dir_name, usage_scenario_path, docker_compose_path=None): shutil.copy2(dockerfile, os.path.join(CURRENT_DIR, 'tmp' ,dir_name)) return dir_name - -def insert_project(uri): - project_name = 'test_' + utils.randomword(12) - pid = DB().fetch_one('INSERT INTO "projects" ("name","uri","email","last_run","created_at") \ - VALUES \ - (%s,%s,\'manual\',NULL,NOW()) RETURNING id;', params=(project_name, uri))[0] - return pid - def replace_include_in_usage_scenario(usage_scenario_path, docker_compose_filename): with open(usage_scenario_path, 'r', encoding='utf-8') as file: data = file.read() @@ -47,10 +33,9 @@ def replace_include_in_usage_scenario(usage_scenario_path, docker_compose_filena file.write(data) -#pylint: disable=too-many-arguments def setup_runner(usage_scenario, docker_compose=None, uri='default', uri_type='folder', branch=None, debug_mode=False, allow_unsafe=False, no_file_cleanup=False, - skip_unsafe=False, verbose_provider_boot=False, dir_name=None, dev_repeat_run=True, skip_config_check=True): + skip_unsafe=False, verbose_provider_boot=False, dir_name=None, dev_repeat_run=True, skip_system_checks=True): usage_scenario_path = os.path.join(CURRENT_DIR, 'data/usage_scenarios/', usage_scenario) if docker_compose is not None: docker_compose_path = os.path.join(CURRENT_DIR, 'data/docker-compose-files/', docker_compose) @@ -63,11 +48,12 @@ def setup_runner(usage_scenario, docker_compose=None, uri='default', uri_type='f make_proj_dir(dir_name=dir_name, usage_scenario_path=usage_scenario_path, docker_compose_path=docker_compose_path) uri = os.path.join(CURRENT_DIR, 'tmp/', dir_name) - pid = insert_project(uri) - return Runner(uri=uri, uri_type=uri_type, pid=pid, filename=usage_scenario, branch=branch, + RUN_NAME = 'test_' + utils.randomword(12) + + return Runner(name=RUN_NAME, uri=uri, uri_type=uri_type, filename=usage_scenario, branch=branch, debug_mode=debug_mode, allow_unsafe=allow_unsafe, no_file_cleanup=no_file_cleanup, skip_unsafe=skip_unsafe, verbose_provider_boot=verbose_provider_boot, dev_repeat_run=dev_repeat_run, - skip_config_check=skip_config_check) + skip_system_checks=skip_system_checks) # This function runs the runner up to and *including* the specified step # remember to catch in try:finally and do cleanup when calling this! @@ -75,17 +61,22 @@ def setup_runner(usage_scenario, docker_compose=None, uri='default', uri_type='f def run_until(runner, step): try: config = GlobalConfig().config + return_run_id = runner.initialize_run() + + # do a meaningless operation on return_run_id so pylint doesn't complain + print(return_run_id) + runner.initialize_folder(runner._tmp_folder) - runner.check_configuration() runner.checkout_repository() runner.initial_parse() + runner.import_metric_providers() runner.populate_image_names() runner.check_running_containers() + runner.check_system() runner.remove_docker_images() runner.download_dependencies() runner.register_machine_id() runner.update_and_insert_specs() - runner.import_metric_providers() runner.start_metric_providers(allow_other=True, allow_container=False) runner.custom_sleep(config['measurement']['idle-time-start']) diff --git a/test/test_runner.py b/tests/test_runner.py similarity index 54% rename from test/test_runner.py rename to tests/test_runner.py index 73033b3b8..f0c4c3ac5 100644 --- a/test/test_runner.py +++ b/tests/test_runner.py @@ -1,6 +1,5 @@ from contextlib import nullcontext as does_not_raise import os -import sys from shutil import copy2 import pytest @@ -8,27 +7,25 @@ CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA_CONFIG_DIR = os.path.join(CURRENT_DIR, "data", "config_files") REPO_ROOT = os.path.realpath(os.path.join(CURRENT_DIR, "..")) -sys.path.append(f"{CURRENT_DIR}/../.") -sys.path.append(f"{CURRENT_DIR}/../lib/") -#pylint:disable=import-error, wrong-import-position, wrong-import-order from runner import Runner -from global_config import GlobalConfig +from lib.global_config import GlobalConfig +from lib.system_checks import ConfigurationCheckError test_data = [ ("two_psu_providers.yml", True, does_not_raise()), - ("two_psu_providers.yml", False, pytest.raises(ValueError)), + ("two_psu_providers.yml", False, pytest.raises(ConfigurationCheckError)), ] -@pytest.mark.parametrize("config_file,skip_config_check,expectation", test_data) -def test_check_configuration(config_file, skip_config_check, expectation): +@pytest.mark.parametrize("config_file,skip_system_checks,expectation", test_data) +def test_check_system(config_file, skip_system_checks, expectation): - runner = Runner("foo", "baz", "bar", skip_config_check=skip_config_check) + runner = Runner("foo", "baz", "bar", skip_system_checks=skip_system_checks) copy2(os.path.join(TEST_DATA_CONFIG_DIR, config_file), os.path.join(REPO_ROOT, config_file)) GlobalConfig().override_config(config_name=config_file) try: with expectation: - runner.check_configuration() + runner.check_system() finally: os.remove(os.path.join(REPO_ROOT, config_file)) diff --git a/test/test_usage_scenario.py b/tests/test_usage_scenario.py similarity index 87% rename from test/test_usage_scenario.py rename to tests/test_usage_scenario.py index b1a601476..4b1789d19 100644 --- a/test/test_usage_scenario.py +++ b/tests/test_usage_scenario.py @@ -2,30 +2,23 @@ # List port mappings or a specific mapping for the container # docker port CONTAINER [PRIVATE_PORT[/PROTO]] - -#pylint: disable=fixme,import-error,wrong-import-position, global-statement, unused-argument, invalid-name, redefined-outer-name -# unused-argument because its not happy with 'module', which is unfortunately necessary for pytest -# also disabled invalid-name because its not happy with single word for d in data , for example - import io import os import re import shutil -import sys import subprocess CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/..") -sys.path.append(f"{CURRENT_DIR}/../lib") from contextlib import redirect_stdout, redirect_stderr from pathlib import Path -from db import DB import pytest -import utils import yaml -from global_config import GlobalConfig -import test_functions as Tests + +from lib.db import DB +from lib import utils +from lib.global_config import GlobalConfig +from tests import test_functions as Tests from runner import Runner GlobalConfig().override_config(config_name='test-config.yml') @@ -36,16 +29,16 @@ # otherwise failing Tests will not run the runner.cleanup() properly # This should be done once per module -@pytest.fixture(autouse=True, scope="module") -def build_image(): +@pytest.fixture(autouse=True, scope="module", name="build_image") +def build_image_fixture(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) subprocess.run(['docker', 'compose', '-f', uri+'/compose.yml', 'build'], check=True) GlobalConfig().override_config(config_name='test-config.yml') # cleanup test/tmp directory after every test run -@pytest.fixture(autouse=True) -def cleanup_tmp_directories(): +@pytest.fixture(autouse=True, name="cleanup_tmp_directories") +def cleanup_tmp_directories_fixture(): yield tmp_dir = os.path.join(CURRENT_DIR, 'tmp/') if os.path.exists(tmp_dir): @@ -301,17 +294,17 @@ def test_cmd_ran(): def test_uri_local_dir(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri ,'--config-override', 'test-config.yml', - '--skip-config-check', '--dev-repeat-run'], + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri ,'--config-override', 'test-config.yml', + '--skip-system-checks', '--dev-repeat-run'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='UTF-8' ) - uri_in_db = utils.get_project_data(project_name)['uri'] + uri_in_db = utils.get_run_data(RUN_NAME)['uri'] assert uri_in_db == uri, Tests.assertion_info(f"uri: {uri}", uri_in_db) assert ps.stderr == '', Tests.assertion_info('no errors', ps.stderr) @@ -326,17 +319,17 @@ def test_uri_local_dir_missing(): # basic positive case def test_uri_github_repo(): uri = 'https://github.com/green-coding-berlin/pytest-dummy-repo' - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri ,'--config-override', 'test-config.yml', - '--skip-config-check', '--dev-repeat-run'], + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri ,'--config-override', 'test-config.yml', + '--skip-system-checks', '--dev-repeat-run'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='UTF-8' ) - uri_in_db = utils.get_project_data(project_name)['uri'] + uri_in_db = utils.get_run_data(RUN_NAME)['uri'] assert uri_in_db == uri, Tests.assertion_info(f"uri: {uri}", uri_in_db) assert ps.stderr == '', Tests.assertion_info('no errors', ps.stderr) @@ -357,18 +350,18 @@ def test_uri_local_branch(): # that makes sure that it really is pulling a different branch def test_uri_github_repo_branch(): uri = 'https://github.com/green-coding-berlin/pytest-dummy-repo' - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri , + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--branch', 'test-branch' , '--filename', 'basic_stress.yml', - '--config-override', 'test-config.yml', '--skip-config-check', '--dev-repeat-run'], + '--config-override', 'test-config.yml', '--skip-system-checks', '--dev-repeat-run'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='UTF-8' ) - branch_in_db = utils.get_project_data(project_name)['branch'] + branch_in_db = utils.get_run_data(RUN_NAME)['branch'] assert branch_in_db == 'test-branch', Tests.assertion_info('branch: test-branch', branch_in_db) assert ps.stderr == '', Tests.assertion_info('no errors', ps.stderr) @@ -392,17 +385,17 @@ def test_uri_github_repo_branch_missing(): def test_name_is_in_db(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri ,'--config-override', 'test-config.yml', - '--skip-config-check'], + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri ,'--config-override', 'test-config.yml', + '--skip-system-checks'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding='UTF-8' ) - name_in_db = utils.get_project_data(project_name)['name'] - assert name_in_db == project_name, Tests.assertion_info(f"name: {project_name}", name_in_db) + name_in_db = utils.get_run_data(RUN_NAME)['name'] + assert name_in_db == RUN_NAME, Tests.assertion_info(f"name: {RUN_NAME}", name_in_db) # --filename FILENAME # An optional alternative filename if you do not want to use "usage_scenario.yml" @@ -413,12 +406,12 @@ def test_different_filename(): compose_path = os.path.abspath(os.path.join(CURRENT_DIR, 'stress-application/compose.yml')) Tests.make_proj_dir(dir_name=dir_name, usage_scenario_path=usage_scenario_path, docker_compose_path=compose_path) uri = os.path.join(CURRENT_DIR, 'tmp/', dir_name) - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri , + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--filename', 'basic_stress.yml', '--config-override', 'test-config.yml', - '--skip-config-check', '--dev-repeat-run'], + '--skip-system-checks', '--dev-repeat-run'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -427,7 +420,7 @@ def test_different_filename(): with open(usage_scenario_path, 'r', encoding='utf-8') as f: usage_scenario_contents = yaml.safe_load(f) - usage_scenario_in_db = utils.get_project_data(project_name)['usage_scenario'] + usage_scenario_in_db = utils.get_run_data(RUN_NAME)['usage_scenario'] assert usage_scenario_in_db == usage_scenario_contents,\ Tests.assertion_info(usage_scenario_contents, usage_scenario_in_db) assert ps.stderr == '', Tests.assertion_info('no errors', ps.stderr) @@ -435,8 +428,9 @@ def test_different_filename(): # if that filename is missing... def test_different_filename_missing(): uri = os.path.abspath(os.path.join(CURRENT_DIR, '..', 'stress-application/')) - pid = Tests.insert_project(uri) - runner = Runner(uri=uri, uri_type='folder', pid=pid, filename='basic_stress.yml', skip_config_check=True) + RUN_NAME = 'test_' + utils.randomword(12) + + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename='basic_stress.yml', skip_system_checks=True) with pytest.raises(FileNotFoundError) as e: runner.run() @@ -449,10 +443,10 @@ def test_different_filename_missing(): def test_no_file_cleanup(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri , - '--no-file-cleanup', '--config-override', 'test-config.yml', '--skip-config-check'], + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , + '--no-file-cleanup', '--config-override', 'test-config.yml', '--skip-system-checks'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -472,10 +466,10 @@ def test_debug(monkeypatch): monkeypatch.setattr('sys.stdin', io.StringIO('Enter')) uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri , - '--debug', '--config-override', 'test-config.yml', '--skip-config-check', + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , + '--debug', '--config-override', 'test-config.yml', '--skip-system-checks', '--dev-repeat-run'], check=True, stderr=subprocess.PIPE, @@ -494,9 +488,9 @@ def test_debug(monkeypatch): def wip_test_verbose_provider_boot(): uri = os.path.abspath(os.path.join( CURRENT_DIR, 'stress-application/')) - project_name = 'test_' + utils.randomword(12) + RUN_NAME = 'test_' + utils.randomword(12) ps = subprocess.run( - ['python3', '../runner.py', '--name', project_name, '--uri', uri , + ['python3', '../runner.py', '--name', RUN_NAME, '--uri', uri , '--verbose-provider-boot', '--config-override', 'test-config.yml', '--dev-repeat-run'], check=True, @@ -504,20 +498,20 @@ def wip_test_verbose_provider_boot(): stdout=subprocess.PIPE, encoding='UTF-8' ) - pid = utils.get_project_data(project_name)['id'] + run_id = utils.get_run_data(RUN_NAME)['id'] query = """ SELECT time, note FROM notes WHERE - project_id = %s + run_id = %s AND note LIKE %s ORDER BY time """ - notes = DB().fetch_all(query, (pid,'Booting%',)) + notes = DB().fetch_all(query, (run_id,'Booting%',)) metric_providers = utils.get_metric_providers_names(config) #for each metric provider, assert there is an an entry in notes diff --git a/test/test_volume_loading.py b/tests/test_volume_loading.py similarity index 92% rename from test/test_volume_loading.py rename to tests/test_volume_loading.py index ff0ccae72..b41fc60c7 100644 --- a/test/test_volume_loading.py +++ b/tests/test_volume_loading.py @@ -1,28 +1,24 @@ -#pylint: disable=wrong-import-position, import-error, invalid-name -# disabling subprocess-run-check because for some of them we *want* the check to fail -#pylint: disable=subprocess-run-check import os import re import shutil -import sys import subprocess import io CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/..") -sys.path.append(f"{CURRENT_DIR}/../lib") -import pytest -import utils from contextlib import redirect_stdout, redirect_stderr -from global_config import GlobalConfig +import pytest + +from tests import test_functions as Tests + +from lib import utils +from lib.global_config import GlobalConfig from runner import Runner -import test_functions as Tests GlobalConfig().override_config(config_name='test-config.yml') -@pytest.fixture(autouse=True) -def cleanup_tmp_directories(): +@pytest.fixture(autouse=True, name="cleanup_tmp_directories") +def cleanup_tmp_directories_fixture(): yield tmp_dir = os.path.join(CURRENT_DIR, 'tmp/') if os.path.exists(tmp_dir): @@ -33,7 +29,8 @@ def check_if_container_running(container_name): ['docker', 'container', 'inspect', '-f', '{{.State.Running}}', container_name], stderr=subprocess.PIPE, stdout=subprocess.PIPE, - encoding='UTF-8' + encoding='UTF-8', + check=False, ) if ps.returncode != 0: return False @@ -97,7 +94,8 @@ def test_load_files_from_within_gmt(): '-c', 'test -f /tmp/test-file && echo "File mounted"'], stderr=subprocess.PIPE, stdout=subprocess.PIPE, - encoding='UTF-8' + encoding='UTF-8', + check=False, ) out = ps.stdout err = ps.stderr @@ -161,7 +159,8 @@ def test_load_volume_references(): '-c', 'test -f /tmp/test-file && echo "File mounted"'], stderr=subprocess.PIPE, stdout=subprocess.PIPE, - encoding='UTF-8' + encoding='UTF-8', + check=False, ) out = ps.stdout err = ps.stderr @@ -171,8 +170,8 @@ def test_load_volume_references(): def test_volume_loading_subdirectories_root(): uri = os.path.join(CURRENT_DIR, 'data/test_cases/subdir_volume_loading') - pid = Tests.insert_project(uri) - runner = Runner(uri=uri, uri_type='folder', pid=pid, skip_config_check=True) + RUN_NAME = 'test_' + utils.randomword(12) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', skip_system_checks=True) out = io.StringIO() err = io.StringIO() @@ -199,8 +198,8 @@ def test_volume_loading_subdirectories_root(): def test_volume_loading_subdirectories_subdir(): uri = os.path.join(CURRENT_DIR, 'data/test_cases/subdir_volume_loading') - pid = Tests.insert_project(uri) - runner = Runner(uri=uri, uri_type='folder', filename="subdir/usage_scenario_subdir.yml", pid=pid, skip_config_check=True) + RUN_NAME = 'test_' + utils.randomword(12) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename="subdir/usage_scenario_subdir.yml", skip_system_checks=True) out = io.StringIO() err = io.StringIO() @@ -218,8 +217,8 @@ def test_volume_loading_subdirectories_subdir(): def test_volume_loading_subdirectories_subdir2(): uri = os.path.join(CURRENT_DIR, 'data/test_cases/subdir_volume_loading') - pid = Tests.insert_project(uri) - runner = Runner(uri=uri, uri_type='folder', filename="subdir/subdir2/usage_scenario_subdir2.yml", pid=pid, skip_config_check=True) + RUN_NAME = 'test_' + utils.randomword(12) + runner = Runner(name=RUN_NAME, uri=uri, uri_type='folder', filename="subdir/subdir2/usage_scenario_subdir2.yml", skip_system_checks=True) out = io.StringIO() err = io.StringIO() diff --git a/test/test_yml_parsing.py b/tests/test_yml_parsing.py similarity index 67% rename from test/test_yml_parsing.py rename to tests/test_yml_parsing.py index a984a5e8c..f72378c2c 100644 --- a/test/test_yml_parsing.py +++ b/tests/test_yml_parsing.py @@ -1,25 +1,22 @@ -#pylint: disable=import-error,wrong-import-position,protected-access import os -import sys import unittest -current_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{current_dir}/../tools") -sys.path.append(f"{current_dir}/../lib") -sys.path.append(f"{current_dir}/..") +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -import uuid -from global_config import GlobalConfig +from lib import utils +from lib.global_config import GlobalConfig from runner import Runner + GlobalConfig().override_config(config_name='test-config.yml') class TestYML(unittest.TestCase): def test_includes(self): - test_dir = os.path.join(current_dir, 'data/usage_scenarios/') + test_dir = os.path.join(CURRENT_DIR, 'data/usage_scenarios/') test_root_file = 'import_one_root.yml' + name = 'test_' + utils.randomword(12) - runner = Runner(uri=test_dir, uri_type='folder', pid=str(uuid.uuid4()), filename=test_root_file) + runner = Runner(name=name, uri=test_dir, uri_type='folder', filename=test_root_file) runner.checkout_repository() # We need to do this to setup the file paths correctly runner.load_yml_file() @@ -31,9 +28,11 @@ def test_includes(self): self.assertEqual(result_obj, runner._usage_scenario) def test_(self): - test_dir = os.path.join(current_dir, 'data/usage_scenarios/') + test_dir = os.path.join(CURRENT_DIR, 'data/usage_scenarios/') test_root_file = 'import_two_root.yml' - runner = Runner(uri=test_dir, uri_type='folder', pid=str(uuid.uuid4()), filename=test_root_file) + name = 'test_' + utils.randomword(12) + + runner = Runner(name=name, uri=test_dir, uri_type='folder', filename=test_root_file) runner.checkout_repository() # We need to do this to setup the file paths correctly runner.load_yml_file() @@ -51,8 +50,9 @@ def test_(self): def test_invalid_path(self): - test_dir = os.path.join(current_dir, 'data/usage_scenarios/') + name = 'test_' + utils.randomword(12) + test_dir = os.path.join(CURRENT_DIR, 'data/usage_scenarios/') test_root_file = 'import_error.yml' - runner = Runner(uri=test_dir, uri_type='folder', pid=str(uuid.uuid4()), filename=test_root_file) + runner = Runner(name=name, uri=test_dir, uri_type='folder', filename=test_root_file) runner.checkout_repository() # We need to do this to setup the file paths correctly self.assertRaises(ValueError, runner.load_yml_file) diff --git a/test/tools/test_jobs.py b/tests/tools/test_jobs.py similarity index 58% rename from test/tools/test_jobs.py rename to tests/tools/test_jobs.py index d9e149d74..0a4bc11a3 100644 --- a/test/tools/test_jobs.py +++ b/tests/tools/test_jobs.py @@ -1,39 +1,30 @@ -# test all functions in jobs.py -#pylint: disable=invalid-name,missing-docstring,too-many-statements,fixme - import os -import sys import subprocess from unittest.mock import patch import pytest import psycopg CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/../../tools") -sys.path.append(f"{CURRENT_DIR}/../../lib") - -#pylint: disable=import-error,wrong-import-position -from db import DB -import jobs -import test_functions as Tests -import utils -from global_config import GlobalConfig + +from lib.db import DB +from lib import utils +from lib.global_config import GlobalConfig +from tools.machine import Machine +from tools.jobs import Job +from tests import test_functions as Tests + GlobalConfig().override_config(config_name='test-config.yml') config = GlobalConfig().config -@pytest.fixture(autouse=True, scope='module') -def cleanup_jobs(): - yield - DB().query('DELETE FROM jobs') +@pytest.fixture(autouse=True, name="register_machine") +def register_machine_fixture(): + machine = Machine(machine_id=1, description='test-machine') + machine.register() -@pytest.fixture(autouse=True, scope='module') -def cleanup_projects(): - yield - DB().query('DELETE FROM projects') # This should be done once per module -@pytest.fixture(autouse=True, scope="module") -def build_image(): +@pytest.fixture(autouse=True, scope="module", name="build_image") +def build_image_fixture(): subprocess.run(['docker', 'compose', '-f', f"{CURRENT_DIR}/../stress-application/compose.yml", 'build'], check=True) def get_job(job_id): @@ -50,9 +41,9 @@ def get_job(job_id): return data -def test_no_project_job(): +def test_no_run_job(): ps = subprocess.run( - ['python3', '../tools/jobs.py', 'project', '--config-override', 'test-config.yml'], + ['python3', '../tools/jobs.py', 'run', '--config-override', 'test-config.yml'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -74,22 +65,20 @@ def test_no_email_job(): Tests.assertion_info('No job to process. Exiting', ps.stdout) def test_insert_job(): - job_id = jobs.insert_job('test') + job_id = Job.insert('Test Name', 'Test URL', 'Test Email', 'Test Branch', 'Test filename', 1) assert job_id is not None - job = get_job(job_id) - assert job['type'] == 'test' + job = Job.get_job('run') + assert job.state == 'WAITING' -def test_simple_project_job(): +def test_simple_run_job(): name = utils.randomword(12) - uri = 'https://github.com/green-coding-berlin/pytest-dummy-repo' + url = 'https://github.com/green-coding-berlin/pytest-dummy-repo' filename = 'usage_scenario.yml' - pid = DB().fetch_one('INSERT INTO "projects" ("name","uri","filename","email","last_run","created_at") \ - VALUES \ - (%s,%s,%s,\'manual\',NULL,NOW()) RETURNING id;', params=(name, uri, filename))[0] - jobs.insert_job('project', pid) + Job.insert(name, url, 'Test Email', 'main', filename, 1) + ps = subprocess.run( - ['python3', '../tools/jobs.py', 'project', '--config-override', 'test-config.yml', '--skip-config-check'], + ['python3', '../tools/jobs.py', 'run', '--config-override', 'test-config.yml', '--skip-system-checks'], check=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, @@ -103,16 +92,15 @@ def test_simple_project_job(): Tests.assertion_info('MEASUREMENT SUCCESSFULLY COMPLETED', ps.stdout) #pylint: disable=unused-variable # for the time being, until I get the mocking to work -def test_simple_email_job(): +## This test doesn't really make sense anymore as is, since we don't have "email jobs" in the same way, +## more that we send an email after a run job is finished. +def todo_test_simple_email_job(): name = utils.randomword(12) - uri = 'https://github.com/green-coding-berlin/pytest-dummy-repo' + url = 'https://github.com/green-coding-berlin/pytest-dummy-repo' email = 'fakeemailaddress' - filename = 'usage_scenario.yml' - pid = DB().fetch_one('INSERT INTO "projects" ("name","uri","email","filename","last_run","created_at") \ - VALUES \ - (%s,%s,%s,%s,NULL,NOW()) RETURNING id;', params=(name, uri, email, filename))[0] + filename = 'usage_scenario.yml' - jobs.insert_job('email', pid) + Job.insert(name, url, email, 'main', filename, 1) # Why is this patch not working :-( with patch('email_helpers.send_report_email') as send_email: @@ -125,5 +113,6 @@ def test_simple_email_job(): ) #send_email.assert_called_with(email, pid) assert ps.stderr == '', Tests.assertion_info('No Error', ps.stderr) - assert ps.stdout == 'Successfully processed jobs queue item.\n',\ - Tests.assertion_info('Successfully processed jobs queue item.', ps.stdout) + job_success_message = 'Successfully processed jobs queue item.' + assert job_success_message in ps.stdout,\ + Tests.assertion_info('Successfully processed jobs queue item.', ps.stdout) diff --git a/tools/client.py b/tools/client.py index 1a601f492..2b49458d4 100644 --- a/tools/client.py +++ b/tools/client.py @@ -1,35 +1,30 @@ -# pylint: disable=import-error -# pylint: disable=wrong-import-position +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- -import sys -import os import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr + +import os import time import subprocess -sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../lib') - -from jobs import get_job, process_job, handle_job_exception -from global_config import GlobalConfig -from db import DB - - -faulthandler.enable() # will catch segfaults and write to STDERR +from tools.jobs import Job, handle_job_exception +from lib.global_config import GlobalConfig +from lib.db import DB # We currently have this dynamically as it will probably change quite a bit STATUS_LIST = ['job_no', 'job_start', 'job_error', 'job_end', 'cleanup_start', 'cleanup_stop'] -# pylint: disable=redefined-outer-name -def set_status(status_code, data=None, project_id=None): +def set_status(status_code, data=None, run_id=None): if status_code not in STATUS_LIST: raise ValueError(f"Status code not valid: '{status_code}'. Should be in: {STATUS_LIST}") query = """ INSERT INTO - client_status (status_code, machine_id, data, project_id) + client_status (status_code, machine_id, data, run_id) VALUES (%s, %s, %s, %s) """ - params = (status_code, GlobalConfig().config['machine']['id'], data, project_id) + params = (status_code, GlobalConfig().config['machine']['id'], data, run_id) DB().query(query=query, params=params) @@ -37,21 +32,20 @@ def set_status(status_code, data=None, project_id=None): if __name__ == '__main__': while True: - job = get_job('project') + job = Job.get_job('run') - if (job is None or job == []): + if not job: set_status('job_no') - time.sleep(GlobalConfig().config['client']['sleep_time']) + time.sleep(GlobalConfig().config['client']['sleep_time_no_job']) else: - project_id = job[2] - set_status('job_start', '', project_id) + set_status('job_start', '', job.run_id) try: - process_job(*job) + job.process(docker_prune=True) except Exception as exc: - set_status('job_error', str(exc), project_id) - handle_job_exception(exc, project_id) + set_status('job_error', str(exc), job.run_id) + handle_job_exception(exc, job) else: - set_status('job_end', '', project_id) + set_status('job_end', '', job.run_id) set_status('cleanup_start') @@ -62,3 +56,5 @@ def set_status(status_code, data=None, project_id=None): check=True,) set_status('cleanup_stop', f"stdout: {result.stdout}, stderr: {result.stderr}") + + time.sleep(GlobalConfig().config['client']['sleep_time_after_job']) diff --git a/tools/dc_converter.py b/tools/dc_converter.py index d1dc7253f..5a4d9b2e7 100644 --- a/tools/dc_converter.py +++ b/tools/dc_converter.py @@ -1,8 +1,13 @@ -#pylint: disable=invalid-name, line-too-long +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr import sys import argparse from io import StringIO + import psycopg import pandas as pd @@ -59,7 +64,7 @@ def main(args): df = df.melt(id_vars=['time'], var_name='detail_name', value_name='value') - df['project_id'] = args.project_id + df['run_id'] = args.run_id df['metric'] = 'atx_energy_dc_channel' f = StringIO(df.to_csv(index=False, header=False)) @@ -79,7 +84,7 @@ def main(args): parser = argparse.ArgumentParser() parser.add_argument('filename', type=str) - parser.add_argument('project_id', type=str) + parser.add_argument('run_id', type=str) parser.add_argument('db_host', type=str) parser.add_argument('db_pw', type=str) diff --git a/tools/import_data.py b/tools/import_data.py index c09a87df5..e2654190f 100644 --- a/tools/import_data.py +++ b/tools/import_data.py @@ -1,10 +1,10 @@ -#pylint: disable=import-error,wrong-import-position +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- -import sys -import os -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib')) +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr -from db import DB +from lib.db import DB if __name__ == '__main__': import argparse diff --git a/tools/jobs.py b/tools/jobs.py index 2a58b2343..73898edf2 100644 --- a/tools/jobs.py +++ b/tools/jobs.py @@ -1,154 +1,213 @@ -#pylint: disable=import-error,wrong-import-position +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr + import sys import os -import faulthandler from datetime import datetime -faulthandler.enable() # will catch segfaults and write to STDERR - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/..") -sys.path.append(f"{CURRENT_DIR}/../lib") - -import email_helpers -import error_helpers -from db import DB -from global_config import GlobalConfig -from phase_stats import build_and_store_phase_stats -from runner import Runner - -def insert_job(job_type, project_id=None, machine_id=None): - query = """ - INSERT INTO - jobs (type, failed, running, created_at, project_id, machine_id) - VALUES - (%s, FALSE, FALSE, NOW(), %s, %s) RETURNING id; - """ - params = (job_type, project_id, machine_id,) - job_id = DB().fetch_one(query, params=params)[0] - return job_id - -# do the first job you get. -def get_job(job_type): - clear_old_jobs() - query = """ - SELECT id, type, project_id - FROM jobs - WHERE failed=false AND type=%s AND (machine_id IS NULL or machine_id = %s) - ORDER BY created_at ASC - LIMIT 1 - """ - - return DB().fetch_one(query, (job_type, GlobalConfig().config['machine']['id'])) - - -def delete_job(job_id): - query = "DELETE FROM jobs WHERE id=%s" - params = (job_id,) - DB().query(query, params=params) - -# if there is no job of that type running, set this job to running - - -def check_job_running(job_type, job_id): - query = "SELECT FROM jobs WHERE running=true AND type=%s" - params = (job_type,) - data = DB().fetch_one(query, params=params) - if data: - # No email here, only debug - error_helpers.log_error('Job was still running: ', job_type, job_id) - sys.exit(1) # is this the right way to exit here? - else: - query_update = "UPDATE jobs SET running=true, last_run=NOW() WHERE id=%s" - params_update = (job_id,) - DB().query(query_update, params=params_update) - - -def clear_old_jobs(): - query = "DELETE FROM jobs WHERE last_run < NOW() - INTERVAL '20 minutes' AND failed=false" - DB().query(query) - -def get_project(project_id): - data = DB().fetch_one( - """SELECT p.name, p.uri, p.email, p.branch, p.filename, m.description - FROM projects as p - LEFT JOIN machines AS m ON p.machine_id = m.id - WHERE p.id = %s LIMIT 1""", (project_id, )) - - if data is None or data == []: - raise RuntimeError(f"couldn't find project w/ id: {project_id}") - - return data +from lib import email_helpers +from lib import error_helpers +from lib.db import DB +from lib.global_config import GlobalConfig +from lib.terminal_colors import TerminalColors +from tools.phase_stats import build_and_store_phase_stats + +""" + The jobs.py file is effectively a state machine that can insert a job in the 'WAITING' + state and then push it through the states 'RUNNING', 'FAILED/FINISHED', 'NOTIFYING' + and 'NOTIFIED'. + + After 14 days all FAILED and NOTIFIED jobs will be deleted. +""" + +class Job: + def __init__(self, state, name, email, url, branch, filename, machine_id, run_id=None, job_id=None, machine_description=None): + self.id = job_id + self.state = state + self.name = name + self.email = email + self.url = url + self.branch = branch + self.filename = filename + self.machine_id = machine_id + self.machine_description = machine_description + self.run_id = run_id + + def check_measurement_job_running(self): + query = "SELECT * FROM jobs WHERE state = 'RUNNING' AND machine_id = %s" + params = (self.machine_id,) + data = DB().fetch_one(query, params=params) + if data: + error_helpers.log_error('Measurement-Job was still running: ', data) + if GlobalConfig().config['admin']['no_emails'] is False: + email_helpers.send_error_email(GlobalConfig().config['admin']['email'], 'Measurement-Job was still running on box!') + return True + return False + + def check_email_job_running(self): + query = "SELECT * FROM jobs WHERE state = 'NOTIFYING'" + data = DB().fetch_one(query) + if data: + error_helpers.log_error('Notifying-Job was still running: ', data) + if GlobalConfig().config['admin']['no_emails'] is False: + email_helpers.send_error_email(GlobalConfig().config['admin']['email'], 'Notifying-Job was still running on box!') + return True + return False + + def update_state(self, state): + query_update = "UPDATE jobs SET state = %s WHERE id=%s" + params_update = (state, self.id,) + DB().query(query_update, params=params_update) -def process_job(job_id, job_type, project_id, skip_config_check=False, docker_prune=True, full_docker_prune=False): - try: - if job_type == 'email': - _do_email_job(job_id, project_id) - elif job_type == 'project': - _do_project_job(job_id, project_id, skip_config_check, docker_prune, full_docker_prune) + def process(self, skip_system_checks=False, docker_prune=False, full_docker_prune=False): + try: + if self.state == 'FINISHED': + self._do_email_job() + elif self.state == 'WAITING': + self._do_run_job(skip_system_checks, docker_prune, full_docker_prune) + else: + raise RuntimeError( + f"Job w/ id {self.id} has unknown state: {self.state}.") + except Exception as exc: + self.update_state('FAILED') + raise exc + + # should not be called without enclosing try-except block + def _do_email_job(self): + if self.check_email_job_running(): + return + self.update_state('NOTIFYING') + + if GlobalConfig().config['admin']['no_emails'] is False and self.email: + email_helpers.send_report_email(self.email, self.run_id, self.name, machine=self.machine_description) + + self.update_state('NOTIFIED') + + # should not be called without enclosing try-except block + def _do_run_job(self, skip_system_checks=False, docker_prune=False, full_docker_prune=False): + + if self.check_measurement_job_running(): + return + self.update_state('RUNNING') + + # We need this exclusion here, as the jobs.py is also included in the API and there the + # import of the Runner will lead to import conflicts. It is also not used there, so this is acceptable. + #pylint: disable=import-outside-toplevel + from runner import Runner + + runner = Runner( + name=self.name, + uri=self.url, + uri_type='URL', + filename=self.filename, + branch=self.branch, + skip_unsafe=True, + skip_system_checks=skip_system_checks, + full_docker_prune=full_docker_prune, + docker_prune=docker_prune, + job_id=self.id, + ) + try: + # Start main code. Only URL is allowed for cron jobs + self.run_id = runner.run() + build_and_store_phase_stats(self.run_id, runner._sci) + self.update_state('FINISHED') + except Exception as exc: + raise exc + + @classmethod + def insert(cls, name, url, email, branch, filename, machine_id): + query = """ + INSERT INTO + jobs (name, url, email, branch, filename, machine_id, state, created_at) + VALUES + (%s, %s, %s, %s, %s, %s, 'WAITING', NOW()) RETURNING id; + """ + params = (name, url, email, branch, filename, machine_id,) + return DB().fetch_one(query, params=params)[0] + + # A static method to get a job object + @classmethod + def get_job(cls, job_type): + cls.clear_old_jobs() + + query = ''' + SELECT + j.id, j.state, j.name, j.email, j.url, j.branch, + j.filename, j.machine_id, m.description, r.id as run_id + FROM jobs as j + LEFT JOIN machines as m on m.id = j.machine_id + LEFT JOIN runs as r on r.job_id = j.id + WHERE + ''' + params = [] + config = GlobalConfig().config + + if job_type == 'run': + query = f"{query} j.state = 'WAITING' AND j.machine_id = %s " + params.append(config['machine']['id']) else: - raise RuntimeError( - f"Job w/ id {job_id} has unknown type: {job_type}.") - except Exception as exc: - DB().query("UPDATE jobs SET failed=true, running=false WHERE id=%s", params=(job_id,)) - raise exc - - -# should not be called without enclosing try-except block -def _do_email_job(job_id, project_id): - check_job_running('email', job_id) - - [name, _, email, _, _, machine] = get_project(project_id) - - config = GlobalConfig().config - if (config['admin']['notify_admin_for_own_project_ready'] or config['admin']['email'] != email): - email_helpers.send_report_email(email, project_id, name, machine=machine) - - delete_job(job_id) - - -# should not be called without enclosing try-except block -def _do_project_job(job_id, project_id, skip_config_check=False, docker_prune=False, full_docker_prune=False): - check_job_running('project', job_id) - - [_, uri, _, branch, filename, _] = get_project(project_id) - - runner = Runner( - uri=uri, - uri_type='URL', - pid=project_id, - filename=filename, - branch=branch, - skip_unsafe=True, - skip_config_check=skip_config_check, - full_docker_prune=full_docker_prune, - docker_prune=docker_prune, - ) - try: - # Start main code. Only URL is allowed for cron jobs - runner.run() - build_and_store_phase_stats(project_id, runner._sci) - insert_job('email', project_id=project_id) - delete_job(job_id) - except Exception as exc: - raise exc - -# pylint: disable=redefined-outer-name -def handle_job_exception(exce, p_id): - project_name = None - client_mail = None - if p_id: - [project_name, _, client_mail, _, _, machine] = get_project(p_id) + query = f"{query} j.state = 'FINISHED' AND j.email IS NOT NULL " + if config['machine']['jobs_processing'] == 'random': + query = f"{query} ORDER BY RANDOM()" + else: + query = f"{query} ORDER BY j.created_at ASC" # default case == 'fifo' + + query = f"{query} LIMIT 1" + + job = DB().fetch_one(query, params=params) + if not job: + return False + + return Job( + job_id=job[0], + state=job[1], + name=job[2], + email=job[3], + url=job[4], + branch=job[5], + filename=job[6], + machine_id=job[7], + machine_description=job[8], + run_id=job[9] + ) + + @classmethod + def clear_old_jobs(cls): + query = ''' + DELETE FROM jobs + WHERE + (state = 'NOTIFIED' AND updated_at < NOW() - INTERVAL '14 DAYS') + OR + (state = 'FAILED' AND updated_at < NOW() - INTERVAL '14 DAYS') + OR + (state = 'FINISHED' AND updated_at < NOW() - INTERVAL '14 DAYS' AND email IS NULL) + ''' + DB().query(query) + + +# a simple helper method unrelated to the class +def handle_job_exception(exce, job): error_helpers.log_error('Base exception occurred in jobs.py: ', exce) - email_helpers.send_error_email(GlobalConfig().config['admin']['email'], error_helpers.format_error( - 'Base exception occurred in jobs.py: ', exce), project_id=p_id, name=project_name, machine=machine) - # reduced error message to client - if client_mail and GlobalConfig().config['admin']['email'] != client_mail: - email_helpers.send_error_email(client_mail, exce, project_id=p_id, name=project_name, machine=machine) + if GlobalConfig().config['admin']['no_emails'] is False: + if job is not None: + email_helpers.send_error_email(GlobalConfig().config['admin']['email'], error_helpers.format_error( + 'Base exception occurred in jobs.py: ', exce), run_id=job.run_id, name=job.name, machine=job.machine_description) + else: + email_helpers.send_error_email(GlobalConfig().config['admin']['email'], error_helpers.format_error( + 'Base exception occurred in jobs.py: ', exce)) + + # reduced error message to client + if job.email and GlobalConfig().config['admin']['email'] != job.email: + email_helpers.send_error_email(job.email, exce, run_id=job.run_id, name=job.name, machine=job.machine_description) if __name__ == '__main__': #pylint: disable=broad-except,invalid-name @@ -157,15 +216,18 @@ def handle_job_exception(exce, p_id): from pathlib import Path parser = argparse.ArgumentParser() - parser.add_argument('type', help='Select the operation mode.', choices=['email', 'project']) + parser.add_argument('type', help='Select the operation mode.', choices=['email', 'run']) parser.add_argument('--config-override', type=str, help='Override the configuration file with the passed in yml file. Must be located in the same directory as the regular configuration file. Pass in only the name.') - parser.add_argument('--skip-config-check', action='store_true', default=False, help='Skip checking the configuration') - parser.add_argument('--full-docker-prune', action='store_true', help='Stop and remove all containers, build caches, volumes and images on the system') + parser.add_argument('--skip-system-checks', action='store_true', default=False, help='Skip system checks') + parser.add_argument('--full-docker-prune', action='store_true', default=False, help='Prune all images and build caches on the system') parser.add_argument('--docker-prune', action='store_true', help='Prune all unassociated build caches, networks volumes and stopped containers on the system') - args = parser.parse_args() # script will exit if type is not present + if args.type == 'run': + print(TerminalColors.WARNING, '\nWarning: Calling Jobs.py with argument "run" directly is deprecated.\nPlease do not use this functionality in a cronjob and only in CLI for testing\n', TerminalColors.ENDC) + + if args.config_override is not None: if args.config_override[-4:] != '.yml': parser.print_help() @@ -178,14 +240,14 @@ def handle_job_exception(exce, p_id): sys.exit(1) GlobalConfig(config_name=args.config_override) - p_id = None + job_main = None try: - job = get_job(args.type) - if job is None or job == []: + job_main = Job.get_job(args.type) + if not job_main: print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'No job to process. Exiting') sys.exit(0) - p_id = job[2] - process_job(job[0], job[1], job[2], args.skip_config_check, args.docker_prune, args.full_docker_prune) + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'Processing Job ID#: ', job_main.id) + job_main.process(args.skip_system_checks, args.docker_prune, args.full_docker_prune) print('Successfully processed jobs queue item.') - except Exception as exce: - handle_job_exception(exce, p_id) + except Exception as exception: + handle_job_exception(exception, job_main) diff --git a/tools/machine.py b/tools/machine.py new file mode 100644 index 000000000..fddcaa51a --- /dev/null +++ b/tools/machine.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr + +import os + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + +from lib.db import DB + +class Machine: + + def __init__(self, machine_id, description): + if machine_id is None or not isinstance(machine_id, int): + raise RuntimeError('You must set machine id.') + if description is None or description == '': + raise RuntimeError('You must set machine description.') + self.id = machine_id + self.description = description + + def register(self): + DB().query(""" + INSERT INTO machines + ("id", "description", "available", "created_at") + VALUES + (%s, %s, TRUE, 'NOW()') + ON CONFLICT (id) DO + UPDATE SET description = %s -- no need to make where clause here for correct row + """, params=(self.id, + self.description, + self.description + ) + ) diff --git a/tools/phase_stats.py b/tools/phase_stats.py index 1f1f51fce..b36358d29 100644 --- a/tools/phase_stats.py +++ b/tools/phase_stats.py @@ -1,40 +1,37 @@ -#pylint: disable=import-error,wrong-import-position -from io import StringIO -import sys -import os -import decimal +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + import faulthandler -faulthandler.enable() # will catch segfaults and write to STDERR +faulthandler.enable() # will catch segfaults and write to stderr -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(f"{CURRENT_DIR}/..") -sys.path.append(f"{CURRENT_DIR}/../lib") +import decimal +from io import StringIO -from db import DB -from global_config import GlobalConfig +from lib.global_config import GlobalConfig +from lib.db import DB -def generate_csv_line(project_id, metric, detail_name, phase_name, value, value_type, max_value, min_value, unit): - return f"{project_id},{metric},{detail_name},{phase_name},{round(value)},{value_type},{round(max_value) if max_value is not None else ''},{round(min_value) if min_value is not None else ''},{unit},NOW()\n" +def generate_csv_line(run_id, metric, detail_name, phase_name, value, value_type, max_value, min_value, unit): + return f"{run_id},{metric},{detail_name},{phase_name},{round(value)},{value_type},{round(max_value) if max_value is not None else ''},{round(min_value) if min_value is not None else ''},{unit},NOW()\n" -def build_and_store_phase_stats(project_id, sci=None): +def build_and_store_phase_stats(run_id, sci=None): config = GlobalConfig().config query = """ SELECT metric, unit, detail_name FROM measurements - WHERE project_id = %s + WHERE run_id = %s GROUP BY metric, unit, detail_name ORDER BY metric ASC -- we need this ordering for later, when we read again """ - metrics = DB().fetch_all(query, (project_id, )) + metrics = DB().fetch_all(query, (run_id, )) query = """ SELECT phases - FROM projects + FROM runs WHERE id = %s """ - phases = DB().fetch_one(query, (project_id, )) + phases = DB().fetch_one(query, (run_id, )) csv_buffer = StringIO() @@ -46,21 +43,21 @@ def build_and_store_phase_stats(project_id, sci=None): select_query = """ SELECT SUM(value), MAX(value), MIN(value), AVG(value), COUNT(value) FROM measurements - WHERE project_id = %s AND metric = %s AND detail_name = %s AND time > %s and time < %s + WHERE run_id = %s AND metric = %s AND detail_name = %s AND time > %s and time < %s """ - # now we go through all metrics in the project and aggregate them + # now we go through all metrics in the run and aggregate them for (metric, unit, detail_name) in metrics: # unpack # -- saved for future if I need lag time query # WITH times as ( # SELECT id, value, time, (time - LAG(time) OVER (ORDER BY detail_name ASC, time ASC)) AS diff, unit # FROM measurements - # WHERE project_id = %s AND metric = %s + # WHERE run_id = %s AND metric = %s # ORDER BY detail_name ASC, time ASC # ) -- Backlog: if we need derivatives / integrations in the future results = DB().fetch_one(select_query, - (project_id, metric, detail_name, phase['start'], phase['end'], )) + (run_id, metric, detail_name, phase['start'], phase['end'], )) value_sum = 0 max_value = 0 @@ -82,34 +79,34 @@ def build_and_store_phase_stats(project_id, sci=None): 'memory_total_cgroup_container', 'cpu_frequency_sysfs_core', ): - csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", avg_value, 'MEAN', max_value, min_value, unit)) + csv_buffer.write(generate_csv_line(run_id, metric, detail_name, f"{idx:03}_{phase['name']}", avg_value, 'MEAN', max_value, min_value, unit)) elif metric == 'network_io_cgroup_container': # These metrics are accumulating already. We only need the max here and deliver it as total - csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", max_value-min_value, 'TOTAL', None, None, unit)) + csv_buffer.write(generate_csv_line(run_id, metric, detail_name, f"{idx:03}_{phase['name']}", max_value-min_value, 'TOTAL', None, None, unit)) # No max here # But we need to build the energy network_io_bytes_total.append(max_value-min_value) elif metric == 'energy_impact_powermetrics_vm': - csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", avg_value, 'MEAN', max_value, min_value, unit)) + csv_buffer.write(generate_csv_line(run_id, metric, detail_name, f"{idx:03}_{phase['name']}", avg_value, 'MEAN', max_value, min_value, unit)) elif "_energy_" in metric and unit == 'mJ': - csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_sum, 'TOTAL', None, None, unit)) + csv_buffer.write(generate_csv_line(run_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_sum, 'TOTAL', None, None, unit)) # for energy we want to deliver an extra value, the watts. # Here we need to calculate the average differently power_sum = (value_sum * 10**6) / (phase['end'] - phase['start']) power_max = (max_value * 10**6) / ((phase['end'] - phase['start']) / value_count) power_min = (min_value * 10**6) / ((phase['end'] - phase['start']) / value_count) - csv_buffer.write(generate_csv_line(project_id, f"{metric.replace('_energy_', '_power_')}", detail_name, f"{idx:03}_{phase['name']}", power_sum, 'MEAN', power_max, power_min, 'mW')) + csv_buffer.write(generate_csv_line(run_id, f"{metric.replace('_energy_', '_power_')}", detail_name, f"{idx:03}_{phase['name']}", power_sum, 'MEAN', power_max, power_min, 'mW')) if metric.endswith('_machine'): machine_co2 = (value_sum / 3_600) * config['sci']['I'] - csv_buffer.write(generate_csv_line(project_id, f"{metric.replace('_energy_', '_co2_')}", detail_name, f"{idx:03}_{phase['name']}", machine_co2, 'TOTAL', None, None, 'ug')) + csv_buffer.write(generate_csv_line(run_id, f"{metric.replace('_energy_', '_co2_')}", detail_name, f"{idx:03}_{phase['name']}", machine_co2, 'TOTAL', None, None, 'ug')) else: - csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_sum, 'TOTAL', max_value, min_value, unit)) + csv_buffer.write(generate_csv_line(run_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_sum, 'TOTAL', max_value, min_value, unit)) # after going through detail metrics, create cumulated ones if network_io_bytes_total: # build the network energy @@ -117,22 +114,22 @@ def build_and_store_phase_stats(project_id, sci=None): # pylint: disable=invalid-name network_io_in_kWh = (sum(network_io_bytes_total) / 1_000_000_000) * 0.00375 network_io_in_mJ = network_io_in_kWh * 3_600_000_000 - csv_buffer.write(generate_csv_line(project_id, 'network_energy_formula_global', '[FORMULA]', f"{idx:03}_{phase['name']}", network_io_in_mJ, 'TOTAL', None, None, 'mJ')) + csv_buffer.write(generate_csv_line(run_id, 'network_energy_formula_global', '[FORMULA]', f"{idx:03}_{phase['name']}", network_io_in_mJ, 'TOTAL', None, None, 'mJ')) # co2 calculations network_io_co2_in_ug = network_io_in_kWh * config['sci']['I'] * 1_000_000 - csv_buffer.write(generate_csv_line(project_id, 'network_co2_formula_global', '[FORMULA]', f"{idx:03}_{phase['name']}", network_io_co2_in_ug, 'TOTAL', None, None, 'ug')) + csv_buffer.write(generate_csv_line(run_id, 'network_co2_formula_global', '[FORMULA]', f"{idx:03}_{phase['name']}", network_io_co2_in_ug, 'TOTAL', None, None, 'ug')) duration = phase['end']-phase['start'] - csv_buffer.write(generate_csv_line(project_id, 'phase_time_syscall_system', '[SYSTEM]', f"{idx:03}_{phase['name']}", duration, 'TOTAL', None, None, 'us')) + csv_buffer.write(generate_csv_line(run_id, 'phase_time_syscall_system', '[SYSTEM]', f"{idx:03}_{phase['name']}", duration, 'TOTAL', None, None, 'us')) duration_in_years = duration / (1_000_000 * 60 * 60 * 24 * 365) embodied_carbon_share_g = (duration_in_years / (config['sci']['EL']) ) * config['sci']['TE'] * config['sci']['RS'] embodied_carbon_share_ug = decimal.Decimal(embodied_carbon_share_g * 1_000_000) - csv_buffer.write(generate_csv_line(project_id, 'embodied_carbon_share_machine', '[SYSTEM]', f"{idx:03}_{phase['name']}", embodied_carbon_share_ug, 'TOTAL', None, None, 'ug')) + csv_buffer.write(generate_csv_line(run_id, 'embodied_carbon_share_machine', '[SYSTEM]', f"{idx:03}_{phase['name']}", embodied_carbon_share_ug, 'TOTAL', None, None, 'ug')) if phase['name'] == '[RUNTIME]' and machine_co2 is not None and sci is not None \ and sci.get('R', None) is not None and sci['R'] != 0: - csv_buffer.write(generate_csv_line(project_id, 'software_carbon_intensity_global', '[SYSTEM]', f"{idx:03}_{phase['name']}", (machine_co2 + embodied_carbon_share_ug) / sci['R'], 'TOTAL', None, None, f"ugCO2e/{sci['R_d']}")) + csv_buffer.write(generate_csv_line(run_id, 'software_carbon_intensity_global', '[SYSTEM]', f"{idx:03}_{phase['name']}", (machine_co2 + embodied_carbon_share_ug) / sci['R'], 'TOTAL', None, None, f"ugCO2e/{sci['R_d']}")) csv_buffer.seek(0) # Reset buffer position to the beginning @@ -140,19 +137,17 @@ def build_and_store_phase_stats(project_id, sci=None): csv_buffer, table='phase_stats', sep=',', - columns=('project_id', 'metric', 'detail_name', 'phase', 'value', 'type', 'max_value', 'min_value', 'unit', 'created_at') + columns=('run_id', 'metric', 'detail_name', 'phase', 'value', 'type', 'max_value', 'min_value', 'unit', 'created_at') ) csv_buffer.close() # Close the buffer if __name__ == '__main__': - #pylint: disable=broad-except,invalid-name - import argparse parser = argparse.ArgumentParser() - parser.add_argument('project_id', help='Project ID', type=str) + parser.add_argument('run_id', help='Run ID', type=str) args = parser.parse_args() # script will exit if type is not present - build_and_store_phase_stats(args.project_id) + build_and_store_phase_stats(args.run_id) diff --git a/tools/prune_db.py b/tools/prune_db.py index da539da76..f83d2d270 100644 --- a/tools/prune_db.py +++ b/tools/prune_db.py @@ -1,10 +1,12 @@ -#pylint: disable=import-error,wrong-import-position +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr import sys -import os -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib')) -from db import DB +from lib.db import DB if __name__ == '__main__': import argparse @@ -15,14 +17,14 @@ args = parser.parse_args() # script will exit if arguments not present if args.all: - print("This will remove ALL projects and measurement data from the DB. Continue? (y/N)") + print("This will remove ALL runs and measurement data from the DB. Continue? (y/N)") answer = sys.stdin.readline() if answer.strip().lower() == 'y': - DB().query('DELETE FROM projects') + DB().query('DELETE FROM runs') print("Done") else: - print("This will remove all failed projects and measurement data from the DB. Continue? (y/N)") + print("This will remove all runs that have not ended, which includes failed ones, but also possibly running, so be sure no measurement is currently active. Continue? (y/N)") answer = sys.stdin.readline() if answer.strip().lower() == 'y': - DB().query('DELETE FROM projects WHERE end_measurement IS NULL') + DB().query('DELETE FROM runs WHERE end_measurement IS NULL') print("Done") diff --git a/tools/rebuild_phase_stats.py b/tools/rebuild_phase_stats.py index c6e1968d1..99e57239c 100644 --- a/tools/rebuild_phase_stats.py +++ b/tools/rebuild_phase_stats.py @@ -1,11 +1,14 @@ -#pylint: disable=import-error,wrong-import-position +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr import sys -import os -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib')) -from phase_stats import build_and_store_phase_stats -from db import DB +from tools.phase_stats import build_and_store_phase_stats + +from lib.db import DB if __name__ == '__main__': print('This will remove ALL phase_stats and completely rebuild them. No data will get lost, but it will take some time. Continue? (y/N)') @@ -13,18 +16,18 @@ if answer.strip().lower() == 'y': print('Deleting old phase_stats ...') DB().query('DELETE FROM phase_stats') - print('Fetching projects ...') + print('Fetching runs ...') query = ''' SELECT id - FROM projects + FROM runs WHERE end_measurement IS NOT NULL AND phases IS NOT NULL ''' - projects = DB().fetch_all(query) + runs = DB().fetch_all(query) - print(f"Fetched {len(projects)} projects. Commencing ...") - for idx, project_id in enumerate(projects): + print(f"Fetched {len(runs)} runs. Commencing ...") + for idx, run_id in enumerate(runs): - print(f"Rebuilding phase_stats for project #{idx} {project_id[0]}") - build_and_store_phase_stats(project_id[0]) + print(f"Rebuilding phase_stats for run #{idx} {run_id[0]}") + build_and_store_phase_stats(run_id[0]) print('Done') diff --git a/tools/timeline_projects.py b/tools/timeline_projects.py new file mode 100644 index 000000000..a3d56665f --- /dev/null +++ b/tools/timeline_projects.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr + +import os +import pprint +from psycopg.rows import dict_row as psycopg_rows_dict_row + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + +from lib.db import DB +from tools.jobs import Job + +""" + This file schedules new Timeline Projects by inserting jobs in the jobs table + + +""" + +class TimelineProject(): + #pylint:disable=redefined-outer-name + @classmethod + def insert(cls, name, url, branch, filename, machine_id, schedule_mode): + # Timeline projects never insert / use emails as they are always premium and made by admin + # So they need no notification on success / add + insert_query = """ + INSERT INTO + timeline_projects (name, url, branch, filename, machine_id, schedule_mode, created_at) + VALUES + (%s, %s, %s, %s, %s, %s, NOW()) RETURNING id; + """ + params = (name, url, branch, filename, machine_id, schedule_mode,) + return DB().fetch_one(insert_query, params=params)[0] + + +if __name__ == '__main__': + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=['show', 'schedule'], help='Show will show all projects. Schedule will insert a job.') + + args = parser.parse_args() # script will exit if arguments not present + + if args.mode == 'show': + query = """ + SELECT + p.id, p.name, p.url, + (SELECT STRING_AGG(t.name, ', ' ) FROM unnest(p.categories) as elements + LEFT JOIN categories as t on t.id = elements) as categories, + p.branch, p.filename, m.description, p.last_scheduled, p.schedule_mode, + p.created_at, p.updated_at + FROM timeline_projects as p + LEFT JOIN machines as m on m.id = p.machine_id + ORDER BY p.url ASC + """ + data = DB().fetch_all(query, row_factory=psycopg_rows_dict_row) + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(data) + + else: + query = """ + SELECT + id, name, url, branch, filename, machine_id, schedule_mode, last_scheduled, + DATE(last_scheduled) >= DATE(NOW()) as "scheduled_today" + FROM timeline_projects + """ + data = DB().fetch_all(query) + + for [project_id, name, url, branch, filename, machine_id, schedule_mode, last_scheduled, scheduled_today] in data: + if not last_scheduled: + print('Project was not scheduled yet ', url, branch, filename, machine_id) + DB().query('UPDATE timeline_projects SET last_scheduled = NOW() WHERE id = %s', params=(project_id,)) + Job.insert(name, url, None, branch, filename, machine_id) + print('\tInserted ') + elif schedule_mode == 'time': + print('Project is on time schedule', url, branch, filename, machine_id) + if scheduled_today is False: + print('\tProject was not scheduled today', scheduled_today) + DB().query('UPDATE timeline_projects SET last_scheduled = NOW() WHERE id = %s', params=(project_id,)) + Job.insert(name, url, None, branch, filename, machine_id) + print('\tInserted') + elif schedule_mode == 'commit': + print('Project is on time schedule', url, branch, filename, machine_id) + print('This functionality is not yet implemented ...') diff --git a/tools/update_commit_data.py b/tools/update_commit_data.py index bb4b3bc9d..8a6bbe053 100644 --- a/tools/update_commit_data.py +++ b/tools/update_commit_data.py @@ -1,16 +1,15 @@ -#pylint: disable=import-error,wrong-import-position +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- -# This script will update the commit_timestamp field in the database -# for old projects where only the commit_hash field was populated - - -import sys -import os -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib')) +import faulthandler +faulthandler.enable() # will catch segfaults and write to stderr +# This script will update the commit_timestamp field in the database +# for old runs where only the commit_hash field was populated import subprocess from datetime import datetime -from db import DB + +from lib.db import DB if __name__ == '__main__': @@ -27,7 +26,7 @@ SELECT id, commit_hash FROM - projects + runs WHERE uri = %s AND commit_hash IS NOT NULL @@ -37,7 +36,7 @@ raise RuntimeError(f"No match found in DB for {args.uri}!") for row in data: - project_id = str(row[0]) + run_id = str(row[0]) commit_hash = row[1] commit_timestamp = subprocess.run( ['git', 'show', '-s', row[1], '--format=%ci'], @@ -50,7 +49,7 @@ parsed_timestamp = datetime.strptime(commit_timestamp, "%Y-%m-%d %H:%M:%S %z") DB().query( - 'UPDATE projects SET commit_timestamp = %s WHERE id = %s', - params=(parsed_timestamp, project_id) + 'UPDATE runs SET commit_timestamp = %s WHERE id = %s', + params=(parsed_timestamp, run_id) ) print(parsed_timestamp)