From fa730789e7ae61ebeece50acc05bbfa3e27622ea Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 12:25:22 +0200 Subject: [PATCH 01/13] Removing old GitHub runners --- scripts/vars.sh | 88 ++----------------------------------------------- 1 file changed, 3 insertions(+), 85 deletions(-) diff --git a/scripts/vars.sh b/scripts/vars.sh index 7adc03d4..2a6ac550 100755 --- a/scripts/vars.sh +++ b/scripts/vars.sh @@ -48,91 +48,9 @@ read_vars() { } function cpu_vars_fill { - if [[ "$model_name" == *"8272CL"* ]]; then - echo "Found 8272CL model" - add_var "MODEL_NAME" "8272CL"; - - add_var "TDP" 195; - add_var "CPU_THREADS" 52; - add_var "CPU_CORES" 26; - add_var "CPU_MAKE" "intel"; - add_var "RELEASE_YEAR" 2019; - add_var "RAM" 182; - add_var "CPU_FREQ" 2600; - add_var "CPU_CHIPS" 1; - add_var "VHOST_RATIO" $(echo "2/52" | bc -l); - - elif [[ "$model_name" == *"8370C"* ]]; then - echo "Found 8370C model" - add_var "MODEL_NAME" "8370C"; - - add_var "TDP" 270; - add_var "CPU_THREADS" 64; - add_var "CPU_CORES" 32; - add_var "CPU_MAKE" "intel"; - add_var "RELEASE_YEAR" 2021; - add_var "RAM" 224; - add_var "CPU_FREQ" 2800; - add_var "CPU_CHIPS" 1; - add_var "VHOST_RATIO" $(echo "2/64" | bc -l); - - elif [[ "$model_name" == *"9880H"* ]]; then - echo "Found 9880H model" - add_var "MODEL_NAME" "9880H" - - add_var "TDP" 45; - add_var "CPU_THREADS" 16; - add_var "CPU_CORES" 8; - add_var "CPU_MAKE" "intel"; - add_var "RELEASE_YEAR" 2019; - add_var "RAM" 128; - add_var "CPU_FREQ" 2300; - add_var "CPU_CHIPS" 1; - - elif [[ "$model_name" == *"E5-2673 v4"* ]]; then - echo "Found E5-2673 v4 model" - add_var "MODEL_NAME" "E5-2673v4"; - - add_var "TDP" 165; - add_var "CPU_THREADS" 52; - add_var "CPU_CORES" 26; - add_var "CPU_MAKE" "intel"; - add_var "RELEASE_YEAR" 2018; - add_var "RAM" 182; - add_var "CPU_FREQ" 2300; - add_var "CPU_CHIPS" 1; - add_var "VHOST_RATIO" $(echo "2/52" | bc -l); - - elif [[ "$model_name" == *"E5-2673 v3"* ]]; then - echo "Found E5-2673 v3 model" - add_var "MODEL_NAME" "E5-2673v3"; - - add_var "TDP" 110; - add_var "CPU_THREADS" 24; - add_var "CPU_CORES" 12; - add_var "CPU_MAKE" "intel"; - add_var "RELEASE_YEAR" 2015; - add_var "RAM" 84; - add_var "CPU_FREQ" 2400; - add_var "CPU_CHIPS" 1; - add_var "VHOST_RATIO" $(echo "2/24" | bc -l); - - # model is underclocked - elif [[ "$model_name" == *"8171M"* ]]; then - echo "Found 8171M model" - add_var "MODEL_NAME" "8171M"; - - add_var "TDP" 165; - add_var "CPU_THREADS" 52; - add_var "CPU_CORES" 26; - add_var "CPU_MAKE" "intel"; - add_var "RELEASE_YEAR" 2018; - add_var "RAM" 182; - add_var "CPU_FREQ" 2600; - add_var "CPU_CHIPS" 1; - add_var "VHOST_RATIO" $(echo "2/52" | bc -l); - elif [[ "$model_name" == *"AMD EPYC 7763"* ]]; then + # Current GitHub default (Q1/2024) + if [[ "$model_name" == *"AMD EPYC 7763"* ]]; then echo "Found EPYC 7763 model"; add_var "MODEL_NAME" "EPYC_7763"; @@ -146,7 +64,7 @@ function cpu_vars_fill { add_var "CPU_CHIPS" 1; add_var "VHOST_RATIO" $(echo "4/128" | bc -l); - # gitlab uses this one + # gitlab uses this one (Q1/2024) elif [[ "$model_name" == *"AMD EPYC 7B12"* ]]; then echo "Found EPYC 7B12 model" add_var "MODEL_NAME" "EPYC_7B12"; From bce769892373f72a1cafdba26e724c443e7274fd Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 13:53:58 +0200 Subject: [PATCH 02/13] Using container instead of build; SCI M value added; --auto used for model if hardware was given --- action.yml | 58 +----------------------- scripts/display_results.sh | 31 ++++++------- scripts/make_measurement.sh | 42 +++++------------ scripts/setup.sh | 90 ++++++++++--------------------------- scripts/vars.sh | 49 +++++++++++++++++--- 5 files changed, 94 insertions(+), 176 deletions(-) diff --git a/action.yml b/action.yml index 07b4a590..fdb6d6e4 100644 --- a/action.yml +++ b/action.yml @@ -75,62 +75,8 @@ runs: name: Setup shell: bash run: | - if command -v python3 &>/dev/null; then - echo "Python is already installed." - else - echo "Python is not installed. Installing..." - apt-get update - apt-get install -y python3.12 python3.12-venv || apt-get install -y python3.10 python3.10-venv - echo "Python has been installed." - fi - - python_version=$(python3 --version 2>&1) - python_major_version=$(python3 -c 'import sys; print(sys.version_info[0])') - python_minor_version=$(python3 -c 'import sys; print(sys.version_info[1])') - python_cache_path="/tmp/eco-ci/venv/lib/python${python_major_version}.${python_minor_version}/site-packages" - echo "python_cache_path=$python_cache_path" >> $GITHUB_OUTPUT - - # call the initialize function of setup.sh - ${{github.action_path}}/scripts/setup.sh initialize -g ${{inputs.display-graph}} - - # To identify the hash for our cache we cannot use the classic mechansim of - # hashFiles('/tmp/eco-ci/spec-power-model/requirements.txt') - # hashFiles is restricted to ONLY work in the GITHUB_WORKSPACE which is for the calling action - # therefore we need to construct the hash ourselfs beforehand and save it to an output variable - - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' - name: Hash requirements file - id: hash-requirements - shell: bash - run: echo "myhash=$(md5sum /tmp/eco-ci/spec-power-model/requirements.txt | cut -d ' ' -f1)" >> $GITHUB_OUTPUT; - - - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' - name: Cache pip packages - id: cache-pip - uses: actions/cache@v4 - env: - cache-name: cache-pip-packages - with: - # npm cache files are stored in `~/.npm` on Linux/macOS - path: ${{ steps.initialize.outputs.python_cache_path }} - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.hash-requirements.outputs.myhash }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.hash-requirements.outputs.myhash }} - - - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' && steps.cache-pip.outputs.cache-hit == 'true' - name: Inform about cache hit - continue-on-error: true - shell: bash - run: | - echo "Cache hit succeeded! 😀" - - - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' && steps.cache-pip.outputs.cache-hit != 'true' - name: Inform about cache hit - continue-on-error: true - shell: bash - run: | - echo "Cache hit failed! ❌" - + ${{github.action_path}}/scripts/setup.sh initialize - if: inputs.task == 'start-measurement' name: Starting measurement @@ -138,8 +84,6 @@ runs: # if measurement is started first time the reporter might not have run already # we prefer this over manual startint / stopping as it is less error prone for users run: | - ${{github.action_path}}/scripts/setup.sh setup_python - if ${{inputs.send-data}}; then curl_response=$(curl -s -H "Authorization: Bearer ${{github.token}}" ${{ github.api_url }}/repos/${{ github.repository }}/actions/workflows) workflow_id=$(echo $curl_response | jq '.workflows[] | select(.name == "${{ github.workflow }}") | .id') diff --git a/scripts/display_results.sh b/scripts/display_results.sh index 982c4370..ea5d1926 100755 --- a/scripts/display_results.sh +++ b/scripts/display_results.sh @@ -18,7 +18,6 @@ function display_results { CPU_FREQ=${CPU_FREQ:-} CPU_CHIPS=${CPU_CHIPS:-} VHOST_RATIO=${VHOST_RATIO:-} - PREVIOUS_VENV=${PREVIOUS_VENV:-} MEASUREMENT_COUNT=${MEASUREMENT_COUNT:-} WORKFLOW_ID=${WORKFLOW_ID:-} API_BASE=${API_BASE:-} @@ -29,30 +28,25 @@ function display_results { if [[ $MEASUREMENT_RAN != true ]]; then echo "Running a measurement to have at least one result to display." - source /tmp/eco-ci/venv/bin/activate if [[ "$MODEL_NAME" == "unknown" ]]; then - cat /tmp/eco-ci/cpu-util.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + $(docker run --rm cloud-energy python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - cat /tmp/eco-ci/cpu-util.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \ + $(docker run --rm cloud-energy python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --vhost-ratio $VHOST_RATIO --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --vhost-ratio $VHOST_RATIO --silent ) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - cat /tmp/eco-ci/cpu-util.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \ + $(docker run --rm cloud-energy python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --silent ) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt fi - # reactivate the old venv, if it was present - if [[ $PREVIOUS_VENV != '' ]]; then - source $PREVIOUS_VENV/bin/activate - fi max_measurement_number=1 fi @@ -116,21 +110,28 @@ function display_results { echo '📈 Energy graph:' | tee -a $output $output_pr echo '```bash' | tee -a $output $output_pr echo ' ' | tee -a $output $output_pr - cat /tmp/eco-ci/energy-total.txt | /home/runner/go/bin/asciigraph -h 10 -c "Watts over time" | tee -a $output $output_pr + $(docker run --rm cloud-energy /home/woker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr echo ' ```' | tee -a $output $output_pr elif [[ $source == 'gitlab' ]]; then echo '📈 Energy graph:' >> $output - cat /tmp/eco-ci/energy-total.txt | /home/runner/go/bin/asciigraph -h 10 -c "Watts over time" >> $output + $(docker run --rm cloud-energy /home/woker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt >> $output fi fi repo_enc=$( echo ${repo} | jq -Rr @uri) branch_enc=$( echo ${branch} | jq -Rr @uri) if [[ ${show_carbon} == 'true' ]]; then - source "$(dirname "$0")/vars.sh" get_co2 "$total_energy" - if [ -n "${CO2EQ-}" ]; then # We only check for co2 as if this is set the others should be set too + source "$(dirname "$0")/vars.sh" get_energy_co2 "$total_energy" + source "$(dirname "$0")/vars.sh" get_embodied_co2 "$time" + + + if [ -n "$CO2EQ_EMBODIED" ] && [ -n "$CO2EQ_ENERGY" ]; then # We only check for co2 as if this is set the others should be set too + CO2EQ=$(echo "$CO2EQ_EMBODIED + $CO2EQ_ENERGY" | bc -l) + echo '🌳 CO2 Data:' | tee -a $output $output_pr echo "City: $CITY, Lat: $LAT, Lon: $LON" | tee -a $output $output_pr + echo "CO₂ from energy is: $CO2EQ_ENERGY" | tee -a $output $output_pr + echo "CO₂ from manufacturing (embodied carbon) is: $CO2EQ_ENERGY" | tee -a $output $output_pr echo "Carbon Intensity for this location: $CO2I gCO₂eq/kWh" | tee -a $output $output_pr printf "SCI: %.6f gCO₂eq / pipeline run emitted\n" $CO2EQ | tee -a $output $output_pr else diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index be68a3de..04d27052 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -30,39 +30,24 @@ function make_measurement() { # capture cpu util cat /tmp/eco-ci/cpu-util.txt > /tmp/eco-ci/cpu-util-temp.txt - # if a previous venv is already active, - if type deactivate &>/dev/null - then - deactivate - fi - # then activate our venv - source /tmp/eco-ci/venv/bin/activate - ## make a note that we cannot use --energy, skew the result as we do not have an input delay. # this works because demo-reporter is 1/second if [[ "$MODEL_NAME" == "unknown" ]]; then - cat /tmp/eco-ci/cpu-util-temp.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + $(docker run --rm cloud-energy python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - cat /tmp/eco-ci/cpu-util-temp.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \ + $(docker run --rm cloud-energy python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --vhost-ratio $VHOST_RATIO --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --vhost-ratio $VHOST_RATIO --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - cat /tmp/eco-ci/cpu-util-temp.txt | python3 /tmp/eco-ci/spec-power-model/xgb.py \ + $(docker run --rm cloud-energy python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --silent | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt - fi - - # now reset to old venv - deactivate - # reactivate the old venv, if it was present - if [[ $PREVIOUS_VENV != '' ]]; then - source $PREVIOUS_VENV/bin/activate + --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt fi if [[ $MEASUREMENT_COUNT == '' ]]; then @@ -88,13 +73,12 @@ function make_measurement() { echo $total_energy >> /tmp/eco-ci/energy-values.txt source "$(dirname "$0")/vars.sh" add_var MEASUREMENT_RAN true - if [ -z "$cb_machine_uuid" ]; then - cb_machine_uuid=$(uuidgen) - fi - if [[ $send_data == 'true' ]]; then - source "$(dirname "$0")/vars.sh" get_co2 "$total_energy" + source "$(dirname "$0")/vars.sh" get_energy_co2 "$total_energy" + source "$(dirname "$0")/vars.sh" get_embodied_co2 "$time" + + CO2EQ=$(echo "$CO2EQ_EMBODIED + $CO2EQ_ENERGY" | bc -l) add_endpoint=$API_BASE"/v1/ci/measurement/add" value_mJ=$(echo "$total_energy*1000" | bc -l | cut -d '.' -f 1) @@ -127,8 +111,6 @@ function make_measurement() { }" fi - - # write data to output lap_data_file="/tmp/eco-ci/lap-data.json" repo_enc=$( echo ${repo} | jq -Rr @uri) @@ -141,10 +123,8 @@ function make_measurement() { source "$(dirname "$0")/create-and-add-meta.sh" --file "${lap_data_file}" --repository "${repo_enc}" --branch "${branch_enc}" --workflow "$WORKFLOW_ID" --run_id "${run_id_enc}" source "$(dirname "$0")/add-data.sh" --file "${lap_data_file}" --label "$label" --cpu "${cpu_avg}" --energy "${total_energy}" --power "${power_avg}" --time "${time}" - # reset timer and cpu capturing - killall -9 -q /tmp/eco-ci/demo-reporter || true - /tmp/eco-ci/demo-reporter | tee -a /tmp/eco-ci/cpu-util-total.txt > /tmp/eco-ci/cpu-util.txt & - date +%s > /tmp/eco-ci/timer.txt + # reset timer and cpu capturing (lap) + source "$(dirname "$0")/setup.sh" lap_measurement else echo "Skipping measurement as no data was collected since last call" diff --git a/scripts/setup.sh b/scripts/setup.sh index f78f81b1..ec47675d 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -5,78 +5,41 @@ set -euo pipefail source "$(dirname "$0")/vars.sh" read_vars function initialize { + if [[ $reset == true ]]; then if [[ -d /tmp/eco-ci ]]; then rm -rf /tmp/eco-ci fi mkdir /tmp/eco-ci fi - - git clone --depth 1 --single-branch --branch main https://github.com/green-coding-solutions/spec-power-model /tmp/eco-ci/spec-power-model - - ## Reimplement ascii graph when we find a better library - # install go ascii - - if [[ $install_go == true ]]; then - go install github.com/guptarohit/asciigraph/cmd/asciigraph@latest - fi - - # check for gcc - if ! command -v gcc &> /dev/null - then - echo "gcc could not be found, please install it" - exit - fi - - # compile - gcc /tmp/eco-ci/spec-power-model/demo-reporter/cpu-utilization.c -o /tmp/eco-ci/demo-reporter - + # call init_variables + source "$(dirname "$0")/vars.sh" cpu_vars + source "$(dirname "$0")/vars.sh" add_var API_BASE "https://api.green-coding.io" } -function setup_python { - # Create a venv, and backup old - python3 -m venv /tmp/eco-ci/venv - - VENV_VALUE=${VIRTUAL_ENV:-} - PREVIOUS_VENV='' - - if [[ $VENV_VALUE != '' ]]; then - PREVIOUS_VENV=$VENV_VALUE - source "$(dirname "$0")/vars.sh" add_var PREVIOUS_VENV $PREVIOUS_VENV - fi - - # Installing requirements - # first activate our venv - source /tmp/eco-ci/venv/bin/activate - python3 -m pip install --upgrade pip - python3 -m pip install -r /tmp/eco-ci/spec-power-model/requirements.txt - # now reset to old venv - deactivate our venv - # reactivate the old one, if it was present - if [[ $PREVIOUS_VENV != '' ]]; then - source $PREVIOUS_VENV/bin/activate - fi +function start_measurement { + # start global timer + date +%s > /tmp/eco-ci/timer-total.txt + lap_measurement } -function start_measurement { - # call init_variables - source "$(dirname "$0")/vars.sh" cpu_vars +function lap_measurement { + # start step timer + date +%s > /tmp/eco-ci/timer.txt - source "$(dirname "$0")/vars.sh" add_var API_BASE "https://api.green-coding.io" - source "$(dirname "$0")/vars.sh" add_var INIT "DONE" + container_exists=$(docker ps -a -q -f name=^/cloud-energy-cpu-utilization$) - # start measurement - killall -9 -q /tmp/eco-ci/demo-reporter || true - /tmp/eco-ci/demo-reporter | tee -a /tmp/eco-ci/cpu-util-total.txt > /tmp/eco-ci/cpu-util.txt & - # start a timer - date +%s > /tmp/eco-ci/timer.txt - date +%s > /tmp/eco-ci/timer-total.txt + if [ -n "$container_exists" ]; then + docker logs cloud-energy-cpu-utilization | tee -a /tmp/eco-ci/cpu-util-total.txt > /tmp/eco-ci/cpu-util.txt + docker rm -f cloud-energy-cpu-utilization + fi + docker run --rm -d --name cloud-energy-cpu-utilization cloud-energy /home/worker/cpu-utilization } # Main script logic if [ $# -eq 0 ]; then - echo "No option provided. Please specify an option: initialize, setup_python, or start_measurement." + echo "No option provided. Please specify an option: initialize, or start_measurement." exit 1 fi @@ -87,29 +50,24 @@ case $option in initialize) func=initialize ;; - setup_python) - func=setup_python - ;; start_measurement) func=start_measurement ;; + lap_measurement) + func=lap_measurement + ;; *) - echo "Invalid option. Please specify an option: initialize, setup_python, or start_measurement." + echo "Invalid option. Please specify an option: initialize, lap_measurement or start_measurement." exit 1 ;; esac -install_go=true reset=true while [[ $# -gt 1 ]]; do opt="$2" case $opt in - -g|--go) - install_go=$3 - shift - ;; -r|--reset) reset=$3 shift @@ -122,8 +80,8 @@ done if [[ $func == initialize ]]; then initialize -elif [[ $func == setup_python ]]; then - setup_python elif [[ $func == start_measurement ]]; then start_measurement +elif [[ $func == lap_measurement ]]; then + lap_measurement fi diff --git a/scripts/vars.sh b/scripts/vars.sh index 2a6ac550..c156d2f7 100755 --- a/scripts/vars.sh +++ b/scripts/vars.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -model_name=$(cat /proc/cpuinfo | grep "model name") +#model_name=$(cat /proc/cpuinfo | grep "model name") +model_name="unknown" add_var() { key=$1 @@ -50,6 +51,7 @@ read_vars() { function cpu_vars_fill { # Current GitHub default (Q1/2024) + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources if [[ "$model_name" == *"AMD EPYC 7763"* ]]; then echo "Found EPYC 7763 model"; add_var "MODEL_NAME" "EPYC_7763"; @@ -63,6 +65,12 @@ function cpu_vars_fill { add_var "CPU_FREQ" 2450; add_var "CPU_CHIPS" 1; add_var "VHOST_RATIO" $(echo "4/128" | bc -l); + # FROM https://datavizta.boavizta.org/serversimpact + # we assume a disk size of 448 GB total. + # This totals to 1151.7 kg. With a 4/128 splitting this is 35,990.625 gCO2e + add_var "SCI_M" 35990.625; + # we use 4 years - 1*60*60*24*365*4 = + add_var "SCI_USAGE_DURATION" 126144000 # gitlab uses this one (Q1/2024) elif [[ "$model_name" == *"AMD EPYC 7B12"* ]]; then @@ -78,11 +86,22 @@ function cpu_vars_fill { add_var "CPU_FREQ" 2250; add_var "CPU_CHIPS" 1; # see if we can find reference for this add_var "VHOST_RATIO" $(echo "1/64" | bc -l); + # we assume a disk size of 1344 GB total according to https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29107 + # which claims runners have 21 GB of disk space with a splitting facttor of 1/64 + # FROM https://datavizta.boavizta.org/serversimpact + # This totals to 1173.7 kg. With a 1/64 splitting this is 18339,0625 gCO2e + add_var "SCI_M" 18339.0625; + # we use 4 years - 1*60*60*24*365*4 = + add_var "SCI_USAGE_DURATION" 126144000 + else - echo "⚠️ Unknown model $model_name for estimation, running default ..." # >> $GITHUB_STEP_SUMMARY + echo "⚠️ Unknown model $model_name for estimation, will use auto detect ..." # >> $GITHUB_STEP_SUMMARY + # we use a default configuration here from https://datavizta.boavizta.org/serversimpact + add_var "SCI_M" 800.3; + # we use 4 years - 1*60*60*24*365*4 = + add_var "SCI_USAGE_DURATION" 126144000 add_var "MODEL_NAME" "unknown"; - fi } @@ -126,8 +145,21 @@ get_carbon_intensity() { echo "$response" | jq '.carbonIntensity' } -get_co2_val (){ +get_embodied_co2_val (){ + time=$1 + + if [ -n "$SCI_M" ]; then + co2_value=$(echo "$SCI_M * ($time/$SCI_USAGE_DURATION)" | bc -l) + export CO2EQ_EMBODIED="$co2_value" + else + echo "SCI_M was not set" >&2 + fi + +} + +get_energy_co2_val (){ total_energy=$1 + geo_data=$(get_geo_ipapi_co) || true if [ -n "$geo_data" ]; then latitude=$(echo "$geo_data" | jq '.lat') @@ -147,7 +179,7 @@ get_co2_val (){ value_kWh=$(echo "$value_mJ * 10^-9" | bc -l) co2_value=$(echo "$value_kWh * $carbon_intensity" | bc -l) - export CO2EQ="$co2_value" + export CO2EQ_ENERGY="$co2_value" else echo "Failed to get carbon intensity data." >&2 @@ -174,8 +206,11 @@ case $option in read_vars) read_vars ;; - get_co2) - get_co2_val $2 + get_energy_co2) + get_energy_co2_val $2 + ;; + get_embodied_co2) + get_embodied_co2_val $2 ;; *) echo "Invalid option ($option). Please specify an option: cpu_vars, or add_var [key] [value]." From c02a3173899c929dc748e340328ba5b85ec3f4cb Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 13:54:10 +0200 Subject: [PATCH 03/13] Migrated gitlab also --- .gitlab-ci.yml.example | 1 + eco-ci-gitlab.yml | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.gitlab-ci.yml.example b/.gitlab-ci.yml.example index b50b407f..14ca74f2 100644 --- a/.gitlab-ci.yml.example +++ b/.gitlab-ci.yml.example @@ -12,6 +12,7 @@ test-job: # Generate one here for example: https://www.freecodeformat.com/validate-uuid-guid.php #- export ECO_CI_COMPANY_UUID="YOUR COMPANY UUID" #- export ECO_CI_PROJECT_UUID="YOUR PROJECT UUID" + #- export ECO_CI_MACHINE_UUID="YOUR MACHINE UUID" - !reference [.initialize_energy_estimator, script] - !reference [.start_measurement, script] diff --git a/eco-ci-gitlab.yml b/eco-ci-gitlab.yml index a4e49ed0..a3c7a4c0 100644 --- a/eco-ci-gitlab.yml +++ b/eco-ci-gitlab.yml @@ -15,15 +15,9 @@ variables: .initialize_energy_estimator: script: - | - apt-get update - apt-get install git uuid-runtime python3.10 python3.10-venv gcc bc jq curl golang-go psmisc -y if [[ -d /tmp/eco-ci ]]; then rm -rf /tmp/eco-ci fi - git clone --depth 1 --single-branch --branch main https://github.com/green-coding-solutions/eco-ci-energy-estimation /tmp/eco-ci/main - /tmp/eco-ci/main/scripts/setup.sh initialize -r false -g false - /tmp/eco-ci/main/scripts/setup.sh setup_python - .start_measurement: script: - | From 97f6947da13620c13e665f02d6283550ff24ce5f Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 15:13:18 +0200 Subject: [PATCH 04/13] Used remote image name --- scripts/display_results.sh | 10 +++++----- scripts/make_measurement.sh | 6 +++--- scripts/setup.sh | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/display_results.sh b/scripts/display_results.sh index ea5d1926..fc7c2616 100755 --- a/scripts/display_results.sh +++ b/scripts/display_results.sh @@ -30,16 +30,16 @@ function display_results { echo "Running a measurement to have at least one result to display." if [[ "$MODEL_NAME" == "unknown" ]]; then - $(docker run --rm cloud-energy python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - $(docker run --rm cloud-energy python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ --vhost-ratio $VHOST_RATIO --silent ) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - $(docker run --rm cloud-energy python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ @@ -110,11 +110,11 @@ function display_results { echo '📈 Energy graph:' | tee -a $output $output_pr echo '```bash' | tee -a $output $output_pr echo ' ' | tee -a $output $output_pr - $(docker run --rm cloud-energy /home/woker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr + $(docker run --rm greencoding/cloud-energy:asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr echo ' ```' | tee -a $output $output_pr elif [[ $source == 'gitlab' ]]; then echo '📈 Energy graph:' >> $output - $(docker run --rm cloud-energy /home/woker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt >> $output + $(docker run --rm greencoding/cloud-energy:asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt >> $output fi fi repo_enc=$( echo ${repo} | jq -Rr @uri) diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index 04d27052..bc763acd 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -33,16 +33,16 @@ function make_measurement() { ## make a note that we cannot use --energy, skew the result as we do not have an input delay. # this works because demo-reporter is 1/second if [[ "$MODEL_NAME" == "unknown" ]]; then - $(docker run --rm cloud-energy python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - $(docker run --rm cloud-energy python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ --vhost-ratio $VHOST_RATIO --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - $(docker run --rm cloud-energy python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ diff --git a/scripts/setup.sh b/scripts/setup.sh index ec47675d..90d44495 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -34,7 +34,7 @@ function lap_measurement { docker logs cloud-energy-cpu-utilization | tee -a /tmp/eco-ci/cpu-util-total.txt > /tmp/eco-ci/cpu-util.txt docker rm -f cloud-energy-cpu-utilization fi - docker run --rm -d --name cloud-energy-cpu-utilization cloud-energy /home/worker/cpu-utilization + docker run --rm -d --name cloud-energy-cpu-utilization greencoding/cloud-energy:asciicharts /home/worker/cpu-utilization } # Main script logic From 24e8869cf7707cb4741cea56b6f13754958f2135 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 15:38:30 +0200 Subject: [PATCH 05/13] Different tag --- scripts/display_results.sh | 10 +++++----- scripts/make_measurement.sh | 6 +++--- scripts/setup.sh | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/display_results.sh b/scripts/display_results.sh index fc7c2616..cdab2368 100755 --- a/scripts/display_results.sh +++ b/scripts/display_results.sh @@ -30,16 +30,16 @@ function display_results { echo "Running a measurement to have at least one result to display." if [[ "$MODEL_NAME" == "unknown" ]]; then - $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ --vhost-ratio $VHOST_RATIO --silent ) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ @@ -110,11 +110,11 @@ function display_results { echo '📈 Energy graph:' | tee -a $output $output_pr echo '```bash' | tee -a $output $output_pr echo ' ' | tee -a $output $output_pr - $(docker run --rm greencoding/cloud-energy:asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr + $(docker run --rm greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr echo ' ```' | tee -a $output $output_pr elif [[ $source == 'gitlab' ]]; then echo '📈 Energy graph:' >> $output - $(docker run --rm greencoding/cloud-energy:asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt >> $output + $(docker run --rm greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt >> $output fi fi repo_enc=$( echo ${repo} | jq -Rr @uri) diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index bc763acd..04ad2c10 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -33,16 +33,16 @@ function make_measurement() { ## make a note that we cannot use --energy, skew the result as we do not have an input delay. # this works because demo-reporter is 1/second if [[ "$MODEL_NAME" == "unknown" ]]; then - $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ --vhost-ratio $VHOST_RATIO --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - $(docker run --rm greencoding/cloud-energy:asciicharts python3 xgb.py \ + $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ diff --git a/scripts/setup.sh b/scripts/setup.sh index 90d44495..b8888901 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -34,7 +34,7 @@ function lap_measurement { docker logs cloud-energy-cpu-utilization | tee -a /tmp/eco-ci/cpu-util-total.txt > /tmp/eco-ci/cpu-util.txt docker rm -f cloud-energy-cpu-utilization fi - docker run --rm -d --name cloud-energy-cpu-utilization greencoding/cloud-energy:asciicharts /home/worker/cpu-utilization + docker run --rm -d --name cloud-energy-cpu-utilization greencoding/cloud-energy:latest-asciicharts /home/worker/cpu-utilization } # Main script logic From b8438efcca1271dbf43f4b91e6bdb2fde28c6b83 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 16:41:41 +0200 Subject: [PATCH 06/13] Different file capture --- scripts/make_measurement.sh | 18 ++++++++++++------ scripts/setup.sh | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index 04ad2c10..be3d567d 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -22,13 +22,18 @@ function make_measurement() { WORKFLOW_ID=${WORKFLOW_ID:-} API_BASE=${API_BASE:-} + + # reset timer and cpu capturing (lap) + source "$(dirname "$0")/setup.sh" lap_measurement + + # capture cpu util + cat /tmp/eco-ci/cpu-util.txt > /tmp/eco-ci/cpu-util-temp.txt + + # capture time + time=$(($(date +%s) - $(cat /tmp/eco-ci/timer.txt))) + # check wc -l of cpu-util is greater than 0 if [[ $(wc -l < /tmp/eco-ci/cpu-util.txt) -gt 0 ]]; then - # capture time - time=$(($(date +%s) - $(cat /tmp/eco-ci/timer.txt))) - - # capture cpu util - cat /tmp/eco-ci/cpu-util.txt > /tmp/eco-ci/cpu-util-temp.txt ## make a note that we cannot use --energy, skew the result as we do not have an input delay. # this works because demo-reporter is 1/second @@ -123,7 +128,8 @@ function make_measurement() { source "$(dirname "$0")/create-and-add-meta.sh" --file "${lap_data_file}" --repository "${repo_enc}" --branch "${branch_enc}" --workflow "$WORKFLOW_ID" --run_id "${run_id_enc}" source "$(dirname "$0")/add-data.sh" --file "${lap_data_file}" --label "$label" --cpu "${cpu_avg}" --energy "${total_energy}" --power "${power_avg}" --time "${time}" - # reset timer and cpu capturing (lap) + # Reset the timers again, so we do not capture the overhead per step + # we want to only caputure the overhead in the totals source "$(dirname "$0")/setup.sh" lap_measurement else diff --git a/scripts/setup.sh b/scripts/setup.sh index b8888901..cc9ee404 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -20,6 +20,7 @@ function initialize { function start_measurement { # start global timer + touch /tmp/eco-ci/cpu-util.txt date +%s > /tmp/eco-ci/timer-total.txt lap_measurement } From 383522d7f7c124e0e93c0250faf041577c3bd8da Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 17:04:05 +0200 Subject: [PATCH 07/13] Different docker run command --- scripts/display_results.sh | 14 +++++++------- scripts/make_measurement.sh | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/display_results.sh b/scripts/display_results.sh index cdab2368..c5c82dba 100755 --- a/scripts/display_results.sh +++ b/scripts/display_results.sh @@ -30,21 +30,21 @@ function display_results { echo "Running a measurement to have at least one result to display." if [[ "$MODEL_NAME" == "unknown" ]]; then - $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ + docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --vhost-ratio $VHOST_RATIO --silent ) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --vhost-ratio $VHOST_RATIO --silent < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ + docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --silent ) < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --silent < /tmp/eco-ci/cpu-util.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt fi max_measurement_number=1 @@ -110,11 +110,11 @@ function display_results { echo '📈 Energy graph:' | tee -a $output $output_pr echo '```bash' | tee -a $output $output_pr echo ' ' | tee -a $output $output_pr - $(docker run --rm greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr + docker run --rm -i greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time" < /tmp/eco-ci/energy-total.txt | tee -a $output $output_pr echo ' ```' | tee -a $output $output_pr elif [[ $source == 'gitlab' ]]; then echo '📈 Energy graph:' >> $output - $(docker run --rm greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time") < /tmp/eco-ci/energy-total.txt >> $output + docker run --rm -i greencoding/cloud-energy:latest-asciicharts /home/worker/go/bin/asciigraph -h 10 -c "Watts over time" < /tmp/eco-ci/energy-total.txt >> $output fi fi repo_enc=$( echo ${repo} | jq -Rr @uri) diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index be3d567d..7f1eb87f 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -38,21 +38,21 @@ function make_measurement() { ## make a note that we cannot use --energy, skew the result as we do not have an input delay. # this works because demo-reporter is 1/second if [[ "$MODEL_NAME" == "unknown" ]]; then - $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py --auto --silent < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt elif [[ -n "$VHOST_RATIO" ]]; then - $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ + docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --vhost-ratio $VHOST_RATIO --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --vhost-ratio $VHOST_RATIO --silent < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt else - $(docker run --rm greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ + docker run --rm -i greencoding/cloud-energy:latest-asciicharts python3 xgb.py \ --tdp $TDP --cpu-threads $CPU_THREADS \ --cpu-cores $CPU_CORES --cpu-make $CPU_MAKE \ --release-year $RELEASE_YEAR --ram $RAM \ --cpu-freq $CPU_FREQ --cpu-chips $CPU_CHIPS \ - --silent) < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt + --silent < /tmp/eco-ci/cpu-util-temp.txt | tee -a /tmp/eco-ci/energy-total.txt > /tmp/eco-ci/energy.txt fi if [[ $MEASUREMENT_COUNT == '' ]]; then From 59e94ecfb1509c69e2ea3052d397f7406fa766f9 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 17:05:54 +0200 Subject: [PATCH 08/13] Embodied wrong var --- scripts/display_results.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/display_results.sh b/scripts/display_results.sh index c5c82dba..29ff678a 100755 --- a/scripts/display_results.sh +++ b/scripts/display_results.sh @@ -131,7 +131,7 @@ function display_results { echo '🌳 CO2 Data:' | tee -a $output $output_pr echo "City: $CITY, Lat: $LAT, Lon: $LON" | tee -a $output $output_pr echo "CO₂ from energy is: $CO2EQ_ENERGY" | tee -a $output $output_pr - echo "CO₂ from manufacturing (embodied carbon) is: $CO2EQ_ENERGY" | tee -a $output $output_pr + echo "CO₂ from manufacturing (embodied carbon) is: $CO2EQ_EMBODIED" | tee -a $output $output_pr echo "Carbon Intensity for this location: $CO2I gCO₂eq/kWh" | tee -a $output $output_pr printf "SCI: %.6f gCO₂eq / pipeline run emitted\n" $CO2EQ | tee -a $output $output_pr else From 00fbb3b0a8d2be61a6100cf979417a606f9c3fae Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 17:18:30 +0200 Subject: [PATCH 09/13] Bringing model name back --- scripts/vars.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/vars.sh b/scripts/vars.sh index c156d2f7..65547fcf 100755 --- a/scripts/vars.sh +++ b/scripts/vars.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -#model_name=$(cat /proc/cpuinfo | grep "model name") -model_name="unknown" +model_name=$(cat /proc/cpuinfo | grep "model name") add_var() { key=$1 From 655aa182002b9944c308b0d62a736fe924478796 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 17:27:11 +0200 Subject: [PATCH 10/13] Fixed time --- scripts/make_measurement.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index 7f1eb87f..60f67027 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -23,14 +23,16 @@ function make_measurement() { API_BASE=${API_BASE:-} + + # capture time + time=$(($(date +%s) - $(cat /tmp/eco-ci/timer.txt))) + # reset timer and cpu capturing (lap) source "$(dirname "$0")/setup.sh" lap_measurement - # capture cpu util + # capture cpu util - can only be done after docker container has written cat /tmp/eco-ci/cpu-util.txt > /tmp/eco-ci/cpu-util-temp.txt - # capture time - time=$(($(date +%s) - $(cat /tmp/eco-ci/timer.txt))) # check wc -l of cpu-util is greater than 0 if [[ $(wc -l < /tmp/eco-ci/cpu-util.txt) -gt 0 ]]; then From 2fc972f4b12e83568273bde8844096ba9932fa29 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 20:00:50 +0200 Subject: [PATCH 11/13] Beefed up test workflow --- .github/workflows/test.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11547c88..61679015 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,22 +31,34 @@ jobs: with: task: start-measurement - - name: Wait before the next step + - name: Sleep step run: sleep 2 - name: Test measurement 1 uses: ./ with: task: get-measurement + label: "Sleep 3s" - - name: Wait before the next step - run: sleep 2 + - name: Filesystem + run: ls -alhR / - name: Test measurement 2 uses: ./ with: task: get-measurement - label: "Sleep #2" + label: "ls -alhR /" + + - name: Sleep 3 + run: sleep 3 + + - name: Test measurement 2 + uses: ./ + with: + task: get-measurement + label: "Sleep 3s" + + - name: Eco CI Energy Estimation uses: ./ From a7a0cd8e108ead9ac632f35f4504ec2e48f2570e Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 20:05:52 +0200 Subject: [PATCH 12/13] Less ls --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61679015..a687d388 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,13 +41,13 @@ jobs: label: "Sleep 3s" - name: Filesystem - run: ls -alhR / + run: ls -alhR /usr - name: Test measurement 2 uses: ./ with: task: get-measurement - label: "ls -alhR /" + label: "ls -alhR /usr" - name: Sleep 3 run: sleep 3 From f2b84f2312f326bd65d6422b488746185b24594e Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Tue, 4 Jun 2024 20:50:20 +0200 Subject: [PATCH 13/13] Added dump --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a687d388..ffbea404 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,6 +59,14 @@ jobs: label: "Sleep 3s" + - name: Dump ECO-CI CPU + run: | + cat /tmp/eco-ci/cpu-util-total.txt + + - name: Dump ECO-CI Energy + run: | + cat /tmp/eco-ci//tmp/eco-ci/energy.txt + - name: Eco CI Energy Estimation uses: ./