Merge pull request #136 from sustainable-computing-io/carbon3

Green Software Foundation Carbon Aware SDK Integration

README.md

@@ -16,8 +16,8 @@ SusQL is an operator that can be deployed in a Kubernetes/OpenShift cluster. You
 By default SusQL calculates carbon dioxide emission in grams of CO2 using a carbon intensity value from 
 [US EPA](https://www.epa.gov/energy/greenhouse-gases-equivalencies-calculator-calculations-and-references).
 
-Detailed information on configuration of CO2 emission calculation in SusQL is available in the SusQL carbon
-calculation [documentation](doc/carbon.md).
+Detailed information on configuration of CO2 emission calculation in SusQL is available in the [SusQL carbon
+calculation documentation.](doc/carbon.md)
 
 ### Prerequisites
 

VERSION

@@ -1 +1 @@
-0.0.30
+0.0.31

bundle/manifests/susql-operator.clusterserviceversion.yaml

@@ -27,15 +27,15 @@ metadata:
       ]
     capabilities: Basic Install
     categories: Monitoring
-    containerImage: quay.io/sustainable_computing_io/susql_operator:0.0.30
-    createdAt: "2024-09-12T07:28:28Z"
+    containerImage: quay.io/sustainable_computing_io/susql_operator:0.0.31
+    createdAt: "2024-09-28T12:08:54Z"
     description: 'Aggregates energy and CO2 emission data for pods tagged with SusQL
       labels '
     operators.operatorframework.io/builder: operator-sdk-v1.36.1
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
     repository: https://github.com/sustainable-computing-io/susql-operator
     support: https://github.com/sustainable-computing-io/susql-operator/issues
-  name: susql-operator.v0.0.30
+  name: susql-operator.v0.0.31
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -301,7 +301,7 @@ spec:
                       key: CARBON-QUERY-CONV-2J
                       name: susql-config
                       optional: true
-                image: quay.io/sustainable_computing_io/susql_operator:0.0.30
+                image: quay.io/sustainable_computing_io/susql_operator:0.0.31
                 imagePullPolicy: Always
                 livenessProbe:
                   httpGet:
@@ -409,4 +409,4 @@ spec:
     name: SusQL Operator Contributors
     url: https://github.com/sustainable-computing-io/susql-operator
   replaces: susql-operator.v0.0.29
-  version: 0.0.30
+  version: 0.0.31

cmd/main.go

@@ -73,7 +73,7 @@ func main() {
 	var samplingRate string = "2"
 	var susqlLogLevel string = "-5"
 	// Carbon Intensity Factor in grams CO2 / Joule
-	var carbonMethod string = "static" // options: static, simpledynamic, sdk
+	var carbonMethod string = "static" // options: static, simpledynamic, casdk
 	var carbonIntensity string = "0.0001158333333333"
 	var carbonIntensityUrl string = "https://api.electricitymap.org/v3/carbon-intensity/latest?zone=%s"
 	var carbonLocation string = "JP-TK"
@@ -202,7 +202,7 @@ func main() {
 	}
 
 	// TODO: verify that carbonMethod is an expected value. If not log warning and set to default value.
-	// (static, simpledynamic, sdk)
+	// (static, simpledynamic, casdk)
 	// Note: assume that carbonIntensityUrl, carbonLocation, and carbonQueryFilter are OK. If not, we will log errors at runtime.
 
 	samplingRateInteger, err := strconv.Atoi(samplingRate)

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: quay.io/sustainable_computing_io/susql_operator
-  newTag: 0.0.30
+  newTag: 0.0.31

doc/carbon.md

@@ -1,30 +1,83 @@
 # Carbon Dioxide Emission Estimation
 
-There are two currently CO2 emission calculation methods available, and an additional method is under development.
+There are three primary CO2 emission calculation methods.
 
-"Out-of-the-box" SusQL reports an estimated CO2 emission value for all measured workloads using the `static` method described below.
+"Out-of-the-box" SusQL reports an estimated CO2 emission value for all measured workloads using the `static` method:
 
-### `static` Method
-- The default CO2 emission calculation method is used when the `CARBON-METHOD` ConfigMap value is set to `static`
-which simply uses a grams of CO2 per Joule 
-of electricity consumed coefficient to calculate grams of CO2 emitted. This value is user tunable by
-modifying the `CARBON-INTENSITY` ConfigMap value.  The default value is based on
-[US EPA](https://www.epa.gov/energy/greenhouse-gases-equivalencies-calculator-calculations-and-references)
-data.
+## `static` Method
+- This `static` method uses a static "carbon intensity value" as a coefficient to calculate grams of CO2 emitted.
+  This calculation method is used when the `CARBON-METHOD` `ConfigMap` value is set to `static`.
 
-### `simpledynamic` Method
+#### `static` Method `ConfigMap` Configurable items
+  - `CARBON-METHOD` - The `static` method is enabled when this is set to `static`.
+  - `CARBON-INTENSITY` - Carbon intensity value. A coefficient used to convert Joules to grams of CO2 per Joule. The unit definition is grams of CO2 per Joule.
+    The default carbon intensity value is based on [US EPA](https://www.epa.gov/energy/greenhouse-gases-equivalencies-calculator-calculations-and-references) data.
+
+## `simpledynamic` Method
 - The `simpledynamic` method periodically queries the carbon intensity value for a user specified location to provide a more accurate estimation of CO2 emission.
-The ConfigMap user configurable items are:
-  - `CARBON-METHOD` - The `simpledynamic` method is enabled when this set to `simpledynamic`.
-  - `CARBON-INTENSITY` - This value is set automatically with `simpledynamic`. User specified values will be overwritten.
-  - `CARBON-INTENSITY-URL` - Specifies a web API that returns carbon intensity. The default value works as of the date of this writing.
-  - `CARBON-LOCATION` - The location identifiers are defined by the API provider, but can be selected by he user.
+  This calculation method is used when the `CARBON-METHOD` `ConfigMap` value is set to `simpledynamic`.
+
+#### `simpledynamic` Method `ConfigMap` Configurable Items
+  - `CARBON-METHOD` - The `simpledynamic` method is enabled when this is set to `simpledynamic`.
+  - `CARBON-INTENSITY` - This value is set automatically with the `simpledynamic` method. User specified values will be overwritten. The unit definition is grams of CO2 per Joule.
+  - `CARBON-INTENSITY-URL` - Specifies a web API that returns carbon intensity. (The default value works as of the date of this writing, but may need to be modified in the future.)
+  - `CARBON-LOCATION` - The location identifiers are defined by the API provider, but can be selected by the user. (The default value works as of the date of this writing.)
   - `CARBON-QUERY-RATE` - Interval in seconds at which the carbon intensity is queried. The data available from the source is updated less than hourly, so an interval of greater than 3600 seconds is recommended.
   - `CARBON-QUERY-FILTER` - When the return value is embedded in a JSON object, this specification enables the extraction of the data. The default value matches the default provider.
   - `CARBON-QUERY-CONV-2J` - If the carbon data provider does not provide data in the standard "grams of CO2 per Joule" then this factor can be specified to normalize the units displayed. The default value ensures that the default provider data is in the correct unit.
 
-### `sdk` Method
-- The third method `sdk` is still under development, and will offer an integration with the Green Software Foundation's [Carbon Aware SDK](https://github.com/Green-Software-Foundation/carbon-aware-sdk).
+## `casdk` Method
+- The `casdk` offers integration with the Green Software Foundation's [Carbon Aware SDK](https://github.com/Green-Software-Foundation/carbon-aware-sdk).
+  This calculation method is used when the `CARBON-METHOD` `ConfigMap` value is set to `casdk`.
+  The user is required to first prepare a local instance of the Carbon Aware SDK that is configured to support carbon intensity queries.
+
+#### Configuring and installing Carbon Aware SDK
+- Following guidance in https://github.com/Green-Software-Foundation/carbon-aware-sdk/blob/dev/casdk-docs/docs/overview/enablement.md,
+the Carbon Aware SDK can be easily installed on a Kubernetes cluster such as OpenShift.
+- Preparation: clone the repository and edit `helm-chart/values.yaml` as needed to reflect private password, configuration, etc.
+(Useful configuration tips available at https://github.com/Green-Software-Foundation/carbon-aware-sdk/blob/dev/casdk-docs/docs/tutorial-extras/configuration.md )
+
+```
+git clone [email protected]:Green-Software-Foundation/carbon-aware-sdk.git
+vi helm-chart/values.yaml
+```
+- Preparation: required software and permission
+  - Ensure that `helm`, and `kubectl` (or `oc`) are installed
+  - Ensure that CLI user is logged in to cluster with sufficient permissions
+- Perform installation
+```
+cd carbon-aware-sdk
+helm upgrade --install --wait carbon-aware-sdk helm-chart --create-namespace gsf
+oc expose svc/webapi -n gsf
+oc get routes
+```
+Note the value reported for "HOST/PORT". This will be used in the next configuration step.
+- Update `susql-config.yaml`:
+Update the following items in the `susql-config.yaml` file:
+```
+  CARBON-METHOD: "casdk"
+  CARBON-INTENSITY-URL: "http://<HOST/PORT-VALUE>/emissions/bylocation?location=%s"
+  CARBON-LOCATION: "<WORKLOAD-LOCATION>"
+  CARBON-QUERY-FILTER: "rating"
+```
+Tip: try this command to verify sdk container functionality and also view available locations: `curl -s "http:<HOST/PORT-VALUE>/locations"`
+
+
+Apply the updated `susql-config.yaml` file:
+```
+oc apply -f susql-config.yaml -n <SUSQL-OPERATOR-NAMESPACE>
+```
+You are now ready to install and use the SusQL operator.
+If the SusQL Operator is alreay installed, then restart the control pod.
+
+#### `casdk` Method `ConfigMap` Configurable Items
+  - `CARBON-METHOD` - The `casdk` method is enabled when this is set to `casdk`.
+  - `CARBON-INTENSITY` - This value is set automatically with the `casdk` method. User specified values will be overwritten. The unit definition is grams of CO2 per Joule.
+  - `CARBON-INTENSITY-URL` - Specifies the web API that returns carbon intensity. This is assumed to be a GSF Carbon Aware SDK prepared and configured by the user.
+  - `CARBON-LOCATION` - The location identifiers are defined by the API provider, and should be specified by the user.
+  - `CARBON-QUERY-RATE` - Interval in seconds at which the carbon intensity is queried.
+  - `CARBON-QUERY-FILTER` - When the return value is embedded in a JSON object, this specification enables the extraction of the data.
+  - `CARBON-QUERY-CONV-2J` - The default values converts "grams of CO2 per KWH" (Carbon Aware SDK standard) to "grams of CO2 per Joule".
 
 ## License
 

doc/helm-installation.md

@@ -65,7 +65,7 @@ The following environment variables will influence the way that the SusQL Operat
 | SUSQL_REGISTRY              | quay.io/sustainable_computing_io | Container registry that SusQL is stored in    |
 | SUSQL_IMAGE_NAME            | susql_operator                | Image name used on SusQL container registry      |
 | SUSQL_IMAGE_TAG             | latest                        | Tag for SusQL container                          |
-| CARBON_METHOD               | static                        | "static", "simpledynamic", "sdk"                 |
+| CARBON_METHOD               | static                        | "static", "simpledynamic", "scadk"               |
 | CARBON_INTENSITY            | "0.00011583333"               | Carbon intensity in grams CO2 / Joule            |
 | CARBON_INTENSITY_URL        |                               | Web API to query carbon intensity                |
 | CARBON_LOCATION             |                               | Location for carbon intensity query              |

internal/controller/simple_dynamic_query.go → internal/controller/carbon_query.go

@@ -26,6 +26,35 @@ import (
 
 func queryCarbonIntensity(url string, location string, filter string, conv2J float64) (float64, error) {
 
+	fmt.Println("CARBON QUERY: url=" + fmt.Sprintf(url, location))
+
+	response, err := http.Get(fmt.Sprintf(url, location))
+	if err != nil {
+		return 0.0, err
+	}
+
+	responseData, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return 0.0, err
+	}
+
+	length := gjson.Get(string(responseData), "#").Int() - 1
+	index := strconv.Itoa(int(length))
+
+	newFilter := index + "." + filter
+	carbonIntensityString := gjson.Get(string(responseData), newFilter).String()
+
+	carbonIntensityFloat, err := strconv.ParseFloat(carbonIntensityString, 64)
+	if err != nil {
+		return 0.0, err
+	}
+
+	// return nil error since no error
+	return carbonIntensityFloat * conv2J, nil
+}
+
+func querySimpleCarbonIntensity(url string, location string, filter string, conv2J float64) (float64, error) {
+
 	response, err := http.Get(fmt.Sprintf(url, location))
 	if err != nil {
 		return 0.0, err

internal/controller/labelgroup_controller.go

@@ -114,6 +114,19 @@ func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	// Is it time to update the Carbon Intensity value?
 	// TODO: put this code only in Reloading and Aggregating cases
 	if r.CarbonMethod == "simpledynamic" {
+		currentEpoch := time.Now().Unix()
+		if (currentEpoch - r.CarbonIntensityTimeStamp) > r.CarbonQueryRate {
+			newCarbonIntensity, err := querySimpleCarbonIntensity(r.CarbonIntensityUrl, r.CarbonLocation, r.CarbonQueryFilter, r.CarbonQueryConv2J)
+			if err == nil {
+				r.CarbonIntensity = newCarbonIntensity
+				r.CarbonIntensityTimeStamp = currentEpoch
+				r.Logger.V(5).Info(fmt.Sprintf("[Reconcile-simpledynamic] Obtained dynamic carbon intensity of %.10f.", newCarbonIntensity))
+			} else {
+				r.Logger.V(0).Error(err, "[Reconcile-simpledynamic] Unable to query carbon intensity.")
+			}
+		}
+	}
+	if r.CarbonMethod == "casdk" {
 		currentEpoch := time.Now().Unix()
 		if (currentEpoch - r.CarbonIntensityTimeStamp) > r.CarbonQueryRate {
 			newCarbonIntensity, err := queryCarbonIntensity(r.CarbonIntensityUrl, r.CarbonLocation, r.CarbonQueryFilter, r.CarbonQueryConv2J)