Merge pull request #137 from sustainable-computing-io/carbon3

carbon3: improve docs & error output and handling
sustainable-computing-io · Sep 29, 2024 · b9c078d · b9c078d
2 parents 6ffb146 + c46f561
commit b9c078d
Show file tree

Hide file tree

Showing 14 changed files with 102 additions and 68 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # SusQL Operator
 
-SusQL is a Kubernetes operator that aggregates energy and estimated carbon dioxide emission data for pods tagged with SusQL specific labels. The energy measurements are taken from [Kepler](https://sustainable-computing.io/) which should be deployed on the cluster before using SusQL. Watch a video with a demonstration by clicking the following link.
+SusQL is a Kubernetes operator that aggregates energy and estimated carbon dioxide emission data for pods tagged with SusQL specific labels. The energy measurements are obtained from [Kepler](https://sustainable-computing.io/) which should be deployed on the cluster before using SusQL. Watch a demonstration video by clicking the following link.
 
 https://github.com/sustainable-computing-io/susql-operator/wiki/files/SusQL-Demo-2024-09.mp4
 
@@ -9,29 +9,32 @@ https://github.com/sustainable-computing-io/susql-operator/wiki/files/SusQL-Demo
 
 ## Getting Started
 
-SusQL is an operator that can be deployed in a Kubernetes/OpenShift cluster. You can use [kind](https://sigs.k8s.io/kind) or [minikube](https://minikube.sigs.k8s.io/) to get a local cluster for testing, or run against a remote cluster.
+SusQL is an operator that can be deployed in a Kubernetes/OpenShift cluster. You can also use [kind](https://sigs.k8s.io/kind) or [minikube](https://minikube.sigs.k8s.io/) as a local cluster for testing, or run against a remote cluster.
 
 ## Carbon Dioxide Emission Calculation
 
 By default SusQL calculates carbon dioxide emission in grams of CO2 using a carbon intensity value from 
 [US EPA](https://www.epa.gov/energy/greenhouse-gases-equivalencies-calculator-calculations-and-references).
+SusQL can be configured to use other static carbon intensity values or query carbon intensity values for a
+given location from web API's such as those provided by
+the Green Software Foundation's [Carbon Aware SDK](https://github.com/Green-Software-Foundation/carbon-aware-sdk).
 
 Detailed information on configuration of CO2 emission calculation in SusQL is available in the [SusQL carbon
 calculation documentation.](doc/carbon.md)
 
-### Prerequisites
+## Prerequisites
 
 Kepler is assumed to be installed in the cluster.
 
-### Installation
+## Installation
 
 - Follow these instructions for easy SusQL installation from the Red Hat Community Operator catalog on an OpenShift cluster.
   - [Installation on OpenShift](doc/openshift-installation.md)
 
 - Follow these instructions to install the SusQL Operator from [OperatorHub.io](https://operatorhub.io) on a Kubernetes cluster including OpenShift.
   - [Installation from OperatorHub.io](doc/operatorhub-installation.md)
 
-- Follow these instructions to install the SusQL Operator with Helm on a Kubernetes cluster including OpenShift.
+- Follow these instructions to install the SusQL Operator from a Helm chart on a Kubernetes cluster, including OpenShift.
   - [Installation with Helm](doc/helm-installation.md)
 
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.0.31
+0.0.32
diff --git a/bundle/manifests/susql-operator.clusterserviceversion.yaml b/bundle/manifests/susql-operator.clusterserviceversion.yaml
@@ -27,15 +27,15 @@ metadata:
       ]
     capabilities: Basic Install
     categories: Monitoring
-    containerImage: quay.io/sustainable_computing_io/susql_operator:0.0.31
-    createdAt: "2024-09-28T12:08:54Z"
+    containerImage: quay.io/sustainable_computing_io/susql_operator:0.0.32
+    createdAt: "2024-09-29T09:27:06Z"
     description: 'Aggregates energy and CO2 emission data for pods tagged with SusQL
       labels '
     operators.operatorframework.io/builder: operator-sdk-v1.36.1
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v4
     repository: https://github.com/sustainable-computing-io/susql-operator
     support: https://github.com/sustainable-computing-io/susql-operator/issues
-  name: susql-operator.v0.0.31
+  name: susql-operator.v0.0.32
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -301,7 +301,7 @@ spec:
                       key: CARBON-QUERY-CONV-2J
                       name: susql-config
                       optional: true
-                image: quay.io/sustainable_computing_io/susql_operator:0.0.31
+                image: quay.io/sustainable_computing_io/susql_operator:0.0.32
                 imagePullPolicy: Always
                 livenessProbe:
                   httpGet:
@@ -408,5 +408,5 @@ spec:
   provider:
     name: SusQL Operator Contributors
     url: https://github.com/sustainable-computing-io/susql-operator
-  replaces: susql-operator.v0.0.29
-  version: 0.0.31
+  replaces: susql-operator.v0.0.30
+  version: 0.0.32
diff --git a/cmd/main.go b/cmd/main.go
@@ -230,22 +230,23 @@ func main() {
 	susqlLog.Info("Setting up labelGroupReconciler.")
 
 	if err = (&controller.LabelGroupReconciler{
-		Client:                     mgr.GetClient(),
-		Scheme:                     mgr.GetScheme(),
-		KeplerPrometheusUrl:        keplerPrometheusUrl,
-		KeplerMetricName:           keplerMetricName,
-		SusQLPrometheusDatabaseUrl: susqlPrometheusDatabaseUrl,
-		SusQLPrometheusMetricsUrl:  susqlPrometheusMetricsUrl,
-		SamplingRate:               time.Duration(samplingRateInteger) * time.Second,
-		CarbonMethod:               carbonMethod,
-		CarbonIntensity:            carbonIntensityFloat,
-		CarbonIntensityUrl:         carbonIntensityUrl,
-		CarbonIntensityTimeStamp:   0,
-		CarbonLocation:             carbonLocation,
-		CarbonQueryRate:            carbonQueryRateInteger,
-		CarbonQueryFilter:          carbonQueryFilter,
-		CarbonQueryConv2J:          carbonQueryConv2JFloat,
-		Logger:                     susqlLog,
+		Client:                        mgr.GetClient(),
+		Scheme:                        mgr.GetScheme(),
+		KeplerPrometheusUrl:           keplerPrometheusUrl,
+		KeplerMetricName:              keplerMetricName,
+		SusQLPrometheusDatabaseUrl:    susqlPrometheusDatabaseUrl,
+		SusQLPrometheusMetricsUrl:     susqlPrometheusMetricsUrl,
+		SamplingRate:                  time.Duration(samplingRateInteger) * time.Second,
+		CarbonMethod:                  carbonMethod,
+		CarbonIntensity:               carbonIntensityFloat,
+		CarbonIntensityUrl:            carbonIntensityUrl,
+		CarbonIntensityTimeStamp:      0,
+		CarbonIntensityErrorTimeStamp: 0,
+		CarbonLocation:                carbonLocation,
+		CarbonQueryRate:               carbonQueryRateInteger,
+		CarbonQueryFilter:             carbonQueryFilter,
+		CarbonQueryConv2J:             carbonQueryConv2JFloat,
+		Logger:                        susqlLog,
 	}).SetupWithManager(mgr); err != nil {
 		susqlLog.Error(err, "unable to create controller", "controller", "LabelGroup")
 		os.Exit(1)

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: quay.io/sustainable_computing_io/susql_operator
-  newTag: 0.0.31
+  newTag: 0.0.32
diff --git a/config/manifests/bases/susql-operator.clusterserviceversion.yaml b/config/manifests/bases/susql-operator.clusterserviceversion.yaml
@@ -116,5 +116,5 @@ spec:
   provider:
     name: SusQL Operator Contributors
     url: https://github.com/sustainable-computing-io/susql-operator
-  replaces: susql-operator.v0.0.29
+  replaces: susql-operator.v0.0.30
   version: 0.0.0
diff --git a/doc/c3-susql-config.yaml b/doc/c3-susql-config.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: susql-config
+data:
+  KEPLER-PROMETHEUS-URL: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9091"
+  KEPLER-METRIC-NAME: "kepler_container_joules_total"
+  SUSQL-PROMETHEUS-DATABASE-URL: "https://thanos-querier.openshift-monitoring.svc.cluster.local:9091"
+  SUSQL-PROMETHEUS-METRICS-URL: "http://0.0.0.0:8082"
+  SAMPLING-RATE: "2"
+  LEADER-ELECT: "false"
+  HEALTH-PROBE-BIND-ADDRESS: ":8081"
+  SUSQL-LOG-LEVEL: "-5"
+  CARBON-METHOD: "casdk"
+  CARBON-INTENSITY: "0.0001158333333333"
+  CARBON-INTENSITY-URL: "https://api.electricitymap.org/v3/carbon-intensity/latest?zone=%s"
+  CARBON-INTENSITY-URL: "http://webapi-green-software-foundation.apps.multi-nic-v2.llmdev.res.ibm.com/emissions/bylocation?location=%s"
+  CARBON-LOCATION: "japanwestERROR"
+  CARBON-QUERY-RATE: "7200"
+  CARBON-QUERY-FILTER: "rating"
+  CARBON-QUERY-CONV-2J: "0.0000002777777778"
diff --git a/doc/carbon.md b/doc/carbon.md
@@ -4,11 +4,14 @@ There are three primary CO2 emission calculation methods.
 
 "Out-of-the-box" SusQL reports an estimated CO2 emission value for all measured workloads using the `static` method:
 
+The behavior of SusQL carbon calculation can be tuned by modyfing the `susql-config` `ConfigMap` in the same namespace that the SusQL operator is running in.
+A sample file is provided in `samples/susql-config.yaml`.
+
 ## `static` Method
 - This `static` method uses a static "carbon intensity value" as a coefficient to calculate grams of CO2 emitted.
   This calculation method is used when the `CARBON-METHOD` `ConfigMap` value is set to `static`.
 
-#### `static` Method `ConfigMap` Configurable items
+#### `static` Method `ConfigMap` Configurable Items
   - `CARBON-METHOD` - The `static` method is enabled when this is set to `static`.
   - `CARBON-INTENSITY` - Carbon intensity value. A coefficient used to convert Joules to grams of CO2 per Joule. The unit definition is grams of CO2 per Joule.
     The default carbon intensity value is based on [US EPA](https://www.epa.gov/energy/greenhouse-gases-equivalencies-calculator-calculations-and-references) data.
@@ -34,7 +37,7 @@ There are three primary CO2 emission calculation methods.
 #### Configuring and installing Carbon Aware SDK
 - Following guidance in https://github.com/Green-Software-Foundation/carbon-aware-sdk/blob/dev/casdk-docs/docs/overview/enablement.md,
 the Carbon Aware SDK can be easily installed on a Kubernetes cluster such as OpenShift.
-- Preparation: clone the repository and edit `helm-chart/values.yaml` as needed to reflect private password, configuration, etc.
+- Preparation: clone the Carbon Aware SDK repository and edit `helm-chart/values.yaml` as needed to reflect private password, configuration, etc.
 (Useful configuration tips available at https://github.com/Green-Software-Foundation/carbon-aware-sdk/blob/dev/casdk-docs/docs/tutorial-extras/configuration.md )
 
 ```
@@ -43,8 +46,8 @@ vi helm-chart/values.yaml
 ```
 - Preparation: required software and permission
   - Ensure that `helm`, and `kubectl` (or `oc`) are installed
-  - Ensure that CLI user is logged in to cluster with sufficient permissions
-- Perform installation
+  - Ensure that CLI user is logged in to the cluster with sufficient permissions
+- Perform installation: (The example installes into namespace `gsf`. However, other namespaces may be used.)
 ```
 cd carbon-aware-sdk
 helm upgrade --install --wait carbon-aware-sdk helm-chart --create-namespace gsf
@@ -56,19 +59,19 @@ Note the value reported for "HOST/PORT". This will be used in the next configura
 Update the following items in the `susql-config.yaml` file:
 ```
   CARBON-METHOD: "casdk"
-  CARBON-INTENSITY-URL: "http://<HOST/PORT-VALUE>/emissions/bylocation?location=%s"
+  CARBON-INTENSITY-URL: "http://<HOST/PORT>/emissions/bylocation?location=%s"
   CARBON-LOCATION: "<WORKLOAD-LOCATION>"
   CARBON-QUERY-FILTER: "rating"
 ```
-Tip: try this command to verify sdk container functionality and also view available locations: `curl -s "http:<HOST/PORT-VALUE>/locations"`
+Tip: try this command to verify sdk container functionality and also view available locations: `curl -s "http:<HOST/PORT>/locations"`
 
 
 Apply the updated `susql-config.yaml` file:
 ```
 oc apply -f susql-config.yaml -n <SUSQL-OPERATOR-NAMESPACE>
 ```
 You are now ready to install and use the SusQL operator.
-If the SusQL Operator is alreay installed, then restart the control pod.
+If the SusQL Operator is already installed, then restart the control pod.
 
 #### `casdk` Method `ConfigMap` Configurable Items
   - `CARBON-METHOD` - The `casdk` method is enabled when this is set to `casdk`.

diff --git a/doc/helm-installation.md b/doc/helm-installation.md
@@ -65,7 +65,7 @@ The following environment variables will influence the way that the SusQL Operat
 | SUSQL_REGISTRY              | quay.io/sustainable_computing_io | Container registry that SusQL is stored in    |
 | SUSQL_IMAGE_NAME            | susql_operator                | Image name used on SusQL container registry      |
 | SUSQL_IMAGE_TAG             | latest                        | Tag for SusQL container                          |
-| CARBON_METHOD               | static                        | "static", "simpledynamic", "scadk"               |
+| CARBON_METHOD               | static                        | "static", "simpledynamic", "casdk"               |
 | CARBON_INTENSITY            | "0.00011583333"               | Carbon intensity in grams CO2 / Joule            |
 | CARBON_INTENSITY_URL        |                               | Web API to query carbon intensity                |
 | CARBON_LOCATION             |                               | Location for carbon intensity query              |

diff --git a/doc/openshift-installation.md b/doc/openshift-installation.md
@@ -24,7 +24,7 @@ Use the OpenShift web console to install the SusQL Operator:
 
 Before deploying the SusQL Operator create a `ConfigMap` called `susql-config` in
 the same namespace that the operator will run in.
-[susql-config.yaml](susql-config.yaml) is a good starting point. If you download it first, you
+[susql-config.yaml](../samples/susql-config.yaml) is a good starting point. If you download it first, you
 could create the ConfigMap with `oc apply -n <NAMESPACE> -f susql-config.yaml`.
 If you update (or create) the ConfigMap after the SusQL Operator has been installed, then restarting the SusQL Operator controller pod will
 enable the changes. (e.g., Delete the pod, and allow it to be recreated automatically.)

diff --git a/doc/operatorhub-installation.md b/doc/operatorhub-installation.md
@@ -49,7 +49,7 @@ Next, use the OpenShift web console to install the SusQL Operator:
 
 Before deploying the SusQL Operator create a `ConfigMap` called `susql-config` in
 the same namespace that the operator will run in.
-[susql-config.yaml](susql-config.yaml) is a good starting point. If you download it first, you
+[susql-config.yaml](../samples/susql-config.yaml) is a good starting point. If you download it first, you
 could create the ConfigMap with `oc apply -n <NAMESPACE> -f susql-config.yaml`.
 If you update (or create) the ConfigMap after the SusQL Operator has been installed, then restarting the SusQL Operator controller pod will
 enable the changes. (e.g., Delete the pod, and allow it to be recreated automatically.)

diff --git a/internal/controller/carbon_query.go b/internal/controller/carbon_query.go
@@ -25,17 +25,16 @@ import (
 )
 
 func queryCarbonIntensity(url string, location string, filter string, conv2J float64) (float64, error) {
+	queryUrl := fmt.Sprintf(url, location)
 
-	fmt.Println("CARBON QUERY: url=" + fmt.Sprintf(url, location))
-
-	response, err := http.Get(fmt.Sprintf(url, location))
+	response, err := http.Get(queryUrl)
 	if err != nil {
-		return 0.0, err
+		return 0.0, fmt.Errorf("queryCarbonIntensity: %w\nURL=%s", err, queryUrl)
 	}
 
 	responseData, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		return 0.0, err
+		return 0.0, fmt.Errorf("queryCarbonIntensity: %w\nURL=%s\nresponse=%s", err, queryUrl, string(responseData))
 	}
 
 	length := gjson.Get(string(responseData), "#").Int() - 1
@@ -46,30 +45,31 @@ func queryCarbonIntensity(url string, location string, filter string, conv2J flo
 
 	carbonIntensityFloat, err := strconv.ParseFloat(carbonIntensityString, 64)
 	if err != nil {
-		return 0.0, err
+		return 0.0, fmt.Errorf("queryCarbonIntensity: %w\nURL=%s\nresponse=%s\nfilter=%s\nresult=%s", err, queryUrl, string(responseData), newFilter, carbonIntensityString)
 	}
 
 	// return nil error since no error
 	return carbonIntensityFloat * conv2J, nil
 }
 
 func querySimpleCarbonIntensity(url string, location string, filter string, conv2J float64) (float64, error) {
+	queryUrl := fmt.Sprintf(url, location)
 
 	response, err := http.Get(fmt.Sprintf(url, location))
 	if err != nil {
-		return 0.0, err
+		return 0.0, fmt.Errorf("querySimpleCarbonIntensity: %w\nURL=%s", err, queryUrl)
 	}
 
 	responseData, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		return 0.0, err
+		return 0.0, fmt.Errorf("queryCarbonSimpleIntensity: %w\nURL=%s\nresponse=%s", err, queryUrl, string(responseData))
 	}
 
 	carbonIntensityString := gjson.Get(string(responseData), filter).String()
 
 	carbonIntensityFloat, err := strconv.ParseFloat(carbonIntensityString, 64)
 	if err != nil {
-		return 0.0, err
+		return 0.0, fmt.Errorf("queryCarbonSimpleIntensity: %w\nURL=%s\nresponse=%s\nfilter=%s\nresult=%s", err, queryUrl, string(responseData), filter, carbonIntensityString)
 	}
 
 	// return nil error since no error

diff --git a/internal/controller/labelgroup_controller.go b/internal/controller/labelgroup_controller.go
@@ -36,21 +36,22 @@ import (
 // LabelGroupReconciler reconciles a LabelGroup object
 type LabelGroupReconciler struct {
 	client.Client
-	Scheme                     *runtime.Scheme
-	KeplerPrometheusUrl        string
-	KeplerMetricName           string
-	SusQLPrometheusDatabaseUrl string
-	SusQLPrometheusMetricsUrl  string
-	SamplingRate               time.Duration // Sampling rate for all LabelGroups
-	CarbonMethod               string
-	CarbonIntensity            float64
-	CarbonIntensityUrl         string
-	CarbonIntensityTimeStamp   int64
-	CarbonLocation             string
-	CarbonQueryRate            int64
-	CarbonQueryFilter          string
-	CarbonQueryConv2J          float64
-	Logger                     logr.Logger
+	Scheme                        *runtime.Scheme
+	KeplerPrometheusUrl           string
+	KeplerMetricName              string
+	SusQLPrometheusDatabaseUrl    string
+	SusQLPrometheusMetricsUrl     string
+	SamplingRate                  time.Duration // Sampling rate for all LabelGroups
+	CarbonMethod                  string
+	CarbonIntensity               float64
+	CarbonIntensityUrl            string
+	CarbonIntensityTimeStamp      int64
+	CarbonIntensityErrorTimeStamp int64
+	CarbonLocation                string
+	CarbonQueryRate               int64
+	CarbonQueryFilter             string
+	CarbonQueryConv2J             float64
+	Logger                        logr.Logger
 }
 
 const (
@@ -59,6 +60,7 @@ const (
 	fixingDelay           = 15 * time.Second                   // Time to wait in the event the LabelGroup was badly constructed
 	nopodDelay            = 15 * time.Second                   // Time to wait in the event no pods are found
 	errorDelay            = 1 * time.Second                    // Time to wait when an error happens due to network connectivity issues
+	carbonRetryDelay      = 300                                // Number of seconds to wait for retry after carbon query failure
 )
 
 var (
@@ -115,27 +117,31 @@ func (r *LabelGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	// TODO: put this code only in Reloading and Aggregating cases
 	if r.CarbonMethod == "simpledynamic" {
 		currentEpoch := time.Now().Unix()
-		if (currentEpoch - r.CarbonIntensityTimeStamp) > r.CarbonQueryRate {
+		if (currentEpoch-r.CarbonIntensityTimeStamp) > r.CarbonQueryRate && (currentEpoch-r.CarbonIntensityErrorTimeStamp) > carbonRetryDelay {
 			newCarbonIntensity, err := querySimpleCarbonIntensity(r.CarbonIntensityUrl, r.CarbonLocation, r.CarbonQueryFilter, r.CarbonQueryConv2J)
 			if err == nil {
 				r.CarbonIntensity = newCarbonIntensity
 				r.CarbonIntensityTimeStamp = currentEpoch
+				r.CarbonIntensityErrorTimeStamp = 0
 				r.Logger.V(5).Info(fmt.Sprintf("[Reconcile-simpledynamic] Obtained dynamic carbon intensity of %.10f.", newCarbonIntensity))
 			} else {
+				r.CarbonIntensityErrorTimeStamp = currentEpoch
 				r.Logger.V(0).Error(err, "[Reconcile-simpledynamic] Unable to query carbon intensity.")
 			}
 		}
 	}
 	if r.CarbonMethod == "casdk" {
 		currentEpoch := time.Now().Unix()
-		if (currentEpoch - r.CarbonIntensityTimeStamp) > r.CarbonQueryRate {
+		if (currentEpoch-r.CarbonIntensityTimeStamp) > r.CarbonQueryRate && (currentEpoch-r.CarbonIntensityErrorTimeStamp) > carbonRetryDelay {
 			newCarbonIntensity, err := queryCarbonIntensity(r.CarbonIntensityUrl, r.CarbonLocation, r.CarbonQueryFilter, r.CarbonQueryConv2J)
 			if err == nil {
 				r.CarbonIntensity = newCarbonIntensity
 				r.CarbonIntensityTimeStamp = currentEpoch
-				r.Logger.V(5).Info(fmt.Sprintf("[Reconcile-simpledynamic] Obtained dynamic carbon intensity of %.10f.", newCarbonIntensity))
+				r.CarbonIntensityErrorTimeStamp = 0
+				r.Logger.V(5).Info(fmt.Sprintf("[Reconcile-casdk] Obtained dynamic carbon intensity of %.10f.", newCarbonIntensity))
 			} else {
-				r.Logger.V(0).Error(err, "[Reconcile-simpledynamic] Unable to query carbon intensity.")
+				r.CarbonIntensityErrorTimeStamp = currentEpoch
+				r.Logger.V(0).Error(err, "[Reconcile-casdk] Unable to query carbon intensity.")
 			}
 		}
 	}

diff --git a/doc/susql-config.yaml → samples/susql-config.yaml b/doc/susql-config.yaml → samples/susql-config.yaml