From d4510941e0addd4ec6e803bbe5a097a603bc5aca Mon Sep 17 00:00:00 2001 From: Flavia Paganelli Date: Mon, 3 Jun 2024 17:24:57 +0200 Subject: [PATCH 1/8] Fix/improvement for issue #81 https://github.com/thegreenwebfoundation/grid-intensity-go/issues/81 - Now sending always the timestamp in the Prometheus metrics. - For ElectricityMaps, we now use the historic endpoint instead of the carbon-intensity one, so we can get both the latest estimated and real values. - Tested that it works with Ember and CarbonIntensityOrgUK, Watttime not yet (I need to register) - The label is_estimated has been added in the Prometheus metrics. For ElectricityMaps it can provide values for both, or for one of the two, depending on the location. - For the other providers it always returns the value as is_estimated=false (the default) because I didn't make any changes. I'm. not 100% sure this is correct for all of them, or if some of the providers also return estimations (to investigate). - The error handling when parsing generates a lot of clutter, not sure if there's a better way to do it in go... --- cmd/exporter.go | 28 +++--- go.mod | 5 +- go.sum | 15 ++++ pkg/provider/electricity_maps.go | 149 +++++++++++++++++++++++++++---- pkg/provider/provider.go | 1 + 5 files changed, 167 insertions(+), 31 deletions(-) diff --git a/cmd/exporter.go b/cmd/exporter.go index 49c011b..fb38dc8 100644 --- a/cmd/exporter.go +++ b/cmd/exporter.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net/http" + "strconv" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -15,14 +16,15 @@ import ( ) const ( - labelLocation = "location" - labelNode = "node" - labelProvider = "provider" - labelRegion = "region" - labelUnits = "units" - namespace = "grid_intensity" - nodeKey = "node" - regionKey = "region" + labelLocation = "location" + labelNode = "node" + labelProvider = "provider" + labelRegion = "region" + labelUnits = "units" + labelIsEstimated = "is_estimated" + namespace = "grid_intensity" + nodeKey = "node" + regionKey = "region" ) func init() { @@ -51,6 +53,7 @@ var ( labelProvider, labelRegion, labelUnits, + labelIsEstimated, }, nil, ) @@ -64,6 +67,7 @@ var ( labelProvider, labelRegion, labelUnits, + labelIsEstimated, }, nil, ) @@ -162,7 +166,10 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { continue } - ch <- prometheus.MustNewConstMetric( + // log.Printf("desc is %s", desc) + + ch <- prometheus.NewMetricWithTimestamp(data.ValidFrom, prometheus.MustNewConstMetric( + // ch <- prometheus.MustNewConstMetric( desc, prometheus.GaugeValue, data.Value, @@ -171,7 +178,8 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { data.Provider, e.region, data.Units, - ) + strconv.FormatBool(data.IsEstimated), + )) } } diff --git a/go.mod b/go.mod index 9a31d6c..8802523 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,13 @@ require ( github.com/Xuanwo/go-locale v1.1.0 github.com/cenkalti/backoff/v4 v4.2.1 github.com/gofrs/flock v0.8.1 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/jellydator/ttlcache/v2 v2.11.1 github.com/prometheus/client_golang v1.15.1 github.com/rodaine/table v1.1.0 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 + golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 ) require ( @@ -34,7 +35,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index a386ae3..ac94f66 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,7 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -114,6 +115,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -144,10 +147,13 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jellydator/ttlcache/v2 v2.11.1 h1:AZGME43Eh2Vv3giG6GeqeLeFXxwxn1/qHItqWZl6U64= github.com/jellydator/ttlcache/v2 v2.11.1/go.mod h1:RtE5Snf0/57e+2cLWFYWCCsLas2Hy3c5Z4n14XmSvTI= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -163,6 +169,9 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -242,6 +251,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck= +golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -321,6 +332,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -425,6 +438,7 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -528,6 +542,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/provider/electricity_maps.go b/pkg/provider/electricity_maps.go index c04382a..62d9020 100644 --- a/pkg/provider/electricity_maps.go +++ b/pkg/provider/electricity_maps.go @@ -41,7 +41,7 @@ func NewElectricityMaps(config ElectricityMapsConfig) (Interface, error) { } func (e *ElectricityMapsClient) GetCarbonIntensity(ctx context.Context, location string) ([]CarbonIntensity, error) { - intensityURL, err := e.intensityURLWithZone(location) + intensityURL, err := e.historicIntensityURLWithZone(location) if err != nil { return nil, err } @@ -64,34 +64,144 @@ func (e *ElectricityMapsClient) GetCarbonIntensity(ctx context.Context, location return nil, errBadStatus(resp) } - data := &electricityMapsData{} - err = json.NewDecoder(resp.Body).Decode(data) + type EMHistoryResponse struct { + Zone string + History []electricityMapsData + } + + historyResponse := &EMHistoryResponse{} + err = json.NewDecoder(resp.Body).Decode(&historyResponse) if err != nil { return nil, err } - validFrom, err := time.Parse(time.RFC3339Nano, data.DateTime) + var carbonIntensityPoints []CarbonIntensity + var recentDatapoints = NewMostRecentDataPoints() + + for _, dataPoint := range historyResponse.History { + + // We get the most recent value (which is usually estimated) + // and the most recent value which is registered (not estimated) + // and they wil end up in the recentDatapoints variable + recentDatapoints.update(dataPoint) + + } + // We need to check if each value exists, because sometimes there are no estimated datapoints + // and sometimes there are only estimated datapoints (depending on the location): + if recentDatapoints.estimatedFound { + estimatedCarbonIntensity, err := toCarbonIntensity(location, recentDatapoints.estimated) + if err == nil { + carbonIntensityPoints = append(carbonIntensityPoints, *estimatedCarbonIntensity) + } + } + if recentDatapoints.realFound { + realCarbonIntensity, err := toCarbonIntensity(location, recentDatapoints.real) + if err == nil { + carbonIntensityPoints = append(carbonIntensityPoints, *realCarbonIntensity) + } + } + + return carbonIntensityPoints, nil +} + +// Helper struct to remove clutter in the calling function +// while finding the latest (and greatest) data points +type mostRecentDatapoints struct { + estimated electricityMapsData + real electricityMapsData + estimatedFound bool + realFound bool +} + +func NewMostRecentDataPoints() mostRecentDatapoints { + dataPoints := mostRecentDatapoints{} + dataPoints.estimatedFound = false + dataPoints.realFound = false + return dataPoints +} + +func (m *mostRecentDatapoints) setEstimated(dataPoint electricityMapsData) { + m.estimated = dataPoint + m.estimatedFound = true +} + +func (m *mostRecentDatapoints) setReal(dataPoint electricityMapsData) { + m.real = dataPoint + m.realFound = true +} + +func (m *mostRecentDatapoints) update(dataPoint electricityMapsData) error { + + dataPointDateTime, err := stringToTime(dataPoint.DateTime) + if err != nil { + return err + } + + // If the current datapoint is estimated, + // update if it's the most recent: + if dataPoint.IsEstimated { + if !m.estimatedFound { + m.setEstimated(dataPoint) + } else { + estimatedDateTime, err := stringToTime(m.estimated.DateTime) + if err != nil { + return err + } + if estimatedDateTime.Before(dataPointDateTime) { + m.setEstimated(dataPoint) + } + } + } + + if !dataPoint.IsEstimated { + if !m.realFound { + m.setReal(dataPoint) + } else { + realDateTime, err := stringToTime(m.real.DateTime) + if err != nil { + return err + } + if realDateTime.Before(dataPointDateTime) { + m.setReal(dataPoint) + } + } + } + + return nil +} + +func stringToTime(dateTimeString string) (time.Time, error) { + return time.Parse(time.RFC3339Nano, dateTimeString) +} + +func toCarbonIntensity(location string, dataPoint electricityMapsData) (*CarbonIntensity, error) { + validFrom, err := time.Parse(time.RFC3339Nano, dataPoint.DateTime) if err != nil { + log.Printf("Error parsing datetime %s", dataPoint.DateTime) return nil, err } validTo := validFrom.Add(60 * time.Minute) - - return []CarbonIntensity{ - { - EmissionsType: AverageEmissionsType, - MetricType: AbsoluteMetricType, - Provider: ElectricityMaps, - Location: location, - Units: GramsCO2EPerkWh, - ValidFrom: validFrom, - ValidTo: validTo, - Value: data.CarbonIntensity, - }, - }, nil + carbonIntensityDataPoint := CarbonIntensity{ + EmissionsType: AverageEmissionsType, + MetricType: AbsoluteMetricType, + Provider: ElectricityMaps, + Location: location, + Units: GramsCO2EPerkWh, + ValidFrom: validFrom, + ValidTo: validTo, + Value: dataPoint.CarbonIntensity, + IsEstimated: dataPoint.IsEstimated, + } + return &carbonIntensityDataPoint, nil } -func (e *ElectricityMapsClient) intensityURLWithZone(zone string) (string, error) { - zoneURL := fmt.Sprintf("/carbon-intensity/latest?zone=%s", zone) +// func (e *ElectricityMapsClient) intensityURLWithZone(zone string) (string, error) { +// zoneURL := fmt.Sprintf("/carbon-intensity/latest?zone=%s", zone) +// return buildURL(e.apiURL, zoneURL) +// } + +func (e *ElectricityMapsClient) historicIntensityURLWithZone(zone string) (string, error) { + zoneURL := fmt.Sprintf("/carbon-intensity/history?zone=%s", zone) return buildURL(e.apiURL, zoneURL) } @@ -100,4 +210,5 @@ type electricityMapsData struct { CarbonIntensity float64 `json:"carbonIntensity"` DateTime string `json:"datetime"` UpdatedAt string `json:"updatedAt"` + IsEstimated bool `json:"isEstimated"` } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 16fbe4b..445f74a 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -37,6 +37,7 @@ type CarbonIntensity struct { ValidFrom time.Time `json:"valid_from"` ValidTo time.Time `json:"valid_to"` Value float64 `json:"value"` + IsEstimated bool `json:"is_estimated"` } type Details struct { From 0904e0c2413b4bf97c18b75d14383f9b75e30f1c Mon Sep 17 00:00:00 2001 From: Flavia Paganelli Date: Fri, 7 Jun 2024 14:42:54 +0200 Subject: [PATCH 2/8] Fix for Watttime - was missing the is_estimated label... for relative carbon intensity Otherwise we see this error, for example: panic: inconsistent label cardinality: expected 5 label values but got 6 in []string{"CAISO_NORTH", "", "WattTime", "", "percent", "false"} --- cmd/exporter.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/exporter.go b/cmd/exporter.go index fb38dc8..e12f476 100644 --- a/cmd/exporter.go +++ b/cmd/exporter.go @@ -81,6 +81,7 @@ var ( labelProvider, labelRegion, labelUnits, + labelIsEstimated, }, nil, ) From 7c0adf278220f465fe87b4c374c938c8514949a9 Mon Sep 17 00:00:00 2001 From: Flavia Paganelli Date: Fri, 7 Jun 2024 15:08:20 +0200 Subject: [PATCH 3/8] Cleaned up some commented code --- cmd/exporter.go | 3 --- pkg/provider/electricity_maps.go | 5 ----- 2 files changed, 8 deletions(-) diff --git a/cmd/exporter.go b/cmd/exporter.go index e12f476..7f07fd4 100644 --- a/cmd/exporter.go +++ b/cmd/exporter.go @@ -167,10 +167,7 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { continue } - // log.Printf("desc is %s", desc) - ch <- prometheus.NewMetricWithTimestamp(data.ValidFrom, prometheus.MustNewConstMetric( - // ch <- prometheus.MustNewConstMetric( desc, prometheus.GaugeValue, data.Value, diff --git a/pkg/provider/electricity_maps.go b/pkg/provider/electricity_maps.go index 62d9020..e119fef 100644 --- a/pkg/provider/electricity_maps.go +++ b/pkg/provider/electricity_maps.go @@ -195,11 +195,6 @@ func toCarbonIntensity(location string, dataPoint electricityMapsData) (*CarbonI return &carbonIntensityDataPoint, nil } -// func (e *ElectricityMapsClient) intensityURLWithZone(zone string) (string, error) { -// zoneURL := fmt.Sprintf("/carbon-intensity/latest?zone=%s", zone) -// return buildURL(e.apiURL, zoneURL) -// } - func (e *ElectricityMapsClient) historicIntensityURLWithZone(zone string) (string, error) { zoneURL := fmt.Sprintf("/carbon-intensity/history?zone=%s", zone) return buildURL(e.apiURL, zoneURL) From 60601790ccba717d122b0dad506689eff01046ef Mon Sep 17 00:00:00 2001 From: Flavia Paganelli Date: Tue, 11 Jun 2024 11:14:23 +0200 Subject: [PATCH 4/8] It turns out that all the other providers return estimated values Thus, we are setting IsEstimated to true. See for more info https://github.com/thegreenwebfoundation/grid-intensity-go/pull/83#issuecomment-2159956389 --- pkg/provider/carbon_intensity_uk.go | 1 + pkg/provider/carbon_intensity_uk_test.go | 1 + pkg/provider/electricity_maps_test.go | 1 + pkg/provider/ember.go | 1 + pkg/provider/ember_test.go | 3 +++ pkg/provider/watt_time.go | 2 ++ pkg/provider/watt_time_test.go | 2 ++ 7 files changed, 11 insertions(+) diff --git a/pkg/provider/carbon_intensity_uk.go b/pkg/provider/carbon_intensity_uk.go index d1c87cb..6e51f89 100644 --- a/pkg/provider/carbon_intensity_uk.go +++ b/pkg/provider/carbon_intensity_uk.go @@ -94,6 +94,7 @@ func (a *CarbonIntensityUKClient) GetCarbonIntensity(ctx context.Context, locati ValidFrom: validFrom, ValidTo: validTo, Value: data.Intensity.Actual, + IsEstimated: true, }, }, nil } diff --git a/pkg/provider/carbon_intensity_uk_test.go b/pkg/provider/carbon_intensity_uk_test.go index 042a5af..1065ec2 100644 --- a/pkg/provider/carbon_intensity_uk_test.go +++ b/pkg/provider/carbon_intensity_uk_test.go @@ -56,6 +56,7 @@ func Test_CarbonIntensityUK_SimpleRequest(t *testing.T) { ValidFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), ValidTo: time.Date(2020, 1, 1, 0, 30, 0, 0, time.UTC), Value: 190, + IsEstimated: true, }, } if !reflect.DeepEqual(expected, res) { diff --git a/pkg/provider/electricity_maps_test.go b/pkg/provider/electricity_maps_test.go index 4105bf8..2423f62 100644 --- a/pkg/provider/electricity_maps_test.go +++ b/pkg/provider/electricity_maps_test.go @@ -50,6 +50,7 @@ func Test_ElectricityMaps_SimpleRequest(t *testing.T) { ValidFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), ValidTo: time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC), Value: 312, + IsEstimated: true, }, } if !reflect.DeepEqual(expected, res) { diff --git a/pkg/provider/ember.go b/pkg/provider/ember.go index ebe8ea5..24f0617 100644 --- a/pkg/provider/ember.go +++ b/pkg/provider/ember.go @@ -50,6 +50,7 @@ func (a *EmberClient) GetCarbonIntensity(ctx context.Context, location string) ( ValidFrom: validFrom, ValidTo: validTo, Value: result.EmissionsIntensityGCO2PerKWH, + IsEstimated: true, }, }, nil } diff --git a/pkg/provider/ember_test.go b/pkg/provider/ember_test.go index 838c72d..1695de4 100644 --- a/pkg/provider/ember_test.go +++ b/pkg/provider/ember_test.go @@ -31,6 +31,7 @@ func Test_GetGridIntensityForCountry(t *testing.T) { ValidFrom: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), ValidTo: time.Date(2021, 12, 31, 23, 59, 0, 0, time.UTC), Value: 193.737, + IsEstimated: true, }, }, }, @@ -47,6 +48,7 @@ func Test_GetGridIntensityForCountry(t *testing.T) { ValidFrom: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), ValidTo: time.Date(2021, 12, 31, 23, 59, 0, 0, time.UTC), Value: 193.737, + IsEstimated: true, }, }, }, @@ -63,6 +65,7 @@ func Test_GetGridIntensityForCountry(t *testing.T) { ValidFrom: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), ValidTo: time.Date(2021, 12, 31, 23, 59, 0, 0, time.UTC), Value: 268.255, + IsEstimated: true, }, }, }, diff --git a/pkg/provider/watt_time.go b/pkg/provider/watt_time.go index 49a21e4..2300867 100644 --- a/pkg/provider/watt_time.go +++ b/pkg/provider/watt_time.go @@ -234,6 +234,7 @@ func parseCarbonIntensityData(ctx context.Context, location string, indexData *w ValidFrom: validFrom, ValidTo: validTo, Value: percent, + IsEstimated: true, } result = append(result, relative) } @@ -252,6 +253,7 @@ func parseCarbonIntensityData(ctx context.Context, location string, indexData *w ValidFrom: validFrom, ValidTo: validTo, Value: moer, + IsEstimated: true, } result = append(result, marginal) } diff --git a/pkg/provider/watt_time_test.go b/pkg/provider/watt_time_test.go index 128f1df..247c383 100644 --- a/pkg/provider/watt_time_test.go +++ b/pkg/provider/watt_time_test.go @@ -65,6 +65,7 @@ func Test_WattTime_SimpleRequest(t *testing.T) { ValidFrom: time.Date(2022, 7, 6, 16, 25, 0, 0, time.UTC), ValidTo: time.Date(2022, 7, 6, 16, 30, 0, 0, time.UTC), Value: 78, + IsEstimated: true, }, { EmissionsType: "marginal", @@ -75,6 +76,7 @@ func Test_WattTime_SimpleRequest(t *testing.T) { ValidFrom: time.Date(2022, 7, 6, 16, 25, 0, 0, time.UTC), ValidTo: time.Date(2022, 7, 6, 16, 30, 0, 0, time.UTC), Value: 916, + IsEstimated: true, }, } if !reflect.DeepEqual(expected, result) { From e5ffba1d2d31a64cb05e03e4e0de97fc55fe3146 Mon Sep 17 00:00:00 2001 From: Flavia Paganelli Date: Tue, 11 Jun 2024 13:40:18 +0200 Subject: [PATCH 5/8] Some renaming as suggested by Chris --- pkg/provider/electricity_maps.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/provider/electricity_maps.go b/pkg/provider/electricity_maps.go index e119fef..d6f4a52 100644 --- a/pkg/provider/electricity_maps.go +++ b/pkg/provider/electricity_maps.go @@ -64,19 +64,14 @@ func (e *ElectricityMapsClient) GetCarbonIntensity(ctx context.Context, location return nil, errBadStatus(resp) } - type EMHistoryResponse struct { - Zone string - History []electricityMapsData - } - - historyResponse := &EMHistoryResponse{} + historyResponse := &electricityMapsHistoryResponse{} err = json.NewDecoder(resp.Body).Decode(&historyResponse) if err != nil { return nil, err } var carbonIntensityPoints []CarbonIntensity - var recentDatapoints = NewMostRecentDataPoints() + var recentDatapoints = NewElectricityMapsDatapoints() for _, dataPoint := range historyResponse.History { @@ -106,31 +101,31 @@ func (e *ElectricityMapsClient) GetCarbonIntensity(ctx context.Context, location // Helper struct to remove clutter in the calling function // while finding the latest (and greatest) data points -type mostRecentDatapoints struct { +type electricityMapsDatapoints struct { estimated electricityMapsData real electricityMapsData estimatedFound bool realFound bool } -func NewMostRecentDataPoints() mostRecentDatapoints { - dataPoints := mostRecentDatapoints{} +func NewElectricityMapsDatapoints() electricityMapsDatapoints { + dataPoints := electricityMapsDatapoints{} dataPoints.estimatedFound = false dataPoints.realFound = false return dataPoints } -func (m *mostRecentDatapoints) setEstimated(dataPoint electricityMapsData) { +func (m *electricityMapsDatapoints) setEstimated(dataPoint electricityMapsData) { m.estimated = dataPoint m.estimatedFound = true } -func (m *mostRecentDatapoints) setReal(dataPoint electricityMapsData) { +func (m *electricityMapsDatapoints) setReal(dataPoint electricityMapsData) { m.real = dataPoint m.realFound = true } -func (m *mostRecentDatapoints) update(dataPoint electricityMapsData) error { +func (m *electricityMapsDatapoints) update(dataPoint electricityMapsData) error { dataPointDateTime, err := stringToTime(dataPoint.DateTime) if err != nil { @@ -207,3 +202,8 @@ type electricityMapsData struct { UpdatedAt string `json:"updatedAt"` IsEstimated bool `json:"isEstimated"` } + +type electricityMapsHistoryResponse struct { + Zone string + History []electricityMapsData +} From 55f4695e575856ff8a4c79b7a30a46010c2f3af9 Mon Sep 17 00:00:00 2001 From: locomundo Date: Tue, 11 Jun 2024 13:41:47 +0200 Subject: [PATCH 6/8] Update pkg/provider/electricity_maps.go Co-authored-by: Ross Fairbanks --- pkg/provider/electricity_maps.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/provider/electricity_maps.go b/pkg/provider/electricity_maps.go index d6f4a52..8279b31 100644 --- a/pkg/provider/electricity_maps.go +++ b/pkg/provider/electricity_maps.go @@ -74,10 +74,9 @@ func (e *ElectricityMapsClient) GetCarbonIntensity(ctx context.Context, location var recentDatapoints = NewElectricityMapsDatapoints() for _, dataPoint := range historyResponse.History { - // We get the most recent value (which is usually estimated) // and the most recent value which is registered (not estimated) - // and they wil end up in the recentDatapoints variable + // and they will end up in the recentDatapoints variable recentDatapoints.update(dataPoint) } From 8e83f618c2d95256e96ef9bb7674197eb5e35359 Mon Sep 17 00:00:00 2001 From: Flavia Paganelli Date: Thu, 13 Jun 2024 13:39:24 +0200 Subject: [PATCH 7/8] Document how to fix "old" samples not being ingested in Prometheus. Related to PR #83 "For the Prometheus exporter, the values based on ElecticityMaps provider are estimations and not the real values" --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eeb2920..c0449a9 100644 --- a/README.md +++ b/README.md @@ -124,9 +124,18 @@ View the metrics with curl. $ curl -s http://localhost:8000/metrics | grep grid # HELP grid_intensity_carbon_average Average carbon intensity for the electricity grid in this location. # TYPE grid_intensity_carbon_average gauge -grid_intensity_carbon_average{provider="Ember",location="FR",units="gCO2 per kWh"} 67.781 +grid_intensity_carbon_average{provider="Ember",location="FR",units="gCO2 per kWh"} 67.781 1718258400000 ``` +**Note about Prometheus and samples in the past** + +If you are using the exporter with ElectricityMaps as provider, it will return a value for estimated, which be the most recent one, and another value for the real value, which can be a few hours in the past. Depending on your Prometheus installation, it could be that the metrics that have a timestamp in the past are not accepted, with an error such as this: + +`Error on ingesting samples that are too old or are too far into the future` + +In that case, you can configure the property `tsdb.outOfOrderTimeWindow` to extend the time window accepted, for example to `3h`. + + ### Docker Image Build the docker image to deploy the exporter. From 9cc5a0bff3e98684f6d4f1b924086590f59829bc Mon Sep 17 00:00:00 2001 From: locomundo Date: Thu, 29 Aug 2024 17:42:11 +0200 Subject: [PATCH 8/8] Corrections suggested by Ross Co-authored-by: Ross Fairbanks --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c0449a9..3984ec0 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ grid_intensity_carbon_average{provider="Ember",location="FR",units="gCO2 per kWh **Note about Prometheus and samples in the past** -If you are using the exporter with ElectricityMaps as provider, it will return a value for estimated, which be the most recent one, and another value for the real value, which can be a few hours in the past. Depending on your Prometheus installation, it could be that the metrics that have a timestamp in the past are not accepted, with an error such as this: +If you are using the exporter with the ElectricityMaps provider, it will return a value for estimated, which will be the most recent one, and another value for the real value, which can be a few hours in the past. Depending on your Prometheus installation, it could be that the metrics that have a timestamp in the past are not accepted, with an error such as this: `Error on ingesting samples that are too old or are too far into the future`