From 5831dc09a1c39b770765b184d9efa13da09aa0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= <61736220+Daniel-WWU-IT@users.noreply.github.com> Date: Thu, 1 Oct 2020 08:40:37 +0200 Subject: [PATCH] Add Blackbox Exporter support to Mentix (#1190) --- changelog/unreleased/mentix-bbe-support.md | 5 + .../config/http/services/mentix/_index.md | 8 +- .../http/services/mentix/promfilesd/_index.md | 19 -- .../http/services/mentix/promsd/_index.md | 27 +++ examples/mentix/mentix.toml | 14 +- pkg/mentix/config/config.go | 7 +- pkg/mentix/config/ids.go | 4 +- pkg/mentix/exporters/prometheus/types.go | 2 +- pkg/mentix/exporters/promfilesd.go | 155 ------------- pkg/mentix/exporters/promsd.go | 214 ++++++++++++++++++ 10 files changed, 265 insertions(+), 190 deletions(-) create mode 100644 changelog/unreleased/mentix-bbe-support.md delete mode 100644 docs/content/en/docs/config/http/services/mentix/promfilesd/_index.md create mode 100644 docs/content/en/docs/config/http/services/mentix/promsd/_index.md delete mode 100755 pkg/mentix/exporters/promfilesd.go create mode 100755 pkg/mentix/exporters/promsd.go diff --git a/changelog/unreleased/mentix-bbe-support.md b/changelog/unreleased/mentix-bbe-support.md new file mode 100644 index 0000000000..512a64efb3 --- /dev/null +++ b/changelog/unreleased/mentix-bbe-support.md @@ -0,0 +1,5 @@ +Enhancement: Add Blackbox Exporter support to Mentix + +This update extends Mentix to export a Prometheus SD file specific to the Blackbox Exporter which will be used for initial health monitoring. Usually, Prometheus requires its targets to only consist of the target's hostname; the BBE though expects a full URL here. This makes exporting two distinct files necessary. + +https://github.com/cs3org/reva/pull/1190 diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index eb3f65ec45..90944cdc91 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -32,7 +32,7 @@ connector = "gocdb" {{< /highlight >}} {{% /dir %}} -{{% dir name="exporters" type="[]string" default="[webapi,prom_filesd]" %}} +{{% dir name="exporters" type="[]string" default="[webapi,cs3api,siteloc,promsd]" %}} Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. Supported values are: @@ -41,8 +41,8 @@ Supported values are: Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future. - **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure. - **siteloc** The Site Locations exporter `siteloc` exposes location information of all sites to be consumed by Grafana at the configured relative endpoint (see [here](siteloc)). -- **prom_filesd** -[Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)): +- **promsd** +[Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `promsd` exporter. To use this exporter, you have to specify the target output files in the configuration (see [here](promsd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)): ``` scrape_configs: - job_name: 'sciencemesh' file_sd_configs: @@ -52,7 +52,7 @@ Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data c {{< highlight toml >}} [http.services.mentix] -exporters = ["webapi", "prom_filesd"] +exporters = ["webapi", "promsd"] {{< /highlight >}} {{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/promfilesd/_index.md b/docs/content/en/docs/config/http/services/mentix/promfilesd/_index.md deleted file mode 100644 index c3fb13cded..0000000000 --- a/docs/content/en/docs/config/http/services/mentix/promfilesd/_index.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: "prom_filesd" -linkTitle: "prom_filesd" -weight: 10 -description: > - Configuration for the Prometheus File SD exporter of the Mentix service ---- - -{{% pageinfo %}} -When using the Prometheus File SD exporter, the output filename has to be configured first. -{{% /pageinfo %}} - -{{% dir name="output_file" type="string" default="" %}} -The target filename of the generated Prometheus File SD scrape config. -{{< highlight toml >}} -[http.services.mentix.prom_filesd] -output_file = "/var/shared/prometheus/sciencemesh.json" -{{< /highlight >}} -{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md new file mode 100644 index 0000000000..58375f7449 --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md @@ -0,0 +1,27 @@ +--- +title: "promsd" +linkTitle: "promsd" +weight: 10 +description: > + Configuration for the Prometheus SD exporter of the Mentix service +--- + +{{% pageinfo %}} +When using the Prometheus SD exporter, the output filenames have to be configured first. +{{% /pageinfo %}} + +{{% dir name="metrics_output_file" type="string" default="" %}} +The target filename of the generated Prometheus File SD scrape config for metrics. +{{< highlight toml >}} +[http.services.mentix.promsd] +metrics_output_file = "/var/shared/prometheus/sciencemesh.json" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="blackbox_output_file" type="string" default="" %}} +The target filename of the generated Prometheus File SD scrape config for the blackbox exporter. +{{< highlight toml >}} +[http.services.mentix.promsd] +blackbox_output_file = "/var/shared/prometheus/blackbox.json" +{{< /highlight >}} +{{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 5e8c6da2d6..5ea0c8d8e8 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -8,8 +8,8 @@ enabled_services = ["mentix"] [http.services.mentix] connector = "gocdb" exporters = ["webapi", "cs3api", "siteloc"] -# Enable the Prometheus File Service Discovery: -# exporters = ["webapi", "cs3api", "siteloc", "prom_filesd"] +# Enable the Prometheus Service Discovery: +# exporters = ["webapi", "cs3api", "siteloc", "promsd"] update_interval = "15m" [http.services.mentix.gocdb] @@ -24,7 +24,9 @@ endpoint = "/cs3" [http.services.mentix.siteloc] endpoint = "/loc" -# Configure the Prometheus File Service Discovery: -# [http.services.mentix.prom_filesd] -# Prometheus must be configured to read the following file: -# output_file = "/usr/share/prom/sciencemesh_services.json" +# Configure the Prometheus Service Discovery: +# [http.services.mentix.promsd] +# The following files must be made available to Prometheus. +# They can then be used as the file_sd source of a job. +# metrics_output_file = "/usr/share/prom/sciencemesh_metrics.json" +# blackbox_output_file = "/usr/share/prom/sciencemesh_blackbox.json" diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 2377feb4d7..6125cebb26 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -43,9 +43,10 @@ type Configuration struct { Endpoint string `mapstructure:"endpoint"` } `yaml:"siteloc"` - PrometheusFileSD struct { - OutputFile string `mapstructure:"output_file"` - } `mapstructure:"prom_filesd"` + PrometheusSD struct { + MetricsOutputFile string `mapstructure:"metrics_output_file"` + BlackboxOutputFile string `mapstructure:"blackbox_output_file"` + } `mapstructure:"promsd"` } // Init sets sane defaults diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index f951b1cbe9..dc5abd8dc4 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -30,6 +30,6 @@ const ( ExporterIDCS3API = "cs3api" // ExporterIDSiteLocations is the identifier for the Site Locations exporter. ExporterIDSiteLocations = "siteloc" - // ExporterIDPrometheusFileSD is the identifier for the Prometheus File SD exporter. - ExporterIDPrometheusFileSD = "prom_filesd" + // ExporterIDPrometheusSD is the identifier for the PrometheusSD SD exporter. + ExporterIDPrometheusSD = "promsd" ) diff --git a/pkg/mentix/exporters/prometheus/types.go b/pkg/mentix/exporters/prometheus/types.go index 1b341b592e..b8061b89c9 100755 --- a/pkg/mentix/exporters/prometheus/types.go +++ b/pkg/mentix/exporters/prometheus/types.go @@ -18,7 +18,7 @@ package prometheus -// ScrapeConfig represents a scrape configuration in Prometheus. +// ScrapeConfig represents a scrape configuration in PrometheusSD. type ScrapeConfig struct { Targets []string `json:"targets"` Labels map[string]string `json:"labels"` diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go deleted file mode 100755 index 01c5974a63..0000000000 --- a/pkg/mentix/exporters/promfilesd.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package exporters - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exporters/prometheus" - "github.com/cs3org/reva/pkg/mentix/meshdata" -) - -// PrometheusFileSDExporter implements the File Service Discovery for Prometheus exporter. -type PrometheusFileSDExporter struct { - BaseExporter - - outputFilename string -} - -// Activate activates the exporter. -func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := exporter.BaseExporter.Activate(conf, log); err != nil { - return err - } - - // Check and store Prometheus File SD specific settings - exporter.outputFilename = conf.PrometheusFileSD.OutputFile - if len(exporter.outputFilename) == 0 { - return fmt.Errorf("no output filename configured") - } - - // Create the output directory - if err := os.MkdirAll(filepath.Dir(exporter.outputFilename), os.ModePerm); err != nil { - return fmt.Errorf("unable to create directory tree") - } - - return nil -} - -// UpdateMeshData is called whenever the mesh data has changed to reflect these changes. -func (exporter *PrometheusFileSDExporter) UpdateMeshData(meshData *meshdata.MeshData) error { - if err := exporter.BaseExporter.UpdateMeshData(meshData); err != nil { - return err - } - - // Perform exporting the data asynchronously - go exporter.exportMeshData() - return nil -} - -func (exporter *PrometheusFileSDExporter) exportMeshData() { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - scrapes := exporter.createScrapeConfigs() - if err := exporter.exportScrapeConfig(scrapes); err != nil { - exporter.log.Err(err).Str("file", exporter.outputFilename).Msg("error exporting Prometheus File SD") - } else { - exporter.log.Debug().Str("file", exporter.outputFilename).Msg("exported Prometheus File SD") - } -} - -func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.ScrapeConfig { - var scrapes []*prometheus.ScrapeConfig - var addScrape = func(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) { - if scrape := exporter.createScrapeConfig(site, host, endpoint); scrape != nil { - scrapes = append(scrapes, scrape) - } - } - - // Create a scrape config for each service alongside any additional endpoints - for _, site := range exporter.meshData.Sites { - for _, service := range site.Services { - if !service.IsMonitored { - continue - } - - // Add the "main" service to the scrapes - addScrape(site, service.Host, service.ServiceEndpoint) - - for _, endpoint := range service.AdditionalEndpoints { - if endpoint.IsMonitored { - addScrape(site, service.Host, endpoint) - } - } - } - } - - return scrapes -} - -func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { - labels := map[string]string{ - "site": site.Name, - "country": site.CountryCode, - "service_type": endpoint.Type.Name, - } - - // If a metrics path was specified as a property, use that one by setting the corresponding label - if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { - labels["__metrics_path__"] = metricsPath - } - - return &prometheus.ScrapeConfig{ - Targets: []string{host}, - Labels: labels, - } -} - -func (exporter *PrometheusFileSDExporter) exportScrapeConfig(v interface{}) error { - // Encode scrape config as JSON - data, err := json.MarshalIndent(v, "", "\t") - if err != nil { - return fmt.Errorf("unable to marshal scrape config: %v", err) - } - - // Write the data to disk - if err := ioutil.WriteFile(exporter.outputFilename, data, os.ModePerm); err != nil { - return fmt.Errorf("unable to write scrape config '%v': %v", exporter.outputFilename, err) - } - - return nil -} - -// GetName returns the display name of the exporter. -func (exporter *PrometheusFileSDExporter) GetName() string { - return "Prometheus File SD" -} - -func init() { - registerExporter(config.ExporterIDPrometheusFileSD, &PrometheusFileSDExporter{}) -} diff --git a/pkg/mentix/exporters/promsd.go b/pkg/mentix/exporters/promsd.go new file mode 100755 index 0000000000..9d68b16f39 --- /dev/null +++ b/pkg/mentix/exporters/promsd.go @@ -0,0 +1,214 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters/prometheus" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +type prometheusSDScrapeCreatorCallback = func(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig +type prometheusSDScrapeCreator struct { + outputFilename string + creatorCallback prometheusSDScrapeCreatorCallback +} + +// PrometheusSDExporter implements various Prometheus Service Discovery scrape config exporters. +type PrometheusSDExporter struct { + BaseExporter + + scrapeCreators map[string]prometheusSDScrapeCreator +} + +func createMetricsSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { + labels := map[string]string{ + "site": site.Name, + "country": site.CountryCode, + "service_type": endpoint.Type.Name, + } + + // If a metrics path was specified as a property, use that one by setting the corresponding label + if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { + labels["__metrics_path__"] = metricsPath + } + + return &prometheus.ScrapeConfig{ + Targets: []string{host}, + Labels: labels, + } +} + +func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { + // The URL of the service is used as the actual target; it must be configured properly + target := endpoint.URL + if target == "" { + return nil + } + + labels := map[string]string{ + "site": site.Name, + "country": site.CountryCode, + "service_type": endpoint.Type.Name, + } + + return &prometheus.ScrapeConfig{ + Targets: []string{target}, + Labels: labels, + } +} + +func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Configuration) error { + exporter.scrapeCreators = make(map[string]prometheusSDScrapeCreator) + + registerCreator := func(name string, outputFilename string, creator prometheusSDScrapeCreatorCallback) error { + if len(outputFilename) > 0 { // Only register the creator if an output filename was configured + exporter.scrapeCreators[name] = prometheusSDScrapeCreator{ + outputFilename: outputFilename, + creatorCallback: creator, + } + + // Create the output directory for the target file so it exists when exporting + if err := os.MkdirAll(filepath.Dir(outputFilename), os.ModePerm); err != nil { + return fmt.Errorf("unable to create output directory tree: %v", err) + } + } + + return nil + } + + // Register all scrape creators + if err := registerCreator("metrics", conf.PrometheusSD.MetricsOutputFile, createMetricsSDScrapeConfig); err != nil { + return fmt.Errorf("unable to register the 'metrics' scrape config creator: %v", err) + } + + if err := registerCreator("blackbox", conf.PrometheusSD.BlackboxOutputFile, createBlackboxSDScrapeConfig); err != nil { + return fmt.Errorf("unable to register the 'blackbox' scrape config creator: %v", err) + } + + return nil +} + +// Activate activates the exporter. +func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseExporter.Activate(conf, log); err != nil { + return err + } + + if err := exporter.registerScrapeCreators(conf); err != nil { + return fmt.Errorf("unable to register the scrape creators: %v", err) + } + + // Create all output directories + for _, creator := range exporter.scrapeCreators { + if err := os.MkdirAll(filepath.Dir(creator.outputFilename), os.ModePerm); err != nil { + return fmt.Errorf("unable to create directory tree: %v", err) + } + } + + return nil +} + +// UpdateMeshData is called whenever the mesh data has changed to reflect these changes. +func (exporter *PrometheusSDExporter) UpdateMeshData(meshData *meshdata.MeshData) error { + if err := exporter.BaseExporter.UpdateMeshData(meshData); err != nil { + return err + } + + // Perform exporting the data asynchronously + go exporter.exportMeshData() + return nil +} + +func (exporter *PrometheusSDExporter) exportMeshData() { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + for name, creator := range exporter.scrapeCreators { + scrapes := exporter.createScrapeConfigs(creator.creatorCallback) + if err := exporter.exportScrapeConfig(creator.outputFilename, scrapes); err != nil { + exporter.log.Err(err).Str("kind", name).Str("file", creator.outputFilename).Msg("error exporting Prometheus SD scrape config") + } else { + exporter.log.Debug().Str("kind", name).Str("file", creator.outputFilename).Msg("exported Prometheus SD scrape config") + } + } +} + +func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback prometheusSDScrapeCreatorCallback) []*prometheus.ScrapeConfig { + var scrapes []*prometheus.ScrapeConfig + var addScrape = func(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) { + if scrape := creatorCallback(site, host, endpoint); scrape != nil { + scrapes = append(scrapes, scrape) + } + } + + // Create a scrape config for each service alongside any additional endpoints + for _, site := range exporter.meshData.Sites { + for _, service := range site.Services { + if !service.IsMonitored { + continue + } + + // Add the "main" service to the scrapes + addScrape(site, service.Host, service.ServiceEndpoint) + + // Add all additional endpoints as well + for _, endpoint := range service.AdditionalEndpoints { + if endpoint.IsMonitored { + addScrape(site, service.Host, endpoint) + } + } + } + } + + return scrapes +} + +func (exporter *PrometheusSDExporter) exportScrapeConfig(outputFilename string, v interface{}) error { + // Encode scrape config as JSON + data, err := json.MarshalIndent(v, "", "\t") + if err != nil { + return fmt.Errorf("unable to marshal scrape config: %v", err) + } + + // Write the data to disk + if err := ioutil.WriteFile(outputFilename, data, os.ModePerm); err != nil { + return fmt.Errorf("unable to write scrape config '%v': %v", outputFilename, err) + } + + return nil +} + +// GetName returns the display name of the exporter. +func (exporter *PrometheusSDExporter) GetName() string { + return "PrometheusSD SD" +} + +func init() { + registerExporter(config.ExporterIDPrometheusSD, &PrometheusSDExporter{}) +}