From 8807ee197c38ccf8db08f9848de7a7092d74cf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 9 Jun 2020 10:23:21 +0200 Subject: [PATCH 01/12] Fixed label naming --- pkg/mentix/exporters/promfilesd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index c2906ac1e4..5dcc90a1ee 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -115,7 +115,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site Targets: []string{path.Join(host, endpoint.Path)}, Labels: map[string]string{ "site": site.Name, - "service-type": endpoint.Type.Name, + "service_type": endpoint.Type.Name, }, } } From 7e0ece2c7e70ffb6aaaf4223005ae20a4673d3f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 9 Jun 2020 13:09:02 +0200 Subject: [PATCH 02/12] Added support for service ports --- pkg/mentix/connectors/gocdb.go | 14 +++++++++++++- pkg/mentix/exporters/promfilesd.go | 3 +-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index d10690f15b..f4975eae06 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -21,6 +21,7 @@ package connectors import ( "encoding/xml" "fmt" + "net/url" "strings" "github.com/rs/zerolog" @@ -146,6 +147,17 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site // Copy retrieved data into the mesh data site.Services = nil for _, service := range services.Services { + host := service.Host + + // If a URL is provided, extract the port from it and append it to the host + if len(service.URL) > 0 { + if hostURL, err := url.Parse(service.URL); err == nil { + if port := hostURL.Port(); len(port) > 0 { + host += ":" + port + } + } + } + // Assemble additional endpoints var endpoints []*meshdata.ServiceEndpoint for _, endpoint := range service.Endpoints.Endpoints { @@ -167,7 +179,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site IsMonitored: strings.EqualFold(service.IsMonitored, "Y"), Properties: connector.extensionsToMap(&service.Extensions), }, - Host: service.Host, + Host: host, AdditionalEndpoints: endpoints, }) } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 5dcc90a1ee..0479c333ff 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -23,7 +23,6 @@ import ( "fmt" "io/ioutil" "os" - "path" "path/filepath" "github.com/rs/zerolog" @@ -112,7 +111,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { return &prometheus.ScrapeConfig{ - Targets: []string{path.Join(host, endpoint.Path)}, + Targets: []string{host}, Labels: map[string]string{ "site": site.Name, "service_type": endpoint.Type.Name, From 2e1ab438a039afbb4ced4b6708ccf3c767246088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Jun 2020 10:50:40 +0200 Subject: [PATCH 03/12] Added support for differing metrics paths when exporting scrapes to Prometheus --- pkg/mentix/config/ids.go | 4 ++++ pkg/mentix/exporters/promfilesd.go | 15 +++++++++++---- pkg/mentix/meshdata/service.go | 13 +++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index f8aed9c0dd..e3e837feb9 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -26,3 +26,7 @@ const ( ExporterIDWebAPI = "webapi" ExporterIDPrometheusFileSD = "prom_filesd" ) + +const ( + PropertyMetricsPath = "metrics_path" +) diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 0479c333ff..39c2772fc3 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -110,12 +110,19 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc } func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { + labels := map[string]string{ + "site": site.Name, + "service_type": endpoint.Type.Name, + } + + // If a metrics path was specified as a property, use that one by setting the corresponding label + if metricsPath := endpoint.GetPropertyValue(config.PropertyMetricsPath, ""); len(metricsPath) > 0 { + labels["__metrics_path__"] = metricsPath + } + return &prometheus.ScrapeConfig{ Targets: []string{host}, - Labels: map[string]string{ - "site": site.Name, - "service_type": endpoint.Type.Name, - }, + Labels: labels, } } diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 4ff18ec63e..240c8b25bb 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -18,6 +18,8 @@ package meshdata +import "strings" + // Service represents a service managed by Mentix. type Service struct { ServiceEndpoint @@ -40,3 +42,14 @@ type ServiceEndpoint struct { IsMonitored bool Properties map[string]string } + +// GetPropertyValue performs a case-insensitive search for the given property. +func (endpoint *ServiceEndpoint) GetPropertyValue(id string, defValue string) string { + for key := range endpoint.Properties { + if strings.EqualFold(key, id) { + return endpoint.Properties[key] + } + } + + return defValue +} From 34413c2f6f7701f76a03f80c5e7c01babe7c39e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Jun 2020 11:47:51 +0200 Subject: [PATCH 04/12] Added support for organization names --- pkg/mentix/config/ids.go | 4 ---- pkg/mentix/connectors/gocdb.go | 9 ++++++-- pkg/mentix/exporters/promfilesd.go | 2 +- pkg/mentix/meshdata/properties.go | 37 ++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 pkg/mentix/meshdata/properties.go diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index e3e837feb9..f8aed9c0dd 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -26,7 +26,3 @@ const ( ExporterIDWebAPI = "webapi" ExporterIDPrometheusFileSD = "prom_filesd" ) - -const ( - PropertyMetricsPath = "metrics_path" -) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index f4975eae06..390463e76d 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -121,16 +121,21 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { // Copy retrieved data into the mesh data meshData.Sites = nil for _, site := range sites.Sites { + properties := connector.extensionsToMap(&site.Extensions) + + // See if an organization has been defined using properties; otherwise, use the official name + organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) + meshsite := &meshdata.Site{ Name: site.ShortName, FullName: site.OfficialName, - Organization: "", + Organization: organization, Domain: site.Domain, Homepage: site.Homepage, Email: site.Email, Description: site.Description, Services: nil, - Properties: connector.extensionsToMap(&site.Extensions), + Properties: properties, } meshData.Sites = append(meshData.Sites, meshsite) } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 39c2772fc3..7c1fe91e40 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -116,7 +116,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site } // If a metrics path was specified as a property, use that one by setting the corresponding label - if metricsPath := endpoint.GetPropertyValue(config.PropertyMetricsPath, ""); len(metricsPath) > 0 { + if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { labels["__metrics_path__"] = metricsPath } diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go new file mode 100644 index 0000000000..2d6f1cf62f --- /dev/null +++ b/pkg/mentix/meshdata/properties.go @@ -0,0 +1,37 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +import "strings" + +const ( + PropertyOrganization = "organization" + PropertyMetricsPath = "metrics_path" +) + +// GetPropertyValue performs a case-insensitive search for the given property. +func GetPropertyValue(props map[string]string, id string, defValue string) string { + for key := range props { + if strings.EqualFold(key, id) { + return props[key] + } + } + + return defValue +} From 7fc02aa4c7a7c581f3e907e81ebb4140cd6454b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Jun 2020 13:09:35 +0200 Subject: [PATCH 05/12] Service URLs are now properly combined and exported --- pkg/mentix/connectors/gocdb.go | 43 ++++++++++++++++++++++++++++++++-- pkg/mentix/meshdata/service.go | 15 +----------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 390463e76d..22c5989620 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -22,6 +22,7 @@ import ( "encoding/xml" "fmt" "net/url" + "path" "strings" "github.com/rs/zerolog" @@ -149,6 +150,14 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site return err } + getServiceURLString := func(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint, host string) string { + urlstr := "https://" + host // Fall back to the provided hostname + if svcURL, err := connector.getServiceURL(service, endpoint); err == nil { + urlstr = svcURL.String() + } + return urlstr + } + // Copy retrieved data into the mesh data site.Services = nil for _, service := range services.Services { @@ -169,7 +178,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site endpoints = append(endpoints, &meshdata.ServiceEndpoint{ Type: connector.findServiceType(meshData, endpoint.Type), Name: endpoint.Name, - Path: endpoint.URL, + URL: getServiceURLString(service, endpoint, host), IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"), Properties: connector.extensionsToMap(&endpoint.Extensions), }) @@ -180,7 +189,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site ServiceEndpoint: meshdata.ServiceEndpoint{ Type: connector.findServiceType(meshData, service.Type), Name: fmt.Sprintf("%v - %v", service.Host, service.Type), - Path: "", + URL: getServiceURLString(service, nil, host), IsMonitored: strings.EqualFold(service.IsMonitored, "Y"), Properties: connector.extensionsToMap(&service.Extensions), }, @@ -211,6 +220,36 @@ func (connector *GOCDBConnector) extensionsToMap(extensions *gocdb.Extensions) m return properties } +func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint) (*url.URL, error) { + urlstr := service.URL + if len(urlstr) == 0 { + // The URL defaults to the hostname using the HTTPS protocol + urlstr = "https://" + service.Host + } + + svcURL, err := url.ParseRequestURI(urlstr) + if err != nil { + return nil, fmt.Errorf("unable to parse URL '%v': %v", urlstr, err) + } + + // If an endpoint was provided, use its path + if endpoint != nil { + // If the endpoint URL is an absolute one, just use that; otherwise, make an absolute one out of it + if endpointURL, err := url.ParseRequestURI(endpoint.URL); err == nil && len(endpointURL.Scheme) > 0 { + svcURL = endpointURL + } else { + // Replace entire URL path if the relative path starts with a slash; otherwise, just append + if strings.HasPrefix(endpoint.URL, "/") { + svcURL.Path = endpoint.URL + } else { + svcURL.Path = path.Join(svcURL.Path, endpoint.URL) + } + } + } + + return svcURL, nil +} + func (connector *GOCDBConnector) GetName() string { return "GOCDB" } diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 240c8b25bb..e0f240653d 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -18,8 +18,6 @@ package meshdata -import "strings" - // Service represents a service managed by Mentix. type Service struct { ServiceEndpoint @@ -38,18 +36,7 @@ type ServiceType struct { type ServiceEndpoint struct { Type *ServiceType Name string - Path string + URL string IsMonitored bool Properties map[string]string } - -// GetPropertyValue performs a case-insensitive search for the given property. -func (endpoint *ServiceEndpoint) GetPropertyValue(id string, defValue string) string { - for key := range endpoint.Properties { - if strings.EqualFold(key, id) { - return endpoint.Properties[key] - } - } - - return defValue -} From 2efc8765921f9d9213591d122579f29d55217ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 1 Jul 2020 16:47:37 +0200 Subject: [PATCH 06/12] Added country information to Mentix --- pkg/mentix/connectors/gocdb.go | 2 ++ pkg/mentix/connectors/gocdb/types.go | 2 ++ pkg/mentix/exporters/promfilesd.go | 1 + pkg/mentix/meshdata/site.go | 3 +++ 4 files changed, 8 insertions(+) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 22c5989620..cd6206bc80 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -135,6 +135,8 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { Homepage: site.Homepage, Email: site.Email, Description: site.Description, + Country: site.Country, + CountryCode: site.CountryCode, Services: nil, Properties: properties, } diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index cc09e6e107..9448b47f23 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -48,6 +48,8 @@ type Site struct { Homepage string `xml:"HOME_URL"` Email string `xml:"CONTACT_EMAIL"` Domain string `xml:"DOMAIN>DOMAIN_NAME"` + Country string `xml:"COUNTRY"` + CountryCode string `xml:"COUNTRY_CODE"` Extensions Extensions `xml:"EXTENSIONS"` } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 7c1fe91e40..ec49dd484f 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -112,6 +112,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { labels := map[string]string{ "site": site.Name, + "country": site.CountryCode, "service_type": endpoint.Type.Name, } diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 6fba7eced0..8c90ace8e9 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -27,6 +27,9 @@ type Site struct { Homepage string Email string Description string + Country string + CountryCode string + Location string Services []*Service Properties map[string]string From 1cdc7259a760461d549aa9a878cfdaf1c5e6f146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 8 Jul 2020 15:40:00 +0200 Subject: [PATCH 07/12] Added a CS3API compliant data exporter to Mentix --- .../config/http/services/mentix/_index.md | 1 + .../http/services/mentix/cs3api/_index.md | 19 +++ .../http/services/mentix/webapi/_index.md | 4 +- examples/mentix/mentix.toml | 7 +- internal/http/services/mentix/mentix.go | 4 + pkg/mentix/config/config.go | 4 + pkg/mentix/config/ids.go | 1 + pkg/mentix/connectors/gocdb.go | 2 +- pkg/mentix/exporters/cs3api.go | 69 +++++++++++ pkg/mentix/exporters/cs3api/query.go | 113 ++++++++++++++++++ pkg/mentix/exporters/promfilesd.go | 2 +- pkg/mentix/meshdata/service.go | 2 +- 12 files changed, 221 insertions(+), 7 deletions(-) create mode 100644 docs/content/en/docs/config/http/services/mentix/cs3api/_index.md create mode 100755 pkg/mentix/exporters/cs3api.go create mode 100755 pkg/mentix/exporters/cs3api/query.go diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 6f8b2a8fc6..5401c27462 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -39,6 +39,7 @@ Supported values are: - **webapi** Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future. +- **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure. - **prom_filesd** [Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)): ``` scrape_configs: diff --git a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md new file mode 100644 index 0000000000..e89b810c4e --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md @@ -0,0 +1,19 @@ +--- +title: "cs3api" +linkTitle: "cs3api" +weight: 10 +description: > + Configuration for the CS3API of the Mentix service +--- + +{{% pageinfo %}} +The CS3API exporter exposes Mentix data in a format that is compliant with the CS3API `ProviderInfo` structure via an HTTP endpoint. +{{% /pageinfo %}} + +{{% dir name="endpoint" type="string" default="/" %}} +The endpoint where the mesh data can be queried. +{{< highlight toml >}} +[http.services.mentix.cs3api] +endpoint = "/data" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index 6f937aa60c..5c190f773b 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -7,13 +7,13 @@ description: > --- {{% pageinfo %}} -The WebAPI exporter supports multiple endpoints for exporting data. As there currently is only one such endpoint, the WebAPI settings should not be modified. +The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. {{% /pageinfo %}} {{% dir name="endpoint" type="string" default="/" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} [http.services.mentix.webapi] -endpoint = "data" +endpoint = "/data" {{< /highlight >}} {{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 760b33d7e6..893b26a5d2 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -7,9 +7,9 @@ enabled_services = ["mentix"] [http.services.mentix] connector = "gocdb" -exporters = ["webapi"] +exporters = ["webapi", "cs3api"] # Enable the Prometheus File Service Discovery: -# exporters = ["webapi", "prom_filesd"] +# exporters = ["webapi", "cs3api", "prom_filesd"] update_interval = "15m" [http.services.mentix.gocdb] @@ -18,6 +18,9 @@ address = "http://sciencemesh-test.uni-muenster.de" [http.services.mentix.webapi] endpoint = "/" +[http.services.mentix.cs3api] +endpoint = "/cs3" + # Configure the Prometheus File Service Discovery: # [http.services.mentix.prom_filesd] # Prometheus must be configured to read the following file: diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 745cc1cbbd..997adf774c 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -116,6 +116,10 @@ func applyDefaultConfig(conf *config.Configuration) { if conf.WebAPI.Endpoint == "" { conf.WebAPI.Endpoint = "/" } + + if conf.CS3API.Endpoint == "" { + conf.CS3API.Endpoint = "/cs3" + } } // New returns a new Mentix service. diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 47da2e82f5..211eedb5b0 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -35,6 +35,10 @@ type Configuration struct { Endpoint string `mapstructure:"endpoint"` } `yaml:"webapi"` + CS3API struct { + Endpoint string `mapstructure:"endpoint"` + } `yaml:"cs3api"` + PrometheusFileSD struct { OutputFile string `mapstructure:"output_file"` } `mapstructure:"prom_filesd"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index f8aed9c0dd..64caac7d50 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -24,5 +24,6 @@ const ( const ( ExporterIDWebAPI = "webapi" + ExporterIDCS3API = "cs3api" ExporterIDPrometheusFileSD = "prom_filesd" ) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index cd6206bc80..fd64bd21b4 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -188,7 +188,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site // Add the service to the site site.Services = append(site.Services, &meshdata.Service{ - ServiceEndpoint: meshdata.ServiceEndpoint{ + ServiceEndpoint: &meshdata.ServiceEndpoint{ Type: connector.findServiceType(meshData, service.Type), Name: fmt.Sprintf("%v - %v", service.Host, service.Type), URL: getServiceURLString(service, nil, host), diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exporters/cs3api.go new file mode 100755 index 0000000000..c6b986a02a --- /dev/null +++ b/pkg/mentix/exporters/cs3api.go @@ -0,0 +1,69 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "fmt" + "net/http" + + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters/cs3api" +) + +type CS3APIExporter struct { + BaseRequestExporter +} + +func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseExporter.Activate(conf, log); err != nil { + return err + } + + // Store CS3API specific settings + exporter.endpoint = conf.CS3API.Endpoint + + return nil +} + +func (exporter *CS3APIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + data, err := cs3api.HandleQuery(exporter.meshData, req.URL.Query()) + if err == nil { + if _, err := resp.Write(data); err != nil { + return fmt.Errorf("error writing the API request response: %v", err) + } + } else { + return fmt.Errorf("error while serving API request: %v", err) + } + + return nil +} + +func (exporter *CS3APIExporter) GetName() string { + return "CS3API" +} + +func init() { + registerExporter(config.ExporterIDCS3API, &CS3APIExporter{}) +} diff --git a/pkg/mentix/exporters/cs3api/query.go b/pkg/mentix/exporters/cs3api/query.go new file mode 100755 index 0000000000..12cff23a4e --- /dev/null +++ b/pkg/mentix/exporters/cs3api/query.go @@ -0,0 +1,113 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cs3api + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + + ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + queryMethodDefault = "" +) + +// HandleQuery handles an HTTP request based on the provided 'method' parameter. +func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + method := params.Get("method") + switch strings.ToLower(method) { + case queryMethodDefault: + return handleDefaultQuery(meshData, params) + + default: + return []byte{}, fmt.Errorf("unknown API method '%v'", method) + } +} + +func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + // Convert the mesh data + ocmData, err := convertMeshDataToOCMData(meshData) + if err != nil { + return []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) + } + + // Marshal the OCM data as JSON + data, err := json.MarshalIndent(ocmData, "", "\t") + if err != nil { + return []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + } + + return data, nil +} + +func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.ProviderInfo, error) { + // Convert the mesh data into the corresponding OCM data structures + providers := make([]*ocmprovider.ProviderInfo, 0, len(meshData.Sites)) + for _, site := range meshData.Sites { + // Gather all services from the site + services := make([]*ocmprovider.Service, 0, len(site.Services)) + for _, service := range site.Services { + // Gather all additional endpoints of the service + addEndpoints := make([]*ocmprovider.ServiceEndpoint, 0, len(service.AdditionalEndpoints)) + for _, endpoint := range service.AdditionalEndpoints { + addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint)) + } + + services = append(services, &ocmprovider.Service{ + Host: service.Host, + Endpoint: convertServiceEndpointToOCMData(service.ServiceEndpoint), + AdditionalEndpoints: addEndpoints, + ApiVersion: meshdata.GetPropertyValue(service.Properties, "API_VERSION", ""), + }) + } + + // Copy the site info into a ProviderInfo + providers = append(providers, &ocmprovider.ProviderInfo{ + Name: site.Name, + FullName: site.FullName, + Description: site.Description, + Organization: site.Organization, + Domain: site.Domain, + Homepage: site.Homepage, + Email: site.Email, + Services: services, + Properties: site.Properties, + }) + } + + return providers, nil +} + +func convertServiceEndpointToOCMData(endpoint *meshdata.ServiceEndpoint) *ocmprovider.ServiceEndpoint { + return &ocmprovider.ServiceEndpoint{ + Type: &ocmprovider.ServiceType{ + Name: endpoint.Type.Name, + Description: endpoint.Type.Description, + }, + Name: endpoint.Name, + Path: endpoint.URL, + IsMonitored: endpoint.IsMonitored, + Properties: endpoint.Properties, + } +} diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index ec49dd484f..d4298a3c98 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -96,7 +96,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc } // Add the "main" service to the scrapes - addScrape(site, service.Host, &service.ServiceEndpoint) + addScrape(site, service.Host, service.ServiceEndpoint) for _, endpoint := range service.AdditionalEndpoints { if endpoint.IsMonitored { diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index e0f240653d..a032c426bb 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -20,7 +20,7 @@ package meshdata // Service represents a service managed by Mentix. type Service struct { - ServiceEndpoint + *ServiceEndpoint Host string AdditionalEndpoints []*ServiceEndpoint From 1f4d1b95e8177428f941b7523c86f0c6318a745d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 8 Jul 2020 16:55:33 +0200 Subject: [PATCH 08/12] Added missing comments for exported funcs/types --- pkg/mentix/config/ids.go | 8 ++++++-- pkg/mentix/connectors/connector.go | 2 ++ pkg/mentix/connectors/gocdb.go | 3 +++ pkg/mentix/exporters/cs3api.go | 4 ++++ pkg/mentix/exporters/cs3api/query.go | 2 +- pkg/mentix/exporters/exporter.go | 4 ++++ pkg/mentix/exporters/promfilesd.go | 4 ++++ pkg/mentix/exporters/reqexporter.go | 2 ++ pkg/mentix/exporters/webapi.go | 4 ++++ pkg/mentix/meshdata/properties.go | 6 +++++- 10 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 64caac7d50..0cc67af2fc 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -19,11 +19,15 @@ package config const ( + // Connector identifier for GOCDB. ConnectorIDGOCDB = "gocdb" ) const ( - ExporterIDWebAPI = "webapi" - ExporterIDCS3API = "cs3api" + // Export identifier for the WebAPI exporter. + ExporterIDWebAPI = "webapi" + // Export identifier for the CS3API exporter. + ExporterIDCS3API = "cs3api" + // Export identifier for the Prometheus File SD exporter. ExporterIDPrometheusFileSD = "prom_filesd" ) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index d5f3d6cd59..635ca0344a 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -49,6 +49,7 @@ type BaseConnector struct { log *zerolog.Logger } +// Activate activates the connector. func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { return fmt.Errorf("no configuration provided") @@ -63,6 +64,7 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo return nil } +// FindConnector searches for the given connector ID in all globally registered connectors. func FindConnector(connectorID string) (Connector, error) { for id, connector := range registeredConnectors { if strings.EqualFold(id, connectorID) { diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index fd64bd21b4..b144f26de4 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -39,6 +39,7 @@ type GOCDBConnector struct { gocdbAddress string } +// Activate activates the connector. func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { if err := connector.BaseConnector.Activate(conf, log); err != nil { return err @@ -53,6 +54,7 @@ func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerol return nil } +// RetrieveMeshData fetches new mesh data. func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) { meshData := new(meshdata.MeshData) @@ -252,6 +254,7 @@ func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint return svcURL, nil } +// GetName returns the display name of the connector. func (connector *GOCDBConnector) GetName() string { return "GOCDB" } diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exporters/cs3api.go index c6b986a02a..5e07dd1fab 100755 --- a/pkg/mentix/exporters/cs3api.go +++ b/pkg/mentix/exporters/cs3api.go @@ -28,10 +28,12 @@ import ( "github.com/cs3org/reva/pkg/mentix/exporters/cs3api" ) +// CS3APIExporter implements the CS3API exporter. type CS3APIExporter struct { BaseRequestExporter } +// Activate activates the exporter. func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { if err := exporter.BaseExporter.Activate(conf, log); err != nil { return err @@ -43,6 +45,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo return nil } +// HandleRequest handles the actual HTTP request. func (exporter *CS3APIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { // Data is read, so acquire a read lock exporter.locker.RLock() @@ -60,6 +63,7 @@ func (exporter *CS3APIExporter) HandleRequest(resp http.ResponseWriter, req *htt return nil } +// GetName returns the display name of the exporter. func (exporter *CS3APIExporter) GetName() string { return "CS3API" } diff --git a/pkg/mentix/exporters/cs3api/query.go b/pkg/mentix/exporters/cs3api/query.go index 12cff23a4e..5937262f8d 100755 --- a/pkg/mentix/exporters/cs3api/query.go +++ b/pkg/mentix/exporters/cs3api/query.go @@ -78,7 +78,7 @@ func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.Provi Host: service.Host, Endpoint: convertServiceEndpointToOCMData(service.ServiceEndpoint), AdditionalEndpoints: addEndpoints, - ApiVersion: meshdata.GetPropertyValue(service.Properties, "API_VERSION", ""), + ApiVersion: meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, ""), }) } diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go index f07baefceb..71a792ef82 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exporters/exporter.go @@ -57,6 +57,7 @@ type BaseExporter struct { locker sync.RWMutex } +// Activate activates the exporter. func (exporter *BaseExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { return fmt.Errorf("no configuration provided") @@ -71,14 +72,17 @@ func (exporter *BaseExporter) Activate(conf *config.Configuration, log *zerolog. return nil } +// Start starts the exporter; only exporters which perform periodical background tasks should do something here. func (exporter *BaseExporter) Start() error { return nil } +// Stop stops any running background activities of the exporter. func (exporter *BaseExporter) Stop() { } +// UpdateMeshData is called whenever the mesh data has changed to reflect these changes. func (exporter *BaseExporter) UpdateMeshData(meshData *meshdata.MeshData) error { // Update the stored mesh data if err := exporter.storeMeshData(meshData); err != nil { diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index d4298a3c98..01c5974a63 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -32,12 +32,14 @@ import ( "github.com/cs3org/reva/pkg/mentix/meshdata" ) +// PrometheusFileSDExporter implements the File Service Discovery for Prometheus exporter. type PrometheusFileSDExporter struct { BaseExporter outputFilename string } +// Activate activates the exporter. func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { if err := exporter.BaseExporter.Activate(conf, log); err != nil { return err @@ -57,6 +59,7 @@ func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration, l return nil } +// UpdateMeshData is called whenever the mesh data has changed to reflect these changes. func (exporter *PrometheusFileSDExporter) UpdateMeshData(meshData *meshdata.MeshData) error { if err := exporter.BaseExporter.UpdateMeshData(meshData); err != nil { return err @@ -142,6 +145,7 @@ func (exporter *PrometheusFileSDExporter) exportScrapeConfig(v interface{}) erro return nil } +// GetName returns the display name of the exporter. func (exporter *PrometheusFileSDExporter) GetName() string { return "Prometheus File SD" } diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go index 6a8d676e16..40621c1643 100644 --- a/pkg/mentix/exporters/reqexporter.go +++ b/pkg/mentix/exporters/reqexporter.go @@ -42,6 +42,7 @@ type BaseRequestExporter struct { endpoint string } +// Endpoint returns the (relative) endpoint of the exporter. func (exporter *BaseRequestExporter) Endpoint() string { // Ensure that the endpoint starts with a / endpoint := exporter.endpoint @@ -51,6 +52,7 @@ func (exporter *BaseRequestExporter) Endpoint() string { return strings.TrimSpace(endpoint) } +// WantsRequest returns whether the exporter wants to handle the incoming request. func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { return r.URL.Path == exporter.Endpoint() } diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index 939bb17739..b91288c8ae 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -28,10 +28,12 @@ import ( "github.com/cs3org/reva/pkg/mentix/exporters/webapi" ) +// WebAPIExporter implements the generic Web API exporter. type WebAPIExporter struct { BaseRequestExporter } +// Activate activates the exporter. func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { if err := exporter.BaseExporter.Activate(conf, log); err != nil { return err @@ -43,6 +45,7 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo return nil } +// HandleRequest handles the actual HTTP request. func (exporter *WebAPIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { // Data is read, so acquire a read lock exporter.locker.RLock() @@ -60,6 +63,7 @@ func (exporter *WebAPIExporter) HandleRequest(resp http.ResponseWriter, req *htt return nil } +// GetName returns the display name of the exporter. func (exporter *WebAPIExporter) GetName() string { return "WebAPI" } diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 2d6f1cf62f..7dfb8f7a57 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -21,8 +21,12 @@ package meshdata import "strings" const ( + // PropertyOrganization identifies the organization property. PropertyOrganization = "organization" - PropertyMetricsPath = "metrics_path" + // PropertyMetricsPath identifies the metrics path property. + PropertyMetricsPath = "metrics_path" + // PropertyAPIVersion identifies the API version property. + PropertyAPIVersion = "api_version" ) // GetPropertyValue performs a case-insensitive search for the given property. From 370bd6313bbf43dab2f53858dfc4a3b87876c2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 8 Jul 2020 16:57:26 +0200 Subject: [PATCH 09/12] Comment fixes --- pkg/mentix/config/ids.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 0cc67af2fc..6173cd2838 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -19,15 +19,15 @@ package config const ( - // Connector identifier for GOCDB. + // ConnectorIDGOCDB is the connector identifier for GOCDB. ConnectorIDGOCDB = "gocdb" ) const ( - // Export identifier for the WebAPI exporter. + // ExporterIDWebAPI is the identifier for the WebAPI exporter. ExporterIDWebAPI = "webapi" - // Export identifier for the CS3API exporter. + // ExporterIDCS3API is the identifier for the CS3API exporter. ExporterIDCS3API = "cs3api" - // Export identifier for the Prometheus File SD exporter. + // ExporterIDPrometheusFileSD is the identifier for the Prometheus File SD exporter. ExporterIDPrometheusFileSD = "prom_filesd" ) From df3dc5e7629014bf253f040898630b7f3f44d37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 14 Jul 2020 10:47:29 +0200 Subject: [PATCH 10/12] Added a site locations exporter --- .../config/http/services/mentix/_index.md | 3 +- .../http/services/mentix/siteloc/_index.md | 19 +++++ examples/mentix/mentix.toml | 7 +- internal/http/services/mentix/mentix.go | 4 + pkg/mentix/config/config.go | 4 + pkg/mentix/config/ids.go | 2 + pkg/mentix/connectors/gocdb.go | 2 + pkg/mentix/connectors/gocdb/types.go | 2 + pkg/mentix/exporters/cs3api/query.go | 2 +- pkg/mentix/exporters/siteloc/query.go | 75 +++++++++++++++++++ pkg/mentix/exporters/siteloc/types.go | 27 +++++++ pkg/mentix/exporters/sitelocations.go | 73 ++++++++++++++++++ pkg/mentix/meshdata/site.go | 2 + 13 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 docs/content/en/docs/config/http/services/mentix/siteloc/_index.md create mode 100755 pkg/mentix/exporters/siteloc/query.go create mode 100644 pkg/mentix/exporters/siteloc/types.go create mode 100755 pkg/mentix/exporters/sitelocations.go diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 5401c27462..eb3f65ec45 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -39,7 +39,8 @@ Supported values are: - **webapi** Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future. -- **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure. +- **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure. +- **siteloc** The Site Locations exporter `siteloc` exposes location information of all sites to be consumed by Grafana at the configured relative endpoint (see [here](siteloc)). - **prom_filesd** [Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)): ``` scrape_configs: diff --git a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md new file mode 100644 index 0000000000..1e71c66fbc --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md @@ -0,0 +1,19 @@ +--- +title: "siteloc" +linkTitle: "siteloc" +weight: 10 +description: > + Configuration for the Site Locations exporter of the Mentix service +--- + +{{% pageinfo %}} +The Site Locations exporter exposes location information of all sites to be consumed by Grafana via an HTTP endpoint. +{{% /pageinfo %}} + +{{% dir name="endpoint" type="string" default="/" %}} +The endpoint where the locations data can be queried. +{{< highlight toml >}} +[http.services.mentix.siteloc] +endpoint = "/loc" +{{< /highlight >}} +{{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 893b26a5d2..5e8c6da2d6 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -7,9 +7,9 @@ enabled_services = ["mentix"] [http.services.mentix] connector = "gocdb" -exporters = ["webapi", "cs3api"] +exporters = ["webapi", "cs3api", "siteloc"] # Enable the Prometheus File Service Discovery: -# exporters = ["webapi", "cs3api", "prom_filesd"] +# exporters = ["webapi", "cs3api", "siteloc", "prom_filesd"] update_interval = "15m" [http.services.mentix.gocdb] @@ -21,6 +21,9 @@ endpoint = "/" [http.services.mentix.cs3api] endpoint = "/cs3" +[http.services.mentix.siteloc] +endpoint = "/loc" + # Configure the Prometheus File Service Discovery: # [http.services.mentix.prom_filesd] # Prometheus must be configured to read the following file: diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 997adf774c..95b234c788 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -120,6 +120,10 @@ func applyDefaultConfig(conf *config.Configuration) { if conf.CS3API.Endpoint == "" { conf.CS3API.Endpoint = "/cs3" } + + if conf.SiteLocations.Endpoint == "" { + conf.SiteLocations.Endpoint = "/loc" + } } // New returns a new Mentix service. diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 211eedb5b0..2377feb4d7 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -39,6 +39,10 @@ type Configuration struct { Endpoint string `mapstructure:"endpoint"` } `yaml:"cs3api"` + SiteLocations struct { + Endpoint string `mapstructure:"endpoint"` + } `yaml:"siteloc"` + PrometheusFileSD struct { OutputFile string `mapstructure:"output_file"` } `mapstructure:"prom_filesd"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 6173cd2838..f951b1cbe9 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,6 +28,8 @@ const ( ExporterIDWebAPI = "webapi" // ExporterIDCS3API is the identifier for the CS3API exporter. ExporterIDCS3API = "cs3api" + // ExporterIDSiteLocations is the identifier for the Site Locations exporter. + ExporterIDSiteLocations = "siteloc" // ExporterIDPrometheusFileSD is the identifier for the Prometheus File SD exporter. ExporterIDPrometheusFileSD = "prom_filesd" ) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index b144f26de4..46ab6c0824 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -139,6 +139,8 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { Description: site.Description, Country: site.Country, CountryCode: site.CountryCode, + Longitude: site.Longitude, + Latitude: site.Latitude, Services: nil, Properties: properties, } diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index 9448b47f23..24712b2319 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -50,6 +50,8 @@ type Site struct { Domain string `xml:"DOMAIN>DOMAIN_NAME"` Country string `xml:"COUNTRY"` CountryCode string `xml:"COUNTRY_CODE"` + Latitude float32 `xml:"LATITUDE"` + Longitude float32 `xml:"LONGITUDE"` Extensions Extensions `xml:"EXTENSIONS"` } diff --git a/pkg/mentix/exporters/cs3api/query.go b/pkg/mentix/exporters/cs3api/query.go index 5937262f8d..367dc0b618 100755 --- a/pkg/mentix/exporters/cs3api/query.go +++ b/pkg/mentix/exporters/cs3api/query.go @@ -55,7 +55,7 @@ func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, // Marshal the OCM data as JSON data, err := json.MarshalIndent(ocmData, "", "\t") if err != nil { - return []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + return []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) } return data, nil diff --git a/pkg/mentix/exporters/siteloc/query.go b/pkg/mentix/exporters/siteloc/query.go new file mode 100755 index 0000000000..9c827331dc --- /dev/null +++ b/pkg/mentix/exporters/siteloc/query.go @@ -0,0 +1,75 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package siteloc + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + queryMethodDefault = "" +) + +// HandleQuery handles an HTTP request based on the provided 'method' parameter. +func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + method := params.Get("method") + switch strings.ToLower(method) { + case queryMethodDefault: + return handleDefaultQuery(meshData, params) + + default: + return []byte{}, fmt.Errorf("unknown API method '%v'", method) + } +} + +func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + // Convert the mesh data + locData, err := convertMeshDataToLocationData(meshData) + if err != nil { + return []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) + } + + // Marshal the location data as JSON + data, err := json.MarshalIndent(locData, "", "\t") + if err != nil { + return []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) + } + + return data, nil +} + +func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) { + // Gather the locations of all sites + locations := make([]*SiteLocation, 0, len(meshData.Sites)) + for _, site := range meshData.Sites { + locations = append(locations, &SiteLocation{ + Site: site.Name, + FullName: site.FullName, + Longitude: site.Longitude, + Latitude: site.Latitude, + }) + } + + return locations, nil +} diff --git a/pkg/mentix/exporters/siteloc/types.go b/pkg/mentix/exporters/siteloc/types.go new file mode 100644 index 0000000000..763e1ff944 --- /dev/null +++ b/pkg/mentix/exporters/siteloc/types.go @@ -0,0 +1,27 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package siteloc + +// SiteLocation represents the location information of a site. +type SiteLocation struct { + Site string `json:"key"` + FullName string `json:"name"` + Longitude float32 `json:"longitude"` + Latitude float32 `json:"latitude"` +} diff --git a/pkg/mentix/exporters/sitelocations.go b/pkg/mentix/exporters/sitelocations.go new file mode 100755 index 0000000000..d98499b126 --- /dev/null +++ b/pkg/mentix/exporters/sitelocations.go @@ -0,0 +1,73 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "fmt" + "net/http" + + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters/siteloc" +) + +// SiteLocationsExporter implements the Site Locations exporter to use with Grafana. +type SiteLocationsExporter struct { + BaseRequestExporter +} + +// Activate activates the exporter. +func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseExporter.Activate(conf, log); err != nil { + return err + } + + // Store SiteLocations specific settings + exporter.endpoint = conf.SiteLocations.Endpoint + + return nil +} + +// HandleRequest handles the actual HTTP request. +func (exporter *SiteLocationsExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + data, err := siteloc.HandleQuery(exporter.meshData, req.URL.Query()) + if err == nil { + if _, err := resp.Write(data); err != nil { + return fmt.Errorf("error writing the API request response: %v", err) + } + } else { + return fmt.Errorf("error while serving API request: %v", err) + } + + return nil +} + +// GetName returns the display name of the exporter. +func (exporter *SiteLocationsExporter) GetName() string { + return "Site Locations" +} + +func init() { + registerExporter(config.ExporterIDSiteLocations, &SiteLocationsExporter{}) +} diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 8c90ace8e9..f799030a3e 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -30,6 +30,8 @@ type Site struct { Country string CountryCode string Location string + Latitude float32 + Longitude float32 Services []*Service Properties map[string]string From 24199fa96bef5cf97909d6c18c2129d83a1b621d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 14 Jul 2020 11:03:38 +0200 Subject: [PATCH 11/12] Simplified the web request handling --- pkg/mentix/exporters/cs3api.go | 24 ++------------ pkg/mentix/exporters/cs3api/query.go | 20 ++---------- pkg/mentix/exporters/reqexporter.go | 45 +++++++++++++++++++++++++++ pkg/mentix/exporters/siteloc/query.go | 20 ++---------- pkg/mentix/exporters/sitelocations.go | 24 ++------------ pkg/mentix/exporters/webapi.go | 24 ++------------ pkg/mentix/exporters/webapi/query.go | 20 ++---------- 7 files changed, 57 insertions(+), 120 deletions(-) diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exporters/cs3api.go index 5e07dd1fab..38e946104e 100755 --- a/pkg/mentix/exporters/cs3api.go +++ b/pkg/mentix/exporters/cs3api.go @@ -19,9 +19,6 @@ package exporters import ( - "fmt" - "net/http" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -39,26 +36,9 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo return err } - // Store CS3API specific settings + // Store CS3API specifics exporter.endpoint = conf.CS3API.Endpoint - - return nil -} - -// HandleRequest handles the actual HTTP request. -func (exporter *CS3APIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - data, err := cs3api.HandleQuery(exporter.meshData, req.URL.Query()) - if err == nil { - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } - } else { - return fmt.Errorf("error while serving API request: %v", err) - } + exporter.defaultMethodHandler = cs3api.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exporters/cs3api/query.go b/pkg/mentix/exporters/cs3api/query.go index 367dc0b618..53e016e28b 100755 --- a/pkg/mentix/exporters/cs3api/query.go +++ b/pkg/mentix/exporters/cs3api/query.go @@ -22,30 +22,14 @@ import ( "encoding/json" "fmt" "net/url" - "strings" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryMethodDefault = "" -) - -// HandleQuery handles an HTTP request based on the provided 'method' parameter. -func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { - method := params.Get("method") - switch strings.ToLower(method) { - case queryMethodDefault: - return handleDefaultQuery(meshData, params) - - default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) - } -} - -func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +// HandleDefaultQuery processes a basic query. +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { // Convert the mesh data ocmData, err := convertMeshDataToOCMData(meshData) if err != nil { diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go index 40621c1643..4091952c08 100644 --- a/pkg/mentix/exporters/reqexporter.go +++ b/pkg/mentix/exporters/reqexporter.go @@ -19,10 +19,20 @@ package exporters import ( + "fmt" "net/http" + "net/url" "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + queryMethodDefault = "" ) +type queryCallback func(*meshdata.MeshData, url.Values) ([]byte, error) + // RequestExporter is the interface implemented by exporters that offer an HTTP endpoint. type RequestExporter interface { Exporter @@ -40,6 +50,8 @@ type BaseRequestExporter struct { BaseExporter endpoint string + + defaultMethodHandler queryCallback } // Endpoint returns the (relative) endpoint of the exporter. @@ -56,3 +68,36 @@ func (exporter *BaseRequestExporter) Endpoint() string { func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { return r.URL.Path == exporter.Endpoint() } + +// HandleRequest handles the actual HTTP request. +func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + data, err := exporter.handleQuery(exporter.meshData, req.URL.Query()) + if err == nil { + if _, err := resp.Write(data); err != nil { + return fmt.Errorf("error writing the API request response: %v", err) + } + } else { + return fmt.Errorf("error while serving API request: %v", err) + } + + return nil +} + +func (exporter *BaseRequestExporter) handleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + method := params.Get("method") + switch strings.ToLower(method) { + case queryMethodDefault: + if exporter.defaultMethodHandler != nil { + return exporter.defaultMethodHandler(meshData, params) + } + + default: + return []byte{}, fmt.Errorf("unknown API method '%v'", method) + } + + return []byte{}, fmt.Errorf("unhandled query for method '%v'", method) +} diff --git a/pkg/mentix/exporters/siteloc/query.go b/pkg/mentix/exporters/siteloc/query.go index 9c827331dc..a4907ac66a 100755 --- a/pkg/mentix/exporters/siteloc/query.go +++ b/pkg/mentix/exporters/siteloc/query.go @@ -22,28 +22,12 @@ import ( "encoding/json" "fmt" "net/url" - "strings" "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryMethodDefault = "" -) - -// HandleQuery handles an HTTP request based on the provided 'method' parameter. -func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { - method := params.Get("method") - switch strings.ToLower(method) { - case queryMethodDefault: - return handleDefaultQuery(meshData, params) - - default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) - } -} - -func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +// HandleDefaultQuery processes a basic query. +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { // Convert the mesh data locData, err := convertMeshDataToLocationData(meshData) if err != nil { diff --git a/pkg/mentix/exporters/sitelocations.go b/pkg/mentix/exporters/sitelocations.go index d98499b126..dde1fd5d69 100755 --- a/pkg/mentix/exporters/sitelocations.go +++ b/pkg/mentix/exporters/sitelocations.go @@ -19,9 +19,6 @@ package exporters import ( - "fmt" - "net/http" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -39,26 +36,9 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log return err } - // Store SiteLocations specific settings + // Store SiteLocations specifics exporter.endpoint = conf.SiteLocations.Endpoint - - return nil -} - -// HandleRequest handles the actual HTTP request. -func (exporter *SiteLocationsExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - data, err := siteloc.HandleQuery(exporter.meshData, req.URL.Query()) - if err == nil { - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } - } else { - return fmt.Errorf("error while serving API request: %v", err) - } + exporter.defaultMethodHandler = siteloc.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index b91288c8ae..71ff2c436e 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -19,9 +19,6 @@ package exporters import ( - "fmt" - "net/http" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -39,26 +36,9 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo return err } - // Store WebAPI specific settings + // Store WebAPI specifics exporter.endpoint = conf.WebAPI.Endpoint - - return nil -} - -// HandleRequest handles the actual HTTP request. -func (exporter *WebAPIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - data, err := webapi.HandleQuery(exporter.meshData, req.URL.Query()) - if err == nil { - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } - } else { - return fmt.Errorf("error while serving API request: %v", err) - } + exporter.defaultMethodHandler = webapi.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exporters/webapi/query.go b/pkg/mentix/exporters/webapi/query.go index 16d4bb6945..e413b8375c 100755 --- a/pkg/mentix/exporters/webapi/query.go +++ b/pkg/mentix/exporters/webapi/query.go @@ -22,28 +22,12 @@ import ( "encoding/json" "fmt" "net/url" - "strings" "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryMethodDefault = "" -) - -// HandleQuery handles an HTTP request based on the provided 'method' parameter. -func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { - method := params.Get("method") - switch strings.ToLower(method) { - case queryMethodDefault: - return handleDefaultQuery(meshData, params) - - default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) - } -} - -func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +// HandleDefaultQuery processes a basic query. +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { // Just return the plain, unfiltered data as JSON data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { From d40fb7afccc1e73b085ade17d735d26e4bdedd2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 14 Jul 2020 11:37:07 +0200 Subject: [PATCH 12/12] Added changelog file --- changelog/unreleased/mentix-site-locations.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/mentix-site-locations.md diff --git a/changelog/unreleased/mentix-site-locations.md b/changelog/unreleased/mentix-site-locations.md new file mode 100644 index 0000000000..56853f7433 --- /dev/null +++ b/changelog/unreleased/mentix-site-locations.md @@ -0,0 +1,5 @@ +Enhancement: Added a site locations exporter to Mentix. + +Mentix now offers an endpoint that exposes location information of all sites in the mesh. This can be used in Grafana's world map view to show the exact location of every site. + +https://github.com/cs3org/reva/pull/972