From 5a9dab8b1c97190bce2da1c2e034b75c1fcc40d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= <61736220+Daniel-WWU-IT@users.noreply.github.com> Date: Thu, 16 Jul 2020 11:04:51 +0200 Subject: [PATCH] Added a site locations exporter to Mentix (#972) --- changelog/unreleased/mentix-site-locations.md | 5 ++ .../config/http/services/mentix/_index.md | 3 +- .../http/services/mentix/siteloc/_index.md | 19 ++++++ examples/mentix/mentix.toml | 7 ++- internal/http/services/mentix/mentix.go | 4 ++ pkg/mentix/config/config.go | 4 ++ pkg/mentix/config/ids.go | 2 + pkg/mentix/connectors/gocdb.go | 2 + pkg/mentix/connectors/gocdb/types.go | 2 + pkg/mentix/exporters/cs3api.go | 24 +------- pkg/mentix/exporters/cs3api/query.go | 22 +------ pkg/mentix/exporters/reqexporter.go | 45 ++++++++++++++ pkg/mentix/exporters/siteloc/query.go | 59 +++++++++++++++++++ pkg/mentix/exporters/siteloc/types.go | 27 +++++++++ pkg/mentix/exporters/sitelocations.go | 53 +++++++++++++++++ pkg/mentix/exporters/webapi.go | 24 +------- pkg/mentix/exporters/webapi/query.go | 20 +------ pkg/mentix/meshdata/site.go | 2 + 18 files changed, 240 insertions(+), 84 deletions(-) create mode 100644 changelog/unreleased/mentix-site-locations.md create mode 100644 docs/content/en/docs/config/http/services/mentix/siteloc/_index.md create mode 100755 pkg/mentix/exporters/siteloc/query.go create mode 100644 pkg/mentix/exporters/siteloc/types.go create mode 100755 pkg/mentix/exporters/sitelocations.go diff --git a/changelog/unreleased/mentix-site-locations.md b/changelog/unreleased/mentix-site-locations.md new file mode 100644 index 0000000000..56853f7433 --- /dev/null +++ b/changelog/unreleased/mentix-site-locations.md @@ -0,0 +1,5 @@ +Enhancement: Added a site locations exporter to Mentix. + +Mentix now offers an endpoint that exposes location information of all sites in the mesh. This can be used in Grafana's world map view to show the exact location of every site. + +https://github.com/cs3org/reva/pull/972 diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 5401c27462..eb3f65ec45 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -39,7 +39,8 @@ Supported values are: - **webapi** Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future. -- **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure. +- **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure. +- **siteloc** The Site Locations exporter `siteloc` exposes location information of all sites to be consumed by Grafana at the configured relative endpoint (see [here](siteloc)). - **prom_filesd** [Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)): ``` scrape_configs: diff --git a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md new file mode 100644 index 0000000000..1e71c66fbc --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md @@ -0,0 +1,19 @@ +--- +title: "siteloc" +linkTitle: "siteloc" +weight: 10 +description: > + Configuration for the Site Locations exporter of the Mentix service +--- + +{{% pageinfo %}} +The Site Locations exporter exposes location information of all sites to be consumed by Grafana via an HTTP endpoint. +{{% /pageinfo %}} + +{{% dir name="endpoint" type="string" default="/" %}} +The endpoint where the locations data can be queried. +{{< highlight toml >}} +[http.services.mentix.siteloc] +endpoint = "/loc" +{{< /highlight >}} +{{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 893b26a5d2..5e8c6da2d6 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -7,9 +7,9 @@ enabled_services = ["mentix"] [http.services.mentix] connector = "gocdb" -exporters = ["webapi", "cs3api"] +exporters = ["webapi", "cs3api", "siteloc"] # Enable the Prometheus File Service Discovery: -# exporters = ["webapi", "cs3api", "prom_filesd"] +# exporters = ["webapi", "cs3api", "siteloc", "prom_filesd"] update_interval = "15m" [http.services.mentix.gocdb] @@ -21,6 +21,9 @@ endpoint = "/" [http.services.mentix.cs3api] endpoint = "/cs3" +[http.services.mentix.siteloc] +endpoint = "/loc" + # Configure the Prometheus File Service Discovery: # [http.services.mentix.prom_filesd] # Prometheus must be configured to read the following file: diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 997adf774c..95b234c788 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -120,6 +120,10 @@ func applyDefaultConfig(conf *config.Configuration) { if conf.CS3API.Endpoint == "" { conf.CS3API.Endpoint = "/cs3" } + + if conf.SiteLocations.Endpoint == "" { + conf.SiteLocations.Endpoint = "/loc" + } } // New returns a new Mentix service. diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 211eedb5b0..2377feb4d7 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -39,6 +39,10 @@ type Configuration struct { Endpoint string `mapstructure:"endpoint"` } `yaml:"cs3api"` + SiteLocations struct { + Endpoint string `mapstructure:"endpoint"` + } `yaml:"siteloc"` + PrometheusFileSD struct { OutputFile string `mapstructure:"output_file"` } `mapstructure:"prom_filesd"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 6173cd2838..f951b1cbe9 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,6 +28,8 @@ const ( ExporterIDWebAPI = "webapi" // ExporterIDCS3API is the identifier for the CS3API exporter. ExporterIDCS3API = "cs3api" + // ExporterIDSiteLocations is the identifier for the Site Locations exporter. + ExporterIDSiteLocations = "siteloc" // ExporterIDPrometheusFileSD is the identifier for the Prometheus File SD exporter. ExporterIDPrometheusFileSD = "prom_filesd" ) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index b144f26de4..46ab6c0824 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -139,6 +139,8 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { Description: site.Description, Country: site.Country, CountryCode: site.CountryCode, + Longitude: site.Longitude, + Latitude: site.Latitude, Services: nil, Properties: properties, } diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index 9448b47f23..24712b2319 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -50,6 +50,8 @@ type Site struct { Domain string `xml:"DOMAIN>DOMAIN_NAME"` Country string `xml:"COUNTRY"` CountryCode string `xml:"COUNTRY_CODE"` + Latitude float32 `xml:"LATITUDE"` + Longitude float32 `xml:"LONGITUDE"` Extensions Extensions `xml:"EXTENSIONS"` } diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exporters/cs3api.go index 5e07dd1fab..38e946104e 100755 --- a/pkg/mentix/exporters/cs3api.go +++ b/pkg/mentix/exporters/cs3api.go @@ -19,9 +19,6 @@ package exporters import ( - "fmt" - "net/http" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -39,26 +36,9 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo return err } - // Store CS3API specific settings + // Store CS3API specifics exporter.endpoint = conf.CS3API.Endpoint - - return nil -} - -// HandleRequest handles the actual HTTP request. -func (exporter *CS3APIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - data, err := cs3api.HandleQuery(exporter.meshData, req.URL.Query()) - if err == nil { - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } - } else { - return fmt.Errorf("error while serving API request: %v", err) - } + exporter.defaultMethodHandler = cs3api.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exporters/cs3api/query.go b/pkg/mentix/exporters/cs3api/query.go index 5937262f8d..53e016e28b 100755 --- a/pkg/mentix/exporters/cs3api/query.go +++ b/pkg/mentix/exporters/cs3api/query.go @@ -22,30 +22,14 @@ import ( "encoding/json" "fmt" "net/url" - "strings" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryMethodDefault = "" -) - -// HandleQuery handles an HTTP request based on the provided 'method' parameter. -func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { - method := params.Get("method") - switch strings.ToLower(method) { - case queryMethodDefault: - return handleDefaultQuery(meshData, params) - - default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) - } -} - -func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +// HandleDefaultQuery processes a basic query. +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { // Convert the mesh data ocmData, err := convertMeshDataToOCMData(meshData) if err != nil { @@ -55,7 +39,7 @@ func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, // Marshal the OCM data as JSON data, err := json.MarshalIndent(ocmData, "", "\t") if err != nil { - return []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + return []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) } return data, nil diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go index 40621c1643..4091952c08 100644 --- a/pkg/mentix/exporters/reqexporter.go +++ b/pkg/mentix/exporters/reqexporter.go @@ -19,10 +19,20 @@ package exporters import ( + "fmt" "net/http" + "net/url" "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + queryMethodDefault = "" ) +type queryCallback func(*meshdata.MeshData, url.Values) ([]byte, error) + // RequestExporter is the interface implemented by exporters that offer an HTTP endpoint. type RequestExporter interface { Exporter @@ -40,6 +50,8 @@ type BaseRequestExporter struct { BaseExporter endpoint string + + defaultMethodHandler queryCallback } // Endpoint returns the (relative) endpoint of the exporter. @@ -56,3 +68,36 @@ func (exporter *BaseRequestExporter) Endpoint() string { func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { return r.URL.Path == exporter.Endpoint() } + +// HandleRequest handles the actual HTTP request. +func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + data, err := exporter.handleQuery(exporter.meshData, req.URL.Query()) + if err == nil { + if _, err := resp.Write(data); err != nil { + return fmt.Errorf("error writing the API request response: %v", err) + } + } else { + return fmt.Errorf("error while serving API request: %v", err) + } + + return nil +} + +func (exporter *BaseRequestExporter) handleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + method := params.Get("method") + switch strings.ToLower(method) { + case queryMethodDefault: + if exporter.defaultMethodHandler != nil { + return exporter.defaultMethodHandler(meshData, params) + } + + default: + return []byte{}, fmt.Errorf("unknown API method '%v'", method) + } + + return []byte{}, fmt.Errorf("unhandled query for method '%v'", method) +} diff --git a/pkg/mentix/exporters/siteloc/query.go b/pkg/mentix/exporters/siteloc/query.go new file mode 100755 index 0000000000..a4907ac66a --- /dev/null +++ b/pkg/mentix/exporters/siteloc/query.go @@ -0,0 +1,59 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package siteloc + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +// HandleDefaultQuery processes a basic query. +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + // Convert the mesh data + locData, err := convertMeshDataToLocationData(meshData) + if err != nil { + return []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) + } + + // Marshal the location data as JSON + data, err := json.MarshalIndent(locData, "", "\t") + if err != nil { + return []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) + } + + return data, nil +} + +func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) { + // Gather the locations of all sites + locations := make([]*SiteLocation, 0, len(meshData.Sites)) + for _, site := range meshData.Sites { + locations = append(locations, &SiteLocation{ + Site: site.Name, + FullName: site.FullName, + Longitude: site.Longitude, + Latitude: site.Latitude, + }) + } + + return locations, nil +} diff --git a/pkg/mentix/exporters/siteloc/types.go b/pkg/mentix/exporters/siteloc/types.go new file mode 100644 index 0000000000..763e1ff944 --- /dev/null +++ b/pkg/mentix/exporters/siteloc/types.go @@ -0,0 +1,27 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package siteloc + +// SiteLocation represents the location information of a site. +type SiteLocation struct { + Site string `json:"key"` + FullName string `json:"name"` + Longitude float32 `json:"longitude"` + Latitude float32 `json:"latitude"` +} diff --git a/pkg/mentix/exporters/sitelocations.go b/pkg/mentix/exporters/sitelocations.go new file mode 100755 index 0000000000..dde1fd5d69 --- /dev/null +++ b/pkg/mentix/exporters/sitelocations.go @@ -0,0 +1,53 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters/siteloc" +) + +// SiteLocationsExporter implements the Site Locations exporter to use with Grafana. +type SiteLocationsExporter struct { + BaseRequestExporter +} + +// Activate activates the exporter. +func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseExporter.Activate(conf, log); err != nil { + return err + } + + // Store SiteLocations specifics + exporter.endpoint = conf.SiteLocations.Endpoint + exporter.defaultMethodHandler = siteloc.HandleDefaultQuery + + return nil +} + +// GetName returns the display name of the exporter. +func (exporter *SiteLocationsExporter) GetName() string { + return "Site Locations" +} + +func init() { + registerExporter(config.ExporterIDSiteLocations, &SiteLocationsExporter{}) +} diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index b91288c8ae..71ff2c436e 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -19,9 +19,6 @@ package exporters import ( - "fmt" - "net/http" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -39,26 +36,9 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo return err } - // Store WebAPI specific settings + // Store WebAPI specifics exporter.endpoint = conf.WebAPI.Endpoint - - return nil -} - -// HandleRequest handles the actual HTTP request. -func (exporter *WebAPIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - data, err := webapi.HandleQuery(exporter.meshData, req.URL.Query()) - if err == nil { - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } - } else { - return fmt.Errorf("error while serving API request: %v", err) - } + exporter.defaultMethodHandler = webapi.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exporters/webapi/query.go b/pkg/mentix/exporters/webapi/query.go index 16d4bb6945..e413b8375c 100755 --- a/pkg/mentix/exporters/webapi/query.go +++ b/pkg/mentix/exporters/webapi/query.go @@ -22,28 +22,12 @@ import ( "encoding/json" "fmt" "net/url" - "strings" "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryMethodDefault = "" -) - -// HandleQuery handles an HTTP request based on the provided 'method' parameter. -func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { - method := params.Get("method") - switch strings.ToLower(method) { - case queryMethodDefault: - return handleDefaultQuery(meshData, params) - - default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) - } -} - -func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +// HandleDefaultQuery processes a basic query. +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { // Just return the plain, unfiltered data as JSON data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 8c90ace8e9..f799030a3e 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -30,6 +30,8 @@ type Site struct { Country string CountryCode string Location string + Latitude float32 + Longitude float32 Services []*Service Properties map[string]string