diff --git a/go.mod b/go.mod index eac42f779..d65b8cab4 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/procfs v0.0.7 // indirect golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 // indirect + gopkg.in/yaml.v2 v2.2.4 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 k8s.io/client-go v12.0.0+incompatible diff --git a/go.sum b/go.sum index a16370c12..a71b63875 100644 --- a/go.sum +++ b/go.sum @@ -817,6 +817,7 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 h1:UXl+Zk3jqqcbEVV7ace5lrt4YdA4tXiz3f/KbmD29Vo= google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -824,6 +825,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/pkg/controller/grafanadatasource/datasource_controller.go b/pkg/controller/grafanadatasource/datasource_controller.go index 38ed841cf..d6cc196a1 100644 --- a/pkg/controller/grafanadatasource/datasource_controller.go +++ b/pkg/controller/grafanadatasource/datasource_controller.go @@ -3,12 +3,18 @@ package grafanadatasource import ( "context" "crypto/md5" + defaultErrors "errors" "fmt" + "io" + "sort" + "time" + grafanav1alpha1 "github.com/integr8ly/grafana-operator/v3/pkg/apis/integreatly/v1alpha1" "github.com/integr8ly/grafana-operator/v3/pkg/controller/common" "github.com/integr8ly/grafana-operator/v3/pkg/controller/config" "github.com/integr8ly/grafana-operator/v3/pkg/controller/model" - "io" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -22,8 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/source" - "sort" - "time" ) const ( @@ -33,6 +37,13 @@ const ( var log = logf.Log.WithName(ControllerName) +// Data sources name list from CM. +type cmDataSourceList struct { + Datasources []struct { + Name string `yaml:"name"` + } `yaml:"datasources"` +} + // Add creates a new GrafanaDataSource Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager, _ chan schema.GroupVersionKind, namespace string) error { @@ -118,9 +129,10 @@ type ReconcileGrafanaDataSource struct { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (reconcile.Result, error) { + client, err := r.getClient() // Read the current state of known and cluster datasources currentState := common.NewDataSourcesState() - err := currentState.Read(r.context, r.client, request.Namespace) + err = currentState.Read(r.context, r.client, request.Namespace) if err != nil { return reconcile.Result{}, err } @@ -131,7 +143,7 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon } // Reconcile all data sources - err = r.reconcileDataSources(currentState) + err = r.reconcileDataSources(currentState, client) if err != nil { return reconcile.Result{}, err } @@ -139,7 +151,7 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon return reconcile.Result{Requeue: false}, nil } -func (r *ReconcileGrafanaDataSource) reconcileDataSources(state *common.DataSourcesState) error { +func (r *ReconcileGrafanaDataSource) reconcileDataSources(state *common.DataSourcesState, grafanaClient GrafanaClient) error { var dataSourcesToAddOrUpdate []grafanav1alpha1.GrafanaDataSource var dataSourcesToDelete []string @@ -168,10 +180,22 @@ func (r *ReconcileGrafanaDataSource) reconcileDataSources(state *common.DataSour } } - // apply dataSourcesToDelete + // apply dataSources config maps to delete. Can be multiple data sources in CM for _, ds := range dataSourcesToDelete { - log.Info(fmt.Sprintf("deleting datasource %v", ds)) + log.Info(fmt.Sprintf("deleting datasource config map %v", ds)) if state.KnownDataSources.Data != nil { + dsList, err := r.fetchDataSourceNames(state.KnownDataSources, ds) + if err != nil { + log.Error(err, fmt.Sprintf("error fetching datasources name from CM %v %v", ds, err)) + return err + } + for _, dsName := range dsList { + resp, err := grafanaClient.DeleteDataSourceByName(dsName) + if err != nil { + return defaultErrors.New(fmt.Sprintf(" %v error deleting datasource %v, ID %d message: %v", + err, dsName, resp.ID, resp.Message)) + } + } delete(state.KnownDataSources.Data, ds) } } @@ -282,3 +306,40 @@ func (r *ReconcileGrafanaDataSource) manageSuccess(datasources []grafanav1alpha1 } } } + +// Get an authenticated grafana API client +func (r *ReconcileGrafanaDataSource) getClient() (GrafanaClient, error) { + url := r.state.AdminUrl + if url == "" { + return nil, defaultErrors.New("cannot get grafana admin url") + } + + username := r.state.AdminUsername + if username == "" { + return nil, defaultErrors.New("invalid credentials (username)") + } + + password := r.state.AdminPassword + if password == "" { + return nil, defaultErrors.New("invalid credentials (password)") + } + + duration := time.Duration(r.state.ClientTimeout) + return NewGrafanaClient(url, username, password, duration), nil +} + +func (r *ReconcileGrafanaDataSource) fetchDataSourceNames(dsCM *v1.ConfigMap, dsKey string) ([]string, error) { + dsNameList := make([]string, 0) + tmpCMList := cmDataSourceList{} + + if dsYAML, ok := dsCM.Data[dsKey]; ok { + if err := yaml.Unmarshal([]byte(dsYAML), &tmpCMList); err != nil { + return dsNameList, err + } + for _, ds := range tmpCMList.Datasources { + dsNameList = append(dsNameList, ds.Name) + } + } + + return dsNameList, nil +} diff --git a/pkg/controller/grafanadatasource/grafana_client.go b/pkg/controller/grafanadatasource/grafana_client.go new file mode 100644 index 000000000..3d531b6ad --- /dev/null +++ b/pkg/controller/grafanadatasource/grafana_client.go @@ -0,0 +1,99 @@ +package grafanadatasource + +import ( + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "time" +) + +const ( + DeleteDataSourceByNameUrl = "%v/api/datasources/name/%v" +) + +type DataSourceDeleteResponse struct { + Message string `json:"message"` + ID int `json:"id"` +} + +type GrafanaClient interface { + DeleteDataSourceByName(dsName string) (DataSourceDeleteResponse, error) +} + +type GrafanaClientImpl struct { + url string + user string + password string + client *http.Client +} + +func setHeaders(req *http.Request) { + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "grafana-operator") +} + +func NewGrafanaClient(url, user, password string, timeoutSeconds time.Duration) GrafanaClient { + transport := http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + client := &http.Client{ + Transport: &transport, + Timeout: time.Second * timeoutSeconds, + } + + return &GrafanaClientImpl{ + url: url, + user: user, + password: password, + client: client, + } +} + +// Delete a datasource given by a Name +func (r *GrafanaClientImpl) DeleteDataSourceByName(dsName string) (DataSourceDeleteResponse, error) { + rawUrl := fmt.Sprintf(DeleteDataSourceByNameUrl, r.url, dsName) + response := DataSourceDeleteResponse{} + + parsed, err := url.Parse(rawUrl) + if err != nil { + return response, err + } + + parsed.User = url.UserPassword(r.user, r.password) + req, err := http.NewRequest("DELETE", parsed.String(), nil) + if err != nil { + return response, err + } + + setHeaders(req) + + resp, err := r.client.Do(req) + if err != nil { + return response, err + } + defer resp.Body.Close() + + // Skip 404 not found because data source can be deleted via UI + if resp.StatusCode != 200 && resp.StatusCode != 404 { + return response, errors.New(fmt.Sprintf( + "error deleting datasource, expected status 200 or 404 but got %v", + resp.StatusCode)) + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return response, err + } + + err = json.Unmarshal(data, &response) + + return response, err +}