From 3742e2be310e670cef175e010a275f5daa53d670 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Thu, 23 Jan 2025 11:09:03 +0100 Subject: [PATCH 1/3] Add prometheusSecret into the Spec for prometheus configuration In order to configure Prometheus datasource in Watcher we are using a secret which will provide the required connection data. This secret will be provided by the telemetry operator at some point although could be any secret which has the expected fields, host, port and ca_secret and ca_key. This patch adds the new field into the spec, validates that it provides the required fields and copy from Watcher to SubCRs. It also adds a watcher on the secret in the main controller (also adds the watcher on the top level .Spec.secret which was misssing). --- .../watcher.openstack.org_watcherapis.yaml | 4 + ...watcher.openstack.org_watcherappliers.yaml | 4 + ....openstack.org_watcherdecisionengines.yaml | 4 + api/bases/watcher.openstack.org_watchers.yaml | 4 + api/v1beta1/common_types.go | 5 + api/v1beta1/conditions.go | 2 + .../watcher.openstack.org_watcherapis.yaml | 4 + ...watcher.openstack.org_watcherappliers.yaml | 4 + ....openstack.org_watcherdecisionengines.yaml | 4 + .../bases/watcher.openstack.org_watchers.yaml | 4 + controllers/watcher_common.go | 12 +- controllers/watcher_controller.go | 131 ++++++++++++++++++ tests/functional/base_test.go | 1 + tests/functional/watcher_controller_test.go | 71 ++++++++++ 14 files changed, 253 insertions(+), 1 deletion(-) diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index 1c25bb1..27c8fbc 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -123,6 +123,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string replicas: default: 1 description: Replicas of Watcher service to run diff --git a/api/bases/watcher.openstack.org_watcherappliers.yaml b/api/bases/watcher.openstack.org_watcherappliers.yaml index f55ce34..611cdbb 100644 --- a/api/bases/watcher.openstack.org_watcherappliers.yaml +++ b/api/bases/watcher.openstack.org_watcherappliers.yaml @@ -78,6 +78,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string replicas: default: 1 description: Replicas of Watcher service to run diff --git a/api/bases/watcher.openstack.org_watcherdecisionengines.yaml b/api/bases/watcher.openstack.org_watcherdecisionengines.yaml index 782ceec..1bb18bf 100644 --- a/api/bases/watcher.openstack.org_watcherdecisionengines.yaml +++ b/api/bases/watcher.openstack.org_watcherdecisionengines.yaml @@ -79,6 +79,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string replicas: default: 1 description: Replicas of Watcher service to run diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index 0cf1cb3..2ab4939 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -221,6 +221,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string rabbitMqClusterName: default: rabbitmq description: |- diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index c760c30..e59cbb7 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -64,6 +64,11 @@ type WatcherCommon struct { // to /etc//.conf.d directory as a custom config file. CustomServiceConfig string `json:"customServiceConfig,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=metric-storage-prometheus-config + // Secret containing prometheus connection parameters + PrometheusSecret string `json:"prometheusSecret"` + // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec // TLS - Parameters related to the TLS diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 3beed3f..6f8e9fb 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -24,4 +24,6 @@ const ( WatcherAPIReadyMessage = "WatcherAPI successfully created" // WatcherAPIReadyErrorMessage - WatcherAPIReadyErrorMessage = "WatcherAPI error occured %s" + // WatcherPrometheusSecretErrorMessage - + WatcherPrometheusSecretErrorMessage = "Error with prometheus config secret" ) diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index 1c25bb1..27c8fbc 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -123,6 +123,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string replicas: default: 1 description: Replicas of Watcher service to run diff --git a/config/crd/bases/watcher.openstack.org_watcherappliers.yaml b/config/crd/bases/watcher.openstack.org_watcherappliers.yaml index f55ce34..611cdbb 100644 --- a/config/crd/bases/watcher.openstack.org_watcherappliers.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherappliers.yaml @@ -78,6 +78,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string replicas: default: 1 description: Replicas of Watcher service to run diff --git a/config/crd/bases/watcher.openstack.org_watcherdecisionengines.yaml b/config/crd/bases/watcher.openstack.org_watcherdecisionengines.yaml index 782ceec..1bb18bf 100644 --- a/config/crd/bases/watcher.openstack.org_watcherdecisionengines.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherdecisionengines.yaml @@ -79,6 +79,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string replicas: default: 1 description: Replicas of Watcher service to run diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index 0cf1cb3..2ab4939 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -221,6 +221,10 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusSecret: + default: metric-storage-prometheus-config + description: Secret containing prometheus connection parameters + type: string rabbitMqClusterName: default: rabbitmq description: |- diff --git a/controllers/watcher_common.go b/controllers/watcher_common.go index b0b3b80..f622bb3 100644 --- a/controllers/watcher_common.go +++ b/controllers/watcher_common.go @@ -24,7 +24,8 @@ import ( ) const ( - passwordSecretField = ".spec.secret" + passwordSecretField = ".spec.secret" + prometheusSecretField = ".spec.prometheusSecret" ) var ( @@ -34,6 +35,10 @@ var ( applierWatchFields = []string{ passwordSecretField, } + watcherWatchFields = []string{ + passwordSecretField, + prometheusSecretField, + } ) const ( @@ -48,6 +53,11 @@ const ( // DatabaseUsername is the name of key in the secret for the database // hostname DatabaseHostname = "database_hostname" + // Prometheus configuration keys in prometheusSecret + PrometheusHost = "host" + PrometheusPort = "port" + PrometheusCaCertSecret = "ca_secret" + PrometheusCaCertKey = "ca_key" // WatcherAPILabelPrefix - a unique, service binary specific prefix for the // labels the WatcherAPI controller uses on children objects diff --git a/controllers/watcher_controller.go b/controllers/watcher_controller.go index d012b2e..94b2e44 100644 --- a/controllers/watcher_controller.go +++ b/controllers/watcher_controller.go @@ -26,10 +26,16 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -215,6 +221,7 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re // end of TransportURL creation // Check we have the required inputs + // Top level secret hash, _, inputSecret, err := ensureSecret( ctx, types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, @@ -230,6 +237,7 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return ctrl.Result{}, errors.New("error retrieving required data from secret") } + // TransportURL Secret hashTransporturl, _, transporturlSecret, err := ensureSecret( ctx, types.NamespacedName{Namespace: instance.Namespace, Name: transportURL.Status.SecretName}, @@ -245,6 +253,44 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return ctrl.Result{}, errors.New("error retrieving required data from transporturl secret") } + // Prometheus config secret + + hashPrometheus, _, prometheusSecret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.PrometheusSecret}, + []string{ + PrometheusHost, + PrometheusPort, + }, + helper.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if err != nil || hashPrometheus == "" { + // Empty hash means that there is some problem retrieving the key from the secret + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityWarning, + watcherv1beta1.WatcherPrometheusSecretErrorMessage)) + return ctrl.Result{}, errors.New("error retrieving required data from prometheus secret") + } + + // Add finalizer to prometheus config secret to prevent it from being deleted now that we're using it + if controllerutil.AddFinalizer(&prometheusSecret, helper.GetFinalizer()) { + err := helper.GetClient().Update(ctx, &prometheusSecret) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityWarning, + watcherv1beta1.WatcherPrometheusSecretErrorMessage)) + return ctrl.Result{}, err + } + } + + // End of Prometheus config secret + subLevelSecretName, err := r.createSubLevelSecret(ctx, helper, instance, transporturlSecret, inputSecret, db) if err != nil { return ctrl.Result{}, nil @@ -787,6 +833,9 @@ func (r *WatcherReconciler) ensureAPI( // We need to have TLS defined in SubCRs to have some values available watcherAPISpec.TLS = instance.Spec.TLS + // We need to have the PrometheusSecret in watcherapi + watcherAPISpec.PrometheusSecret = instance.Spec.PrometheusSecret + apiDeployment := &watcherv1beta1.WatcherAPI{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-api", instance.Name), @@ -862,6 +911,24 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch } } + // Remove the finalizer from our Prometheus Secret + prometheusSecret := &corev1.Secret{} + reader := helper.GetClient() + err = reader.Get(ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.PrometheusSecret}, + prometheusSecret) + + if err == nil { + if controllerutil.RemoveFinalizer(prometheusSecret, helper.GetFinalizer()) { + err = helper.GetClient().Update(ctx, prometheusSecret) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from prometheus config secret", instance) + } + } + // + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) return ctrl.Result{}, nil @@ -869,6 +936,31 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch // SetupWithManager sets up the controller with the Manager. func (r *WatcherReconciler) SetupWithManager(mgr ctrl.Manager) error { + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &watcherv1beta1.Watcher{}, passwordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*watcherv1beta1.Watcher) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index prometheusSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &watcherv1beta1.Watcher{}, prometheusSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*watcherv1beta1.Watcher) + if cr.Spec.PrometheusSecret == "" { + return nil + } + return []string{cr.Spec.PrometheusSecret} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&watcherv1beta1.Watcher{}). Owns(&watcherv1beta1.WatcherAPI{}). @@ -883,5 +975,44 @@ func (r *WatcherReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rbacv1.RoleBinding{}). Owns(&batchv1.Job{}). Owns(&corev1.Secret{}). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } + +func (r *WatcherReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("Watcher") + + for _, field := range watcherWatchFields { + crList := &watcherv1beta1.WatcherList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index f9c62b0..1185090 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -53,6 +53,7 @@ func GetNonDefaultWatcherSpec() map[string]interface{} { "tls": map[string]interface{}{ "caBundleSecretName": "combined-ca-bundle", }, + "prometheusSecret": "custom-prometheus-config", } } diff --git a/tests/functional/watcher_controller_test.go b/tests/functional/watcher_controller_test.go index c0b9b7a..0fb0fb6 100644 --- a/tests/functional/watcher_controller_test.go +++ b/tests/functional/watcher_controller_test.go @@ -53,7 +53,9 @@ var _ = Describe("Watcher controller with minimal spec values", func() { Expect(Watcher.Spec.PreserveJobs).Should(BeFalse()) Expect(Watcher.Spec.TLS.CaBundleSecretName).Should(Equal("")) Expect(Watcher.Spec.CustomServiceConfig).Should(Equal("")) + Expect(Watcher.Spec.PrometheusSecret).Should(Equal("metric-storage-prometheus-config")) Expect(Watcher.Spec.APIServiceTemplate.CustomServiceConfig).Should(Equal("")) + }) It("should have the Status fields initialized", func() { @@ -210,6 +212,14 @@ var _ = Describe("Watcher controller", func() { ), ) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) + DeferCleanup( + k8sClient.Delete, ctx, th.CreateSecret( + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "metric-storage-prometheus-config"}, + map[string][]byte{ + "host": []byte("prometheus.example.com"), + "port": []byte("9090"), + }, + )) }) It("Should set DBReady Condition Status when DB is Created", func() { @@ -360,6 +370,7 @@ var _ = Describe("Watcher controller", func() { Expect(int(*WatcherAPI.Spec.Replicas)).To(Equal(1)) Expect(WatcherAPI.Spec.NodeSelector).To(BeNil()) Expect(WatcherAPI.Spec.CustomServiceConfig).To(Equal("")) + Expect(WatcherAPI.Spec.PrometheusSecret).Should(Equal("metric-storage-prometheus-config")) // Assert that the watcher deployment is created deployment := th.GetDeployment(watcherTest.WatcherAPIDeployment) @@ -598,6 +609,7 @@ var _ = Describe("Watcher controller", func() { ) }) }) + When("Watcher with non-default values are created", func() { BeforeEach(func() { DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, GetNonDefaultWatcherSpec())) @@ -620,6 +632,14 @@ var _ = Describe("Watcher controller", func() { ), ) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) + DeferCleanup( + k8sClient.Delete, ctx, th.CreateSecret( + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "custom-prometheus-config"}, + map[string][]byte{ + "host": []byte("customprometheus.example.com"), + "port": []byte("9092"), + }, + )) }) It("should have the Spec fields with the expected values", func() { @@ -632,6 +652,7 @@ var _ = Describe("Watcher controller", func() { Expect(*(Watcher.Spec.RabbitMqClusterName)).Should(Equal("rabbitmq")) Expect(Watcher.Spec.TLS.CaBundleSecretName).Should(Equal("combined-ca-bundle")) Expect(Watcher.Spec.CustomServiceConfig).Should(Equal("# Global config")) + Expect(Watcher.Spec.PrometheusSecret).Should(Equal("custom-prometheus-config")) Expect(Watcher.Spec.APIServiceTemplate.CustomServiceConfig).Should(Equal("# Service config")) }) @@ -785,6 +806,7 @@ var _ = Describe("Watcher controller", func() { Expect(*WatcherAPI.Spec.NodeSelector).To(Equal(map[string]string{"foo": "bar"})) Expect(WatcherAPI.Spec.TLS.CaBundleSecretName).Should(Equal("combined-ca-bundle")) Expect(WatcherAPI.Spec.CustomServiceConfig).Should(Equal("# Service config")) + Expect(WatcherAPI.Spec.PrometheusSecret).Should(Equal("custom-prometheus-config")) // Assert that the watcher deployment is created deployment := th.GetDeployment(watcherTest.WatcherAPIDeployment) @@ -804,4 +826,53 @@ var _ = Describe("Watcher controller", func() { }) + When("The prometheus secret does not exist", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, GetNonDefaultWatcherSpec())) + DeferCleanup(k8sClient.Delete, ctx, CreateWatcherMessageBusSecret(watcherTest.Instance.Namespace, "rabbitmq-secret")) + memcachedSpec := memcachedv1.MemcachedSpec{ + MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ + Replicas: ptr.To(int32(1)), + }, + } + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.Watcher.Namespace, MemcachedInstance, memcachedSpec)) + infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + watcherTest.Instance.Namespace, + *GetWatcher(watcherTest.Instance).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + DeferCleanup( + k8sClient.Delete, ctx, th.CreateSecret( + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: SecretName}, + map[string][]byte{ + "WatcherPassword": []byte("password"), + }, + )) + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) + mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount) + mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName) + infra.SimulateTransportURLReady(watcherTest.WatcherTransportURL) + + }) + + It("Should set Input Ready to False", func() { + // Input Ready (secrets) + th.ExpectConditionWithDetails( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + condition.RequestedReason, + "Error with prometheus config secret", + ) + }) + + }) + }) From aecb03d9d2415a26ad158d9a22310bfb29d1946c Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Thu, 23 Jan 2025 14:05:42 +0100 Subject: [PATCH 2/3] Manage prometheus configuration in watcherapi controller This patch includes the prometheus configuration data from the provided secret into the config files in the watcherapi controller. --- controllers/watcher_common.go | 1 + controllers/watcherapi_controller.go | 73 ++++++++++++++++++- .../watcherdecisionengine_controller.go | 12 +++ pkg/watcher/constants.go | 3 + pkg/watcherapi/deployment.go | 24 ++++++ templates/watcher/config/00-default.conf | 9 +++ tests/functional/watcher_controller_test.go | 8 +- tests/functional/watcher_test_data.go | 5 ++ .../functional/watcherapi_controller_test.go | 65 +++++++++++++++++ 9 files changed, 194 insertions(+), 6 deletions(-) diff --git a/controllers/watcher_common.go b/controllers/watcher_common.go index f622bb3..9e4a81e 100644 --- a/controllers/watcher_common.go +++ b/controllers/watcher_common.go @@ -31,6 +31,7 @@ const ( var ( apiWatchFields = []string{ passwordSecretField, + prometheusSecretField, } applierWatchFields = []string{ passwordSecretField, diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index f67e36c..e7bb971 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -18,8 +18,10 @@ package controllers import ( "context" + "errors" "fmt" "net/url" + "path/filepath" "strings" "time" @@ -176,6 +178,31 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) configVars[instance.Spec.Secret] = env.SetValue(secretHash) + // Prometheus config secret + + hashPrometheus, _, prometheusSecret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.PrometheusSecret}, + []string{ + PrometheusHost, + PrometheusPort, + }, + helper.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if err != nil || hashPrometheus == "" { + // Empty hash means that there is some problem retrieving the key from the secret + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityWarning, + watcherv1beta1.WatcherPrometheusSecretErrorMessage)) + return ctrl.Result{}, errors.New("error retrieving required data from prometheus secret") + } + + configVars[instance.Spec.PrometheusSecret] = env.SetValue(hashPrometheus) + // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) @@ -198,7 +225,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - err = r.generateServiceConfigs(ctx, instance, secret, memcached, helper, &configVars) + err = r.generateServiceConfigs(ctx, instance, secret, prometheusSecret, memcached, helper, &configVars) if err != nil { return ctrl.Result{}, err } @@ -226,7 +253,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - result, err = r.createDeployment(ctx, helper, instance, inputHash) + result, err = r.createDeployment(ctx, helper, instance, prometheusSecret, inputHash) if err != nil { return result, err } @@ -260,6 +287,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) func (r *WatcherAPIReconciler) generateServiceConfigs( ctx context.Context, instance *watcherv1beta1.WatcherAPI, secret corev1.Secret, + prometheusSecret corev1.Secret, memcachedInstance *memcachedv1.Memcached, helper *helper.Helper, envVars *map[string]env.Setter, ) error { @@ -305,6 +333,16 @@ func (r *WatcherAPIReconciler) generateServiceConfigs( databaseHostname := string(secret.Data[DatabaseHostname]) databasePassword := string(secret.Data[DatabasePassword]) + prometheusHost := string(prometheusSecret.Data[PrometheusHost]) + prometheusPort := string(prometheusSecret.Data[PrometheusPort]) + prometheusCaCertSecret := string(prometheusSecret.Data[PrometheusCaCertSecret]) + prometheusCaCertKey := string(prometheusSecret.Data[PrometheusCaCertKey]) + + var prometheusCaCertPath string + if prometheusCaCertSecret != "" && prometheusCaCertKey != "" { + prometheusCaCertPath = filepath.Join(watcher.PrometheusCaCertFolderPath, prometheusCaCertKey) + } + var CaFilePath string if instance.Spec.TLS.CaBundleSecretName != "" { CaFilePath = tls.DownstreamTLSCABundlePath @@ -326,6 +364,9 @@ func (r *WatcherAPIReconciler) generateServiceConfigs( "LogFile": fmt.Sprintf("%s%s.log", watcher.WatcherLogPath, instance.Name), "APIPublicPort": fmt.Sprintf("%d", watcher.WatcherPublicPort), "CaFilePath": CaFilePath, + "PrometheusHost": prometheusHost, + "PrometheusPort": prometheusPort, + "PrometheusCaCertPath": prometheusCaCertPath, } // create httpd vhost template parameters @@ -346,13 +387,26 @@ func (r *WatcherAPIReconciler) createDeployment( ctx context.Context, helper *helper.Helper, instance *watcherv1beta1.WatcherAPI, + prometheusSecret corev1.Secret, configHash string, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Defining WatcherAPI deployment '%s'", instance.Name)) + // If there prometheus config is providing CA cert a volume must be mounted + prometheusCaCertSecret := string(prometheusSecret.Data[PrometheusCaCertSecret]) + prometheusCaCertKey := string(prometheusSecret.Data[PrometheusCaCertKey]) + prometheusCaCert := make(map[string]string) + if prometheusCaCertSecret != "" && prometheusCaCertKey != "" { + prometheusCaCert = map[string]string{ + "casecret_key": prometheusCaCertKey, + "casecret_name": prometheusCaCertSecret, + } + + } + // define a new Deployment object - deploymentDef, err := watcherapi.Deployment(instance, configHash, getAPIServiceLabels()) + deploymentDef, err := watcherapi.Deployment(instance, configHash, prometheusCaCert, getAPIServiceLabels()) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, @@ -362,6 +416,7 @@ func (r *WatcherAPIReconciler) createDeployment( err.Error())) return ctrl.Result{}, err } + Log.Info(fmt.Sprintf("Getting deployment '%s'", instance.Name)) deploymentObject := deployment.NewDeployment(deploymentDef, time.Duration(5)*time.Second) Log.Info(fmt.Sprintf("Got deployment '%s'", instance.Name)) @@ -611,6 +666,18 @@ func (r *WatcherAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + // index prometheusSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &watcherv1beta1.WatcherAPI{}, prometheusSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*watcherv1beta1.WatcherAPI) + if cr.Spec.PrometheusSecret == "" { + return nil + } + return []string{cr.Spec.PrometheusSecret} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&watcherv1beta1.WatcherAPI{}). Owns(&corev1.Secret{}). diff --git a/controllers/watcherdecisionengine_controller.go b/controllers/watcherdecisionengine_controller.go index cce4ba6..3ef6cae 100644 --- a/controllers/watcherdecisionengine_controller.go +++ b/controllers/watcherdecisionengine_controller.go @@ -231,6 +231,18 @@ func (r *WatcherDecisionEngineReconciler) SetupWithManager(mgr ctrl.Manager) err return err } + // index prometheusSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &watcherv1beta1.WatcherDecisionEngine{}, prometheusSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*watcherv1beta1.WatcherDecisionEngine) + if cr.Spec.PrometheusSecret == "" { + return nil + } + return []string{cr.Spec.PrometheusSecret} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&watcherv1beta1.WatcherDecisionEngine{}). Owns(&corev1.Secret{}). diff --git a/pkg/watcher/constants.go b/pkg/watcher/constants.go index 36a7eb5..51e17b0 100644 --- a/pkg/watcher/constants.go +++ b/pkg/watcher/constants.go @@ -40,4 +40,7 @@ const ( // ConfigVolume is the default volume name used to mount service config ConfigVolume = "config-data" + + // Path to deploy the Prometheus CaCert if needed + PrometheusCaCertFolderPath = "/etc/pki/ca-trust/extracted/pem/prometheus/" ) diff --git a/pkg/watcherapi/deployment.go b/pkg/watcherapi/deployment.go index 7581573..3b042c8 100644 --- a/pkg/watcherapi/deployment.go +++ b/pkg/watcherapi/deployment.go @@ -1,6 +1,8 @@ package watcherapi import ( + "path/filepath" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -20,6 +22,7 @@ const ( func Deployment( instance *watcherv1beta1.WatcherAPI, configHash string, + prometheusCaCertSecret map[string]string, labels map[string]string, ) (*appsv1.Deployment, error) { @@ -71,6 +74,27 @@ func Deployment( apiVolumeMounts = append(apiVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) } + if len(prometheusCaCertSecret) != 0 { + apiVolumes = append(apiVolumes, + corev1.Volume{ + Name: "custom-prometheus-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: prometheusCaCertSecret["casecret_name"], + }, + }, + }, + ) + apiVolumeMounts = append(apiVolumeMounts, + corev1.VolumeMount{ + Name: "custom-prometheus-ca", + MountPath: filepath.Join(watcher.PrometheusCaCertFolderPath, prometheusCaCertSecret["casecret_key"]), + SubPath: prometheusCaCertSecret["casecret_key"], + ReadOnly: true, + }, + ) + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, diff --git a/templates/watcher/config/00-default.conf b/templates/watcher/config/00-default.conf index be77d48..d02d6f6 100644 --- a/templates/watcher/config/00-default.conf +++ b/templates/watcher/config/00-default.conf @@ -68,3 +68,12 @@ memcache_servers={{ .MemcachedServersWithInet }} enabled=true tls_enabled={{ .MemcachedTLS }} {{end}} + +{{ if (index . "PrometheusHost") }} +[prometheus_client] +host = {{ .PrometheusHost }} +port = {{ .PrometheusPort }} +{{ if (index . "PrometheusCaCertPath") }} +cafile = {{ .PrometheusCaCertPath }} +{{ end }} +{{ end }} diff --git a/tests/functional/watcher_controller_test.go b/tests/functional/watcher_controller_test.go index 0fb0fb6..9d456ba 100644 --- a/tests/functional/watcher_controller_test.go +++ b/tests/functional/watcher_controller_test.go @@ -636,8 +636,10 @@ var _ = Describe("Watcher controller", func() { k8sClient.Delete, ctx, th.CreateSecret( types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "custom-prometheus-config"}, map[string][]byte{ - "host": []byte("customprometheus.example.com"), - "port": []byte("9092"), + "host": []byte("customprometheus.example.com"), + "port": []byte("9092"), + "ca_secret": []byte("combined-ca-bundle"), + "ca_key": []byte("internal-ca-bundle.pem"), }, )) }) @@ -812,7 +814,7 @@ var _ = Describe("Watcher controller", func() { deployment := th.GetDeployment(watcherTest.WatcherAPIDeployment) Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal("watcher-watcher")) Expect(int(*deployment.Spec.Replicas)).To(Equal(2)) - Expect(deployment.Spec.Template.Spec.Volumes).To(HaveLen(4)) + Expect(deployment.Spec.Template.Spec.Volumes).To(HaveLen(5)) Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(2)) Expect(deployment.Spec.Selector.MatchLabels).To(Equal(map[string]string{"service": "watcher-api"})) diff --git a/tests/functional/watcher_test_data.go b/tests/functional/watcher_test_data.go index a385f54..bf041df 100644 --- a/tests/functional/watcher_test_data.go +++ b/tests/functional/watcher_test_data.go @@ -38,6 +38,7 @@ type WatcherTestData struct { WatcherDatabaseAccount types.NamespacedName WatcherDatabaseAccountSecret types.NamespacedName InternalTopLevelSecretName types.NamespacedName + PrometheusSecretName types.NamespacedName WatcherTransportURL types.NamespacedName KeystoneServiceName types.NamespacedName WatcherAPI types.NamespacedName @@ -85,6 +86,10 @@ func GetWatcherTestData(watcherName types.NamespacedName) WatcherTestData { Namespace: watcherName.Namespace, Name: "test-osp-secret", }, + PrometheusSecretName: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "metric-storage-prometheus-config", + }, RabbitMqClusterName: "rabbitmq", WatcherTransportURL: types.NamespacedName{ Namespace: watcherName.Namespace, diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go index 7fc8543..6a07395 100644 --- a/tests/functional/watcherapi_controller_test.go +++ b/tests/functional/watcherapi_controller_test.go @@ -34,6 +34,7 @@ var _ = Describe("WatcherAPI controller with minimal spec values", func() { Expect(WatcherAPI.Spec.Secret).Should(Equal("osp-secret")) Expect(WatcherAPI.Spec.MemcachedInstance).Should(Equal("memcached")) Expect(WatcherAPI.Spec.PasswordSelectors).Should(Equal(watcherv1beta1.PasswordSelector{Service: "WatcherPassword"})) + Expect(WatcherAPI.Spec.PrometheusSecret).Should(Equal("metric-storage-prometheus-config")) }) It("should have the Status fields initialized", func() { @@ -62,6 +63,7 @@ var _ = Describe("WatcherAPI controller", func() { WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) Expect(WatcherAPI.Spec.Secret).Should(Equal("test-osp-secret")) Expect(WatcherAPI.Spec.MemcachedInstance).Should(Equal("memcached")) + Expect(WatcherAPI.Spec.PrometheusSecret).Should(Equal("metric-storage-prometheus-config")) }) It("should have the Status fields initialized", func() { @@ -119,6 +121,14 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) + prometheusSecret := th.CreateSecret( + watcherTest.PrometheusSecretName, + map[string][]byte{ + "host": []byte("prometheus.example.com"), + "port": []byte("9090"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, prometheusSecret) DeferCleanup( mariadb.DeleteDBService, mariadb.CreateDBService( @@ -301,6 +311,14 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) + prometheusSecret := th.CreateSecret( + watcherTest.PrometheusSecretName, + map[string][]byte{ + "host": []byte("prometheus.example.com"), + "port": []byte("9090"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, prometheusSecret) DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) @@ -323,6 +341,37 @@ var _ = Describe("WatcherAPI controller", func() { ) }) }) + When("prometheus config secret is not created", func() { + BeforeEach(func() { + secret := th.CreateSecret( + watcherTest.InternalTopLevelSecretName, + map[string][]byte{ + "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), + "database_username": []byte("username"), + "database_password": []byte("password"), + "database_hostname": []byte("hostname"), + "database_account": []byte("watcher"), + "01-global-custom.conf": []byte(""), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, secret) + + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) + }) + + It("should have input ready false", func() { + th.ExpectConditionWithDetails( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + condition.RequestedReason, + watcherv1beta1.WatcherPrometheusSecretErrorMessage, + ) + }) + }) + When("secret, db and memcached are created, but there is no keystoneapi", func() { BeforeEach(func() { secret := th.CreateSecret( @@ -338,6 +387,14 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) + prometheusSecret := th.CreateSecret( + watcherTest.PrometheusSecretName, + map[string][]byte{ + "host": []byte("prometheus.example.com"), + "port": []byte("9090"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, prometheusSecret) memcachedSpec := memcachedv1.MemcachedSpec{ MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ Replicas: ptr.To(int32(1)), @@ -391,6 +448,14 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) + prometheusSecret := th.CreateSecret( + watcherTest.PrometheusSecretName, + map[string][]byte{ + "host": []byte("prometheus.example.com"), + "port": []byte("9090"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, prometheusSecret) spec := GetDefaultWatcherAPISpec() apiOverrideSpec := map[string]interface{}{} endpoint := map[string]interface{}{} From 5bb5ffd588386d5d32d5d1da39ff8c321c1397b7 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Thu, 23 Jan 2025 16:33:29 +0100 Subject: [PATCH 3/3] Add kuttl tests for prometheus configuration This patch adds some required prometheus secret for kuttl jobs to run and adds validation checks. Also, it modifies the watcher_deploy make target to create a secret so that the watcher can be deployed until the secret is created by the telemetry operator. --- Makefile | 5 +++ ci/olm.sh | 12 +++++++ config/samples/watcher_requirements.yaml | 10 ++++++ .../default/common/cleanup-errors.yaml | 10 ++++++ .../default/common/cleanup-watcher.yaml | 6 ++++ .../default/common/deploy-with-defaults.yaml | 11 ++++++ .../kuttl/test-suites/default/deps/infra.yaml | 2 +- .../default/deps/kustomization.yaml | 3 ++ .../test-suites/default/deps/telemetry.yaml | 35 +++++++++++++++++++ .../01-deploy-with-defaults.yaml | 11 ++++++ .../default/watcher/01-assert.yaml | 9 +++++ .../default/watcher/04-assert.yaml | 9 +++++ .../04-deploy-with-precreated-account.yaml | 12 +++++++ 13 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 config/samples/watcher_requirements.yaml create mode 100644 tests/kuttl/test-suites/default/deps/telemetry.yaml diff --git a/Makefile b/Makefile index 5f7ff3d..cd969a5 100644 --- a/Makefile +++ b/Makefile @@ -381,15 +381,20 @@ watcher: export CATALOG_IMG=${CATALOG_IMAGE} watcher: ## Install watcher operator via olm bash ci/olm.sh oc apply -f ci/olm.yaml + timeout 300s bash -c "while ! (oc get csv -n openshift-operators -l operators.coreos.com/cluster-observability-operator.openshift-operators -o jsonpath='{.items[*].status.phase}' | grep Succeeded); do sleep 10; done" timeout 300s bash -c "while ! (oc get csv -n openstack-operators -l operators.coreos.com/watcher-operator.openstack-operators -o jsonpath='{.items[*].status.phase}' | grep Succeeded); do sleep 1; done" .PHONY: watcher_deploy watcher_deploy: ## Deploy watcher service + oc apply -f config/samples/watcher_requirements.yaml oc apply -f config/samples/watcher_v1beta1_watcher.yaml + oc wait watcher watcher --for condition=Ready --timeout=600s .PHONY: watcher_deploy_cleanup watcher_deploy_cleanup: ## Undeploy watcher service oc delete -f config/samples/watcher_v1beta1_watcher.yaml + oc delete -f config/samples/watcher_requirements.yaml + timeout 300s bash -c "while (oc get watcher watcher); do sleep 10; done" .PHONY: watcher_cleanup watcher_cleanup: export CATALOG_IMG=${CATALOG_IMAGE} diff --git a/ci/olm.sh b/ci/olm.sh index 2c71c2e..77cd8a8 100644 --- a/ci/olm.sh +++ b/ci/olm.sh @@ -1,5 +1,17 @@ cat > ci/olm.yaml <