From 983cab0b362ccfce41676920af66badea157694f Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Thu, 22 Feb 2024 12:25:18 +0100 Subject: [PATCH] [tlse] TLS database connection The my.cnf file gets added to the secret holding the service configs. The content of my.cnf is centrally managed in the mariadb-operator and retrieved calling db.GetDatabaseClientConfig(tlsCfg) Depends-On: https://github.com/openstack-k8s-operators/mariadb-operator/pull/190 Depends-On: https://github.com/openstack-k8s-operators/mariadb-operator/pull/191 Jira: OSPRH-4547 --- controllers/heat_controller.go | 147 ++++++++++++---------- controllers/heatapi_controller.go | 15 ++- controllers/heatcfnapi_controller.go | 15 ++- controllers/heatengine_controller.go | 15 ++- pkg/heat/volumes.go | 12 ++ templates/heat/bin/init.sh | 2 +- tests/functional/heat_controller_test.go | 18 ++- tests/kuttl/tests/heat_tls/01-assert.yaml | 24 ++++ 8 files changed, 179 insertions(+), 69 deletions(-) diff --git a/controllers/heat_controller.go b/controllers/heat_controller.go index ef89e707..bbf1e511 100644 --- a/controllers/heat_controller.go +++ b/controllers/heat_controller.go @@ -25,6 +25,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/job" "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/fields" @@ -498,6 +499,13 @@ func (r *HeatReconciler) reconcileNormal(ctx context.Context, instance *heatv1be instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + db, result, err := r.ensureDB(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + // // Create ConfigMaps and Secrets required as input for the Service and calculate an overall hash of hashes // @@ -508,7 +516,7 @@ func (r *HeatReconciler) reconcileNormal(ctx context.Context, instance *heatv1be // - %-config configmap holding minimal heat config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached) + err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached, db) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -665,66 +673,6 @@ func (r *HeatReconciler) reconcileInit(ctx context.Context, serviceLabels map[string]string, ) (ctrl.Result, error) { r.Log.Info("Reconciling Heat init") - // - // create service DB instance - // - db := mariadbv1.NewDatabaseWithNamespace( - heat.DatabaseName, - instance.Spec.DatabaseUser, - instance.Spec.Secret, - map[string]string{ - "dbName": instance.Spec.DatabaseInstance, - }, - "heat", - instance.Namespace, - ) - // create or patch the DB - ctrlResult, err := db.CreateOrPatchDBByName( - ctx, - helper, - instance.Spec.DatabaseInstance, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DBReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err - } - if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DBReadyRunningMessage)) - return ctrlResult, nil - } - // wait for the DB to be setup - ctrlResult, err = db.WaitForDBCreated(ctx, helper) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DBReadyErrorMessage, - err.Error())) - return ctrlResult, err - } - if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DBReadyRunningMessage)) - return ctrlResult, nil - } - // update Status.DatabaseHostname, used to bootstrap/config the service - instance.Status.DatabaseHostname = db.GetDatabaseHostname() - instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) - - // create service DB - end // // run Heat db sync @@ -738,7 +686,7 @@ func (r *HeatReconciler) reconcileInit(ctx context.Context, time.Second*10, dbSyncHash, ) - ctrlResult, err = dbSyncjob.DoJob( + ctrlResult, err := dbSyncjob.DoJob( ctx, helper, ) @@ -883,6 +831,7 @@ func (r *HeatReconciler) generateServiceConfigMaps( h *helper.Helper, envVars *map[string]env.Setter, mc *memcachedv1.Memcached, + db *mariadbv1.Database, ) error { // // create Configmap/Secret required for heat input @@ -893,11 +842,19 @@ func (r *HeatReconciler) generateServiceConfigMaps( cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(heat.ServiceName), map[string]string{}) + var tlsCfg *tls.Service + if instance.Spec.HeatAPI.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + // customData hold any customization for the service. // custom.conf is going to /etc/heat/heat.conf.d // all other files get placed into /etc/heat to allow overwrite of e.g. policy.json // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{ + common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig, + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data } @@ -1102,3 +1059,67 @@ func (r *HeatReconciler) getHeatMemcached( } return memcached, err } + +func (r *HeatReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *heatv1beta1.Heat, +) (*mariadbv1.Database, ctrl.Result, error) { + db := mariadbv1.NewDatabaseWithNamespace( + heat.DatabaseName, + instance.Spec.DatabaseUser, + instance.Spec.Secret, + map[string]string{ + "dbName": instance.Spec.DatabaseInstance, + }, + "heat", + instance.Namespace, + ) + // create or patch the DB + ctrlResult, err := db.CreateOrPatchDBByName( + ctx, + h, + instance.Spec.DatabaseInstance, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + // wait for the DB to be setup + ctrlResult, err = db.WaitForDBCreated(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + return db, ctrlResult, nil +} diff --git a/controllers/heatapi_controller.go b/controllers/heatapi_controller.go index 7f463797..0ea8d160 100644 --- a/controllers/heatapi_controller.go +++ b/controllers/heatapi_controller.go @@ -55,6 +55,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" ) // HeatAPIReconciler reconciles a Heat object @@ -820,10 +821,22 @@ func (r *HeatAPIReconciler) generateServiceConfigMaps( cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(heat.ServiceName), map[string]string{}) + db, err := mariadbv1.GetDatabaseByName(ctx, h, heat.DatabaseName) + if err != nil { + return err + } + var tlsCfg *tls.Service + if instance.Spec.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + // customData hold any customization for the service. // custom.conf is going to /etc/heat/heat.conf.d // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{ + common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig, + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data diff --git a/controllers/heatcfnapi_controller.go b/controllers/heatcfnapi_controller.go index e33a74a1..46805a88 100644 --- a/controllers/heatcfnapi_controller.go +++ b/controllers/heatcfnapi_controller.go @@ -55,6 +55,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" ) // HeatCfnAPIReconciler reconciles a Heat object @@ -824,10 +825,22 @@ func (r *HeatCfnAPIReconciler) generateServiceConfigMaps( cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(heat.CfnServiceName), map[string]string{}) + db, err := mariadbv1.GetDatabaseByName(ctx, h, heat.DatabaseName) + if err != nil { + return err + } + var tlsCfg *tls.Service + if instance.Spec.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + // customData hold any customization for the service. // custom.conf is going to /etc/heat/heat.conf.d // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{ + common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig, + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data diff --git a/controllers/heatengine_controller.go b/controllers/heatengine_controller.go index 0b3b9ecb..38073329 100644 --- a/controllers/heatengine_controller.go +++ b/controllers/heatengine_controller.go @@ -51,6 +51,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" ) // HeatEngineReconciler reconciles a Heat object @@ -542,10 +543,22 @@ func (r *HeatEngineReconciler) generateServiceConfigMaps( cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(heat.ServiceName), map[string]string{}) + db, err := mariadbv1.GetDatabaseByName(ctx, h, heat.DatabaseName) + if err != nil { + return err + } + var tlsCfg *tls.Service + if instance.Spec.TLS.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + // customData hold any customization for the service. // custom.conf is going to /etc/heat/heat.conf.d // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{ + common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig, + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data diff --git a/pkg/heat/volumes.go b/pkg/heat/volumes.go index 9f400378..39346707 100644 --- a/pkg/heat/volumes.go +++ b/pkg/heat/volumes.go @@ -74,6 +74,12 @@ func GetInitVolumeMounts() []corev1.VolumeMount { MountPath: "/var/lib/config-data/merged", ReadOnly: false, }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, } } @@ -91,6 +97,12 @@ func GetVolumeMounts() []corev1.VolumeMount { MountPath: "/var/lib/config-data/merged", ReadOnly: false, }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, } } diff --git a/templates/heat/bin/init.sh b/templates/heat/bin/init.sh index fd356207..2a2dac86 100755 --- a/templates/heat/bin/init.sh +++ b/templates/heat/bin/init.sh @@ -73,7 +73,7 @@ if [ -n "$AUTH_ENCRYPTION_KEY" ]; then crudini --set ${SVC_CFG_MERGED} DEFAULT auth_encryption_key $AUTH_ENCRYPTION_KEY fi -crudini --set ${SVC_CFG_MERGED} database connection mysql+pymysql://${DBUSER}:${DBPASSWORD}@${DBHOST}/${DB} +crudini --set ${SVC_CFG_MERGED} database connection mysql+pymysql://${DBUSER}:${DBPASSWORD}@${DBHOST}/${DB}?read_default_file=/etc/my.cnf crudini --set ${SVC_CFG_MERGED} keystone_authtoken password $PASSWORD crudini --set ${SVC_CFG_MERGED} DEFAULT stack_domain_admin_password $PASSWORD crudini --set ${SVC_CFG_MERGED} trustee password $PASSWORD diff --git a/tests/functional/heat_controller_test.go b/tests/functional/heat_controller_test.go index 71c2da1b..fbaec59e 100644 --- a/tests/functional/heat_controller_test.go +++ b/tests/functional/heat_controller_test.go @@ -246,7 +246,7 @@ var _ = Describe("Heat controller", func() { heatName, ConditionGetterFunc(HeatConditionGetter), condition.ServiceConfigReadyCondition, - corev1.ConditionFalse, + corev1.ConditionUnknown, ) th.ExpectCondition( heatName, @@ -277,6 +277,18 @@ var _ = Describe("Heat controller", func() { keystoneAPIName := keystone.CreateKeystoneAPI(namespace) keystoneAPI = keystone.GetKeystoneAPI(keystoneAPIName) DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPIName) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + namespace, + GetHeat(heatName).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + mariadb.SimulateMariaDBAccountCompleted(heatName) + mariadb.SimulateMariaDBDatabaseCompleted(heatName) }) It("should have service config ready", func() { @@ -300,7 +312,7 @@ var _ = Describe("Heat controller", func() { ) }) - It("should create a ConfigMap for heat.conf", func() { + It("should create a ConfigMap for heat.conf and my.cnf", func() { cm := th.GetConfigMap(heatConfigMapName) Expect(cm.Data["heat.conf"]).Should( @@ -315,6 +327,8 @@ var _ = Describe("Heat controller", func() { ContainSubstring("memcache_servers=memcached-0.memcached:11211,memcached-1.memcached:11211,memcached-2.memcached:11211")) Expect(cm.Data["heat.conf"]).Should( ContainSubstring("memcached_servers=inet:[memcached-0.memcached]:11211,inet:[memcached-1.memcached]:11211,inet:[memcached-2.memcached]:11211")) + Expect(cm.Data["my.cnf"]).To( + ContainSubstring("[client]\nssl=0")) }) }) diff --git a/tests/kuttl/tests/heat_tls/01-assert.yaml b/tests/kuttl/tests/heat_tls/01-assert.yaml index baf4d05d..ecb5eba8 100644 --- a/tests/kuttl/tests/heat_tls/01-assert.yaml +++ b/tests/kuttl/tests/heat_tls/01-assert.yaml @@ -95,6 +95,10 @@ spec: readOnly: true - mountPath: /var/lib/config-data/merged name: config-data-merged + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/kolla/config_files/config.json name: config-data-merged readOnly: true @@ -131,6 +135,10 @@ spec: readOnly: true - mountPath: /var/lib/config-data/merged name: config-data-merged + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/config-data/custom name: config-data-custom readOnly: true @@ -184,6 +192,10 @@ spec: readOnly: true - mountPath: /var/lib/config-data/merged name: config-data-merged + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/kolla/config_files/config.json name: config-data-merged readOnly: true @@ -220,6 +232,10 @@ spec: readOnly: true - mountPath: /var/lib/config-data/merged name: config-data-merged + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/config-data/custom name: config-data-custom readOnly: true @@ -273,6 +289,10 @@ spec: readOnly: true - mountPath: /var/lib/config-data/merged name: config-data-merged + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/kolla/config_files/config.json name: config-data-merged readOnly: true @@ -293,6 +313,10 @@ spec: readOnly: true - mountPath: /var/lib/config-data/merged name: config-data-merged + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/config-data/custom name: config-data-custom readOnly: true