[tlse] TLS database connection

moves requesting the DB before rendering the service configuration.
The my.cnf file gets added to the secret holding the service configs.
The content of my.cnf is centrally managed in the mariadb-operator
and retrieved calling db.GetDatabaseClientConfig(tlsCfg)

Depends-On: openstack-k8s-operators/mariadb-operator#190
Depends-On: openstack-k8s-operators/mariadb-operator#191

Jira: OSPRH-4547

controllers/placementapi_controller.go

@@ -349,7 +349,14 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	// all our input checks out so report InputReady
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
 
-	err = r.generateServiceConfigMaps(ctx, h, instance, secret, &configMapVars)
+	db, result, err := r.ensureDB(ctx, h, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	} else if (result != ctrl.Result{}) {
+		return result, nil
+	}
+
+	err = r.generateServiceConfigMaps(ctx, h, instance, secret, &configMapVars, db)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -425,11 +432,6 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return result, err
 	}
 
-	result, err = r.ensureDB(ctx, h, instance)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-
 	apiEndpoints, result, err := r.ensureServiceExposed(ctx, h, instance)
 
 	if (err != nil || result != ctrl.Result{}) {
@@ -971,7 +973,7 @@ func (r *PlacementAPIReconciler) ensureDB(
 	ctx context.Context,
 	h *helper.Helper,
 	instance *placementv1.PlacementAPI,
-) (ctrl.Result, error) {
+) (*mariadbv1.Database, ctrl.Result, error) {
 	// (ksambor) should we use  NewDatabaseWithNamespace instead?
 	db := mariadbv1.NewDatabaseWithNamespace(
 		placement.DatabaseName,
@@ -996,15 +998,15 @@ func (r *PlacementAPIReconciler) ensureDB(
 			condition.SeverityWarning,
 			condition.DBReadyErrorMessage,
 			err.Error()))
-		return ctrl.Result{}, err
+		return db, ctrl.Result{}, err
 	}
 	if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DBReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			condition.DBReadyRunningMessage))
-		return ctrlResult, nil
+		return db, ctrlResult, nil
 	}
 	// wait for the DB to be setup
 	// (ksambor) should we use WaitForDBCreatedWithTimeout instead?
@@ -1016,21 +1018,21 @@ func (r *PlacementAPIReconciler) ensureDB(
 			condition.SeverityWarning,
 			condition.DBReadyErrorMessage,
 			err.Error()))
-		return ctrlResult, err
+		return db, ctrlResult, err
 	}
 	if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DBReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			condition.DBReadyRunningMessage))
-		return ctrlResult, nil
+		return db, ctrlResult, nil
 	}
 
 	// update Status.DatabaseHostname, used to config the service
 	instance.Status.DatabaseHostname = db.GetDatabaseHostname()
 	instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
-	return ctrlResult, nil
+	return db, ctrlResult, nil
 }
 
 func (r *PlacementAPIReconciler) ensureDbSync(
@@ -1174,6 +1176,7 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 	instance *placementv1.PlacementAPI,
 	ospSecret corev1.Secret,
 	envVars *map[string]env.Setter,
+	db *mariadbv1.Database,
 ) error {
 	//
 	// create Secret required for placement input
@@ -1184,10 +1187,19 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 
 	cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(placement.ServiceName), map[string]string{})
 
+	var tlsCfg *tls.Service
+	if instance.Spec.TLS.Ca.CaBundleSecretName != "" {
+		tlsCfg = &tls.Service{}
+	}
+
 	// customData hold any customization for the service.
 	// custom.conf is going to /etc/<service>/<service>.conf.d
+	// my.cnf is going to /etc/my.cnf
 	// all other files get placed into /etc/<service> to allow overwrite of e.g. policy.json
-	customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
+	customData := map[string]string{
+		common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig,
+		"my.cnf":                           db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
+	}
 	for key, data := range instance.Spec.DefaultConfigOverwrite {
 		customData[key] = data
 	}
@@ -1209,11 +1221,13 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 		"KeystoneInternalURL": keystoneInternalURL,
 		"KeystonePublicURL":   keystonePublicURL,
 		"PlacementPassword":   string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]),
-		"DBUser":              instance.Spec.DatabaseUser,
-		"DBPassword":          string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
-		"DBAddress":           instance.Status.DatabaseHostname,
-		"DBName":              placement.DatabaseName,
 		"log_file":            "/var/log/placement/placement-api.log",
+		"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
+			instance.Spec.DatabaseUser,
+			string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
+			instance.Status.DatabaseHostname,
+			placement.DatabaseName,
+		),
 	}
 
 	// create httpd  vhost template parameters

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240219094943-9bbb46c9afba
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240216173409-86913e6d5885
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885
-	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798
+	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e
 	github.com/openstack-k8s-operators/placement-operator/api v0.3.1-0.20240216174613-3d349f26e681
 	go.uber.org/zap v1.26.0
 	k8s.io/api v0.28.6

go.sum

@@ -99,8 +99,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.2024021
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:8QsCFttAm+X6A8I8EQThGjNjeMAYt2hK7ivbvnR3434=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885 h1:ioJ2MO3vAcBkLM+0UBu5IuKW/DPXcyiNSOLq0Xvn+Nw=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:82nzS+DbBe1tzaMvNHH8FctmZzQ14ZAJysFGsMJiivo=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798 h1:zL4DdQ5HPXCLHeRMAWC2zI7ypbkZVYg3UkyEFSnzeow=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e h1:6vqp5HZwcGvPH0MII/23iCd97T3/1HJZlONKW6LyNio=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

templates/placementapi/config/placement-api-config.json

@@ -47,6 +47,12 @@
             "owner": "placement",
             "perm": "0600",
             "optional": true
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "placement",
+            "perm": "0644"
         }
     ],
     "permissions": [

templates/placementapi/config/placement-dbsync-config.json

@@ -12,6 +12,12 @@
             "dest": "/etc/placement/placement.conf.d/custom.conf",
             "owner": "placement",
             "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "placement",
+            "perm": "0644"
         }
     ]
 }

templates/placementapi/config/placement.conf

@@ -9,7 +9,7 @@ log_file = {{ .log_file }}
 debug = true
 
 [placement_database]
-connection = mysql+pymysql://{{ .DBUser }}:{{ .DBPassword }}@{{ .DBAddress }}/{{ .DBName }}
+connection = {{ .DatabaseConnection }}
 
 [api]
 auth_strategy = keystone

tests/functional/placementapi_controller_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package functional_test
 
 import (
+	"fmt"
 	"os"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -200,6 +201,17 @@ var _ = Describe("PlacementAPI controller", func() {
 			)
 			DeferCleanup(
 				k8sClient.Delete, ctx, CreatePlacementAPISecret(namespace, SecretName))
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 		})
 
 		It("should have input ready", func() {
@@ -239,15 +251,62 @@ var _ = Describe("PlacementAPI controller", func() {
 			DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPIName)
 		})
 
+		It("creates MariaDB database", func() {
+			th.ExpectCondition(
+				names.PlacementAPIName,
+				ConditionGetterFunc(PlacementConditionGetter),
+				condition.DBReadyCondition,
+				corev1.ConditionFalse,
+			)
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
+			th.ExpectCondition(
+				names.PlacementAPIName,
+				ConditionGetterFunc(PlacementConditionGetter),
+				condition.DBReadyCondition,
+				corev1.ConditionTrue,
+			)
+		})
+
 		It("should have config ready", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			th.ExpectCondition(
 				names.PlacementAPIName,
 				ConditionGetterFunc(PlacementConditionGetter),
 				condition.ServiceConfigReadyCondition,
 				corev1.ConditionTrue,
 			)
 		})
+
 		It("should create a configuration Secret", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			cm := th.GetSecret(names.ConfigMapName)
 
 			conf := cm.Data["placement.conf"]
@@ -260,18 +319,28 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(conf).Should(
 				ContainSubstring("password = 12345678"))
 			Expect(conf).Should(
-				ContainSubstring("connection = mysql+pymysql://placement:12345678@/placement"))
-
+				ContainSubstring(fmt.Sprintf("connection = mysql+pymysql://placement:12345678@hostname-for-openstack.%s.svc/placement?read_default_file=/etc/my.cnf", namespace)))
 			custom := cm.Data["custom.conf"]
 			Expect(custom).Should(ContainSubstring("foo = bar"))
 
 			policy := cm.Data["policy.yaml"]
 			Expect(policy).Should(
 				ContainSubstring("\"placement:resource_providers:list\": \"!\""))
 
+			myCnf := cm.Data["my.cnf"]
+			Expect(myCnf).To(
+				ContainSubstring("[client]\nssl=0"))
 		})
 
 		It("creates service account, role and rolebindig", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			th.ExpectCondition(
 				names.PlacementAPIName,
 				ConditionGetterFunc(PlacementConditionGetter),
@@ -302,33 +371,6 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(binding.Subjects).To(HaveLen(1))
 			Expect(binding.Subjects[0].Name).To(Equal(sa.Name))
 		})
-
-		It("creates MariaDB database", func() {
-			th.ExpectCondition(
-				names.PlacementAPIName,
-				ConditionGetterFunc(PlacementConditionGetter),
-				condition.DBReadyCondition,
-				corev1.ConditionFalse,
-			)
-
-			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
-			DeferCleanup(
-				mariadb.DeleteDBService,
-				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
-			)
-			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
-			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
-
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
-			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
-
-			th.ExpectCondition(
-				names.PlacementAPIName,
-				ConditionGetterFunc(PlacementConditionGetter),
-				condition.DBReadyCondition,
-				corev1.ConditionTrue,
-			)
-		})
 		It("creates keystone service", func() {
 			th.ExpectCondition(
 				names.PlacementAPIName,
@@ -757,7 +799,7 @@ var _ = Describe("PlacementAPI controller", func() {
 				mariadb.DeleteDBService,
 				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
 			)
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(names.MariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
 			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)
@@ -795,6 +837,10 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile   \"/etc/pki/tls/private/internal.key\""))
 			Expect(configData).Should(ContainSubstring("SSLCertificateFile      \"/etc/pki/tls/certs/public.crt\""))
 			Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile   \"/etc/pki/tls/private/public.key\""))
+
+			configData = string(configDataMap.Data["my.cnf"])
+			Expect(configData).To(
+				ContainSubstring("[client]\nssl-ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nssl=1"))
 		})
 	})
 })
@@ -823,7 +869,7 @@ var _ = Describe("PlacementAPI reconfiguration", func() {
 				mariadb.DeleteDBService,
 				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
 			)
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(names.MariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
 			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)