[tlse] TLS database connection

moves requesting the DB before rendering the service configuration.
The my.cnf file gets added to the secret holding the service configs.
The content of my.cnf is centrally managed in the mariadb-operator
and retrieved calling db.GetDatabaseClientConfig(tlsCfg)

Depends-On: openstack-k8s-operators/mariadb-operator#190
Depends-On: openstack-k8s-operators/mariadb-operator#191

Jira: OSPRH-4547

controllers/placementapi_controller.go

@@ -349,6 +349,13 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	// all our input checks out so report InputReady
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
 
+	result, err = r.ensureDB(ctx, h, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	} else if (result != ctrl.Result{}) {
+		return result, nil
+	}
+
 	err = r.generateServiceConfigMaps(ctx, h, instance, secret, &configMapVars)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
@@ -425,11 +432,6 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return result, err
 	}
 
-	result, err = r.ensureDB(ctx, h, instance)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-
 	apiEndpoints, result, err := r.ensureServiceExposed(ctx, h, instance)
 
 	if (err != nil || result != ctrl.Result{}) {
@@ -1184,10 +1186,19 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 
 	cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(placement.ServiceName), map[string]string{})
 
+	db, err := mariadbv1.GetDatabaseByName(ctx, h, placement.DatabaseName)
+	if err != nil {
+		return err
+	}
+
 	// customData hold any customization for the service.
 	// custom.conf is going to /etc/<service>/<service>.conf.d
+	// my.cnf is going to /etc/my.cnf
 	// all other files get placed into /etc/<service> to allow overwrite of e.g. policy.json
-	customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
+	customData := map[string]string{
+		common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig,
+		"my.cnf":                           db.GetDatabaseClientConfig(&tls.Service{}), //(mschuppert) for now just get the default my.cnf
+	}
 	for key, data := range instance.Spec.DefaultConfigOverwrite {
 		customData[key] = data
 	}
@@ -1209,11 +1220,13 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 		"KeystoneInternalURL": keystoneInternalURL,
 		"KeystonePublicURL":   keystonePublicURL,
 		"PlacementPassword":   string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]),
-		"DBUser":              instance.Spec.DatabaseUser,
-		"DBPassword":          string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
-		"DBAddress":           instance.Status.DatabaseHostname,
-		"DBName":              placement.DatabaseName,
 		"log_file":            "/var/log/placement/placement-api.log",
+		"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
+			instance.Spec.DatabaseUser,
+			string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
+			instance.Status.DatabaseHostname,
+			placement.DatabaseName,
+		),
 	}
 
 	// create httpd  vhost template parameters

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240219094943-9bbb46c9afba
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240216173409-86913e6d5885
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885
-	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798
+	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e
 	github.com/openstack-k8s-operators/placement-operator/api v0.3.1-0.20240216174613-3d349f26e681
 	go.uber.org/zap v1.26.0
 	k8s.io/api v0.28.6

go.sum

@@ -99,8 +99,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.2024021
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:8QsCFttAm+X6A8I8EQThGjNjeMAYt2hK7ivbvnR3434=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885 h1:ioJ2MO3vAcBkLM+0UBu5IuKW/DPXcyiNSOLq0Xvn+Nw=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:82nzS+DbBe1tzaMvNHH8FctmZzQ14ZAJysFGsMJiivo=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798 h1:zL4DdQ5HPXCLHeRMAWC2zI7ypbkZVYg3UkyEFSnzeow=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e h1:6vqp5HZwcGvPH0MII/23iCd97T3/1HJZlONKW6LyNio=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

templates/placementapi/config/placement-api-config.json

@@ -47,6 +47,12 @@
             "owner": "placement",
             "perm": "0600",
             "optional": true
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "placement",
+            "perm": "0600"
         }
     ],
     "permissions": [

templates/placementapi/config/placement-dbsync-config.json

@@ -12,6 +12,12 @@
             "dest": "/etc/placement/placement.conf.d/custom.conf",
             "owner": "placement",
             "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "placement",
+            "perm": "0600"
         }
     ]
 }

templates/placementapi/config/placement.conf

@@ -9,7 +9,7 @@ log_file = {{ .log_file }}
 debug = true
 
 [placement_database]
-connection = mysql+pymysql://{{ .DBUser }}:{{ .DBPassword }}@{{ .DBAddress }}/{{ .DBName }}
+connection = {{ .DatabaseConnection }}
 
 [api]
 auth_strategy = keystone

tests/functional/placementapi_controller_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package functional_test
 
 import (
+	"fmt"
 	"os"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -200,6 +201,17 @@ var _ = Describe("PlacementAPI controller", func() {
 			)
 			DeferCleanup(
 				k8sClient.Delete, ctx, CreatePlacementAPISecret(namespace, SecretName))
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 		})
 
 		It("should have input ready", func() {
@@ -239,15 +251,62 @@ var _ = Describe("PlacementAPI controller", func() {
 			DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPIName)
 		})
 
+		It("creates MariaDB database", func() {
+			th.ExpectCondition(
+				names.PlacementAPIName,
+				ConditionGetterFunc(PlacementConditionGetter),
+				condition.DBReadyCondition,
+				corev1.ConditionFalse,
+			)
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
+			th.ExpectCondition(
+				names.PlacementAPIName,
+				ConditionGetterFunc(PlacementConditionGetter),
+				condition.DBReadyCondition,
+				corev1.ConditionTrue,
+			)
+		})
+
 		It("should have config ready", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			th.ExpectCondition(
 				names.PlacementAPIName,
 				ConditionGetterFunc(PlacementConditionGetter),
 				condition.ServiceConfigReadyCondition,
 				corev1.ConditionTrue,
 			)
 		})
+
 		It("should create a configuration Secret", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			cm := th.GetSecret(names.ConfigMapName)
 
 			conf := cm.Data["placement.conf"]
@@ -260,18 +319,28 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(conf).Should(
 				ContainSubstring("password = 12345678"))
 			Expect(conf).Should(
-				ContainSubstring("connection = mysql+pymysql://placement:12345678@/placement"))
-
+				ContainSubstring(fmt.Sprintf("connection = mysql+pymysql://placement:12345678@hostname-for-openstack.%s.svc/placement?read_default_file=/etc/my.cnf", namespace)))
 			custom := cm.Data["custom.conf"]
 			Expect(custom).Should(ContainSubstring("foo = bar"))
 
 			policy := cm.Data["policy.yaml"]
 			Expect(policy).Should(
 				ContainSubstring("\"placement:resource_providers:list\": \"!\""))
 
+			myCnf := cm.Data["my.cnf"]
+			Expect(myCnf).To(
+				ContainSubstring("[client]\nssl=0"))
 		})
 
 		It("creates service account, role and rolebindig", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			th.ExpectCondition(
 				names.PlacementAPIName,
 				ConditionGetterFunc(PlacementConditionGetter),
@@ -302,33 +371,6 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(binding.Subjects).To(HaveLen(1))
 			Expect(binding.Subjects[0].Name).To(Equal(sa.Name))
 		})
-
-		It("creates MariaDB database", func() {
-			th.ExpectCondition(
-				names.PlacementAPIName,
-				ConditionGetterFunc(PlacementConditionGetter),
-				condition.DBReadyCondition,
-				corev1.ConditionFalse,
-			)
-
-			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
-			DeferCleanup(
-				mariadb.DeleteDBService,
-				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
-			)
-			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
-			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
-
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
-			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
-
-			th.ExpectCondition(
-				names.PlacementAPIName,
-				ConditionGetterFunc(PlacementConditionGetter),
-				condition.DBReadyCondition,
-				corev1.ConditionTrue,
-			)
-		})
 		It("creates keystone service", func() {
 			th.ExpectCondition(
 				names.PlacementAPIName,
@@ -757,7 +799,7 @@ var _ = Describe("PlacementAPI controller", func() {
 				mariadb.DeleteDBService,
 				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
 			)
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(names.MariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
 			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)
@@ -795,6 +837,10 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile   \"/etc/pki/tls/private/internal.key\""))
 			Expect(configData).Should(ContainSubstring("SSLCertificateFile      \"/etc/pki/tls/certs/public.crt\""))
 			Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile   \"/etc/pki/tls/private/public.key\""))
+
+			configData = string(configDataMap.Data["my.cnf"])
+			Expect(configData).To(
+				ContainSubstring("[client]\nssl-ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nssl=1"))
 		})
 	})
 })
@@ -823,7 +869,7 @@ var _ = Describe("PlacementAPI reconfiguration", func() {
 				mariadb.DeleteDBService,
 				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
 			)
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(names.MariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
 			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)