TLS connection to database service

This adds the ability to configure oslo.db/pymysql to connect to the database service over TLS. It requires adding TLS options to bind-mount a CA that can validate the TLS certificate exposed by the database service.
openstack-k8s-operators · Oct 5, 2023 · 1e32653 · 1e32653
1 parent 508b207
commit 1e32653
Show file tree

Hide file tree

Showing 18 changed files with 173 additions and 30 deletions.
diff --git a/api/bases/keystone.openstack.org_keystoneapis.yaml b/api/bases/keystone.openstack.org_keystoneapis.yaml
@@ -381,6 +381,29 @@ spec:
                 description: Secret containing OpenStack password information for
                   keystone KeystoneDatabasePassword, AdminPassword
                 type: string
+              tls:
+                description: TLS certificate and CA for internal TLS traffic
+                properties:
+                  ca:
+                    description: Ca contains CA-specific settings, which could be
+                      used both by services (to define their own CA certificates)
+                      and by clients (to verify the server's certificate)
+                    properties:
+                      caSecretName:
+                        type: string
+                    type: object
+                  service:
+                    description: Service contains server-specific TLS secret
+                    properties:
+                      disableNonTLSListeners:
+                        type: boolean
+                      secretName:
+                        type: string
+                    type: object
+                required:
+                - ca
+                - service
+                type: object
               trustFlushArgs:
                 default: ""
                 description: TrustFlushArgs - Arguments added to keystone-manage trust_flush

diff --git a/api/go.mod b/api/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/go-logr/logr v1.2.4
 	github.com/google/uuid v1.3.1
 	github.com/onsi/gomega v1.28.0
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231001084618-12369665b166
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e
 	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20231001084618-12369665b166
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.1.2-0.20231001084618-12369665b166
 	k8s.io/api v0.26.9

diff --git a/api/go.sum b/api/go.sum
@@ -228,8 +228,8 @@ github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c=
 github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8=
 github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxCMwNRnMjhhIDOWHJowi6q8G6koI=
 github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231001084618-12369665b166 h1:pZPl9njjTchHSFf2OSuzyRxrRW8LLb1OscPxVNoATrk=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231001084618-12369665b166/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e h1:/bKZdCAsu73wscdiMsmctAmh0Jz432WxVQe4h1+ipzQ=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20231001084618-12369665b166 h1:zOnRGMdgq2XvOCCtF1lY4tFhKx3jXrcrtOiZZ1PR6M8=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20231001084618-12369665b166/go.mod h1:LOXXvTQCwhOBNd+0FTlgllpa3wqlkI6Vf3Q5QVRVPlw=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.1.2-0.20231001084618-12369665b166 h1:lh3WHM+3DcPlXK4I3QWHmvV+cPCy+dmiMdfImHF/Nqc=

diff --git a/api/v1beta1/keystoneapi_types.go b/api/v1beta1/keystoneapi_types.go
@@ -23,6 +23,7 @@ import (
 	"github.com/openstack-k8s-operators/lib-common/modules/common/endpoint"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -153,6 +154,10 @@ type KeystoneAPISpec struct {
 	// +kubebuilder:validation:Optional
 	// Override, provides the ability to override the generated manifest of several child resources.
 	Override APIOverrideSpec `json:"override,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// TLS certificate and CA for internal TLS traffic
+	TLS *tls.TLS `json:"tls,omitempty"`
 }
 
 // APIOverrideSpec to override the generated manifest of several child resources.

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml
@@ -381,6 +381,29 @@ spec:
                 description: Secret containing OpenStack password information for
                   keystone KeystoneDatabasePassword, AdminPassword
                 type: string
+              tls:
+                description: TLS certificate and CA for internal TLS traffic
+                properties:
+                  ca:
+                    description: Ca contains CA-specific settings, which could be
+                      used both by services (to define their own CA certificates)
+                      and by clients (to verify the server's certificate)
+                    properties:
+                      caSecretName:
+                        type: string
+                    type: object
+                  service:
+                    description: Service contains server-specific TLS secret
+                    properties:
+                      disableNonTLSListeners:
+                        type: boolean
+                      secretName:
+                        type: string
+                    type: object
+                required:
+                - ca
+                - service
+                type: object
               trustFlushArgs:
                 default: ""
                 description: TrustFlushArgs - Arguments added to keystone-manage trust_flush

diff --git a/controllers/keystoneapi_controller.go b/controllers/keystoneapi_controller.go
@@ -949,8 +949,15 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
 		customData[key] = data
 	}
 
+	var mysqlTLSConfig string
+	if instance.Spec.TLS != nil {
+		mysqlTLSConfig = instance.Spec.TLS.CreateDatabaseClientConfig()
+	} else {
+		mysqlTLSConfig = ""
+	}
 	templateParameters := map[string]interface{}{
 		"memcachedServers": strings.Join(mc.Status.ServerList, ","),
+		"mysqlTLSConfig":   mysqlTLSConfig,
 	}
 
 	cms := []util.Template{

diff --git a/go.mod b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/onsi/gomega v1.28.0
 	github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20231001103054-f74a88ed4971
 	github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230920085319-92ae0260bbf3
-	github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231001084618-12369665b166
+	github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e
 	github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20231001084618-12369665b166
 	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20231001084618-12369665b166
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.1.2-0.20231001084618-12369665b166
@@ -97,3 +97,7 @@ replace github.com/openshift/api => github.com/openshift/api v0.0.0-202304141430
 // needed to to cert-manager v1.11.4 see https://github.com/cert-manager/cert-manager/blob/v1.11.4/go.mod#L263C1-L264C104
 // remove this once we bump to cert-manager v1.12.x
 replace github.com/Venafi/vcert/v4 => github.com/jetstack/vcert/v4 v4.9.6-0.20230519122548-219f317ae107 //allow-merging
+
+// WIP
+replace github.com/openstack-k8s-operators/lib-common/modules/database => ../../dciabrin/lib-common/modules/database
+replace github.com/openstack-k8s-operators/lib-common/modules/common => ../../dciabrin/lib-common/modules/common
diff --git a/go.sum b/go.sum
@@ -238,10 +238,6 @@ github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxC
 github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4=
 github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20231001103054-f74a88ed4971 h1:5kNS+gseixkgRhYPukQVJyewPTM0lfUOmKP0A03wFFY=
 github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20231001103054-f74a88ed4971/go.mod h1:zqFs5MrBKeaE4HQroUgMWwIkBwmmcygg6sghcidSdCA=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231001084618-12369665b166 h1:pZPl9njjTchHSFf2OSuzyRxrRW8LLb1OscPxVNoATrk=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231001084618-12369665b166/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM=
-github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20231001084618-12369665b166 h1:63I4+qBYwm0Q2txAZRfJ2TRo5HyBJwHxDYSpOr+jOOo=
-github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20231001084618-12369665b166/go.mod h1:RroLfB6Wstc+z7JVJY9o+6YPu+wBIzTAAfMpwhv7pDI=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20231001084618-12369665b166 h1:zOnRGMdgq2XvOCCtF1lY4tFhKx3jXrcrtOiZZ1PR6M8=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20231001084618-12369665b166/go.mod h1:LOXXvTQCwhOBNd+0FTlgllpa3wqlkI6Vf3Q5QVRVPlw=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.1.2-0.20231001084618-12369665b166 h1:lh3WHM+3DcPlXK4I3QWHmvV+cPCy+dmiMdfImHF/Nqc=

diff --git a/pkg/keystone/bootstrap.go b/pkg/keystone/bootstrap.go
@@ -102,15 +102,15 @@ func BootstrapJob(
 									},
 								},
 							},
-							VolumeMounts: getVolumeMounts(),
+							VolumeMounts: getVolumeMounts(instance),
 						},
 					},
 				},
 			},
 		},
 	}
 	job.Spec.Template.Spec.Containers[0].Env = env.MergeEnvs(job.Spec.Template.Spec.Containers[0].Env, envVars)
-	job.Spec.Template.Spec.Volumes = getVolumes(instance.Name)
+	job.Spec.Template.Spec.Volumes = getVolumes(instance)
 
 	initContainerDetails := APIDetails{
 		ContainerImage:       instance.Spec.ContainerImage,
@@ -120,7 +120,7 @@ func BootstrapJob(
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,
-		VolumeMounts:         getInitVolumeMounts(),
+		VolumeMounts:         getInitVolumeMounts(instance),
 	}
 	job.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails)
 

diff --git a/pkg/keystone/cronjob.go b/pkg/keystone/cronjob.go
@@ -79,13 +79,13 @@ func CronJob(
 									},
 									Args:         args,
 									Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-									VolumeMounts: getVolumeMounts(),
+									VolumeMounts: getVolumeMounts(instance),
 									SecurityContext: &corev1.SecurityContext{
 										RunAsUser: &runAsUser,
 									},
 								},
 							},
-							Volumes:            getVolumes(instance.Name),
+							Volumes:            getVolumes(instance),
 							RestartPolicy:      corev1.RestartPolicyNever,
 							ServiceAccountName: instance.RbacResourceName(),
 						},
@@ -106,7 +106,7 @@ func CronJob(
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,
-		VolumeMounts:         getInitVolumeMounts(),
+		VolumeMounts:         getInitVolumeMounts(instance),
 	}
 	cronjob.Spec.JobTemplate.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails)
 

diff --git a/pkg/keystone/dbsync.go b/pkg/keystone/dbsync.go
@@ -75,15 +75,15 @@ func DbSyncJob(
 								RunAsUser: &runAsUser,
 							},
 							Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-							VolumeMounts: getVolumeMounts(),
+							VolumeMounts: getVolumeMounts(instance),
 						},
 					},
 				},
 			},
 		},
 	}
 
-	job.Spec.Template.Spec.Volumes = getVolumes(ServiceName)
+	job.Spec.Template.Spec.Volumes = getVolumes(instance)
 	initContainerDetails := APIDetails{
 		ContainerImage:       instance.Spec.ContainerImage,
 		DatabaseHost:         instance.Status.DatabaseHostname,
@@ -92,7 +92,7 @@ func DbSyncJob(
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,
-		VolumeMounts:         getInitVolumeMounts(),
+		VolumeMounts:         getInitVolumeMounts(instance),
 	}
 	job.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails)
 

diff --git a/pkg/keystone/deployment.go b/pkg/keystone/deployment.go
@@ -105,7 +105,7 @@ func Deployment(
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: instance.RbacResourceName(),
-					Volumes:            getVolumes(instance.Name),
+					Volumes:            getVolumes(instance),
 					Containers: []corev1.Container{
 						{
 							Name: ServiceName + "-api",
@@ -118,7 +118,7 @@ func Deployment(
 								RunAsUser: &runAsUser,
 							},
 							Env:            env.MergeEnvs([]corev1.EnvVar{}, envVars),
-							VolumeMounts:   getVolumeMounts(),
+							VolumeMounts:   getVolumeMounts(instance),
 							Resources:      instance.Spec.Resources,
 							ReadinessProbe: readinessProbe,
 							LivenessProbe:  livenessProbe,
@@ -150,7 +150,7 @@ func Deployment(
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,
-		VolumeMounts:         getInitVolumeMounts(),
+		VolumeMounts:         getInitVolumeMounts(instance),
 	}
 	deployment.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails)
 

diff --git a/pkg/keystone/initcontainer.go b/pkg/keystone/initcontainer.go
@@ -90,7 +90,7 @@ func initContainer(init APIDetails) []corev1.Container {
 			},
 			Args:         args,
 			Env:          envs,
-			VolumeMounts: getInitVolumeMounts(),
+			VolumeMounts: init.VolumeMounts,
 		},
 	}
 }
diff --git a/pkg/keystone/volumes.go b/pkg/keystone/volumes.go
@@ -16,22 +16,24 @@ limitations under the License.
 package keystone
 
 import (
+	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
+
 	corev1 "k8s.io/api/core/v1"
 )
 
 // getVolumes - service volumes
-func getVolumes(name string) []corev1.Volume {
+func getVolumes(instance *keystonev1.KeystoneAPI) []corev1.Volume {
 	var scriptsVolumeDefaultMode int32 = 0755
 	var config0640AccessMode int32 = 0640
 
-	return []corev1.Volume{
+	volumes := []corev1.Volume{
 		{
 			Name: "scripts",
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
 					DefaultMode: &scriptsVolumeDefaultMode,
 					LocalObjectReference: corev1.LocalObjectReference{
-						Name: name + "-scripts",
+						Name: instance.Name + "-scripts",
 					},
 				},
 			},
@@ -42,7 +44,18 @@ func getVolumes(name string) []corev1.Volume {
 				ConfigMap: &corev1.ConfigMapVolumeSource{
 					DefaultMode: &config0640AccessMode,
 					LocalObjectReference: corev1.LocalObjectReference{
-						Name: name + "-config-data",
+						Name: instance.Name + "-config-data",
+					},
+				},
+			},
+		},
+		{
+			Name: "mysql-config-data",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					DefaultMode: &config0640AccessMode,
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: "openstack-config-data",
 					},
 				},
 			},
@@ -91,11 +104,17 @@ func getVolumes(name string) []corev1.Volume {
 		},
 	}
 
+	if instance.Spec.TLS != nil {
+		caVolumes := instance.Spec.TLS.CreateVolumes()
+		volumes = append(volumes, caVolumes...)
+	}
+
+	return volumes
 }
 
 // getInitVolumeMounts - general init task VolumeMounts
-func getInitVolumeMounts() []corev1.VolumeMount {
-	return []corev1.VolumeMount{
+func getInitVolumeMounts(instance *keystonev1.KeystoneAPI) []corev1.VolumeMount {
+	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "scripts",
 			MountPath: "/usr/local/bin/container-scripts",
@@ -112,11 +131,18 @@ func getInitVolumeMounts() []corev1.VolumeMount {
 			ReadOnly:  false,
 		},
 	}
+
+	if instance.Spec.TLS != nil {
+		caVolumeMounts := instance.Spec.TLS.CreateVolumeMounts()
+		volumeMounts = append(volumeMounts, caVolumeMounts...)
+	}
+
+	return volumeMounts
 }
 
 // getVolumeMounts - general VolumeMounts
-func getVolumeMounts() []corev1.VolumeMount {
-	return []corev1.VolumeMount{
+func getVolumeMounts(instance *keystonev1.KeystoneAPI) []corev1.VolumeMount {
+	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "scripts",
 			MountPath: "/usr/local/bin/container-scripts",
@@ -143,5 +169,17 @@ func getVolumeMounts() []corev1.VolumeMount {
 			ReadOnly:  true,
 			Name:      "credential-keys",
 		},
+		{
+			Name:      "mysql-config-data",
+			MountPath: "/var/lib/mysql-config-data",
+			ReadOnly:  true,
+		},
 	}
+
+	if instance.Spec.TLS != nil {
+		caVolumeMounts := instance.Spec.TLS.CreateVolumeMounts()
+		volumeMounts = append(volumeMounts, caVolumeMounts...)
+	}
+
+	return volumeMounts
 }