From 05370dff64e31a3c71061f65127382f2a9bfe650 Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Tue, 5 Sep 2023 09:15:39 +0000 Subject: [PATCH] WIP direct TLS connection to database service This adds the ability to configure oslo.db/pymysql to connect to the database service over TLS. It requires adding TLS options to bind-mount a CA that can validate the TLS certificate exposed by the database service. --- .../keystone.openstack.org_keystoneapis.yaml | 12 ++++ api/v1beta1/keystoneapi_types.go | 12 ++++ api/v1beta1/zz_generated.deepcopy.go | 16 +++++ .../keystone.openstack.org_keystoneapis.yaml | 12 ++++ controllers/keystoneapi_controller.go | 48 +++++++++++++- controllers/keystoneendpoint_controller.go | 2 +- pkg/keystone/bootstrap.go | 5 +- pkg/keystone/cronjob.go | 5 +- pkg/keystone/dbsync.go | 5 +- pkg/keystone/deployment.go | 5 +- pkg/keystone/initcontainer.go | 9 +++ pkg/keystone/volumes.go | 62 +++++++++++++++++-- templates/keystoneapi/bin/init.sh | 8 +++ .../config/keystone-api-config.json | 14 +++++ 14 files changed, 199 insertions(+), 16 deletions(-) diff --git a/api/bases/keystone.openstack.org_keystoneapis.yaml b/api/bases/keystone.openstack.org_keystoneapis.yaml index f76496553..92fbd1e46 100644 --- a/api/bases/keystone.openstack.org_keystoneapis.yaml +++ b/api/bases/keystone.openstack.org_keystoneapis.yaml @@ -249,6 +249,18 @@ spec: description: Secret containing OpenStack password information for keystone KeystoneDatabasePassword, AdminPassword type: string + tls: + description: TLS certificate and CA for Keystone + properties: + caSecretName: + description: Secret in the same namespace containing the CA cert + (ca.crt) for client certificate validation + type: string + secretName: + description: Secret in the same namespace containing the server + private key (tls.key) and public cert (tls.crt) for TLS + type: string + type: object trustFlushArgs: default: "" description: TrustFlushArgs - Arguments added to keystone-manage trust_flush diff --git a/api/v1beta1/keystoneapi_types.go b/api/v1beta1/keystoneapi_types.go index bb43d2ed4..71aa0361f 100644 --- a/api/v1beta1/keystoneapi_types.go +++ b/api/v1beta1/keystoneapi_types.go @@ -152,8 +152,20 @@ type KeystoneAPISpec struct { // +kubebuilder:validation:Optional // ExternalEndpoints, expose a VIP using a pre-created IPAddressPool ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` + + // TLS certificate and CA for Keystone + TLS TLSSpec `json:"tls,omitempty"` +} + +// TLSSpec defines the TLS options +type TLSSpec struct { + // Secret in the same namespace containing the server private key (tls.key) and public cert (tls.crt) for TLS + SecretName string `json:"secretName,omitempty"` + // Secret in the same namespace containing the CA cert (ca.crt) for client certificate validation + CaSecretName string `json:"caSecretName,omitempty"` } + // MetalLBConfig to configure the MetalLB loadbalancer service type MetalLBConfig struct { // +kubebuilder:validation:Required diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5daa63c72..d4dd7e51a 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -137,6 +137,7 @@ func (in *KeystoneAPISpec) DeepCopyInto(out *KeystoneAPISpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneAPISpec. @@ -516,3 +517,18 @@ func (in *PasswordSelector) DeepCopy() *PasswordSelector { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSSpec. +func (in *TLSSpec) DeepCopy() *TLSSpec { + if in == nil { + return nil + } + out := new(TLSSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml index f76496553..92fbd1e46 100644 --- a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml +++ b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml @@ -249,6 +249,18 @@ spec: description: Secret containing OpenStack password information for keystone KeystoneDatabasePassword, AdminPassword type: string + tls: + description: TLS certificate and CA for Keystone + properties: + caSecretName: + description: Secret in the same namespace containing the CA cert + (ca.crt) for client certificate validation + type: string + secretName: + description: Secret in the same namespace containing the server + private key (tls.key) and public cert (tls.crt) for TLS + type: string + type: object trustFlushArgs: default: "" description: TrustFlushArgs - Arguments added to keystone-manage trust_flush diff --git a/controllers/keystoneapi_controller.go b/controllers/keystoneapi_controller.go index 0ebc7b85d..36154fa16 100644 --- a/controllers/keystoneapi_controller.go +++ b/controllers/keystoneapi_controller.go @@ -38,6 +38,7 @@ import ( nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" database "github.com/openstack-k8s-operators/lib-common/modules/database" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -405,7 +406,8 @@ func (r *KeystoneAPIReconciler) reconcileInit( return ctrlResult, nil } // update Status.DatabaseHostname, used to bootstrap/config the service - instance.Status.DatabaseHostname = db.GetDatabaseHostname() + // TODO lib-common should return the FQDN for service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + "." + instance.GetNamespace() + ".svc" instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) // create service DB - end @@ -1100,3 +1102,47 @@ func (r *KeystoneAPIReconciler) getKeystoneMemcached( } return memcached, err } + +// getClientConfigForDatabase - client config flags to access a database +// (e.g. enforce TLS connections if the database uses TLS) +// +// returns a string of mysql config statements, and any error +// NOTE: currently unused +func getClientConfigForDatabase( + ctx context.Context, + h *helper.Helper, + databaseName string, + namespace string, +) (string, error) { + // . A MariaDBDatabase CR has a label `dbName` that references the + // DB server CR. When the server CR is configured to expose TLS, an + // optional mysql client config is read by oslo.db to connect via TLS + // . To check whether the server is configured with TLS, look for + // for a secret CR that has a label `mariadb-ref` that references the + // server CR. + db := &mariadbv1.MariaDBDatabase{} + err := h.GetClient().Get(ctx, types.NamespacedName{Name: databaseName, Namespace: namespace}, db) + if err != nil { + return "", client.IgnoreNotFound(err) + } + serverCRName := db.Labels["dbName"] + + selector := map[string]string{ + "mariadb-ref": serverCRName, + } + secretList, err := secret.GetSecrets( + ctx, + h, + h.GetBeforeObject().GetNamespace(), + selector, + ) + if err != nil || len(secretList.Items) == 0 { + return "", fmt.Errorf("Error getting the DB certificate secrets using label %v: %w", + selector, err) + } + + if len(secretList.Items) > 0 { + return "ssl=1\nssl-ca=/etc/ipa/ca.crt'", nil + } + return "", nil +} diff --git a/controllers/keystoneendpoint_controller.go b/controllers/keystoneendpoint_controller.go index 181495953..78c230a3c 100644 --- a/controllers/keystoneendpoint_controller.go +++ b/controllers/keystoneendpoint_controller.go @@ -280,7 +280,7 @@ func (r *KeystoneEndpointReconciler) reconcileDelete( return ctrl.Result{}, err } } - } else if ! k8s_errors.IsNotFound(err) { + } else if !k8s_errors.IsNotFound(err) { return ctrl.Result{}, err } diff --git a/pkg/keystone/bootstrap.go b/pkg/keystone/bootstrap.go index 29d69b372..4758c9507 100644 --- a/pkg/keystone/bootstrap.go +++ b/pkg/keystone/bootstrap.go @@ -102,7 +102,7 @@ func BootstrapJob( }, }, }, - VolumeMounts: getVolumeMounts(), + VolumeMounts: getVolumeMounts(instance), }, }, }, @@ -110,13 +110,14 @@ func BootstrapJob( }, } job.Spec.Template.Spec.Containers[0].Env = env.MergeEnvs(job.Spec.Template.Spec.Containers[0].Env, envVars) - job.Spec.Template.Spec.Volumes = getVolumes(instance.Name) + job.Spec.Template.Spec.Volumes = getVolumes(instance) initContainerDetails := APIDetails{ ContainerImage: instance.Spec.ContainerImage, DatabaseHost: instance.Status.DatabaseHostname, DatabaseUser: instance.Spec.DatabaseUser, DatabaseName: DatabaseName, + DatabaseUseTLS: instance.Spec.TLS.CaSecretName != "", OSPSecret: instance.Spec.Secret, DBPasswordSelector: instance.Spec.PasswordSelectors.Database, UserPasswordSelector: instance.Spec.PasswordSelectors.Admin, diff --git a/pkg/keystone/cronjob.go b/pkg/keystone/cronjob.go index 5dc2e405d..5ae4c47c1 100644 --- a/pkg/keystone/cronjob.go +++ b/pkg/keystone/cronjob.go @@ -79,13 +79,13 @@ func CronJob( }, Args: args, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: getVolumeMounts(), + VolumeMounts: getVolumeMounts(instance), SecurityContext: &corev1.SecurityContext{ RunAsUser: &runAsUser, }, }, }, - Volumes: getVolumes(instance.Name), + Volumes: getVolumes(instance), RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: instance.RbacResourceName(), }, @@ -103,6 +103,7 @@ func CronJob( DatabaseHost: instance.Status.DatabaseHostname, DatabaseUser: instance.Spec.DatabaseUser, DatabaseName: DatabaseName, + DatabaseUseTLS: instance.Spec.TLS.CaSecretName != "", OSPSecret: instance.Spec.Secret, DBPasswordSelector: instance.Spec.PasswordSelectors.Database, UserPasswordSelector: instance.Spec.PasswordSelectors.Admin, diff --git a/pkg/keystone/dbsync.go b/pkg/keystone/dbsync.go index 22b49f4f7..65a298da2 100644 --- a/pkg/keystone/dbsync.go +++ b/pkg/keystone/dbsync.go @@ -75,7 +75,7 @@ func DbSyncJob( RunAsUser: &runAsUser, }, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: getVolumeMounts(), + VolumeMounts: getVolumeMounts(instance), }, }, }, @@ -83,12 +83,13 @@ func DbSyncJob( }, } - job.Spec.Template.Spec.Volumes = getVolumes(ServiceName) + job.Spec.Template.Spec.Volumes = getVolumes(instance) initContainerDetails := APIDetails{ ContainerImage: instance.Spec.ContainerImage, DatabaseHost: instance.Status.DatabaseHostname, DatabaseUser: instance.Spec.DatabaseUser, DatabaseName: DatabaseName, + DatabaseUseTLS: instance.Spec.TLS.CaSecretName != "", OSPSecret: instance.Spec.Secret, DBPasswordSelector: instance.Spec.PasswordSelectors.Database, UserPasswordSelector: instance.Spec.PasswordSelectors.Admin, diff --git a/pkg/keystone/deployment.go b/pkg/keystone/deployment.go index 1f4d2f24d..0bfda49c0 100644 --- a/pkg/keystone/deployment.go +++ b/pkg/keystone/deployment.go @@ -105,7 +105,7 @@ func Deployment( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.RbacResourceName(), - Volumes: getVolumes(instance.Name), + Volumes: getVolumes(instance), Containers: []corev1.Container{ { Name: ServiceName + "-api", @@ -118,7 +118,7 @@ func Deployment( RunAsUser: &runAsUser, }, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: getVolumeMounts(), + VolumeMounts: getVolumeMounts(instance), Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, @@ -147,6 +147,7 @@ func Deployment( DatabaseHost: instance.Status.DatabaseHostname, DatabaseUser: instance.Spec.DatabaseUser, DatabaseName: DatabaseName, + DatabaseUseTLS: instance.Spec.TLS.CaSecretName != "", OSPSecret: instance.Spec.Secret, DBPasswordSelector: instance.Spec.PasswordSelectors.Database, UserPasswordSelector: instance.Spec.PasswordSelectors.Admin, diff --git a/pkg/keystone/initcontainer.go b/pkg/keystone/initcontainer.go index 5559a5911..fd0ca2030 100644 --- a/pkg/keystone/initcontainer.go +++ b/pkg/keystone/initcontainer.go @@ -27,6 +27,7 @@ type APIDetails struct { DatabaseHost string DatabaseUser string DatabaseName string + DatabaseUseTLS bool OSPSecret string DBPasswordSelector string UserPasswordSelector string @@ -52,6 +53,14 @@ func initContainer(init APIDetails) []corev1.Container { envVars["DatabaseUser"] = env.SetValue(init.DatabaseUser) envVars["DatabaseName"] = env.SetValue(init.DatabaseName) + var clientConfig string + if init.DatabaseUseTLS { + clientConfig = "ssl=1\nssl-ca=/etc/ipa/ca.crt'" + } else { + clientConfig = "" + } + envVars["DatabaseClientConfig"] = env.SetValue(clientConfig) + envs := []corev1.EnvVar{ { Name: "DatabasePassword", diff --git a/pkg/keystone/volumes.go b/pkg/keystone/volumes.go index fe901e3c6..f396e5510 100644 --- a/pkg/keystone/volumes.go +++ b/pkg/keystone/volumes.go @@ -16,22 +16,24 @@ limitations under the License. package keystone import ( + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" ) // getVolumes - service volumes -func getVolumes(name string) []corev1.Volume { +func getVolumes(instance *keystonev1.KeystoneAPI) []corev1.Volume { var scriptsVolumeDefaultMode int32 = 0755 var config0640AccessMode int32 = 0640 - return []corev1.Volume{ + volumes := []corev1.Volume{ { Name: "scripts", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ DefaultMode: &scriptsVolumeDefaultMode, LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-scripts", + Name: instance.Name + "-scripts", }, }, }, @@ -42,7 +44,18 @@ func getVolumes(name string) []corev1.Volume { ConfigMap: &corev1.ConfigMapVolumeSource{ DefaultMode: &config0640AccessMode, LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-config-data", + Name: instance.Name + "-config-data", + }, + }, + }, + }, + { + Name: "mysql-config-data", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &config0640AccessMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: "openstack-config-data", }, }, }, @@ -91,6 +104,26 @@ func getVolumes(name string) []corev1.Volume { }, } + if instance.Spec.TLS.CaSecretName != "" { + volumes = append(volumes, []corev1.Volume{ + { + Name: "tls-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: instance.Spec.TLS.CaSecretName, + Items: []corev1.KeyToPath{ + { + Key: "ca.crt", + Path: "ca.crt", + }, + }, + }, + }, + }, + }...) + } + + return volumes } // getInitVolumeMounts - general init task VolumeMounts @@ -115,8 +148,8 @@ func getInitVolumeMounts() []corev1.VolumeMount { } // getVolumeMounts - general VolumeMounts -func getVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ +func getVolumeMounts(instance *keystonev1.KeystoneAPI) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ { Name: "scripts", MountPath: "/usr/local/bin/container-scripts", @@ -143,5 +176,22 @@ func getVolumeMounts() []corev1.VolumeMount { ReadOnly: true, Name: "credential-keys", }, + { + Name: "mysql-config-data", + MountPath: "/var/lib/mysql-config-data", + ReadOnly: true, + }, + } + + if instance.Spec.TLS.CaSecretName != "" { + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ + { + MountPath: "/var/lib/tls-ca", + ReadOnly: true, + Name: "tls-ca", + }, + }...) } + + return volumeMounts } diff --git a/templates/keystoneapi/bin/init.sh b/templates/keystoneapi/bin/init.sh index bc9c7935b..9005b370c 100755 --- a/templates/keystoneapi/bin/init.sh +++ b/templates/keystoneapi/bin/init.sh @@ -21,9 +21,11 @@ export DBHOST=${DatabaseHost:?"Please specify a DatabaseHost variable."} export DBUSER=${DatabaseUser:?"Please specify a DatabaseUser variable."} export DBPASSWORD=${DatabasePassword:?"Please specify a DatabasePassword variable."} export DB=${DatabaseName:-"keystone"} +export DBCLIENTCONFIG=${DatabaseClientConfig:-} SVC_CFG=/etc/keystone/keystone.conf SVC_CFG_MERGED=/var/lib/config-data/merged/keystone.conf +SVC_DB_CFG=/var/lib/config-data/merged/client.cnf # expect that the common.sh is in the same dir as the calling script SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" @@ -39,3 +41,9 @@ done # set secrets crudini --set ${SVC_CFG_MERGED} database connection mysql+pymysql://${DBUSER}:${DBPASSWORD}@${DBHOST}/${DB} + +# DB-specific config +cat >${SVC_DB_CFG} <