Support TLS for galera

Ability to specify a certificate and a CA to be used for galera
cluster communication (GCOMM, SST).

When the Galera CR is configured to use TLS, the mariadbdatabase
CR creates DB users with grants that only allow connecting to the
DB over TLS.

api/bases/mariadb.openstack.org_galeras.yaml

@@ -84,6 +84,18 @@ spec:
               storageRequest:
                 description: Storage size allocated for the mariadb databases
                 type: string
+              tls:
+                description: TLS settings to use for MySQL and Galera replication
+                properties:
+                  caSecretName:
+                    description: Secret in the same namespace containing the CA cert
+                      (ca.crt) for client certificate validation
+                    type: string
+                  secretName:
+                    description: Secret in the same namespace containing the server
+                      private key (tls.key) and public cert (tls.crt) for TLS
+                    type: string
+                type: object
             required:
             - containerImage
             - replicas

api/v1beta1/galera_types.go

@@ -55,6 +55,17 @@ type GaleraSpec struct {
 	// +kubebuilder:validation:Optional
 	// Adoption configuration
 	AdoptionRedirect AdoptionRedirectSpec `json:"adoptionRedirect"`
+	// +kubebuilder:validation:Optional
+	// TLS settings to use for MySQL and Galera replication
+	TLS TLSSpec `json:"tls,omitempty"`
+}
+
+// TLSSpec defines the TLS options
+type TLSSpec struct {
+	// Secret in the same namespace containing the server private key (tls.key) and public cert (tls.crt) for TLS
+	SecretName string `json:"secretName,omitempty"`
+	// Secret in the same namespace containing the CA cert (ca.crt) for client certificate validation
+	CaSecretName string `json:"caSecretName,omitempty"`
 }
 
 // GaleraAttributes holds startup information for a Galera host

config/crd/bases/mariadb.openstack.org_galeras.yaml

@@ -84,6 +84,18 @@ spec:
               storageRequest:
                 description: Storage size allocated for the mariadb databases
                 type: string
+              tls:
+                description: TLS settings to use for MySQL and Galera replication
+                properties:
+                  caSecretName:
+                    description: Secret in the same namespace containing the CA cert
+                      (ca.crt) for client certificate validation
+                    type: string
+                  secretName:
+                    description: Secret in the same namespace containing the server
+                      private key (tls.key) and public cert (tls.crt) for TLS
+                    type: string
+                type: object
             required:
             - containerImage
             - replicas

config/samples/mariadb_v1beta1_galera.yaml

@@ -3,7 +3,7 @@ kind: Galera
 metadata:
   name: openstack
 spec:
-  secret: mariadb-secret
+  secret: osp-secret
   storageClass: local-storage
   storageRequest: 500M
   containerImage: quay.io/podified-antelope-centos9/openstack-mariadb:current-podified

config/samples/mariadb_v1beta1_galera_custom_config.yaml

@@ -3,7 +3,7 @@ kind: Galera
 metadata:
   name: openstack
 spec:
-  secret: mariadb-secret
+  secret: osp-secret
   storageClass: local-storage
   storageRequest: 500M
   containerImage: quay.io/podified-antelope-centos9/openstack-mariadb:current-podified

config/samples/mariadb_v1beta1_galera_tls.yaml

@@ -0,0 +1,13 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  containerImage: quay.io/podified-antelope-centos9/openstack-mariadb:current-podified
+  replicas: 3
+  tls:
+    secretName: galera-tls
+    caSecretName: galera-tls

controllers/galera_controller.go

@@ -22,12 +22,14 @@ import (
 	env "github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
+	secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
+	k8s_labels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/kubectl/pkg/util/podutils"
@@ -44,15 +46,21 @@ import (
 
 	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 	mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg"
 )
 
+const mariaDBReconcileLabel = "mariadb-ref"
+
 // GaleraReconciler reconciles a Galera object
 type GaleraReconciler struct {
 	client.Client
@@ -91,7 +99,7 @@ func buildGcommURI(instance *mariadbv1.Galera) string {
 	res := []string{}
 
 	for i := 0; i < replicas; i++ {
-		res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename)
+		res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename+"."+instance.Namespace+".svc")
 	}
 	uri := "gcomm://" + strings.Join(res, ",")
 	return uri
@@ -250,15 +258,17 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal
 		if !found {
 			continue
 		}
-		ci := instance.Status.Attributes[pod.Name].ContainerID
-		pci := pod.Status.ContainerStatuses[0].ContainerID
-		if ci != pci {
+		// A node can have various attributes depending on its known state.
+		// A ContainerID attribute is only present if the node is being started.
+		attrCID := instance.Status.Attributes[pod.Name].ContainerID
+		podCID := pod.Status.ContainerStatuses[0].ContainerID
+		if attrCID != "" && attrCID != podCID {
 			// This gcomm URI was pushed in a pod which was restarted
 			// before the attribute got cleared, which means the pod
 			// failed to start galera. Clear the attribute here, and
 			// reprobe the pod's state in the next reconcile loop
 			clearPodAttributes(instance, pod.Name)
-			util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name)
+			util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name, "current pod ID", podCID, "recorded ID", attrCID)
 		}
 	}
 }
@@ -454,7 +464,7 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		r.Log.Info(fmt.Sprintf("%s %s database service %s - operation: %s", instance.Kind, instance.Name, service.Name, string(op)))
 	}
 
-	// Generate the config maps for the various services
+	// Generate the config maps
 	configMapVars := make(map[string]env.Setter)
 	err = r.generateConfigMaps(ctx, helper, instance, &configMapVars)
 	if err != nil {
@@ -466,16 +476,62 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 			err.Error()))
 		return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err)
 	}
+
+	//
+	// Extend the config maps with hashes for TLS secrets if any
+	//
+	var tlsres ctrl.Result
+	var tlserror error
+	if instance.Spec.TLS.SecretName != "" {
+		tlsres, err = r.checkAndHashTLSSecret(ctx, helper, instance, instance.Spec.TLS.SecretName, &configMapVars)
+	}
+	if tlserror == nil && instance.Spec.TLS.CaSecretName != "" {
+		tlsres, err = r.checkAndHashTLSSecret(ctx, helper, instance, instance.Spec.TLS.CaSecretName, &configMapVars)
+	}
+
+	if tlserror != nil {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.ServiceConfigReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.ServiceConfigReadyErrorMessage,
+			tlserror.Error()))
+
+		if tlsres.RequeueAfter > 0 {
+			// TLS secret might be recreated, requeue as this is not fatal
+			util.LogForObject(helper, "Requeuing until TLS configuration is available", instance, "error", tlserror.Error())
+			return tlsres, nil
+		}
+
+		return tlsres, fmt.Errorf("error calculating configmap hash: %w", tlserror)
+	}
+
 	// From hereon, configMapVars holds a hash of the config generated for this instance
+	// as well as a hash and of the current TLS certificate and CA used if any.
 	// This is used in an envvar in the statefulset to restart it on config change
-	envHash := &corev1.EnvVar{}
-	configMapVars[configMapNameForConfig(instance)](envHash)
-	instance.Status.ConfigHash = envHash.Value
+
+	keys := make([]string, 0, len(configMapVars))
+	for k := range configMapVars {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	hash := ""
+	for _, k := range keys {
+		envVar := &corev1.EnvVar{}
+		configMapVars[k](envVar)
+		hash = hash + envVar.Value
+	}
+	// TODO maybe hash the resulting string to shorten it
+	instance.Status.ConfigHash = hash
 	instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
 
 	commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance), 5)
 	sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper)
 	if sferr != nil {
+		if k8s_errors.IsNotFound(sferr) {
+			return ctrl.Result{RequeueAfter: time.Second * 10}, nil
+		}
 		return sfres, sferr
 	}
 	statefulset := commonstatefulset.GetStatefulSet()
@@ -585,7 +641,7 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 	// a chance to react to all pod's transitions.
 	if statefulset.Status.AvailableReplicas != statefulset.Status.Replicas {
 		util.LogForObject(helper, "Requeuing until all replicas are available", instance)
-		return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil
+		return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil
 	}
 
 	return ctrl.Result{}, err
@@ -644,6 +700,44 @@ func (r *GaleraReconciler) generateConfigMaps(
 	return nil
 }
 
+func (r *GaleraReconciler) checkAndHashTLSSecret(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *mariadbv1.Galera,
+	secretName string,
+	envVars *map[string]env.Setter,
+) (ctrl.Result, error) {
+	tlsSecret, tlsHash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			return ctrl.Result{RequeueAfter: time.Second * 10}, nil
+		}
+		return ctrl.Result{}, fmt.Errorf("secret %s not found", secretName)
+	}
+
+	if value, ok := tlsSecret.Labels[mariaDBReconcileLabel]; !ok || value != instance.Name {
+		tlsSecret.GetObjectMeta().SetLabels(
+			k8s_labels.Merge(
+				tlsSecret.GetObjectMeta().GetLabels(),
+				map[string]string{
+					mariaDBReconcileLabel: instance.Name,
+				},
+			),
+		)
+		err = r.Client.Update(ctx, tlsSecret)
+		if err != nil {
+			if k8s_errors.IsConflict(err) || k8s_errors.IsNotFound(err) {
+				return ctrl.Result{Requeue: true}, err
+			}
+			return ctrl.Result{}, err
+		}
+	}
+
+	(*envVars)[secretName] = env.SetValue(tlsHash)
+
+	return ctrl.Result{}, nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.config = mgr.GetConfig()
@@ -653,5 +747,22 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.Service{}).
 		Owns(&corev1.Endpoints{}).
 		Owns(&corev1.ConfigMap{}).
+		Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(
+			func(o client.Object) []reconcile.Request {
+				labels := o.GetLabels()
+
+				reconcileCR, hasLabel := labels[mariaDBReconcileLabel]
+				if !hasLabel {
+					return []reconcile.Request{}
+				}
+
+				return []reconcile.Request{
+					{NamespacedName: types.NamespacedName{
+						Name:      reconcileCR,
+						Namespace: o.GetNamespace(),
+					}},
+				}
+			},
+		)).
 		Complete(r)
 }

controllers/mariadbdatabase_controller.go

@@ -135,6 +135,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	// Non-deletion (normal) flow follows
 	//
 	var dbName, dbSecret, dbContainerImage, serviceAccount string
+	var useTLS bool
 
 	// It is impossible to reach here without either dbGalera or dbMariadb not being nil, due to the checks above
 	if dbGalera != nil {
@@ -147,6 +148,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		dbSecret = dbGalera.Spec.Secret
 		dbContainerImage = dbGalera.Spec.ContainerImage
 		serviceAccount = dbGalera.RbacResourceName()
+		useTLS = (dbGalera.Spec.TLS.SecretName != "")
 	} else if dbMariadb != nil {
 		if dbMariadb.Status.DbInitHash == "" {
 			r.Log.Info("DB initialization not complete. Requeue...")
@@ -157,10 +159,11 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		dbSecret = dbMariadb.Spec.Secret
 		dbContainerImage = dbMariadb.Spec.ContainerImage
 		serviceAccount = dbMariadb.RbacResourceName()
+		useTLS = false
 	}
 
 	// Define a new Job object (hostname, password, containerImage)
-	jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount)
+	jobDef, err := mariadb.DbDatabaseJobExt(instance, dbName, dbSecret, dbContainerImage, serviceAccount, useTLS)
 	if err != nil {
 		return ctrl.Result{}, err
 	}

pkg/database.go

@@ -14,12 +14,23 @@ type dbCreateOptions struct {
 	DatabaseName          string
 	DatabaseHostname      string
 	DatabaseAdminUsername string
+	DatabaseUserTLS       string
 }
 
 // DbDatabaseJob -
 func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) {
+	return DbDatabaseJobExt(database, databaseHostName, databaseSecret, containerImage, serviceAccountName, false)
+}
 
-	opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root"}
+// DbDatabaseJobExt -
+func DbDatabaseJobExt(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, useTLS bool) (*batchv1.Job, error) {
+	var tlsStatement string
+	if useTLS {
+		tlsStatement = " REQUIRE SSL"
+	} else {
+		tlsStatement = ""
+	}
+	opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root", tlsStatement}
 	dbCmd, err := util.ExecuteTemplateFile("database.sh", &opts)
 	if err != nil {
 		return nil, err
@@ -83,7 +94,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s
 // DeleteDbDatabaseJob -
 func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) {
 
-	opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root"}
+	opts := dbCreateOptions{database.Spec.Name, databaseHostName, "root", ""}
 	delCmd, err := util.ExecuteTemplateFile("delete_database.sh", &opts)
 	if err != nil {
 		return nil, err