Support TLS for galera

Ability to specify a certificate and a CA to be used for galera
cluster communication (GCOMM, SST).

Updates to the certificate used for galera automatically triggers
a rolling restart of the galera pods, without service disruption.

When the Galera CR is configured to use TLS, the mariadbdatabase
CR creates DB users that still allow connection to the DB without
using TLS. This is because Openstack clients currently cannot be
configured to connect via TLS or via plain TCP. This specific
part will be addressed in a subsequent commit.

api/bases/mariadb.openstack.org_galeras.yaml

@@ -91,6 +91,18 @@ spec:
               storageRequest:
                 description: Storage size allocated for the mariadb databases
                 type: string
+              tls:
+                description: TLS settings to use for MySQL and Galera replication
+                properties:
+                  caSecretName:
+                    description: Secret in the same namespace containing the CA cert
+                      (ca.crt) for client certificate validation
+                    type: string
+                  secretName:
+                    description: Secret in the same namespace containing the server
+                      private key (tls.key) and public cert (tls.crt) for TLS
+                    type: string
+                type: object
             required:
             - containerImage
             - replicas

api/v1beta1/galera_types.go

@@ -62,6 +62,16 @@ type GaleraSpec struct {
 	// +kubebuilder:validation:Enum=rsync;mariabackup
 	// Snapshot State Transfer method to use for full node synchronization
 	SST GaleraSST `json:"sst"`
+	// TLS settings to use for MySQL and Galera replication
+	TLS TLSSpec `json:"tls,omitempty"`
+}
+
+// TLSSpec defines the TLS options
+type TLSSpec struct {
+	// Secret in the same namespace containing the server private key (tls.key) and public cert (tls.crt) for TLS
+	SecretName string `json:"secretName,omitempty"`
+	// Secret in the same namespace containing the CA cert (ca.crt) for client certificate validation
+	CaSecretName string `json:"caSecretName,omitempty"`
 }
 
 // Supported SST type

config/crd/bases/mariadb.openstack.org_galeras.yaml

@@ -91,6 +91,18 @@ spec:
               storageRequest:
                 description: Storage size allocated for the mariadb databases
                 type: string
+              tls:
+                description: TLS settings to use for MySQL and Galera replication
+                properties:
+                  caSecretName:
+                    description: Secret in the same namespace containing the CA cert
+                      (ca.crt) for client certificate validation
+                    type: string
+                  secretName:
+                    description: Secret in the same namespace containing the server
+                      private key (tls.key) and public cert (tls.crt) for TLS
+                    type: string
+                type: object
             required:
             - containerImage
             - replicas

config/samples/cert-manager-galera-cert.yaml

@@ -0,0 +1,74 @@
+# the cluster-wide issuer, used to generate a root certificate
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  selfSigned: {}
+---
+# The root certificate. they cert/key/ca will be generated in the secret 'root-secret'
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: my-selfsigned-ca
+  namespace: openstack
+spec:
+  isCA: true
+  commonName: my-selfsigned-ca
+  secretName: root-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+# The CA issuer for galera, uses the certificate from `my-selfsigned-ca`
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: my-ca-issuer
+  namespace: openstack
+spec:
+  ca:
+    secretName: root-secret
+---
+# The certificate used by all galera replicas for GCOMM and SST.
+# The replicas in the galera statefulset all share the same
+# certificate, so the latter requires wildcard in dnsNames for TLS
+# validation.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: galera-cert
+spec:
+  secretName: galera-tls
+  secretTemplate:
+    labels:
+       mariadb-ref: openstack
+  duration: 6h
+  renewBefore: 1h
+  subject:
+    organizations:
+      - cluster.local
+  commonName: openstack-galera
+  isCA: false
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS8
+    size: 2048
+  usages:
+    - server auth
+    - client auth
+  dnsNames:
+    - "openstack-galera"
+    - "*.openstack-galera"
+    - "*.openstack-galera.openstack"
+    - "*.openstack-galera.openstack.svc"
+    - "*.openstack-galera.openstack.svc.cluster"
+    - "*.openstack-galera.openstack.svc.cluster.local"
+  issuerRef:
+    name: my-ca-issuer
+    group: cert-manager.io
+    kind: Issuer

config/samples/mariadb_v1beta1_galera_tls.yaml

@@ -0,0 +1,12 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  tls:
+    secretName: galera-tls
+    caSecretName: galera-tls

controllers/galera_controller.go

@@ -22,12 +22,14 @@ import (
 	env "github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
+	secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
+	k8s_labels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/kubectl/pkg/util/podutils"
@@ -44,15 +46,22 @@ import (
 
 	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 	mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg/mariadb"
 )
 
+// Label used in a k8s secret to reference its corresponding galera CR
+const mariaDBReconcileLabel = "mariadb-ref"
+
 // GaleraReconciler reconciles a Galera object
 type GaleraReconciler struct {
 	client.Client
@@ -91,7 +100,8 @@ func buildGcommURI(instance *mariadbv1.Galera) string {
 	res := []string{}
 
 	for i := 0; i < replicas; i++ {
-		res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename)
+		// Generate Gcomm with FQDN for TLS validation
+		res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename+"."+instance.Namespace+".svc")
 	}
 	uri := "gcomm://" + strings.Join(res, ",")
 	return uri
@@ -250,15 +260,17 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal
 		if !found {
 			continue
 		}
-		ci := instance.Status.Attributes[pod.Name].ContainerID
-		pci := pod.Status.ContainerStatuses[0].ContainerID
-		if ci != pci {
+		// A node can have various attributes depending on its known state.
+		// A ContainerID attribute is only present if the node is being started.
+		attrCID := instance.Status.Attributes[pod.Name].ContainerID
+		podCID := pod.Status.ContainerStatuses[0].ContainerID
+		if attrCID != "" && attrCID != podCID {
 			// This gcomm URI was pushed in a pod which was restarted
 			// before the attribute got cleared, which means the pod
 			// failed to start galera. Clear the attribute here, and
 			// reprobe the pod's state in the next reconcile loop
 			clearPodAttributes(instance, pod.Name)
-			util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name)
+			util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name, "current pod ID", podCID, "recorded ID", attrCID)
 		}
 	}
 }
@@ -461,7 +473,7 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		r.Log.Info(fmt.Sprintf("%s %s database service %s - operation: %s", instance.Kind, instance.Name, service.Name, string(op)))
 	}
 
-	// Generate the config maps for the various services
+	// Generate the config maps
 	configMapVars := make(map[string]env.Setter)
 	err = r.generateConfigMaps(ctx, helper, instance, &configMapVars)
 	if err != nil {
@@ -473,16 +485,62 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 			err.Error()))
 		return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err)
 	}
+
+	//
+	// Extend the config maps with hashes for TLS secrets if any
+	//
+	var tlsres ctrl.Result
+	var tlserror error
+	if instance.Spec.TLS.SecretName != "" {
+		tlsres, tlserror = r.checkAndHashTLSSecret(ctx, helper, instance, instance.Spec.TLS.SecretName, &configMapVars)
+	}
+	if tlserror == nil && instance.Spec.TLS.CaSecretName != "" {
+		tlsres, tlserror = r.checkAndHashTLSSecret(ctx, helper, instance, instance.Spec.TLS.CaSecretName, &configMapVars)
+	}
+
+	if tlserror != nil {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.ServiceConfigReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.ServiceConfigReadyErrorMessage,
+			tlserror.Error()))
+
+		if tlsres.RequeueAfter > 0 {
+			// TLS secret might be recreated, requeue as this is not fatal
+			util.LogForObject(helper, "Requeuing until TLS configuration is available", instance, "error", tlserror.Error())
+			return tlsres, nil
+		}
+
+		return tlsres, fmt.Errorf("error calculating configmap hash: %w", tlserror)
+	}
+
 	// From hereon, configMapVars holds a hash of the config generated for this instance
+	// as well as a hash and of the current TLS certificate and CA used if any.
 	// This is used in an envvar in the statefulset to restart it on config change
-	envHash := &corev1.EnvVar{}
-	configMapVars[configMapNameForConfig(instance)](envHash)
-	instance.Status.ConfigHash = envHash.Value
+
+	keys := make([]string, 0, len(configMapVars))
+	for k := range configMapVars {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	hash := ""
+	for _, k := range keys {
+		envVar := &corev1.EnvVar{}
+		configMapVars[k](envVar)
+		hash = hash + envVar.Value
+	}
+	// TODO maybe hash the resulting string to shorten it
+	instance.Status.ConfigHash = hash
 	instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
 
 	commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance), 5)
 	sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper)
 	if sferr != nil {
+		if k8s_errors.IsNotFound(sferr) {
+			return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil
+		}
 		return sfres, sferr
 	}
 	statefulset := commonstatefulset.GetStatefulSet()
@@ -651,6 +709,44 @@ func (r *GaleraReconciler) generateConfigMaps(
 	return nil
 }
 
+func (r *GaleraReconciler) checkAndHashTLSSecret(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *mariadbv1.Galera,
+	secretName string,
+	envVars *map[string]env.Setter,
+) (ctrl.Result, error) {
+	tlsSecret, tlsHash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil
+		}
+		return ctrl.Result{}, fmt.Errorf("secret %s not found", secretName)
+	}
+
+	if value, ok := tlsSecret.Labels[mariaDBReconcileLabel]; !ok || value != instance.Name {
+		tlsSecret.GetObjectMeta().SetLabels(
+			k8s_labels.Merge(
+				tlsSecret.GetObjectMeta().GetLabels(),
+				map[string]string{
+					mariaDBReconcileLabel: instance.Name,
+				},
+			),
+		)
+		err = r.Client.Update(ctx, tlsSecret)
+		if err != nil {
+			if k8s_errors.IsConflict(err) || k8s_errors.IsNotFound(err) {
+				return ctrl.Result{Requeue: true}, err
+			}
+			return ctrl.Result{}, err
+		}
+	}
+
+	(*envVars)[secretName] = env.SetValue(tlsHash)
+
+	return ctrl.Result{}, nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.config = mgr.GetConfig()
@@ -663,5 +759,22 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.Role{}).
 		Owns(&rbacv1.RoleBinding{}).
+		Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(
+			func(o client.Object) []reconcile.Request {
+				labels := o.GetLabels()
+
+				reconcileCR, hasLabel := labels[mariaDBReconcileLabel]
+				if !hasLabel {
+					return []reconcile.Request{}
+				}
+
+				return []reconcile.Request{
+					{NamespacedName: types.NamespacedName{
+						Name:      reconcileCR,
+						Namespace: o.GetNamespace(),
+					}},
+				}
+			},
+		)).
 		Complete(r)
 }