diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index e4860d42..6c696964 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -84,6 +84,29 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings for MySQL service and internal Galera replication + properties: + ca: + description: Ca contains CA-specific settings, which could be + used both by services (to define their own CA certificates) + and by clients (to verify the server's certificate) + properties: + caSecretName: + type: string + type: object + service: + description: Service contains server-specific TLS secret + properties: + disableNonTLSListeners: + type: boolean + secretName: + type: string + type: object + required: + - ca + - service + type: object required: - containerImage - replicas @@ -162,16 +185,16 @@ spec: - type type: object type: array - configHash: - default: "" - description: Hash of the configuration files - type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string required: - bootstrapped - - configHash type: object type: object served: true diff --git a/api/go.mod b/api/go.mod index 84f1133d..d575ae55 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,9 +4,9 @@ go 1.19 require ( github.com/go-logr/logr v1.2.4 - github.com/onsi/ginkgo/v2 v2.12.0 - github.com/onsi/gomega v1.27.10 - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 + github.com/onsi/ginkgo/v2 v2.12.1 + github.com/onsi/gomega v1.28.0 + github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e k8s.io/api v0.26.12 k8s.io/apimachinery v0.26.12 k8s.io/client-go v0.26.12 @@ -32,7 +32,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -50,14 +50,14 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.25.0 // indirect + go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.13.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect @@ -66,7 +66,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.12 // indirect k8s.io/component-base v0.26.12 // indirect - k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/api/go.sum b/api/go.sum index 318036a2..440b22e1 100644 --- a/api/go.sum +++ b/api/go.sum @@ -41,7 +41,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -172,8 +171,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -227,14 +226,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e h1:/bKZdCAsu73wscdiMsmctAmh0Jz432WxVQe4h1+ipzQ= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -312,8 +311,8 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -480,8 +479,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -613,8 +612,8 @@ k8s.io/client-go v0.26.12 h1:kPpTpIeFNqwo4UyvoqzNp3DNK2mbGcdGv23eS1U8VMo= k8s.io/client-go v0.26.12/go.mod h1:V7thEnIFroyNZOU30dKLiiVeqQmJz45shJG1mu7nONQ= k8s.io/component-base v0.26.12 h1:OyYjCtruv4/Yau5Z1v6e59N+JRDTj8JnW95W9w9AMpg= k8s.io/component-base v0.26.12/go.mod h1:X98Et5BxJ8i4TcDusUcKS8EYxCujBU1lCL3pc/CUtHQ= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index b00981b5..98b14b69 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -69,6 +69,9 @@ const ( // MariaDBInitializedErrorMessage MariaDBInitializedErrorMessage = "MariaDB dbinit error occured %s" + // MariaDBInputSecretNotFoundMessage + MariaDBInputSecretNotFoundMessage = "Input secret not found: %s" + MariaDBDatabaseReadyInitMessage = "MariaDBDatabase not yet available" MariaDBDatabaseReadyMessage = "MariaDBDatabase ready" diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index da3abaec..4f646785 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -56,6 +57,9 @@ type GaleraSpec struct { // +kubebuilder:validation:Optional // Adoption configuration AdoptionRedirect AdoptionRedirectSpec `json:"adoptionRedirect"` + // +kubebuilder:validation:Optional + // TLS settings for MySQL service and internal Galera replication + TLS *tls.TLS `json:"tls,omitempty"` } // GaleraAttributes holds startup information for a Galera host @@ -77,9 +81,8 @@ type GaleraStatus struct { // Is the galera cluster currently running // +kubebuilder:default=false Bootstrapped bool `json:"bootstrapped"` - // Hash of the configuration files - // +kubebuilder:default="" - ConfigHash string `json:"configHash"` + // Map of hashes to track input changes + Hash map[string]string `json:"hash,omitempty"` // Deployment Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` } diff --git a/api/v1beta1/mariadbdatabase_funcs.go b/api/v1beta1/mariadbdatabase_funcs.go index 8f5e818d..7238e6f0 100644 --- a/api/v1beta1/mariadbdatabase_funcs.go +++ b/api/v1beta1/mariadbdatabase_funcs.go @@ -106,7 +106,8 @@ func (d *Database) setDatabaseHostname( err, ) } - d.databaseHostname = serviceList.Items[0].GetName() + svc := serviceList.Items[0] + d.databaseHostname = svc.GetName() + "." + svc.GetNamespace() + ".svc" return nil } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index eb1cd555..1c44c979 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "k8s.io/apimachinery/pkg/runtime" ) @@ -158,6 +159,11 @@ func (in *GaleraSpec) DeepCopyInto(out *GaleraSpec) { } } out.AdoptionRedirect = in.AdoptionRedirect + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(tls.TLS) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GaleraSpec. @@ -180,6 +186,13 @@ func (in *GaleraStatus) DeepCopyInto(out *GaleraStatus) { (*out)[key] = val } } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(condition.Conditions, len(*in)) diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index e4860d42..6c696964 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -84,6 +84,29 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings for MySQL service and internal Galera replication + properties: + ca: + description: Ca contains CA-specific settings, which could be + used both by services (to define their own CA certificates) + and by clients (to verify the server's certificate) + properties: + caSecretName: + type: string + type: object + service: + description: Service contains server-specific TLS secret + properties: + disableNonTLSListeners: + type: boolean + secretName: + type: string + type: object + required: + - ca + - service + type: object required: - containerImage - replicas @@ -162,16 +185,16 @@ spec: - type type: object type: array - configHash: - default: "" - description: Hash of the configuration files - type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string required: - bootstrapped - - configHash type: object type: object served: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d8dbef82..597e4d8d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -121,7 +121,9 @@ rules: - delete - get - list + - patch - update + - watch - apiGroups: - "" resources: diff --git a/config/samples/cert-manager-galera-cert.yaml b/config/samples/cert-manager-galera-cert.yaml new file mode 100644 index 00000000..0982b9a9 --- /dev/null +++ b/config/samples/cert-manager-galera-cert.yaml @@ -0,0 +1,75 @@ +# the cluster-wide issuer, used to generate a root certificate +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} +--- +# The root certificate. they cert/key/ca will be generated in the secret 'root-secret' +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: my-selfsigned-ca + namespace: openstack +spec: + isCA: true + commonName: my-selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: ClusterIssuer + group: cert-manager.io +--- +# The CA issuer for galera, uses the certificate from `my-selfsigned-ca` +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: my-ca-issuer + namespace: openstack +spec: + ca: + secretName: root-secret +--- +# The certificate used by all galera replicas for GCOMM and SST. +# The replicas in the galera statefulset all share the same +# certificate, so the latter requires wildcard in dnsNames for TLS +# validation. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: galera-cert +spec: + secretName: galera-tls + secretTemplate: + labels: + mariadb-ref: openstack + duration: 6h + renewBefore: 1h + subject: + organizations: + - cluster.local + commonName: openstack-galera + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS8 + size: 2048 + usages: + - server auth + - client auth + dnsNames: + - "openstack.openstack.svc" + - "openstack.openstack.svc.cluster.local" + - "*.openstack-galera" + - "*.openstack-galera.openstack" + - "*.openstack-galera.openstack.svc" + - "*.openstack-galera.openstack.svc.cluster" + - "*.openstack-galera.openstack.svc.cluster.local" + issuerRef: + name: my-ca-issuer + group: cert-manager.io + kind: Issuer diff --git a/config/samples/mariadb_v1beta1_galera_tls.yaml b/config/samples/mariadb_v1beta1_galera_tls.yaml new file mode 100644 index 00000000..e92995d9 --- /dev/null +++ b/config/samples/mariadb_v1beta1_galera_tls.yaml @@ -0,0 +1,14 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + service: + secretName: galera-tls + ca: + caSecretName: galera-tls diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index 59d1b28a..9e2b2afe 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -17,11 +17,13 @@ limitations under the License. package controllers import ( + "github.com/openstack-k8s-operators/lib-common/modules/common" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" configmap "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" @@ -29,6 +31,8 @@ import ( rbacv1 "k8s.io/api/rbac/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/util/podutils" @@ -46,15 +50,33 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" databasev1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg/mariadb" ) +// fields to index to reconcile on CR change +const ( + serviceSecretNameField = ".spec.tls.service.SecretName" + caSecretNameField = ".spec.tls.ca.caSecretName" +) + +var ( + allWatchFields = []string{ + serviceSecretNameField, + caSecretNameField, + } +) + // GaleraReconciler reconciles a Galera object type GaleraReconciler struct { client.Client @@ -97,7 +119,8 @@ func buildGcommURI(instance *mariadbv1.Galera) string { res := []string{} for i := 0; i < replicas; i++ { - res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename) + // Generate Gcomm with FQDN for TLS validation + res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename+"."+instance.Namespace+".svc") } uri := "gcomm://" + strings.Join(res, ",") return uri @@ -257,15 +280,17 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal if !found { continue } - ci := instance.Status.Attributes[pod.Name].ContainerID - pci := pod.Status.ContainerStatuses[0].ContainerID - if ci != pci { + // A node can have various attributes depending on its known state. + // A ContainerID attribute is only present if the node is being started. + attrCID := instance.Status.Attributes[pod.Name].ContainerID + podCID := pod.Status.ContainerStatuses[0].ContainerID + if attrCID != "" && attrCID != podCID { // This gcomm URI was pushed in a pod which was restarted // before the attribute got cleared, which means the pod // failed to start galera. Clear the attribute here, and // reprobe the pod's state in the next reconcile loop clearPodAttributes(instance, pod.Name) - util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name) + util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name, "current pod ID", podCID, "recorded ID", attrCID) } } } @@ -283,6 +308,9 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// RBAC for secrets +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; + // RBAC for services and endpoints // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;create;update;patch;delete; @@ -359,6 +387,8 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res instance.Status.Conditions = condition.Conditions{} // initialize conditions used later as Status=Unknown cl := condition.CreateList( + // DB Root password and TLS secrets + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), // endpoint for adoption redirect condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), // configmap generation @@ -468,9 +498,50 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res log.Info("", "Kind", instance.Kind, "Name", instance.Name, "database service", service.Name, "operation", string(op)) } - // Generate the config maps for the various services - configMapVars := make(map[string]env.Setter) - err = r.generateConfigMaps(ctx, helper, instance, &configMapVars) + // Hash of all resources that may cause a service restart + inputHashEnv := make(map[string]env.Setter) + + // Check and hash inputs + secretName := instance.Spec.Secret + // NOTE do not hash the db root password, as its change requires + // more orchestration than a simple rolling restart + _, _, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + + var certHash, caHash string + tls := instance.Spec.TLS + if err == nil && tls != nil && tls.Service.SecretName != "" { + secretName := tls.Service.SecretName + _, certHash, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + inputHashEnv["Cert"] = env.SetValue(certHash) + } + if err == nil && tls != nil && tls.Ca.CaSecretName != "" { + secretName := tls.Ca.CaSecretName + _, caHash, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + inputHashEnv["CA"] = env.SetValue(caHash) + } + + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + mariadbv1.MariaDBInputSecretNotFoundMessage, + secretName)) + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("error calculating input hash: %w", err) + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // Generate and hash config maps + err = r.generateConfigMaps(ctx, helper, instance, &inputHashEnv) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -480,16 +551,36 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res err.Error())) return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err) } - // From hereon, configMapVars holds a hash of the config generated for this instance - // This is used in an envvar in the statefulset to restart it on config change - envHash := &corev1.EnvVar{} - configMapVars[configMapNameForConfig(instance)](envHash) - instance.Status.ConfigHash = envHash.Value instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance), 5) + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + hashOfHashes, err := util.HashOfInputHashes(inputHashEnv) + if err != nil { + return ctrl.Result{}, err + } + if hashMap, changed := util.SetHash(instance.Status.Hash, common.InputHashName, hashOfHashes); changed { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so update all the input hashes and return to reconcile again + instance.Status.Hash = hashMap + for k, s := range inputHashEnv { + var envVar corev1.EnvVar + s(&envVar) + instance.Status.Hash[k] = envVar.Value + } + util.LogForObject(helper, fmt.Sprintf("Input hash changed %s", hashOfHashes), instance) + return ctrl.Result{}, nil + } + + // The statefulset that manages galera pod + commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance, hashOfHashes), 5) sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper) if sferr != nil { + if k8s_errors.IsNotFound(sferr) { + return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + } return sfres, sferr } statefulset := commonstatefulset.GetStatefulSet() @@ -662,6 +753,34 @@ func (r *GaleraReconciler) generateConfigMaps( // SetupWithManager sets up the controller with the Manager. func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { r.config = mgr.GetConfig() + + // Various CR fields need to be indexed to filter watch events + // for the secret changes we want to be notified of + // index caSecretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &mariadbv1.Galera{}, caSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*mariadbv1.Galera) + tls := cr.Spec.TLS + if tls != nil && tls.Ca != nil && tls.Ca.CaSecretName == "" { + return []string{tls.Ca.CaSecretName} + } + return nil + }); err != nil { + return err + } + // index serviceSecretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &mariadbv1.Galera{}, serviceSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*mariadbv1.Galera) + tls := cr.Spec.TLS + if tls != nil && tls.Service != nil && tls.Service.SecretName == "" { + return []string{cr.Spec.TLS.Service.SecretName} + } + return nil + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&mariadbv1.Galera{}). Owns(&appsv1.StatefulSet{}). @@ -671,6 +790,11 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } @@ -715,3 +839,33 @@ func GetDatabaseObject(clientObj client.Client, ctx context.Context, name string return dbGalera, nil } + +// findObjectsForSrc - returns a reconcile request if the object is referenced by a Galera CR +func (r *GaleraReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + for _, field := range allWatchFields { + crList := &mariadbv1.GaleraList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go index 92430262..1ce43ee6 100644 --- a/controllers/mariadbdatabase_controller.go +++ b/controllers/mariadbdatabase_controller.go @@ -132,6 +132,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Non-deletion (normal) flow follows // var dbName, dbSecret, dbContainerImage, serviceAccount string + var useTLS bool // It is impossible to reach here without either dbGalera or dbMariadb not being nil, due to the checks above @@ -149,6 +150,11 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbSecret = dbGalera.Spec.Secret dbContainerImage = dbGalera.Spec.ContainerImage serviceAccount = dbGalera.RbacResourceName() + // NOTE(dciabrin) When configured to only allow TLS connections, all clients + // accessing this DB must support client connection via TLS. + useTLS = (dbGalera.Spec.TLS != nil && + dbGalera.Spec.TLS.Service != nil && + dbGalera.Spec.TLS.Service.DisableNonTLSListeners) } else if isMariaDB { if dbMariadb.Status.DbInitHash == "" { log.Info("DB initialization not complete. Requeue...") @@ -159,10 +165,11 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ dbSecret = dbMariadb.Spec.Secret dbContainerImage = dbMariadb.Spec.ContainerImage serviceAccount = dbMariadb.RbacResourceName() + useTLS = false } // Define a new Job object (hostname, password, containerImage) - jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount) + jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount, useTLS) if err != nil { return ctrl.Result{}, err } diff --git a/go.mod b/go.mod index fbba3d53..087c55d1 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.19 require ( github.com/go-logr/logr v1.2.4 github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega v1.27.10 - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230927082538-4f614f333d17 + github.com/onsi/gomega v1.28.0 + github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e github.com/openstack-k8s-operators/mariadb-operator/api v0.1.1-0.20230823144333-b9363c5be8d2 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.26.12 diff --git a/go.sum b/go.sum index 304b7186..0906bee0 100644 --- a/go.sum +++ b/go.sum @@ -231,10 +231,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230927082538-4f614f333d17 h1:n5QmZLJfPtKbNnPVqqSQkLU1X/NMmW3CbML3yjBUjyY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230927082538-4f614f333d17/go.mod h1:kZS5rqVWBZeCyYor2PeQB9IEZ19mGaeL/to3x8F9OJg= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e h1:/bKZdCAsu73wscdiMsmctAmh0Jz432WxVQe4h1+ipzQ= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20231004075925-7a2ccbf0ea0e/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/pkg/mariadb/database.go b/pkg/mariadb/database.go index 59b45719..59980d20 100644 --- a/pkg/mariadb/database.go +++ b/pkg/mariadb/database.go @@ -16,10 +16,17 @@ type dbCreateOptions struct { DatabaseAdminUsername string DefaultCharacterSet string DefaultCollation string + DatabaseUserTLS string } // DbDatabaseJob - -func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) { +func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, useTLS bool) (*batchv1.Job, error) { + var tlsStatement string + if useTLS { + tlsStatement = " REQUIRE SSL" + } else { + tlsStatement = "" + } opts := dbCreateOptions{ database.Spec.Name, @@ -27,6 +34,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s "root", database.Spec.DefaultCharacterSet, database.Spec.DefaultCollation, + tlsStatement, } dbCmd, err := util.ExecuteTemplateFile("database.sh", &opts) if err != nil { @@ -97,6 +105,7 @@ func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHost "root", database.Spec.DefaultCharacterSet, database.Spec.DefaultCollation, + "", } delCmd, err := util.ExecuteTemplateFile("delete_database.sh", &opts) if err != nil { diff --git a/pkg/mariadb/statefulset.go b/pkg/mariadb/statefulset.go index 2abcfff8..b945e555 100644 --- a/pkg/mariadb/statefulset.go +++ b/pkg/mariadb/statefulset.go @@ -11,9 +11,12 @@ import ( ) // StatefulSet returns a StatefulSet object for the galera cluster -func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { +func StatefulSet(g *mariadbv1.Galera, configHash string) *appsv1.StatefulSet { ls := StatefulSetLabels(g) name := StatefulSetName(g.Name) + replicas := g.Spec.Replicas + storage := g.Spec.StorageClass + storageRequest := resource.MustParse(g.Spec.StorageRequest) sts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -21,7 +24,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, Spec: appsv1.StatefulSetSpec{ ServiceName: name, - Replicas: g.Spec.Replicas, + Replicas: replicas, Selector: &metav1.LabelSelector{ MatchLabels: ls, }, @@ -53,25 +56,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraInitVolumeMounts(g), }}, Containers: []corev1.Container{{ Image: g.Spec.ContainerImage, @@ -80,7 +65,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { Command: []string{"/usr/bin/dumb-init", "--", "/usr/local/bin/kolla_start"}, Env: []corev1.EnvVar{{ Name: "CR_CONFIG_HASH", - Value: g.Status.ConfigHash, + Value: configHash, }, { Name: "KOLLA_CONFIG_STRATEGY", Value: "COPY_ALWAYS", @@ -102,29 +87,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { ContainerPort: 4567, Name: "galera", }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/secrets", - ReadOnly: true, - Name: "secrets", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraVolumeMounts(g), StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ @@ -149,92 +112,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - Volumes: []corev1.Volume{ - { - Name: "secrets", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: g.Spec.Secret, - Items: []corev1.KeyToPath{ - { - Key: "DbRootPassword", - Path: "dbpassword", - }, - }, - }, - }, - }, - { - Name: "kolla-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "pod-config-data", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "config-data", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "galera.cnf.in", - Path: "galera.cnf.in", - }, - { - Key: mariadbv1.CustomServiceConfigFile, - Path: mariadbv1.CustomServiceConfigFile, - }, - }, - }, - }, - }, - { - Name: "operator-scripts", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-scripts", - }, - Items: []corev1.KeyToPath{ - { - Key: "mysql_bootstrap.sh", - Path: "mysql_bootstrap.sh", - }, - { - Key: "mysql_probe.sh", - Path: "mysql_probe.sh", - }, - { - Key: "detect_last_commit.sh", - Path: "detect_last_commit.sh", - }, - { - Key: "detect_gcomm_and_start.sh", - Path: "detect_gcomm_and_start.sh", - }, - }, - }, - }, - }, - }, + Volumes: getGaleraVolumes(g), }, }, VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ @@ -247,10 +125,10 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, - StorageClassName: &g.Spec.StorageClass, + StorageClassName: &storage, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - "storage": resource.MustParse(g.Spec.StorageRequest), + "storage": storageRequest, }, }, }, diff --git a/pkg/mariadb/volumes.go b/pkg/mariadb/volumes.go index 459131f3..a6f2cbb5 100644 --- a/pkg/mariadb/volumes.go +++ b/pkg/mariadb/volumes.go @@ -1,6 +1,7 @@ package mariadb import ( + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) @@ -113,3 +114,181 @@ func getInitVolumeMounts() []corev1.VolumeMount { } } + +func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { + configTemplates := []corev1.KeyToPath{ + { + Key: "galera.cnf.in", + Path: "galera.cnf.in", + }, + { + Key: mariadbv1.CustomServiceConfigFile, + Path: mariadbv1.CustomServiceConfigFile, + }, + } + + if g.Spec.TLS != nil && g.Spec.TLS.Service.SecretName != "" { + if g.Spec.TLS.Ca.CaSecretName != "" { + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_tls.cnf.in", + Path: "galera_tls.cnf.in", + }) + } else { + // Without a CA, WSREP is unencrypted. Only SQL traffic is. + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_external_tls.cnf.in", + Path: "galera_external_tls.cnf.in", + }) + } + } + + volumes := []corev1.Volume{ + { + Name: "secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.Secret, + Items: []corev1.KeyToPath{ + { + Key: "DbRootPassword", + Path: "dbpassword", + }, + }, + }, + }, + }, + { + Name: "kolla-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: []corev1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + }, + }, + }, + }, + }, + { + Name: "pod-config-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: configTemplates, + }, + }, + }, + { + Name: "operator-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-scripts", + }, + Items: []corev1.KeyToPath{ + { + Key: "mysql_bootstrap.sh", + Path: "mysql_bootstrap.sh", + }, + { + Key: "mysql_probe.sh", + Path: "mysql_probe.sh", + }, + { + Key: "detect_last_commit.sh", + Path: "detect_last_commit.sh", + }, + { + Key: "detect_gcomm_and_start.sh", + Path: "detect_gcomm_and_start.sh", + }, + }, + }, + }, + }, + } + + if g.Spec.TLS != nil { + caVolumes := g.Spec.TLS.CreateVolumes() + volumes = append(volumes, caVolumes...) + } + + return volumes +} + +func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config", + }, + } + + if g.Spec.TLS != nil { + caVolumeMounts := g.Spec.TLS.CreateVolumeMounts() + volumeMounts = append(volumeMounts, caVolumeMounts...) + } + + return volumeMounts +} + +func getGaleraInitVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config", + }, + } + + return volumeMounts +} diff --git a/templates/database.sh b/templates/database.sh index 96484af9..7b933395 100755 --- a/templates/database.sh +++ b/templates/database.sh @@ -1,4 +1,4 @@ #!/bin/bash export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} -mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}'; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword';GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword';" +mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}'; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};" diff --git a/templates/galera/config/config.json b/templates/galera/config/config.json index 185e33fa..bc2b6f33 100644 --- a/templates/galera/config/config.json +++ b/templates/galera/config/config.json @@ -7,6 +7,20 @@ "owner": "root", "perm": "0644" }, + { + "source": "/var/lib/pod-config-data/galera_tls.cnf", + "dest": "/etc/my.cnf.d/galera_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, + { + "source": "/var/lib/pod-config-data/galera_external_tls.cnf", + "dest": "/etc/my.cnf.d/galera_external_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, { "source": "/var/lib/pod-config-data/galera_custom.cnf", "dest": "/etc/my.cnf.d/galera_custom.cnf", @@ -20,6 +34,27 @@ "owner": "root", "perm": "0755", "merge": "true" + }, + { + "source": "/var/lib/config-data/tls-certificates/tls.key", + "dest": "/etc/pki/tls/private/mysql.key", + "owner": "mysql", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/config-data/tls-certificates/tls.crt", + "dest": "/etc/pki/tls/certs/mysql.crt", + "owner": "mysql", + "perm": "0755", + "optional": true + }, + { + "source": "/var/lib/config-data/ca-certificates/ca.crt", + "dest": "/etc/ipa/ca.crt", + "owner": "mysql", + "perm": "0444", + "optional": true } ], "permissions": [ diff --git a/templates/galera/config/galera_external_tls.cnf.in b/templates/galera/config/galera_external_tls.cnf.in new file mode 100644 index 00000000..20a099f0 --- /dev/null +++ b/templates/galera/config/galera_external_tls.cnf.in @@ -0,0 +1,8 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/mysql.crt +ssl-key = /etc/pki/tls/private/mysql.key +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 + +[sst] +ssl-mode = DISABLED diff --git a/templates/galera/config/galera_tls.cnf.in b/templates/galera/config/galera_tls.cnf.in new file mode 100644 index 00000000..040c4e60 --- /dev/null +++ b/templates/galera/config/galera_tls.cnf.in @@ -0,0 +1,15 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/mysql.crt +ssl-key = /etc/pki/tls/private/mysql.key +ssl-ca = /etc/ipa/ca.crt +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +wsrep_provider_options = gcache.recover=no;gmcast.listen_addr=tcp://{ PODIP }:4567;socket.ssl_key=/etc/pki/tls/private/mysql.key;socket.ssl_cert=/etc/pki/tls/certs/mysql.crt;socket.ssl_cipher=AES128-SHA256;socket.ssl_ca=/etc/ipa/ca.crt; + +[sst] +sockopt = cipher=!SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +tcert = /etc/pki/tls/certs/mysql.crt +tkey = /etc/pki/tls/private/mysql.key +tca = /etc/ipa/ca.crt +encrypt = 3 +ssl-mode = REQUIRED diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index 33d9720c..2df58a6c 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -28,6 +28,10 @@ status: reason: Ready status: "True" type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml new file mode 100644 index 00000000..4e0eaec3 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml @@ -0,0 +1,44 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 1 + secret: osp-secret + storageRequest: 500M + tls: + service: + secretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml new file mode 100644 index 00000000..d94f3ba8 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml @@ -0,0 +1,45 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: kuttldb + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource with external TLS only (no TLS for galera WSREP traffic) +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 1 + tls: + service: + secretName: kuttl-galera-tls + disableNonTLSListeners: true + ca: + caSecretName: kuttl-galera-tls +--- +# a database cr associated with the db above will create users with grants +# set up to only allow TLS connection +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: kuttldb + labels: + dbName: openstack +spec: + secret: kuttldb-secret + name: kuttldb +--- +apiVersion: v1 +kind: Secret +metadata: + name: kuttldb-secret +data: + DatabasePassword: MTIzNDU2Nzg= diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml new file mode 120000 index 00000000..d04979c5 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml @@ -0,0 +1 @@ +../galera_deploy_tls/01-tls-certificate.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml b/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml new file mode 100644 index 00000000..ecb23685 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure db users are configured to TLS restriction + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -e "show grants for \`kuttldb\`@\`%\`;"' | grep 'REQUIRE SSL' diff --git a/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml b/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml new file mode 100644 index 00000000..0d81c742 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: kuttldb + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml b/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml index 5d073a88..d87f62c4 100644 --- a/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml +++ b/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml @@ -7,3 +7,4 @@ delete: commands: - script: | oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml new file mode 100644 index 00000000..7b80b88b --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml @@ -0,0 +1,54 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + tls: + service: + secretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml new file mode 100644 index 00000000..902f6d35 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource with external TLS only (no TLS for galera WSREP traffic) +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + service: + secretName: kuttl-galera-tls + ca: + caSecretName: diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml new file mode 120000 index 00000000..d04979c5 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml @@ -0,0 +1 @@ +../galera_deploy_tls/01-tls-certificate.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml new file mode 100644 index 00000000..e59da5b7 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure galera does not encrypt WSREP traffic + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.wsrep_provider_options;"' | grep -o -w 'gmcast.listen_addr = tcp' + # ensure mysql/SQL traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.ssl_cipher;"' | grep -v '^NULL$' diff --git a/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml b/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml new file mode 100644 index 00000000..68c7f897 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml new file mode 100644 index 00000000..ae35e3e2 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml @@ -0,0 +1,56 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + tls: + service: + secretName: kuttl-galera-tls + ca: + caSecretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml b/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml new file mode 100644 index 00000000..870aaa22 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource using the TLS certs +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + service: + secretName: kuttl-galera-tls + ca: + caSecretName: kuttl-galera-tls diff --git a/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml new file mode 100644 index 00000000..f50c4cde --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml @@ -0,0 +1,49 @@ +# hardcode the secret generated by a certificate CR here, +# to avoid kuttl from depending on cert-manager at runtime +# --- +# apiVersion: cert-manager.io/v1 +# kind: Certificate +# metadata: +# name: kuttl-galera-cert +# spec: +# secretName: kuttl-galera-tls +# secretTemplate: +# labels: +# mariadb-ref: openstack +# duration: 12720h +# renewBefore: 1h +# subject: +# organizations: +# - cluster.local +# commonName: openstack-galera +# isCA: false +# privateKey: +# algorithm: RSA +# encoding: PKCS8 +# size: 1024 +# usages: +# - server auth +# - client auth +# dnsNames: +# - "openstack.openstack.svc" +# - "openstack.openstack.svc.cluster.local" +# - "*.openstack-galera" +# - "*.openstack-galera.openstack" +# - "*.openstack-galera.openstack.svc" +# - "*.openstack-galera.openstack.svc.cluster" +# - "*.openstack-galera.openstack.svc.cluster.local" +# issuerRef: +# name: kuttl-ca-issuer +# group: cert-manager.io +# kind: Issuer +# --- +apiVersion: v1 +kind: Secret +metadata: + name: kuttl-galera-tls + labels: + mariadb-ref: openstack +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmVENDQVNLZ0F3SUJBZ0lRUHhtRFFscmxjNTNhb215RVU5MU9pakFLQmdncWhrak9QUVFEQWpBZU1Sd3cKR2dZRFZRUURFeE5yZFhSMGJDMXpaV3htYzJsbmJtVmtMV05oTUI0WERUSXpNVEF4T0RFeU1EazFNMW9YRFRJMApNREV4TmpFeU1EazFNMW93SGpFY01Cb0dBMVVFQXhNVGEzVjBkR3d0YzJWc1puTnBaMjVsWkMxallUQlpNQk1HCkJ5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCSVdJY0JiR0cveEg4Lzlkc2lMbkJCdnRqcEZoQ2JRM3U4R0EKZXBVcnhTY25XM0hrZ2hrc1BCVE12M3NCeGdnVFQwL0Eva0dtazRYTkJ0dElnbUZJaFBpalFqQkFNQTRHQTFVZApEd0VCL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlFKaDd3VklFYjgxcFlsCkl3RDAraTBwSnlCTjNqQUtCZ2dxaGtqT1BRUURBZ05KQURCR0FpRUF2a3h5RzZjNzltSDlRWHRIVWFSM014REkKUUVRRGVtL1hZR3VGY1ZCUDJpQUNJUUNFeEZqeStQUTBkNFU5dEJacTVOd1gzdmxibnQxVlNCYWE5VFIrNkNkbAozdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQXhDZ0F3SUJBZ0lRUWw2bjhlS21BbXZlbDhZTGhIaFVjREFLQmdncWhrak9QUVFEQWpBZU1Sd3cKR2dZRFZRUURFeE5yZFhSMGJDMXpaV3htYzJsbmJtVmtMV05oTUI0WERUSXpNVEF5TXpBNE16UTBNRm9YRFRJMQpNRFF3TlRBNE16UTBNRm93TXpFV01CUUdBMVVFQ2hNTlkyeDFjM1JsY2k1c2IyTmhiREVaTUJjR0ExVUVBeE1RCmIzQmxibk4wWVdOckxXZGhiR1Z5WVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUIKQU9QMnFUVkhvUmFsb1VCb0RSS1NrZExsR2ZqV2U2WEFFQ21HZGpRdzNjY0NHWUNKc3U3akROSFMxalJCa0ZUKwpnVERUNGk5MndPelI2WGpWQ2Fld1pCSXlyU1d4N3Y3SndaZFkvVkQzVWdRSFdpWEpNOFJlRFZUV0J6TjdqL3JTCktGY0lIcURGYjFGRy9jS0gzcEJrTVRReTZSZXRRdHRkTVl0bE1tVEdFdFJnQVZQdGlLTXZnMXZadThUY053VEwKZ2h0cjFOdURBYzFpVXhzL1p5S0o0ZjBHNEJMVlNHNVgwRXRvVUZxcWF3R2EySGhMZld5TXEyRWdoWGE1QzZHeQoxSkdVYUtCZHdCMjRLd2loSVR0U1Z3SGxrK2FGQ2Nac3FzT1BuR2c4WTZRSkM1bERUeE1UWXpHN1pxaGtMVmgwClFJTUpVMnNtMEd3aENYWVlGOXF5c2JNQ0F3RUFBYU9DQVU0d2dnRktNQjBHQTFVZEpRUVdNQlFHQ0NzR0FRVUYKQndNQkJnZ3JCZ0VGQlFjREFqQU1CZ05WSFJNQkFmOEVBakFBTUI4R0ExVWRJd1FZTUJhQUZBbUh2QlVnUnZ6VwpsaVVqQVBUNkxTa25JRTNlTUlINUJnTlZIUkVFZ2ZFd2dlNkNGMjl3Wlc1emRHRmpheTV2Y0dWdWMzUmhZMnN1CmMzWmpnaVZ2Y0dWdWMzUmhZMnN1YjNCbGJuTjBZV05yTG5OMll5NWpiSFZ6ZEdWeUxteHZZMkZzZ2hJcUxtOXcKWlc1emRHRmpheTFuWVd4bGNtR0NIQ291YjNCbGJuTjBZV05yTFdkaGJHVnlZUzV2Y0dWdWMzUmhZMnVDSUNvdQpiM0JsYm5OMFlXTnJMV2RoYkdWeVlTNXZjR1Z1YzNSaFkyc3VjM1pqZ2lncUxtOXdaVzV6ZEdGamF5MW5ZV3hsCmNtRXViM0JsYm5OMFlXTnJMbk4yWXk1amJIVnpkR1Z5Z2k0cUxtOXdaVzV6ZEdGamF5MW5ZV3hsY21FdWIzQmwKYm5OMFlXTnJMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNNQW9HQ0NxR1NNNDlCQU1DQTBrQU1FWUNJUUNIbllHMApxWFhqVGxPSFFpaXpwdVN0NVVQc3F5UXY3eTVNVDhaZ3RxYmd3Z0loQVBRdUNWeFBMb3daOGVWV3BTc09DVnNICjM0ZmFhWFFmbjVEZXE0eGFGSXFmCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRGo5cWsxUjZFV3BhRkEKYUEwU2twSFM1Um40MW51bHdCQXBoblkwTU4zSEFobUFpYkx1NHd6UjB0WTBRWkJVL29FdzArSXZkc0RzMGVsNAoxUW1uc0dRU01xMGxzZTcreWNHWFdQMVE5MUlFQjFvbHlUUEVYZzFVMWdjemU0LzYwaWhYQ0I2Z3hXOVJSdjNDCmg5NlFaREUwTXVrWHJVTGJYVEdMWlRKa3hoTFVZQUZUN1lpakw0TmIyYnZFM0RjRXk0SWJhOVRiZ3dITllsTWIKUDJjaWllSDlCdUFTMVVodVY5QkxhRkJhcW1zQm10aDRTMzFzakt0aElJVjJ1UXVoc3RTUmxHaWdYY0FkdUNzSQpvU0U3VWxjQjVaUG1oUW5HYktyRGo1eG9QR09rQ1F1WlEwOFRFMk14dTJhb1pDMVlkRUNEQ1ZOckp0QnNJUWwyCkdCZmFzckd6QWdNQkFBRUNnZ0VBQlBwQzBaWUJ6M1BJWkphaENuSEcyMXNEU1NxdHduM0NISllDQ0x1MGEvOEEKdll6RUtuRXN1M3dtRHpnTFJiL1U1Rk1NRWtzK2hTMUVNczQ4Q3J4UVF4RjJOR2VCTjNFQ1dDS3FUVlZKSml0ZQo3WXBLSmhTRHFBR1FZTEJuQVd3TFUzTXE4QW0vbDMrOFZMNFF2d2hoQnFuQjZ6RnVUMG5tcGd1TW9TU3VEcE1MCmh3akN6QlU3cHJ1UEJIOVNrWHNNKzRGeHVkOTlJMUo4YkdDeWN5Qm9OaFc4OHgwYVJ2ZU9DdVVXaHVHdjBCT3oKYkRkZXE2cEI0S0hjL3QyUmg4S05lcjVETE9YdDBaeU96a3VMWEFQM2xzTE9mOHBOWWtHVjJ0dzBJQ3pkMndmNQphSTk4c2FDSDRuQXUyTHBEUCtkM0xmTXlML2EwamszOFVpbUs5WXBTZ1FLQmdRRHNndi91WGw2bTR3Qm1LemphCitLNzVXLzlnc25pK3U3MXl2NkhwZ3l3T2FsTWxIczV5bTg3blhoUFJtc3FyQlhBeG5lNW1Sb29TWjVhKzlLak8Ka3IwT1VXSnkrb2JQbFdmdUtVTXdLbmplVy93UUsrWlpOTGJlM0ZsRHVxM2NOczRGMHFXN2J2dHN0MURXekl4QQp5WHZUaEhxL3ltdGlDMXBYVENHZHN2bXV3UUtCZ1FEMnYxaEJOT09Ea1NRb1ZGZzJLQlplYmNWUzFEUjZ2VVcvCjByMEIyNjRYRzVFRXIxSkNabFUxRG1jR1c2ZHowVkdQeWt6VXhaa2hmZmV2SHdoNXBUZStnVndEd0huMjRFTkkKV1NIaytzbHhBUlFMbFVRakpEczdvaVhqK0ZZcjJhd0JrTWxjVjN4cEorN2syYTFaMVlHZnVmait3Z2FuUStDRwpJUXJaZmVaeGN3S0JnUUNFSXlCb0JkTnQrKzl5SlgzTFlSc28rQXd5OHlOZ3RMVkFrZTRjNzR0Q3RvYWplNVd1CkZIekJhUjg3Z1Bid0c4YTJBZDE2eUxoRlZoaXpzUzlLMGxMTDJBNWYzTFpLN2RjMkNWbWZaR3RKR0MwNHY3NGYKWXNNMHVma3BUUGZyeTdiSkxBb3FNUFJKcGhXRkhKelRhWDVFQzRVaytDdU1pSGs5d3F3WlZqQUhRUUtCZ1FEeQpVZDJ4UStoL1o4V1RtYWw3bmdqQnVabkVpMFB5bWZocXR1SUpkaDFVOGl5TDVrV3BrZWZ5RCsvYmpaMDRuNE15CjdDQmlBS0F2QlE5K09zTE8vQk52a29pejRvMmtobjl0Q0tQQnhReVpYczY1eFdCQ0JxM3BGWWVTaDJyUTY1Y0IKZnBoUmlGVlV5L1BQYVhyVnMxajJiT1A4Qkw0Vi9CSCtoMnFwUnZ4Tnd3S0JnQk82c3dJeURBYmFKc0lOS2ZBZQp3R3g5d0tkcVUyM052TG9oaEFkTVpoZ0dEcHM0Sm13TEJxYTZUb05NSzVLQUZMQzJsenBWT1RvWlZCQUpNVWFpCmlVblJmT0hoMjhhQWhGdHlTZ3dNcWg2M2xqWnNuWXEzdmVzQmJoSTVYQWhDY2xNUVR4b0k2RDhUNTZEWmNEY04KZGx3bkVJamFMV205aE5XMXlPQ0EvUXFVCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml b/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml new file mode 100644 index 00000000..cd069330 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure galera/WSREP traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.wsrep_provider_options;"' | grep -o -w 'socket.ssl = YES' + # ensure mysql/SQL traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.ssl_cipher;"' | grep -v '^NULL$' diff --git a/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml b/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml new file mode 100644 index 00000000..68c7f897 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done