From 16a7312717f86f42234c03aa567ab296401f2521 Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Tue, 20 Jun 2023 12:25:32 +0000 Subject: [PATCH] Support TLS for galera Ability to specify a certificate and a CA to be used for galera cluster communication (GCOMM, SST). Updates to the certificate used for galera automatically triggers a rolling restart of the galera pods, without service disruption. The original volume mount names is changed to accomodate the TLS subpath mounts. They now follow the naming convention found in other Openstack services. When the Galera CR is configured to use TLS, the mariadbdatabase CR creates DB users that still allow connection to the DB without using TLS. This is because Openstack clients currently cannot be configured to connect via TLS or via plain TCP. This specific part will be addressed in a subsequent commit. --- api/bases/mariadb.openstack.org_galeras.yaml | 25 ++- api/go.mod | 30 +-- api/go.sum | 63 +++--- api/v1beta1/conditions.go | 3 + api/v1beta1/galera_types.go | 13 +- api/v1beta1/mariadbdatabase_funcs.go | 3 +- api/v1beta1/zz_generated.deepcopy.go | 8 + .../bases/mariadb.openstack.org_galeras.yaml | 25 ++- config/rbac/role.yaml | 2 + config/samples/cert-manager-galera-cert.yaml | 75 +++++++ .../samples/mariadb_v1beta1_galera_tls.yaml | 12 ++ controllers/galera_controller.go | 184 +++++++++++++++-- controllers/mariadbdatabase_controller.go | 5 +- pkg/mariadb/database.go | 11 +- pkg/mariadb/statefulset.go | 144 +------------ pkg/mariadb/volumes.go | 190 +++++++++++++----- templates/database.sh | 2 +- templates/galera/bin/mysql_bootstrap.sh | 6 +- templates/galera/config/config.json | 32 ++- .../galera/config/galera_external_tls.cnf.in | 8 + templates/galera/config/galera_tls.cnf.in | 15 ++ .../common/assert_sample_deployment.yaml | 4 + .../kuttl/tests/account_create/01-assert.yaml | 4 + .../tests/database_create/01-assert.yaml | 4 + .../01-assert.yaml | 45 +++++ .../01-deploy-galera.yaml | 43 ++++ .../01-tls-certificate.yaml | 1 + .../02-assert.yaml | 6 + .../03-teardown.yaml | 19 ++ .../galera_deploy/03-cleanup_galera.yaml | 1 + .../galera_deploy_external_tls/01-assert.yaml | 53 +++++ .../01-deploy_external_tls_galera.yaml | 19 ++ .../01-tls-certificate.yaml | 1 + .../galera_deploy_external_tls/02-assert.yaml | 8 + .../03-teardown.yaml | 16 ++ .../tests/galera_deploy_tls/01-assert.yaml | 54 +++++ .../01-deploy_tls_galera.yaml | 20 ++ .../galera_deploy_tls/01-tls-certificate.yaml | 47 +++++ .../tests/galera_deploy_tls/02-assert.yaml | 8 + .../tests/galera_deploy_tls/03-teardown.yaml | 16 ++ 40 files changed, 961 insertions(+), 264 deletions(-) create mode 100644 config/samples/cert-manager-galera-cert.yaml create mode 100644 config/samples/mariadb_v1beta1_galera_tls.yaml create mode 100644 templates/galera/config/galera_external_tls.cnf.in create mode 100644 templates/galera/config/galera_tls.cnf.in create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml create mode 120000 tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml create mode 120000 tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/01-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index e4860d42..a5a0d86c 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -60,6 +60,10 @@ spec: defaults, or overwrite rendered information using raw MariaDB config format. The content gets added to /etc/my.cnf.d/galera_custom.cnf type: string + disableNonTLSListeners: + description: When TLS is used, only allow connections to the DB over + TLS + type: boolean nodeSelector: additionalProperties: type: string @@ -84,6 +88,17 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings for MySQL service and internal Galera replication + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - containerImage - replicas @@ -162,16 +177,16 @@ spec: - type type: object type: array - configHash: - default: "" - description: Hash of the configuration files - type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string required: - bootstrapped - - configHash type: object type: object served: true diff --git a/api/go.mod b/api/go.mod index db20ad2b..acfa21a4 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,14 +3,14 @@ module github.com/openstack-k8s-operators/mariadb-operator/api go 1.19 require ( - github.com/go-logr/logr v1.2.4 - github.com/onsi/ginkgo/v2 v2.12.0 - github.com/onsi/gomega v1.27.10 - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 + github.com/go-logr/logr v1.4.1 + github.com/onsi/ginkgo/v2 v2.13.2 + github.com/onsi/gomega v1.30.0 + github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240110111528-21db14521cda k8s.io/api v0.26.12 k8s.io/apimachinery v0.26.12 k8s.io/client-go v0.26.12 - sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/controller-runtime v0.14.7 ) require ( @@ -29,10 +29,10 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -50,14 +50,14 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.25.0 // indirect - golang.org/x/net v0.17.0 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.16.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect @@ -66,9 +66,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.12 // indirect k8s.io/component-base v0.26.12 // indirect - k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect - k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/api/go.sum b/api/go.sum index 281f3935..176815b7 100644 --- a/api/go.sum +++ b/api/go.sum @@ -41,7 +41,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -94,8 +93,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -154,8 +153,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -172,8 +171,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -227,14 +226,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= +github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= +github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240110111528-21db14521cda h1:7SW338JOe8T8Kgl3hK7607qSU9YPW5Jcu15qwCPZ8kE= +github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240110111528-21db14521cda/go.mod h1:ov4lAbniNUsLqZCBp1RTixpqXc8JlzA5B+yTcCkJXQg= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -311,8 +310,8 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -349,7 +348,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -384,8 +383,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -447,12 +446,12 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -461,8 +460,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -511,8 +510,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -644,17 +643,17 @@ k8s.io/client-go v0.26.12 h1:kPpTpIeFNqwo4UyvoqzNp3DNK2mbGcdGv23eS1U8VMo= k8s.io/client-go v0.26.12/go.mod h1:V7thEnIFroyNZOU30dKLiiVeqQmJz45shJG1mu7nONQ= k8s.io/component-base v0.26.12 h1:OyYjCtruv4/Yau5Z1v6e59N+JRDTj8JnW95W9w9AMpg= k8s.io/component-base v0.26.12/go.mod h1:X98Et5BxJ8i4TcDusUcKS8EYxCujBU1lCL3pc/CUtHQ= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= -k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= -k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/controller-runtime v0.14.7 h1:Vrnm2vk9ZFlRkXATHz0W0wXcqNl7kPat8q2JyxVy0Q8= +sigs.k8s.io/controller-runtime v0.14.7/go.mod h1:ErTs3SJCOujNUnTz4AS+uh8hp6DHMo1gj6fFndJT1X8= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index b00981b5..98b14b69 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -69,6 +69,9 @@ const ( // MariaDBInitializedErrorMessage MariaDBInitializedErrorMessage = "MariaDB dbinit error occured %s" + // MariaDBInputSecretNotFoundMessage + MariaDBInputSecretNotFoundMessage = "Input secret not found: %s" + MariaDBDatabaseReadyInitMessage = "MariaDBDatabase not yet available" MariaDBDatabaseReadyMessage = "MariaDBDatabase ready" diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index 251df8b9..6e95c02c 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -66,6 +67,13 @@ type GaleraSpec struct { // +kubebuilder:validation:Optional // Adoption configuration AdoptionRedirect AdoptionRedirectSpec `json:"adoptionRedirect"` + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS settings for MySQL service and internal Galera replication + TLS tls.SimpleService `json:"tls,omitempty"` + // +kubebuilder:validation:Optional + // When TLS is used, only allow connections to the DB over TLS + DisableNonTLSListeners bool `json:"disableNonTLSListeners,omitempty"` } // GaleraAttributes holds startup information for a Galera host @@ -87,9 +95,8 @@ type GaleraStatus struct { // Is the galera cluster currently running // +kubebuilder:default=false Bootstrapped bool `json:"bootstrapped"` - // Hash of the configuration files - // +kubebuilder:default="" - ConfigHash string `json:"configHash"` + // Map of hashes to track input changes + Hash map[string]string `json:"hash,omitempty"` // Deployment Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` } diff --git a/api/v1beta1/mariadbdatabase_funcs.go b/api/v1beta1/mariadbdatabase_funcs.go index 8f5e818d..7238e6f0 100644 --- a/api/v1beta1/mariadbdatabase_funcs.go +++ b/api/v1beta1/mariadbdatabase_funcs.go @@ -106,7 +106,8 @@ func (d *Database) setDatabaseHostname( err, ) } - d.databaseHostname = serviceList.Items[0].GetName() + svc := serviceList.Items[0] + d.databaseHostname = svc.GetName() + "." + svc.GetNamespace() + ".svc" return nil } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 575c84a9..f738ea01 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -173,6 +173,7 @@ func (in *GaleraSpec) DeepCopyInto(out *GaleraSpec) { } } out.AdoptionRedirect = in.AdoptionRedirect + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GaleraSpec. @@ -195,6 +196,13 @@ func (in *GaleraStatus) DeepCopyInto(out *GaleraStatus) { (*out)[key] = val } } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(condition.Conditions, len(*in)) diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index e4860d42..a5a0d86c 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -60,6 +60,10 @@ spec: defaults, or overwrite rendered information using raw MariaDB config format. The content gets added to /etc/my.cnf.d/galera_custom.cnf type: string + disableNonTLSListeners: + description: When TLS is used, only allow connections to the DB over + TLS + type: boolean nodeSelector: additionalProperties: type: string @@ -84,6 +88,17 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + tls: + description: TLS settings for MySQL service and internal Galera replication + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - containerImage - replicas @@ -162,16 +177,16 @@ spec: - type type: object type: array - configHash: - default: "" - description: Hash of the configuration files - type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object safeToBootstrap: description: Name of the node that can safely bootstrap a cluster type: string required: - bootstrapped - - configHash type: object type: object served: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 3cd2733f..9f9d6748 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -121,7 +121,9 @@ rules: - delete - get - list + - patch - update + - watch - apiGroups: - "" resources: diff --git a/config/samples/cert-manager-galera-cert.yaml b/config/samples/cert-manager-galera-cert.yaml new file mode 100644 index 00000000..0982b9a9 --- /dev/null +++ b/config/samples/cert-manager-galera-cert.yaml @@ -0,0 +1,75 @@ +# the cluster-wide issuer, used to generate a root certificate +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} +--- +# The root certificate. they cert/key/ca will be generated in the secret 'root-secret' +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: my-selfsigned-ca + namespace: openstack +spec: + isCA: true + commonName: my-selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: ClusterIssuer + group: cert-manager.io +--- +# The CA issuer for galera, uses the certificate from `my-selfsigned-ca` +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: my-ca-issuer + namespace: openstack +spec: + ca: + secretName: root-secret +--- +# The certificate used by all galera replicas for GCOMM and SST. +# The replicas in the galera statefulset all share the same +# certificate, so the latter requires wildcard in dnsNames for TLS +# validation. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: galera-cert +spec: + secretName: galera-tls + secretTemplate: + labels: + mariadb-ref: openstack + duration: 6h + renewBefore: 1h + subject: + organizations: + - cluster.local + commonName: openstack-galera + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS8 + size: 2048 + usages: + - server auth + - client auth + dnsNames: + - "openstack.openstack.svc" + - "openstack.openstack.svc.cluster.local" + - "*.openstack-galera" + - "*.openstack-galera.openstack" + - "*.openstack-galera.openstack.svc" + - "*.openstack-galera.openstack.svc.cluster" + - "*.openstack-galera.openstack.svc.cluster.local" + issuerRef: + name: my-ca-issuer + group: cert-manager.io + kind: Issuer diff --git a/config/samples/mariadb_v1beta1_galera_tls.yaml b/config/samples/mariadb_v1beta1_galera_tls.yaml new file mode 100644 index 00000000..067b1be8 --- /dev/null +++ b/config/samples/mariadb_v1beta1_galera_tls.yaml @@ -0,0 +1,12 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + secretName: galera-tls + caBundleSecretName: galera-tls diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index 13270a91..14b23ea2 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -17,18 +17,23 @@ limitations under the License. package controllers import ( + "github.com/openstack-k8s-operators/lib-common/modules/common" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" configmap "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/kubectl/pkg/util/podutils" @@ -46,15 +51,33 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" databasev1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" mariadb "github.com/openstack-k8s-operators/mariadb-operator/pkg/mariadb" ) +// fields to index to reconcile on CR change +const ( + serviceSecretNameField = ".spec.tls.genericService.SecretName" + caSecretNameField = ".spec.tls.ca.caBundleSecretName" +) + +var ( + allWatchFields = []string{ + serviceSecretNameField, + caSecretNameField, + } +) + // GaleraReconciler reconciles a Galera object type GaleraReconciler struct { client.Client @@ -97,7 +120,8 @@ func buildGcommURI(instance *mariadbv1.Galera) string { res := []string{} for i := 0; i < replicas; i++ { - res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename) + // Generate Gcomm with FQDN for TLS validation + res = append(res, basename+"-"+strconv.Itoa(i)+"."+basename+"."+instance.Namespace+".svc") } uri := "gcomm://" + strings.Join(res, ",") return uri @@ -257,15 +281,17 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal if !found { continue } - ci := instance.Status.Attributes[pod.Name].ContainerID - pci := pod.Status.ContainerStatuses[0].ContainerID - if ci != pci { + // A node can have various attributes depending on its known state. + // A ContainerID attribute is only present if the node is being started. + attrCID := instance.Status.Attributes[pod.Name].ContainerID + podCID := pod.Status.ContainerStatuses[0].ContainerID + if attrCID != "" && attrCID != podCID { // This gcomm URI was pushed in a pod which was restarted // before the attribute got cleared, which means the pod // failed to start galera. Clear the attribute here, and // reprobe the pod's state in the next reconcile loop clearPodAttributes(instance, pod.Name) - util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name) + util.LogForObject(helper, "Pod restarted while galera was starting", instance, "pod", pod.Name, "current pod ID", podCID, "recorded ID", attrCID) } } } @@ -283,6 +309,9 @@ func assertPodsAttributesValidity(helper *helper.Helper, instance *mariadbv1.Gal // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// RBAC for secrets +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; + // RBAC for services and endpoints // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;create;update;patch;delete; @@ -361,6 +390,8 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res instance.Status.Conditions = condition.Conditions{} // initialize conditions used later as Status=Unknown cl := condition.CreateList( + // DB Root password and TLS secrets + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), // endpoint for adoption redirect condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), // configmap generation @@ -470,9 +501,51 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res log.Info("", "Kind", instance.Kind, "Name", instance.Name, "database service", service.Name, "operation", string(op)) } - // Generate the config maps for the various services - configMapVars := make(map[string]env.Setter) - err = r.generateConfigMaps(ctx, helper, instance, &configMapVars) + // Hash of all resources that may cause a service restart + inputHashEnv := make(map[string]env.Setter) + + // Check and hash inputs + secretName := instance.Spec.Secret + // NOTE do not hash the db root password, as its change requires + // more orchestration than a simple rolling restart + _, _, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace) + + var certHash, caHash string + specTLS := &instance.Spec.TLS + if err == nil && specTLS.Enabled() { + certHash, _, err = specTLS.GenericService.ValidateCertSecret(ctx, helper, instance.Namespace) + inputHashEnv["Cert"] = env.SetValue(certHash) + } + if err == nil && specTLS.Ca.CaBundleSecretName != "" { + caName := types.NamespacedName{ + Name: specTLS.Ca.CaBundleSecretName, + Namespace: instance.Namespace, + } + caHash, _, err = tls.ValidateCACertSecret(ctx, helper.GetClient(), caName) + inputHashEnv["CA"] = env.SetValue(caHash) + } + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + mariadbv1.MariaDBInputSecretNotFoundMessage, + secretName)) + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("error calculating input hash: %w", err) + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // Generate and hash config maps + err = r.generateConfigMaps(ctx, helper, instance, &inputHashEnv) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -482,16 +555,36 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res err.Error())) return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err) } - // From hereon, configMapVars holds a hash of the config generated for this instance - // This is used in an envvar in the statefulset to restart it on config change - envHash := &corev1.EnvVar{} - configMapVars[configMapNameForConfig(instance)](envHash) - instance.Status.ConfigHash = envHash.Value instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance), 5) + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + hashOfHashes, err := util.HashOfInputHashes(inputHashEnv) + if err != nil { + return ctrl.Result{}, err + } + if hashMap, changed := util.SetHash(instance.Status.Hash, common.InputHashName, hashOfHashes); changed { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so update all the input hashes and return to reconcile again + instance.Status.Hash = hashMap + for k, s := range inputHashEnv { + var envVar corev1.EnvVar + s(&envVar) + instance.Status.Hash[k] = envVar.Value + } + util.LogForObject(helper, fmt.Sprintf("Input hash changed %s", hashOfHashes), instance) + return ctrl.Result{}, nil + } + + // The statefulset that manages galera pod + commonstatefulset := commonstatefulset.NewStatefulSet(mariadb.StatefulSet(instance, hashOfHashes), 5) sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper) if sferr != nil { + if k8s_errors.IsNotFound(sferr) { + return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + } return sfres, sferr } statefulset := commonstatefulset.GetStatefulSet() @@ -664,6 +757,34 @@ func (r *GaleraReconciler) generateConfigMaps( // SetupWithManager sets up the controller with the Manager. func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { r.config = mgr.GetConfig() + + // Various CR fields need to be indexed to filter watch events + // for the secret changes we want to be notified of + // index caBundleSecretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &mariadbv1.Galera{}, caSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*mariadbv1.Galera) + tls := &cr.Spec.TLS + if tls.Ca.CaBundleSecretName != "" { + return []string{tls.Ca.CaBundleSecretName} + } + return nil + }); err != nil { + return err + } + // index secretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &mariadbv1.Galera{}, serviceSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*mariadbv1.Galera) + tls := &cr.Spec.TLS + if tls.Enabled() { + return []string{*tls.GenericService.SecretName} + } + return nil + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&mariadbv1.Galera{}). Owns(&appsv1.StatefulSet{}). @@ -673,6 +794,11 @@ func (r *GaleraReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } @@ -699,3 +825,33 @@ func GetDatabaseObject(clientObj client.Client, ctx context.Context, name string } } + +// findObjectsForSrc - returns a reconcile request if the object is referenced by a Galera CR +func (r *GaleraReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + for _, field := range allWatchFields { + crList := &mariadbv1.GaleraList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go index 1cb466c2..79315245 100644 --- a/controllers/mariadbdatabase_controller.go +++ b/controllers/mariadbdatabase_controller.go @@ -130,6 +130,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Non-deletion (normal) flow follows // var dbName, dbSecret, dbContainerImage, serviceAccount string + // NOTE(dciabrin) When configured to only allow TLS connections, all clients + // accessing this DB must support client connection via TLS. + useTLS := dbGalera.Spec.TLS.Enabled() && dbGalera.Spec.DisableNonTLSListeners if !dbGalera.Status.Bootstrapped { log.Info("DB bootstrap not complete. Requeue...") @@ -142,7 +145,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ serviceAccount = dbGalera.RbacResourceName() // Define a new Job object (hostname, password, containerImage) - jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount) + jobDef, err := mariadb.DbDatabaseJob(instance, dbName, dbSecret, dbContainerImage, serviceAccount, useTLS) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/mariadb/database.go b/pkg/mariadb/database.go index 59b45719..59980d20 100644 --- a/pkg/mariadb/database.go +++ b/pkg/mariadb/database.go @@ -16,10 +16,17 @@ type dbCreateOptions struct { DatabaseAdminUsername string DefaultCharacterSet string DefaultCollation string + DatabaseUserTLS string } // DbDatabaseJob - -func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string) (*batchv1.Job, error) { +func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName string, databaseSecret string, containerImage string, serviceAccountName string, useTLS bool) (*batchv1.Job, error) { + var tlsStatement string + if useTLS { + tlsStatement = " REQUIRE SSL" + } else { + tlsStatement = "" + } opts := dbCreateOptions{ database.Spec.Name, @@ -27,6 +34,7 @@ func DbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHostName s "root", database.Spec.DefaultCharacterSet, database.Spec.DefaultCollation, + tlsStatement, } dbCmd, err := util.ExecuteTemplateFile("database.sh", &opts) if err != nil { @@ -97,6 +105,7 @@ func DeleteDbDatabaseJob(database *databasev1beta1.MariaDBDatabase, databaseHost "root", database.Spec.DefaultCharacterSet, database.Spec.DefaultCollation, + "", } delCmd, err := util.ExecuteTemplateFile("delete_database.sh", &opts) if err != nil { diff --git a/pkg/mariadb/statefulset.go b/pkg/mariadb/statefulset.go index 2abcfff8..b945e555 100644 --- a/pkg/mariadb/statefulset.go +++ b/pkg/mariadb/statefulset.go @@ -11,9 +11,12 @@ import ( ) // StatefulSet returns a StatefulSet object for the galera cluster -func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { +func StatefulSet(g *mariadbv1.Galera, configHash string) *appsv1.StatefulSet { ls := StatefulSetLabels(g) name := StatefulSetName(g.Name) + replicas := g.Spec.Replicas + storage := g.Spec.StorageClass + storageRequest := resource.MustParse(g.Spec.StorageRequest) sts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -21,7 +24,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, Spec: appsv1.StatefulSetSpec{ ServiceName: name, - Replicas: g.Spec.Replicas, + Replicas: replicas, Selector: &metav1.LabelSelector{ MatchLabels: ls, }, @@ -53,25 +56,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraInitVolumeMounts(g), }}, Containers: []corev1.Container{{ Image: g.Spec.ContainerImage, @@ -80,7 +65,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { Command: []string{"/usr/bin/dumb-init", "--", "/usr/local/bin/kolla_start"}, Env: []corev1.EnvVar{{ Name: "CR_CONFIG_HASH", - Value: g.Status.ConfigHash, + Value: configHash, }, { Name: "KOLLA_CONFIG_STRATEGY", Value: "COPY_ALWAYS", @@ -102,29 +87,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { ContainerPort: 4567, Name: "galera", }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/secrets", - ReadOnly: true, - Name: "secrets", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraVolumeMounts(g), StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ @@ -149,92 +112,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - Volumes: []corev1.Volume{ - { - Name: "secrets", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: g.Spec.Secret, - Items: []corev1.KeyToPath{ - { - Key: "DbRootPassword", - Path: "dbpassword", - }, - }, - }, - }, - }, - { - Name: "kolla-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "pod-config-data", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "config-data", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "galera.cnf.in", - Path: "galera.cnf.in", - }, - { - Key: mariadbv1.CustomServiceConfigFile, - Path: mariadbv1.CustomServiceConfigFile, - }, - }, - }, - }, - }, - { - Name: "operator-scripts", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-scripts", - }, - Items: []corev1.KeyToPath{ - { - Key: "mysql_bootstrap.sh", - Path: "mysql_bootstrap.sh", - }, - { - Key: "mysql_probe.sh", - Path: "mysql_probe.sh", - }, - { - Key: "detect_last_commit.sh", - Path: "detect_last_commit.sh", - }, - { - Key: "detect_gcomm_and_start.sh", - Path: "detect_gcomm_and_start.sh", - }, - }, - }, - }, - }, - }, + Volumes: getGaleraVolumes(g), }, }, VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ @@ -247,10 +125,10 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, - StorageClassName: &g.Spec.StorageClass, + StorageClassName: &storage, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - "storage": resource.MustParse(g.Spec.StorageRequest), + "storage": storageRequest, }, }, }, diff --git a/pkg/mariadb/volumes.go b/pkg/mariadb/volumes.go index 459131f3..d1cc3db9 100644 --- a/pkg/mariadb/volumes.go +++ b/pkg/mariadb/volumes.go @@ -1,19 +1,63 @@ package mariadb import ( + tls "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) -func getVolumes(name string) []corev1.Volume { +const ( + GaleraCertPrefix = "galera" +) + +func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { + configTemplates := []corev1.KeyToPath{ + { + Key: "galera.cnf.in", + Path: "galera.cnf.in", + }, + { + Key: mariadbv1.CustomServiceConfigFile, + Path: mariadbv1.CustomServiceConfigFile, + }, + } - return []corev1.Volume{ + if g.Spec.TLS.Enabled() { + if g.Spec.TLS.Ca.CaBundleSecretName != "" { + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_tls.cnf.in", + Path: "galera_tls.cnf.in", + }) + } else { + // Without a CA, WSREP is unencrypted. Only SQL traffic is. + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_external_tls.cnf.in", + Path: "galera_external_tls.cnf.in", + }) + } + } + volumes := []corev1.Volume{ + { + Name: "secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.Secret, + Items: []corev1.KeyToPath{ + { + Key: "DbRootPassword", + Path: "dbpassword", + }, + }, + }, + }, + }, { Name: "kolla-config", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "mariadb-" + name, + Name: g.Name + "-config-data", }, Items: []corev1.KeyToPath{ { @@ -25,91 +69,141 @@ func getVolumes(name string) []corev1.Volume { }, }, { - Name: "kolla-config-init", + Name: "config-data-generated", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "config-data-default", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "mariadb-" + name, - }, - Items: []corev1.KeyToPath{ - { - Key: "init_config.json", - Path: "config.json", - }, + Name: g.Name + "-config-data", }, + Items: configTemplates, }, }, }, { - Name: "config-data", + Name: "operator-scripts", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "mariadb-" + name, + Name: g.Name + "-scripts", }, Items: []corev1.KeyToPath{ { - Key: "galera.cnf", - Path: "galera.cnf", + Key: "mysql_bootstrap.sh", + Path: "mysql_bootstrap.sh", + }, + { + Key: "mysql_probe.sh", + Path: "mysql_probe.sh", }, { - Key: "mariadb_init.sh", - Path: "mariadb_init.sh", + Key: "detect_last_commit.sh", + Path: "detect_last_commit.sh", + }, + { + Key: "detect_gcomm_and_start.sh", + Path: "detect_gcomm_and_start.sh", }, }, }, }, }, - { - Name: "lib-data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "mariadb-" + name, - }, - }, - }, } + if g.Spec.TLS.Enabled() { + svc := tls.Service{ + SecretName: *g.Spec.TLS.GenericService.SecretName, + CertMount: nil, + KeyMount: nil, + CaMount: nil, + } + serviceVolume := svc.CreateVolume(GaleraCertPrefix) + volumes = append(volumes, serviceVolume) + if g.Spec.TLS.Ca.CaBundleSecretName != "" { + caVolume := g.Spec.TLS.Ca.CreateVolume() + volumes = append(volumes, caVolume) + } + } + + return volumes } -func getVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ +func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ { - MountPath: "/var/lib/config-data", + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data/default", ReadOnly: true, - Name: "config-data", - }, - { + Name: "config-data-default", + }, { + MountPath: "/var/lib/config-data/generated", + Name: "config-data-generated", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { MountPath: "/var/lib/kolla/config_files", ReadOnly: true, Name: "kolla-config", }, - { - MountPath: "/var/lib/mysql", - ReadOnly: false, - Name: "lib-data", - }, } + if g.Spec.TLS.Enabled() { + svc := tls.Service{ + SecretName: *g.Spec.TLS.GenericService.SecretName, + CertMount: nil, + KeyMount: nil, + CaMount: nil, + } + serviceVolumeMounts := svc.CreateVolumeMounts(GaleraCertPrefix) + volumeMounts = append(volumeMounts, serviceVolumeMounts...) + if g.Spec.TLS.Ca.CaBundleSecretName != "" { + caVolumeMounts := g.Spec.TLS.Ca.CreateVolumeMounts(nil) + volumeMounts = append(volumeMounts, caVolumeMounts...) + } + } + + return volumeMounts } -func getInitVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ +func getGaleraInitVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ { - MountPath: "/var/lib/config-data", + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data/default", ReadOnly: true, - Name: "config-data", - }, - { + Name: "config-data-default", + }, { + MountPath: "/var/lib/config-data/generated", + Name: "config-data-generated", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { MountPath: "/var/lib/kolla/config_files", ReadOnly: true, - Name: "kolla-config-init", - }, - { - MountPath: "/var/lib/mysql", - ReadOnly: false, - Name: "lib-data", + Name: "kolla-config", }, } + return volumeMounts } diff --git a/templates/database.sh b/templates/database.sh index 96484af9..7b933395 100755 --- a/templates/database.sh +++ b/templates/database.sh @@ -1,4 +1,4 @@ #!/bin/bash export DatabasePassword=${DatabasePassword:?"Please specify a DatabasePassword variable."} -mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}'; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword';GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword';" +mysql -h {{.DatabaseHostname}} -u {{.DatabaseAdminUsername}} -P 3306 -e "CREATE DATABASE IF NOT EXISTS {{.DatabaseName}}; ALTER DATABASE {{.DatabaseName}} CHARACTER SET '{{.DefaultCharacterSet}}' COLLATE '{{.DefaultCollation}}'; GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'localhost' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};GRANT ALL PRIVILEGES ON {{.DatabaseName}}.* TO '{{.DatabaseName}}'@'%' IDENTIFIED BY '$DatabasePassword'{{.DatabaseUserTLS}};" diff --git a/templates/galera/bin/mysql_bootstrap.sh b/templates/galera/bin/mysql_bootstrap.sh index efefebc9..14c4aceb 100755 --- a/templates/galera/bin/mysql_bootstrap.sh +++ b/templates/galera/bin/mysql_bootstrap.sh @@ -7,7 +7,7 @@ else echo -e "Creating new mariadb database." # we need the right perm on the persistent directory, # so use Kolla to set it up before bootstrapping the DB - cat </var/lib/pod-config-data/galera.cnf + cat </var/lib/config-data/generated/galera.cnf [mysqld] bind_address=localhost wsrep_provider=none @@ -22,7 +22,7 @@ PODNAME=$(hostname -f | cut -d. -f1,2) PODIPV4=$(grep "${PODNAME}" /etc/hosts | grep -v ':' | cut -d$'\t' -f1) PODIPV6=$(grep "${PODNAME}" /etc/hosts | grep ':' | cut -d$'\t' -f1) -cd /var/lib/config-data +cd /var/lib/config-data/default for cfg in *.cnf.in; do if [ -s "${cfg}" ]; then @@ -35,6 +35,6 @@ for cfg in *.cnf.in; do fi echo "Generating config file from template ${cfg}, will use ${IPSTACK} listen address of ${PODIP}" - sed -e "s/{ PODNAME }/${PODNAME}/" -e "s/{ PODIP }/${PODIP}/" "/var/lib/config-data/${cfg}" > "/var/lib/pod-config-data/${cfg%.in}" + sed -e "s/{ PODNAME }/${PODNAME}/" -e "s/{ PODIP }/${PODIP}/" "/var/lib/config-data/default/${cfg}" > "/var/lib/config-data/generated/${cfg%.in}" fi done diff --git a/templates/galera/config/config.json b/templates/galera/config/config.json index 185e33fa..19f5da6d 100644 --- a/templates/galera/config/config.json +++ b/templates/galera/config/config.json @@ -2,13 +2,27 @@ "command": "/usr/local/bin/detect_gcomm_and_start.sh", "config_files": [ { - "source": "/var/lib/pod-config-data/galera.cnf", + "source": "/var/lib/config-data/generated/galera.cnf", "dest": "/etc/my.cnf.d/galera.cnf", "owner": "root", "perm": "0644" }, { - "source": "/var/lib/pod-config-data/galera_custom.cnf", + "source": "/var/lib/config-data/generated/galera_tls.cnf", + "dest": "/etc/my.cnf.d/galera_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, + { + "source": "/var/lib/config-data/generated/galera_external_tls.cnf", + "dest": "/etc/my.cnf.d/galera_external_tls.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, + { + "source": "/var/lib/config-data/generated/galera_custom.cnf", "dest": "/etc/my.cnf.d/galera_custom.cnf", "owner": "root", "perm": "0644", @@ -20,6 +34,20 @@ "owner": "root", "perm": "0755", "merge": "true" + }, + { + "source": "/var/lib/config-data/tls/private/galera.key", + "dest": "/etc/pki/tls/private/galera.key", + "owner": "mysql", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/config-data/tls/certs/galera.crt", + "dest": "/etc/pki/tls/certs/galera.crt", + "owner": "mysql", + "perm": "0755", + "optional": true } ], "permissions": [ diff --git a/templates/galera/config/galera_external_tls.cnf.in b/templates/galera/config/galera_external_tls.cnf.in new file mode 100644 index 00000000..e3efac9e --- /dev/null +++ b/templates/galera/config/galera_external_tls.cnf.in @@ -0,0 +1,8 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/galera.crt +ssl-key = /etc/pki/tls/private/galera.key +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 + +[sst] +ssl-mode = DISABLED diff --git a/templates/galera/config/galera_tls.cnf.in b/templates/galera/config/galera_tls.cnf.in new file mode 100644 index 00000000..24fcbc82 --- /dev/null +++ b/templates/galera/config/galera_tls.cnf.in @@ -0,0 +1,15 @@ +[mysqld] +ssl +ssl-cert = /etc/pki/tls/certs/galera.crt +ssl-key = /etc/pki/tls/private/galera.key +ssl-ca = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +ssl-cipher = !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +wsrep_provider_options = gcache.recover=no;gmcast.listen_addr=tcp://{ PODIP }:4567;socket.ssl_key=/etc/pki/tls/private/galera.key;socket.ssl_cert=/etc/pki/tls/certs/galera.crt;socket.ssl_cipher=AES128-SHA256;socket.ssl_ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem; + +[sst] +sockopt = cipher=!SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES:!SSLv3:!TLSv1 +tcert = /etc/pki/tls/certs/galera.crt +tkey = /etc/pki/tls/private/galera.key +tca = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +encrypt = 3 +ssl-mode = REQUIRED diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index 33d9720c..2df58a6c 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -28,6 +28,10 @@ status: reason: Ready status: "True" type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/account_create/01-assert.yaml b/tests/kuttl/tests/account_create/01-assert.yaml index 3b721bbd..640e8316 100644 --- a/tests/kuttl/tests/account_create/01-assert.yaml +++ b/tests/kuttl/tests/account_create/01-assert.yaml @@ -28,6 +28,10 @@ status: reason: Ready status: "True" type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/database_create/01-assert.yaml b/tests/kuttl/tests/database_create/01-assert.yaml index 3b721bbd..640e8316 100644 --- a/tests/kuttl/tests/database_create/01-assert.yaml +++ b/tests/kuttl/tests/database_create/01-assert.yaml @@ -28,6 +28,10 @@ status: reason: Ready status: "True" type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady - message: RoleBinding created reason: Ready status: "True" diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml new file mode 100644 index 00000000..3e69c403 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-assert.yaml @@ -0,0 +1,45 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 1 + secret: osp-secret + storageRequest: 500M + tls: + secretName: kuttl-galera-tls + caBundleSecretName: kuttl-galera-tls + disableNonTLSListeners: true +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml new file mode 100644 index 00000000..dcbdfc6b --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-deploy-galera.yaml @@ -0,0 +1,43 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: kuttldb + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource with external TLS only (no TLS for galera WSREP traffic) +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 1 + tls: + secretName: kuttl-galera-tls + caBundleSecretName: kuttl-galera-tls + disableNonTLSListeners: true +--- +# a database cr associated with the db above will create users with grants +# set up to only allow TLS connection +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: kuttldb + labels: + dbName: openstack +spec: + secret: kuttldb-secret + name: kuttldb +--- +apiVersion: v1 +kind: Secret +metadata: + name: kuttldb-secret +data: + DatabasePassword: MTIzNDU2Nzg= diff --git a/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml new file mode 120000 index 00000000..d04979c5 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/01-tls-certificate.yaml @@ -0,0 +1 @@ +../galera_deploy_tls/01-tls-certificate.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml b/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml new file mode 100644 index 00000000..ecb23685 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/02-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure db users are configured to TLS restriction + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -e "show grants for \`kuttldb\`@\`%\`;"' | grep 'REQUIRE SSL' diff --git a/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml b/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml new file mode 100644 index 00000000..0d81c742 --- /dev/null +++ b/tests/kuttl/tests/galera_create_user_require_tls/03-teardown.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: kuttldb + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml b/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml index 5d073a88..d87f62c4 100644 --- a/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml +++ b/tests/kuttl/tests/galera_deploy/03-cleanup_galera.yaml @@ -7,3 +7,4 @@ delete: commands: - script: | oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml new file mode 100644 index 00000000..f45203e4 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-assert.yaml @@ -0,0 +1,53 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + tls: + secretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml new file mode 100644 index 00000000..1ede3af2 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-deploy_external_tls_galera.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource with external TLS only (no TLS for galera WSREP traffic) +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + secretName: kuttl-galera-tls diff --git a/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml new file mode 120000 index 00000000..d04979c5 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/01-tls-certificate.yaml @@ -0,0 +1 @@ +../galera_deploy_tls/01-tls-certificate.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml b/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml new file mode 100644 index 00000000..e59da5b7 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/02-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure galera does not encrypt WSREP traffic + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.wsrep_provider_options;"' | grep -o -w 'gmcast.listen_addr = tcp' + # ensure mysql/SQL traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.ssl_cipher;"' | grep -v '^NULL$' diff --git a/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml b/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml new file mode 100644 index 00000000..68c7f897 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_external_tls/03-teardown.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml b/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml new file mode 100644 index 00000000..42442534 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-assert.yaml @@ -0,0 +1,54 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + tls: + secretName: kuttl-galera-tls + caBundleSecretName: kuttl-galera-tls +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openstack-galera diff --git a/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml b/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml new file mode 100644 index 00000000..bfa1957b --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-deploy_tls_galera.yaml @@ -0,0 +1,20 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +--- +# galera resource using the TLS certs +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + tls: + secretName: kuttl-galera-tls + caBundleSecretName: kuttl-galera-tls diff --git a/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml b/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml new file mode 100644 index 00000000..1c6738b4 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/01-tls-certificate.yaml @@ -0,0 +1,47 @@ +# hardcode the secret generated by the certificate CR below, +# to avoid kuttl from depending on cert-manager at runtime +# the ca.crt key has been renamed to tls-ca-bundle.pem +# --- +# apiVersion: cert-manager.io/v1 +# kind: Certificate +# metadata: +# name: kuttl-galera-cert +# spec: +# secretName: kuttl-galera-tls +# duration: 12720h +# renewBefore: 1h +# subject: +# organizations: +# - cluster.local +# commonName: openstack-galera +# isCA: false +# privateKey: +# algorithm: RSA +# encoding: PKCS8 +# size: 2048 +# usages: +# - server auth +# - client auth +# dnsNames: +# - "openstack.openstack.svc" +# - "openstack.openstack.svc.cluster.local" +# - "*.openstack-galera" +# - "*.openstack-galera.openstack" +# - "*.openstack-galera.openstack.svc" +# - "*.openstack-galera.openstack.svc.cluster" +# - "*.openstack-galera.openstack.svc.cluster.local" +# issuerRef: +# name: kuttl-ca-issuer +# group: cert-manager.io +# kind: Issuer +# --- +apiVersion: v1 +kind: Secret +metadata: + name: kuttl-galera-tls + labels: + mariadb-ref: openstack +data: + tls-ca-bundle.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmRENDQVNLZ0F3SUJBZ0lRSlUrQTRBYUpDRFowRDYzbDF5UXNFVEFLQmdncWhrak9QUVFEQWpBZU1Sd3cKR2dZRFZRUURFeE5yZFhSMGJDMXpaV3htYzJsbmJtVmtMV05oTUI0WERUSTBNREV4T0RFek1UazBNbG9YRFRJMQpNRGN3TVRFek1UazBNbG93SGpFY01Cb0dBMVVFQXhNVGEzVjBkR3d0YzJWc1puTnBaMjVsWkMxallUQlpNQk1HCkJ5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCSTJKUWdYME1oZUNHSjQ2OHFSNE9wMGJYWGFuTWZSMWRpd3EKR1VtcXlrM20vdHVNZ2hxZlJNNmdWYXFpekNLMjQyNjJUL2dIamdsaDNJTEQ4UnByQXFlalFqQkFNQTRHQTFVZApEd0VCL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlFMbThjamY3dmc5ZjRxCjdsMzVmN1YxUXNsRDlqQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpQUZkdEhUbkdiMWtQVlJlZmcvbmNHaThoR2UKVlh5UVZycFJjRStNSXZMeUpRSWhBS2VLZHNleE9LUElQSDVOT0VBUHNxOTQ5cWlFVHU4ZlJEVUdkanozSkZSKwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQXhHZ0F3SUJBZ0lSQUoranowQ1p5aGV5Nk5tVWxTZnBoVUF3Q2dZSUtvWkl6ajBFQXdJd0hqRWMKTUJvR0ExVUVBeE1UYTNWMGRHd3RjMlZzWm5OcFoyNWxaQzFqWVRBZUZ3MHlOREF4TVRneE16STFNekJhRncweQpOVEEzTURFeE16STFNekJhTURNeEZqQVVCZ05WQkFvVERXTnNkWE4wWlhJdWJHOWpZV3d4R1RBWEJnTlZCQU1UCkVHOXdaVzV6ZEdGamF5MW5ZV3hsY21Fd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFEUTBmeFVYZVFvRTVhOGNmb1hTYXBpbmM2dDhHaTI2UFNrQW5kSDVXWG5jQVVmWFU1dEVqdkpWTVlPSmc2Rwp0amtyaHBTbjljL0JNTFlGUldKN2YxYVE1WXlpUTFKMkNYYjN1VlJPNTV3a3VZbmtJQ2Z0QjdERDVWTWVNQ3VDCkhZV0JqZm1vci94L3FLcGcrZ1BBV3NsY25VS245bFBudGRSQXFrU0hFM2lWRzNoR3R2ZDR4YWZ6eGt5bnJheGMKaU9pLysyVVUyeXlnTG0wM1AxRno0RU5BaUhNaVk4RTgxcGUrSit1OC9WMXNxaVNBdkg2b2RlR1Q2S2hrS2kwbworVnlwcGx2dnM4RnFTdUswRk9HbmhnbFhTWDRwZWM4V2ZCdVJXY2djMU9MbGtLMWJPb25ERzlIeVFabi85VWJwCi9wSTdZem5nU3R4TWhEMW56TmRnUTJJVEFnTUJBQUdqZ2dGT01JSUJTakFkQmdOVkhTVUVGakFVQmdnckJnRUYKQlFjREFRWUlLd1lCQlFVSEF3SXdEQVlEVlIwVEFRSC9CQUl3QURBZkJnTlZIU01FR0RBV2dCUUxtOGNqZjd2Zwo5ZjRxN2wzNWY3VjFRc2xEOWpDQitRWURWUjBSQklIeE1JSHVnaGR2Y0dWdWMzUmhZMnN1YjNCbGJuTjBZV05yCkxuTjJZNElsYjNCbGJuTjBZV05yTG05d1pXNXpkR0ZqYXk1emRtTXVZMngxYzNSbGNpNXNiMk5oYklJU0tpNXYKY0dWdWMzUmhZMnN0WjJGc1pYSmhnaHdxTG05d1pXNXpkR0ZqYXkxbllXeGxjbUV1YjNCbGJuTjBZV05yZ2lBcQpMbTl3Wlc1emRHRmpheTFuWVd4bGNtRXViM0JsYm5OMFlXTnJMbk4yWTRJb0tpNXZjR1Z1YzNSaFkyc3RaMkZzClpYSmhMbTl3Wlc1emRHRmpheTV6ZG1NdVkyeDFjM1JsY29JdUtpNXZjR1Z1YzNSaFkyc3RaMkZzWlhKaExtOXcKWlc1emRHRmpheTV6ZG1NdVkyeDFjM1JsY2k1c2IyTmhiREFLQmdncWhrak9QUVFEQWdOSUFEQkZBaUJ6Q1RGbgoyc1BVRHNvTFdYNy9nMlZDbjBsVEtjWVRLdkg3OGtDbUhXK3R6d0loQUxMa1hENWUvZ1hwNVF2UlYvNlhIbk15CjJ2YU5UQStLQXZicnFaR3JLRCtzCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRFEwZnhVWGVRb0U1YTgKY2ZvWFNhcGluYzZ0OEdpMjZQU2tBbmRINVdYbmNBVWZYVTV0RWp2SlZNWU9KZzZHdGprcmhwU245Yy9CTUxZRgpSV0o3ZjFhUTVZeWlRMUoyQ1hiM3VWUk81NXdrdVlua0lDZnRCN0RENVZNZU1DdUNIWVdCamZtb3IveC9xS3BnCitnUEFXc2xjblVLbjlsUG50ZFJBcWtTSEUzaVZHM2hHdHZkNHhhZnp4a3lucmF4Y2lPaS8rMlVVMnl5Z0xtMDMKUDFGejRFTkFpSE1pWThFODFwZStKK3U4L1Yxc3FpU0F2SDZvZGVHVDZLaGtLaTBvK1Z5cHBsdnZzOEZxU3VLMApGT0duaGdsWFNYNHBlYzhXZkJ1UldjZ2MxT0xsa0sxYk9vbkRHOUh5UVpuLzlVYnAvcEk3WXpuZ1N0eE1oRDFuCnpOZGdRMklUQWdNQkFBRUNnZ0VBSVNxSTRqbDdhR1ljRmRnZ2VaeG9wSHNkL1lCbWtVNlV0SXZ6dUFhV1ZkanoKNTFSaGhXOVU4WmcyVUw3NHBhckJqOGt6U1M0QWpsV1hRMlVPekVDZ1Zpa3ZXZ0pKVVpnVlBpbEFXN3ZyaitXcwpJZ0I5ZHVjMnl4WmNTam9xWHVNamRqMC9mdXhjdFVYcnNiVmo3U1ErUVNoMUVzcEw3MHh3NXNoa1UvUGl2Z095Ck4rbHh5UFhCK0RxVGdYSDRpVmQxbHNsaXpCbjB6d0VDb2d5V0RoVEg2d1pVYTBLWDNSVk1CcEhZTEZpdXBGZmgKWU9nbExIa2JDTmtzRXg3Zlg3Rks5eUVmQUxEeGZXSkVpQXRheGpEeng2NmVucFoyVWQzNU91MkNwYVdZbHNjNQpoOUNZRkRjeTVnYmZrL0xKdHFmZ2E0OVg4OUtBeXJrcDlUMS9McTNkU1FLQmdRRFU3dGRMUzVMeDJvUUx2Qi9vCk43ZFlWVGxXZW1BUWpqMFk0RHhVZmNOVE1oUmV6N01FM1dTcFlHYS9CWjNwQjROVGdhcG1qeEdqL2MveGZ3MHUKSG5pd2dFQU5vby9jNHNGVHJlWFBqNnYwbUdvbTZFcHd5UC9GL1NWNnFMVWNGS3piUFVtRTRnd3FtRWRiUjVZZwpsVCtzdHdOZ2czS3gwQW5sR1VucEcvODVIUUtCZ1FEN0RqSzFyejBFbW5aNU1OOXhrdUNYdzcxMzJxMTB1NWpsCjR1WEVqUG92cjZsamM4c3lMOEROUGNMSGxLWkpOUTZyL011T1ovekpDVGRvNWw1dDB0UnNnSFhyZktIN2NXN20KbUpocFk0amZQT0g5VlYveHFaVllGd2JXcVFVaXFtM290OHlTdmhzcE0rbFZmNnhiNHRYOUVVZWl5TWJXTEE1bwp4bGpwNG5SUTd3S0JnUUMrRUJyNFNKTDNjbmNmQ21Mb29xTHpJODgwVTdOZjA3YlJkNFlpWE1kMmdXTVJaZytECkxpTGwxUGloVldBb1d0NXNNWGRxYUJYMDdWOHBUcUR6STV2UzRBZE1wR2dKWUJYMG5XcGVKUDMySy8zRWtOK3gKWUpoOW40Sk94RHcwdm5lMGtqWUhlTVluVnhtS2JwR2dyOWZRVU9PZ3lIUWVKM1pObW84UWxqN3dPUUtCZ1FEQgpXbHYveGFqdTVLK2VBdC8wTHJTKzdjZjhpUFRTVkxFYlREYTl1LzNyd0JSclBnRWU4OXcvdGZOUGx0TEN5eFF6CnJZeHdidkluT3V2cjVKQ1JjTENkcUFvcGhXR1RyL2REcmY2a0hENkwvKzNsR0YyK1YyZG40c1FuaXlFalk3TW0KYW5ncUJEUVM5YUlkY1NrajAzNFBXOEdhUTV1djAxcDlvMVZUUEU5dERRS0JnSGlsbDhpMXIydFdrQUsxTERRbwpYWlRJUnNrb0dUaGFLbVNkNnNLMkxQK0xOMFZ0R3NZem5salhSYldWOE9ubHdUMThTWWhCbFRaNnVrN29GMTFKCmdJYzIwZEs1UUkzcVJoa3JoT21FaGlqTHI3ZkJIei9XRTVvbFppR1locHZJcDZ4QWlpN3BHNlIxYzJZUmtZUnkKY0lvanpYNm5zVmpDeUYrNTNabjJNdExKCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml b/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml new file mode 100644 index 00000000..cd069330 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/02-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure galera/WSREP traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.wsrep_provider_options;"' | grep -o -w 'socket.ssl = YES' + # ensure mysql/SQL traffic uses encryption + oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -Nse "select @@global.ssl_cipher;"' | grep -v '^NULL$' diff --git a/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml b/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml new file mode 100644 index 00000000..68c7f897 --- /dev/null +++ b/tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: v1 + kind: Secret + name: kuttl-galera-tls + - apiVersion: v1 + kind: Secret + name: kuttldb-secret +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done