Skip to content

Commit

Permalink
*: Save Xenon's metadata to config map. radondb#406
Browse files Browse the repository at this point in the history
subpath -> configmap
subpath may not work in some conditions.
  • Loading branch information
runkecheng committed Apr 12, 2022
1 parent 3375b1c commit c38634b
Show file tree
Hide file tree
Showing 16 changed files with 195 additions and 49 deletions.
8 changes: 5 additions & 3 deletions controllers/mysqlcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
}
}()

configMapSyncer := clustersyncer.NewConfigMapSyncer(r.Client, instance)
if err = syncer.Sync(ctx, configMapSyncer, r.Recorder); err != nil {
mysqlCMSyncer := clustersyncer.NewMysqlCMSyncer(r.Client, instance)
if err = syncer.Sync(ctx, mysqlCMSyncer, r.Recorder); err != nil {
return ctrl.Result{}, err
}

Expand All @@ -109,7 +109,8 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, err
}

cmRev := configMapSyncer.Object().(*corev1.ConfigMap).ResourceVersion
// Need fix: modify mysql cm will trigger rolling update but it will not be applied.
cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion

r.XenonExecutor.SetRootPassword(instance.Spec.MysqlOpts.RootPassword)
Expand All @@ -124,6 +125,7 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
clustersyncer.NewFollowerSVCSyncer(r.Client, instance),
clustersyncer.NewStatefulSetSyncer(r.Client, instance, cmRev, sctRev, r.SQLRunnerFactory, r.XenonExecutor),
clustersyncer.NewPDBSyncer(r.Client, instance),
clustersyncer.NewXenonCMSyncer(r.Client, instance),
}

if instance.Spec.MetricsOpts.Enabled {
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/container/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func (c *backupSidecar) getVolumeMounts() []corev1.VolumeMount {
return []corev1.VolumeMount{
{
Name: utils.ConfVolumeName,
MountPath: utils.ConfVolumeMountPath,
MountPath: utils.MysqlConfVolumeMountPath,
},
{
Name: utils.DataVolumeName,
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/container/init_mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (c *initMysql) getVolumeMounts() []corev1.VolumeMount {
return []corev1.VolumeMount{
{
Name: utils.ConfVolumeName,
MountPath: utils.ConfVolumeMountPath,
MountPath: utils.MysqlConfVolumeMountPath,
},
{
Name: utils.DataVolumeName,
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/container/init_mysql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ var (
initMysqlVolumeMounts = []corev1.VolumeMount{
{
Name: utils.ConfVolumeName,
MountPath: utils.ConfVolumeMountPath,
MountPath: utils.MysqlConfVolumeMountPath,
},
{
Name: utils.DataVolumeName,
Expand Down
14 changes: 11 additions & 3 deletions mysqlcluster/container/init_sidecar.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,19 @@ func (c *initSidecar) getVolumeMounts() []corev1.VolumeMount {
volumeMounts := []corev1.VolumeMount{
{
Name: utils.ConfVolumeName,
MountPath: utils.ConfVolumeMountPath,
MountPath: utils.MysqlConfVolumeMountPath,
},
{
Name: utils.ConfMapVolumeName,
MountPath: utils.ConfMapVolumeMountPath,
Name: utils.MysqlCMVolumeName,
MountPath: utils.MysqlCMVolumeMountPath,
},
{
Name: utils.XenonCMVolumeName,
MountPath: utils.XenonCMVolumeMountPath,
},
{
Name: utils.XenonMetaVolumeName,
MountPath: utils.XenonConfVolumeMountPath,
},
{
Name: utils.ScriptsVolumeName,
Expand Down
14 changes: 9 additions & 5 deletions mysqlcluster/container/init_sidecar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,11 +273,15 @@ var (
defaultInitsidecarVolumeMounts = []corev1.VolumeMount{
{
Name: utils.ConfVolumeName,
MountPath: utils.ConfVolumeMountPath,
MountPath: utils.MysqlConfVolumeMountPath,
},
{
Name: utils.ConfMapVolumeName,
MountPath: utils.ConfMapVolumeMountPath,
Name: utils.MysqlCMVolumeName,
MountPath: utils.MysqlCMVolumeMountPath,
},
{
Name: utils.XenonCMVolumeName,
MountPath: utils.XenonCMVolumeMountPath,
},
{
Name: utils.ScriptsVolumeName,
Expand Down Expand Up @@ -437,7 +441,7 @@ func TestGetInitSidecarVolumeMounts(t *testing.T) {
MysqlCluster: &testToKuDBMysqlCluster,
}
tokudbCase := EnsureContainer("init-sidecar", &testTokuDBCluster)
tokuDBVolumeMounts := make([]corev1.VolumeMount, 6, 7)
tokuDBVolumeMounts := make([]corev1.VolumeMount, 7, 8)
copy(tokuDBVolumeMounts, defaultInitsidecarVolumeMounts)
tokuDBVolumeMounts = append(tokuDBVolumeMounts, corev1.VolumeMount{
Name: utils.SysVolumeName,
Expand All @@ -453,7 +457,7 @@ func TestGetInitSidecarVolumeMounts(t *testing.T) {
MysqlCluster: &testPersistenceMysqlCluster,
}
persistenceCase := EnsureContainer("init-sidecar", &testPersistenceCluster)
persistenceVolumeMounts := make([]corev1.VolumeMount, 6, 7)
persistenceVolumeMounts := make([]corev1.VolumeMount, 7, 8)
copy(persistenceVolumeMounts, defaultInitsidecarVolumeMounts)
persistenceVolumeMounts = append(persistenceVolumeMounts, corev1.VolumeMount{
Name: utils.DataVolumeName,
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/container/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func (c *mysql) getVolumeMounts() []corev1.VolumeMount {
return []corev1.VolumeMount{
{
Name: utils.ConfVolumeName,
MountPath: utils.ConfVolumeMountPath,
MountPath: utils.MysqlConfVolumeMountPath,
},
{
Name: utils.DataVolumeName,
Expand Down
4 changes: 4 additions & 0 deletions mysqlcluster/container/xenon.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ func (c *xenon) getVolumeMounts() []corev1.VolumeMount {
Name: utils.XenonVolumeName,
MountPath: utils.XenonVolumeMountPath,
},
{
Name: utils.XenonMetaVolumeName,
MountPath: utils.XenonConfVolumeMountPath,
},
{
Name: utils.SysLocalTimeZone,
MountPath: utils.SysLocalTimeZoneMountPath,
Expand Down
20 changes: 19 additions & 1 deletion mysqlcluster/mysqlcluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
},
},
corev1.Volume{
Name: utils.ConfMapVolumeName,
Name: utils.MysqlCMVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Expand All @@ -214,6 +214,22 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
},
},
},
corev1.Volume{
Name: utils.XenonCMVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: c.GetNameForResource(utils.XenonMetaData),
},
},
},
},
corev1.Volume{
Name: utils.XenonMetaVolumeName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
corev1.Volume{
Name: utils.ScriptsVolumeName,
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -294,6 +310,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string {
return fmt.Sprintf("%s-metrics", c.Name)
case utils.Secret:
return fmt.Sprintf("%s-secret", c.Name)
case utils.XenonMetaData:
return fmt.Sprintf("%s-xenon", c.Name)
default:
return c.Name
}
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/mysqlcluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func TestEnsureVolumes(t *testing.T) {
},
},
{
Name: utils.ConfMapVolumeName,
Name: utils.MysqlCMVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
)

// NewConfigMapSyncer returns configmap syncer.
func NewConfigMapSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
func NewMysqlCMSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
cm := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Expand Down
84 changes: 84 additions & 0 deletions mysqlcluster/syncer/xenon_cm.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
Copyright 2021 RadonDB.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package syncer

import (
"encoding/json"
"fmt"

"github.com/presslabs/controller-util/syncer"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/radondb/radondb-mysql-kubernetes/mysqlcluster"
"github.com/radondb/radondb-mysql-kubernetes/utils"
)

// NewConfigMapSyncer returns configmap syncer.
func NewXenonCMSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
cm := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: c.GetNameForResource(utils.XenonMetaData),
Namespace: c.Namespace,
Labels: c.GetLabels(),
},
}

return syncer.NewObjectSyncer("ConfigMap", c.Unwrap(), cm, cli, func() error {
// If the cluster is closed, need not to modify the xenon metadata.
if int(*c.Spec.Replicas) == 0 {
return nil
}

metaData, err := buildXenonMetaData(c)
if err != nil {
return fmt.Errorf("failed to build xenon metadata: %s", err)
}

cm.Data = map[string]string{
"peers.json": metaData,
}

return nil
})
}

type XenonMetaData struct {
Idlepeers []string `json:"idlepeers"`
Peers []string `json:"peers"`
}

// buildXenonMetaData build the mysql config.
func buildXenonMetaData(c *mysqlcluster.MysqlCluster) (string, error) {
replicas := c.Spec.Replicas
xenonMetaData := XenonMetaData{}
for i := 0; i < int(*replicas); i++ {
// sample-mysql-0.sample-mysql.default:8801
xenonMetaData.Peers = append(xenonMetaData.Peers,
fmt.Sprintf("%s-%d.%s.%s:8801", c.GetNameForResource(utils.StatefulSet), i, c.GetNameForResource(utils.HeadlessSVC), c.Namespace))
}
metaJson, err := json.Marshal(xenonMetaData)
if err != nil {
return "", err
}
return string(metaJson), nil
}
16 changes: 8 additions & 8 deletions sidecar/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ func (cfg *Config) buildXenonConf() []byte {
"election-timeout": %d,
"admit-defeat-hearbeat-count": %d,
"heartbeat-timeout": %d,
"meta-datadir": "/var/lib/xenon/",
"meta-datadir": "%s",
"semi-sync-degrade": true,
"purge-binlog-disabled": true,
"super-idle": false
Expand All @@ -353,7 +353,7 @@ func (cfg *Config) buildXenonConf() []byte {
`, hostName, utils.XenonPort, hostName, utils.XenonPeerPort, cfg.ReplicationPassword, cfg.ReplicationUser,
cfg.GtidPurged, requestTimeout,
pingTimeout, cfg.RootPassword, version, srcSysVars, replicaSysVars, cfg.ElectionTimeout,
cfg.AdmitDefeatHearbeatCount, heartbeatTimeout)
cfg.AdmitDefeatHearbeatCount, heartbeatTimeout, xenonConfigPath)

return utils.StringToBytes(str)
}
Expand Down Expand Up @@ -533,19 +533,19 @@ func (cfg *Config) executeS3Restore(path string) error {
}
}
// Xtrabackup prepare and apply-log-only.
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.ConfVolumeMountPath+"/my.cnf", "--prepare", "--apply-log-only", "--target-dir=/root/backup")
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--prepare", "--apply-log-only", "--target-dir=/root/backup")
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare and apply-log-only : %s", err)
}
// Xtrabackup prepare.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.ConfVolumeMountPath+"/my.cnf", "--prepare", "--target-dir=/root/backup")
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--prepare", "--target-dir=/root/backup")
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare : %s", err)
}
// Xtrabackup copy-back to /var/lib/mysql.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.ConfVolumeMountPath+"/my.cnf", "--datadir="+utils.DataVolumeMountPath, "--copy-back", "--copy-back", "--target-dir=/root/backup")
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--datadir="+utils.DataVolumeMountPath, "--copy-back", "--copy-back", "--target-dir=/root/backup")
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup copy-back : %s", err)
Expand Down Expand Up @@ -577,13 +577,13 @@ func (cfg *Config) executeCloneRestore() error {
os.RemoveAll(path.Join([]string{utils.DataVolumeMountPath, d.Name()}...))
}
// Xtrabackup prepare and apply-log-only.
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.ConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--apply-log-only", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--apply-log-only", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare apply-log-only : %s", err)
}
// Xtrabackup Prepare.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.ConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare : %s", err)
Expand All @@ -595,7 +595,7 @@ func (cfg *Config) executeCloneRestore() error {
}
log.Info("get master gtid purged :", "gtid purged", cfg.GtidPurged)
// Xtrabackup copy-back.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.ConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--datadir="+utils.DataVolumeMountPath, "--copy-back", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--datadir="+utils.DataVolumeMountPath, "--copy-back", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup copy-back : %s", err)
Expand Down
18 changes: 16 additions & 2 deletions sidecar/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,12 @@ func runInitCommand(cfg *Config) error {
}

// copy appropriate my.cnf from config-map to config mount.
if err = copyFile(path.Join(configMapPath, "my.cnf"), path.Join(configPath, "my.cnf")); err != nil {
if err = copyFile(path.Join(mysqlCMPath, "my.cnf"), path.Join(mysqlConfigPath, "my.cnf")); err != nil {
return fmt.Errorf("failed to copy my.cnf: %s", err)
}

buildDefaultXenonMeta(uid, gid)

// build client.conf.
clientConfig, err := cfg.buildClientConfig()
if err != nil {
Expand Down Expand Up @@ -266,7 +268,7 @@ func RunRequestBackup(cfg *Config, host string) error {

// Save plugin.cnf and extra.cnf to specified path.
func saveCnfTo(targetPath string, extraCnf *ini.File) error {
if err := copyFile(path.Join(configMapPath, utils.PluginConfigs), path.Join(targetPath, utils.PluginConfigs)); err != nil {
if err := copyFile(path.Join(mysqlCMPath, utils.PluginConfigs), path.Join(targetPath, utils.PluginConfigs)); err != nil {
return fmt.Errorf("failed to copy plugin.cnf: %s", err)
}
if err := extraCnf.SaveTo(path.Join(targetPath, "extra.cnf")); err != nil {
Expand All @@ -290,3 +292,15 @@ chown -R mysql.mysql %s`,
// chown -R mysql.mysql extra.cnf
path.Join(extraConfPath, "extra.cnf"))
}

func buildDefaultXenonMeta(uid, gid int) error {
// mkdir var/lib/xenon.
if err := os.MkdirAll(xenonConfigPath, 0777); err != nil {
return fmt.Errorf("failed to mkdir %s: %s", xenonConfigPath, err)
}
// copy appropriate peers.json from config-map to config mount.
if err := copyFile(path.Join(xenonCMPath, "peers.json"), path.Join(xenonConfigPath, "peers.json")); err != nil {
return fmt.Errorf("failed to copy peers.json: %s", err)
}
return nil
}
Loading

0 comments on commit c38634b

Please sign in to comment.