From 458143b48b0f779960cb9674faa921cece64eaa7 Mon Sep 17 00:00:00 2001
From: runkecheng <1131648942@qq.com>
Date: Wed, 13 Apr 2022 14:33:52 +0800
Subject: [PATCH] *: Save Xenon's metadata to config map.

subpath -> configmap
subpath may not work in some conditions.
1. Add a Xenon Configmap which is used to save xenon data.
2. Build peers.json according to the replicas.
3. Make peers.json in init-sidecar.
4. Xenon boot using existing peers.json.
---
 controllers/mysqlcluster_controller.go      |  1 +
 mysqlcluster/container/init_sidecar.go      | 12 ++-
 mysqlcluster/container/init_sidecar_test.go | 16 +++-
 mysqlcluster/container/xenon.go             |  8 +-
 mysqlcluster/container/xenon_test.go        |  6 +-
 mysqlcluster/mysqlcluster.go                | 20 ++++-
 mysqlcluster/mysqlcluster_test.go           | 18 ++++-
 mysqlcluster/syncer/xenon_cm.go             | 88 +++++++++++++++++++++
 sidecar/config.go                           |  4 +-
 sidecar/init.go                             | 20 +++++
 sidecar/util.go                             |  8 +-
 utils/constants.go                          |  9 ++-
 12 files changed, 194 insertions(+), 16 deletions(-)
 create mode 100644 mysqlcluster/syncer/xenon_cm.go

diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go
index 78d0167c..297b3430 100644
--- a/controllers/mysqlcluster_controller.go
+++ b/controllers/mysqlcluster_controller.go
@@ -125,6 +125,7 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		clustersyncer.NewFollowerSVCSyncer(r.Client, instance),
 		clustersyncer.NewStatefulSetSyncer(r.Client, instance, cmRev, sctRev, r.SQLRunnerFactory, r.XenonExecutor),
 		clustersyncer.NewPDBSyncer(r.Client, instance),
+		clustersyncer.NewXenonCMSyncer(r.Client, instance),
 	}
 
 	if instance.Spec.MetricsOpts.Enabled {
diff --git a/mysqlcluster/container/init_sidecar.go b/mysqlcluster/container/init_sidecar.go
index f90eb2de..7193c0da 100644
--- a/mysqlcluster/container/init_sidecar.go
+++ b/mysqlcluster/container/init_sidecar.go
@@ -170,13 +170,21 @@ func (c *initSidecar) getVolumeMounts() []corev1.VolumeMount {
 			Name:      utils.MysqlCMVolumeName,
 			MountPath: utils.MysqlCMVolumeMountPath,
 		},
+		{
+			Name:      utils.XenonCMVolumeName,
+			MountPath: utils.XenonCMVolumeMountPath,
+		},
+		{
+			Name:      utils.XenonMetaVolumeName,
+			MountPath: utils.XenonMetaVolumeMountPath,
+		},
 		{
 			Name:      utils.ScriptsVolumeName,
 			MountPath: utils.ScriptsVolumeMountPath,
 		},
 		{
-			Name:      utils.XenonVolumeName,
-			MountPath: utils.XenonVolumeMountPath,
+			Name:      utils.XenonConfVolumeName,
+			MountPath: utils.XenonConfVolumeMountPath,
 		},
 		{
 			Name:      utils.InitFileVolumeName,
diff --git a/mysqlcluster/container/init_sidecar_test.go b/mysqlcluster/container/init_sidecar_test.go
index 1ae6a175..21d08537 100644
--- a/mysqlcluster/container/init_sidecar_test.go
+++ b/mysqlcluster/container/init_sidecar_test.go
@@ -279,13 +279,21 @@ var (
 			Name:      utils.MysqlCMVolumeName,
 			MountPath: utils.MysqlCMVolumeMountPath,
 		},
+		{
+			Name:      utils.XenonCMVolumeName,
+			MountPath: utils.XenonCMVolumeMountPath,
+		},
+		{
+			Name:      utils.XenonMetaVolumeName,
+			MountPath: utils.XenonMetaVolumeMountPath,
+		},
 		{
 			Name:      utils.ScriptsVolumeName,
 			MountPath: utils.ScriptsVolumeMountPath,
 		},
 		{
-			Name:      utils.XenonVolumeName,
-			MountPath: utils.XenonVolumeMountPath,
+			Name:      utils.XenonConfVolumeName,
+			MountPath: utils.XenonConfVolumeMountPath,
 		},
 		{
 			Name:      utils.InitFileVolumeName,
@@ -437,7 +445,7 @@ func TestGetInitSidecarVolumeMounts(t *testing.T) {
 			MysqlCluster: &testToKuDBMysqlCluster,
 		}
 		tokudbCase := EnsureContainer("init-sidecar", &testTokuDBCluster)
-		tokuDBVolumeMounts := make([]corev1.VolumeMount, 6, 7)
+		tokuDBVolumeMounts := make([]corev1.VolumeMount, 8, 9)
 		copy(tokuDBVolumeMounts, defaultInitsidecarVolumeMounts)
 		tokuDBVolumeMounts = append(tokuDBVolumeMounts, corev1.VolumeMount{
 			Name:      utils.SysVolumeName,
@@ -453,7 +461,7 @@ func TestGetInitSidecarVolumeMounts(t *testing.T) {
 			MysqlCluster: &testPersistenceMysqlCluster,
 		}
 		persistenceCase := EnsureContainer("init-sidecar", &testPersistenceCluster)
-		persistenceVolumeMounts := make([]corev1.VolumeMount, 6, 7)
+		persistenceVolumeMounts := make([]corev1.VolumeMount, 8, 9)
 		copy(persistenceVolumeMounts, defaultInitsidecarVolumeMounts)
 		persistenceVolumeMounts = append(persistenceVolumeMounts, corev1.VolumeMount{
 			Name:      utils.DataVolumeName,
diff --git a/mysqlcluster/container/xenon.go b/mysqlcluster/container/xenon.go
index eab0b1d2..a0be9447 100644
--- a/mysqlcluster/container/xenon.go
+++ b/mysqlcluster/container/xenon.go
@@ -115,8 +115,12 @@ func (c *xenon) getVolumeMounts() []corev1.VolumeMount {
 			MountPath: utils.ScriptsVolumeMountPath,
 		},
 		{
-			Name:      utils.XenonVolumeName,
-			MountPath: utils.XenonVolumeMountPath,
+			Name:      utils.XenonConfVolumeName,
+			MountPath: utils.XenonConfVolumeMountPath,
+		},
+		{
+			Name:      utils.XenonMetaVolumeName,
+			MountPath: utils.XenonMetaVolumeMountPath,
 		},
 		{
 			Name:      utils.SysLocalTimeZone,
diff --git a/mysqlcluster/container/xenon_test.go b/mysqlcluster/container/xenon_test.go
index 322d7e2b..1d57e5ec 100644
--- a/mysqlcluster/container/xenon_test.go
+++ b/mysqlcluster/container/xenon_test.go
@@ -135,9 +135,13 @@ func TestGetXenonVolumeMounts(t *testing.T) {
 			MountPath: "/scripts",
 		},
 		{
-			Name:      "xenon",
+			Name:      "xenon-conf",
 			MountPath: "/etc/xenon",
 		},
+		{
+			Name:      "xenon-meta",
+			MountPath: "/var/lib/xenon",
+		},
 		{
 			Name:      utils.SysLocalTimeZone,
 			MountPath: "/etc/localtime",
diff --git a/mysqlcluster/mysqlcluster.go b/mysqlcluster/mysqlcluster.go
index 3edf70a6..44a1f742 100644
--- a/mysqlcluster/mysqlcluster.go
+++ b/mysqlcluster/mysqlcluster.go
@@ -214,6 +214,22 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
 				},
 			},
 		},
+		corev1.Volume{
+			Name: utils.XenonCMVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: c.GetNameForResource(utils.XenonMetaData),
+					},
+				},
+			},
+		},
+		corev1.Volume{
+			Name: utils.XenonMetaVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
 		corev1.Volume{
 			Name: utils.ScriptsVolumeName,
 			VolumeSource: corev1.VolumeSource{
@@ -221,7 +237,7 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
 			},
 		},
 		corev1.Volume{
-			Name: utils.XenonVolumeName,
+			Name: utils.XenonConfVolumeName,
 			VolumeSource: corev1.VolumeSource{
 				EmptyDir: &corev1.EmptyDirVolumeSource{},
 			},
@@ -294,6 +310,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string {
 		return fmt.Sprintf("%s-metrics", c.Name)
 	case utils.Secret:
 		return fmt.Sprintf("%s-secret", c.Name)
+	case utils.XenonMetaData:
+		return fmt.Sprintf("%s-xenon", c.Name)
 	default:
 		return c.Name
 	}
diff --git a/mysqlcluster/mysqlcluster_test.go b/mysqlcluster/mysqlcluster_test.go
index 232369ce..e479869b 100644
--- a/mysqlcluster/mysqlcluster_test.go
+++ b/mysqlcluster/mysqlcluster_test.go
@@ -266,6 +266,22 @@ func TestEnsureVolumes(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: utils.XenonCMVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: "sample-xenon",
+					},
+				},
+			},
+		},
+		{
+			Name: utils.XenonMetaVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
 		{
 			Name: utils.ScriptsVolumeName,
 			VolumeSource: corev1.VolumeSource{
@@ -273,7 +289,7 @@ func TestEnsureVolumes(t *testing.T) {
 			},
 		},
 		{
-			Name: utils.XenonVolumeName,
+			Name: utils.XenonConfVolumeName,
 			VolumeSource: corev1.VolumeSource{
 				EmptyDir: &corev1.EmptyDirVolumeSource{},
 			},
diff --git a/mysqlcluster/syncer/xenon_cm.go b/mysqlcluster/syncer/xenon_cm.go
new file mode 100644
index 00000000..6cdd84f7
--- /dev/null
+++ b/mysqlcluster/syncer/xenon_cm.go
@@ -0,0 +1,88 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package syncer
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/presslabs/controller-util/syncer"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/radondb/radondb-mysql-kubernetes/mysqlcluster"
+	"github.com/radondb/radondb-mysql-kubernetes/utils"
+)
+
+// NewXenonCMSyncer returns xenon configmap syncer.
+func NewXenonCMSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Interface {
+	cm := &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      c.GetNameForResource(utils.XenonMetaData),
+			Namespace: c.Namespace,
+			Labels:    c.GetLabels(),
+		},
+	}
+
+	return syncer.NewObjectSyncer("ConfigMap", c.Unwrap(), cm, cli, func() error {
+		if int(*c.Spec.Replicas) == 0 {
+			return nil
+		}
+
+		metaData, err := buildXenonMetaData(c)
+		if err != nil {
+			return fmt.Errorf("failed to build xenon metadata: %s", err)
+		}
+
+		cm.Data = map[string]string{
+			"peers.json": metaData,
+		}
+
+		return nil
+	})
+}
+
+type XenonMetaData struct {
+	Idlepeers []string `json:"idlepeers"`
+	Peers     []string `json:"peers"`
+}
+
+// buildXenonMetaData build the default metadata of xenon.
+func buildXenonMetaData(c *mysqlcluster.MysqlCluster) (string, error) {
+	replicas := c.Spec.Replicas
+	xenonMetaData := XenonMetaData{}
+	for i := 0; i < int(*replicas); i++ {
+		xenonMetaData.Peers = append(xenonMetaData.Peers,
+			fmt.Sprintf("%s-%d.%s.%s:%d",
+				c.GetNameForResource(utils.StatefulSet),
+				i,
+				c.GetNameForResource(utils.HeadlessSVC),
+				c.Namespace,
+				utils.XenonPort,
+			))
+	}
+	metaJson, err := json.Marshal(xenonMetaData)
+	if err != nil {
+		return "", err
+	}
+	return string(metaJson), nil
+}
diff --git a/sidecar/config.go b/sidecar/config.go
index 986ac33e..f303e051 100644
--- a/sidecar/config.go
+++ b/sidecar/config.go
@@ -344,7 +344,7 @@ func (cfg *Config) buildXenonConf() []byte {
 			"election-timeout": %d,
 			"admit-defeat-hearbeat-count": %d,
 			"heartbeat-timeout": %d,
-			"meta-datadir": "/var/lib/xenon/",
+			"meta-datadir": "%s",
 			"semi-sync-degrade": true,
 			"purge-binlog-disabled": true,
 			"super-idle": false
@@ -353,7 +353,7 @@ func (cfg *Config) buildXenonConf() []byte {
 	`, hostName, utils.XenonPort, hostName, utils.XenonPeerPort, cfg.ReplicationPassword, cfg.ReplicationUser,
 		cfg.GtidPurged, requestTimeout,
 		pingTimeout, cfg.RootPassword, version, srcSysVars, replicaSysVars, cfg.ElectionTimeout,
-		cfg.AdmitDefeatHearbeatCount, heartbeatTimeout)
+		cfg.AdmitDefeatHearbeatCount, heartbeatTimeout, xenonConfigPath)
 
 	return utils.StringToBytes(str)
 }
diff --git a/sidecar/init.go b/sidecar/init.go
index 6826970f..aa7c2a2a 100644
--- a/sidecar/init.go
+++ b/sidecar/init.go
@@ -141,6 +141,8 @@ func runInitCommand(cfg *Config) error {
 		return fmt.Errorf("failed to copy my.cnf: %s", err)
 	}
 
+	buildDefaultXenonMeta(uid, gid)
+
 	// build client.conf.
 	clientConfig, err := cfg.buildClientConfig()
 	if err != nil {
@@ -290,3 +292,21 @@ chown -R mysql.mysql %s`,
 		// chown -R mysql.mysql extra.cnf
 		path.Join(extraConfPath, "extra.cnf"))
 }
+
+func buildDefaultXenonMeta(uid, gid int) error {
+	metaFile := fmt.Sprintf("%s/peers.json", xenonConfigPath)
+	// mkdir var/lib/xenon.
+	// https://github.com/radondb/xenon/blob/master/src/raft/raft.go#L118
+	if err := os.MkdirAll(xenonConfigPath, 0777); err != nil {
+		return fmt.Errorf("failed to mkdir %s: %s", xenonConfigPath, err)
+	}
+	// copy appropriate peers.json from config-map to config mount.
+	if err := copyFile(path.Join(xenonCMPath, "peers.json"), path.Join(xenonConfigPath, "peers.json")); err != nil {
+		return fmt.Errorf("failed to copy peers.json: %s", err)
+	}
+	// chown -R mysql:mysql /var/lib/xenon/peers.json.
+	if err := os.Chown(metaFile, uid, gid); err != nil {
+		return fmt.Errorf("failed to chown %s: %s", metaFile, err)
+	}
+	return nil
+}
diff --git a/sidecar/util.go b/sidecar/util.go
index 181450c1..1787929d 100644
--- a/sidecar/util.go
+++ b/sidecar/util.go
@@ -34,6 +34,9 @@ var (
 	// mysqlConfigPath is the mysql configs path.
 	mysqlConfigPath = utils.MysqlConfVolumeMountPath
 
+	// xenonConfigPath is the xenon meta path.
+	xenonConfigPath = utils.XenonMetaVolumeMountPath
+
 	// clientConfPath is the client.cnf path.
 	clientConfPath = utils.ConfClientPath
 
@@ -43,6 +46,9 @@ var (
 	// mysqlCMPath is the mounted configmap.
 	mysqlCMPath = utils.MysqlCMVolumeMountPath
 
+	// xenonCMPath is the mounted configmap.
+	xenonCMPath = utils.XenonCMVolumeMountPath
+
 	// dataPath is the mysql data path.
 	dataPath = utils.DataVolumeMountPath
 
@@ -53,7 +59,7 @@ var (
 	sysPath = utils.SysVolumeMountPath
 
 	// xenonPath is the xenon configs path.
-	xenonPath = utils.XenonVolumeMountPath
+	xenonPath = utils.XenonConfVolumeMountPath
 
 	// initFilePath is the init files path for mysql.
 	initFilePath = utils.InitFileVolumeMountPath
diff --git a/utils/constants.go b/utils/constants.go
index 7f9f3b6c..6e90971d 100644
--- a/utils/constants.go
+++ b/utils/constants.go
@@ -93,22 +93,25 @@ const (
 	// volumes names.
 	MysqlConfVolumeName = "mysql-conf"
 	MysqlCMVolumeName   = "mysql-cm"
+	XenonMetaVolumeName = "xenon-meta"
+	XenonCMVolumeName   = "xenon-cm"
 	LogsVolumeName      = "logs"
 	DataVolumeName      = "data"
 	SysVolumeName       = "host-sys"
 	ScriptsVolumeName   = "scripts"
-	XenonVolumeName     = "xenon"
+	XenonConfVolumeName = "xenon-conf"
 	InitFileVolumeName  = "init-mysql"
 
 	// volumes mount path.
 	MysqlConfVolumeMountPath = "/etc/mysql"
 	MysqlCMVolumeMountPath   = "/mnt/mysql-cm"
 	XenonMetaVolumeMountPath = "/var/lib/xenon"
+	XenonCMVolumeMountPath   = "/mnt/xenon-cm"
 	LogsVolumeMountPath      = "/var/log/mysql"
 	DataVolumeMountPath      = "/var/lib/mysql"
 	SysVolumeMountPath       = "/host-sys"
 	ScriptsVolumeMountPath   = "/scripts"
-	XenonVolumeMountPath     = "/etc/xenon"
+	XenonConfVolumeMountPath = "/etc/xenon"
 	InitFileVolumeMountPath  = "/docker-entrypoint-initdb.d"
 
 	// Volume timezone name.
@@ -160,6 +163,8 @@ const (
 	ServiceAccount ResourceName = "service-account"
 	// PodDisruptionBudget is the name of pod disruption budget for the statefulset.
 	PodDisruptionBudget ResourceName = "pdb"
+	// XenonMetaData is the name of the configmap that contains xenon metadata.
+	XenonMetaData ResourceName = "xenon-metadata"
 )
 
 // JobType