*: add xenon post-start.sh radondb#135

cluster/container/init_sidecar.go

@@ -17,6 +17,7 @@ limitations under the License.
 package container
 
 import (
+	"fmt"
 	"strconv"
 
 	corev1 "k8s.io/api/core/v1"
@@ -69,6 +70,14 @@ func (c *initSidecar) getEnvVars() []corev1.EnvVar {
 			Name:  "SERVICE_NAME",
 			Value: c.GetNameForResource(utils.HeadlessSVC),
 		},
+		{
+			Name:  "STATEFULSET_NAME",
+			Value: c.GetNameForResource(utils.StatefulSet),
+		},
+		{
+			Name:  "REPLICAS",
+			Value: fmt.Sprintf("%d", *c.Spec.Replicas),
+		},
 		{
 			Name:  "ADMIT_DEFEAT_HEARBEAT_COUNT",
 			Value: strconv.Itoa(int(*c.Spec.XenonOpts.AdmitDefeatHearbeatCount)),

cluster/container/xenon.go

@@ -17,8 +17,6 @@ limitations under the License.
 package container
 
 import (
-	"fmt"
-
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/radondb/radondb-mysql-kubernetes/cluster"
@@ -55,11 +53,10 @@ func (c *xenon) getEnvVars() []corev1.EnvVar {
 
 // getLifecycle get the container lifecycle.
 func (c *xenon) getLifecycle() *corev1.Lifecycle {
-	arg := fmt.Sprintf("until (xenoncli xenon ping && xenoncli cluster add %s) > /dev/null 2>&1; do sleep 2; done", c.CreatePeers())
 	return &corev1.Lifecycle{
 		PostStart: &corev1.Handler{
 			Exec: &corev1.ExecAction{
-				Command: []string{"sh", "-c", arg},
+				Command: []string{"sh", "-c", "/scripts/post-start.sh"},
 			},
 		},
 	}

cluster/syncer/config_map.go

@@ -52,9 +52,7 @@ func NewConfigMapSyncer(cli client.Client, c *cluster.Cluster) syncer.Interface
 		}
 
 		cm.Data = map[string]string{
-			"my.cnf":          data,
-			"leader-start.sh": buildLeaderStart(c),
-			"leader-stop.sh":  buildLeaderStop(c),
+			"my.cnf": data,
 		}
 
 		return nil
@@ -117,21 +115,3 @@ func writeConfigs(cfg *ini.File) (string, error) {
 	}
 	return buf.String(), nil
 }
-
-// buildLeaderStart build the leader-start.sh.
-func buildLeaderStart(c *cluster.Cluster) string {
-	return fmt.Sprintf(`#!/usr/bin/env bash
-curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \
---cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \
--d '[{"op": "replace", "path": "/metadata/labels/role", "value": "leader"}]'
-`, c.Namespace)
-}
-
-// buildLeaderStop build the leader-stop.sh.
-func buildLeaderStop(c *cluster.Cluster) string {
-	return fmt.Sprintf(`#!/usr/bin/env bash
-curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \
---cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \
--d '[{"op": "replace", "path": "/metadata/labels/role", "value": "follower"}]'
-`, c.Namespace)
-}

sidecar/config.go

@@ -34,6 +34,10 @@ type Config struct {
 	NameSpace string
 	// The name of the headless service.
 	ServiceName string
+	// The name of the statefulset.
+	StatefulSetName string
+	// Replicas is the number of pods.
+	Replicas int32
 
 	// The password of the root user.
 	RootPassword string
@@ -70,6 +74,9 @@ type Config struct {
 	AdmitDefeatHearbeatCount int32
 	// The parameter in xenon means election timeout(ms)。
 	ElectionTimeout int32
+
+	// Whether the MySQL data exists.
+	existMySQLData bool
 }
 
 // NewConfig returns a pointer to Config.
@@ -80,6 +87,11 @@ func NewConfig() *Config {
 		mysqlVersion, _ = semver.Parse(utils.MySQLDefaultVersion)
 	}
 
+	replicas, err := strconv.ParseInt(getEnvValue("REPLICAS"), 10, 32)
+	if err != nil {
+		panic(err)
+	}
+
 	initTokuDB := false
 	if len(getEnvValue("INIT_TOKUDB")) > 0 {
 		initTokuDB = true
@@ -94,10 +106,14 @@ func NewConfig() *Config {
 		electionTimeout = 10000
 	}
 
+	existMySQLData, _ := checkIfPathExists(fmt.Sprintf("%s/mysql", dataPath))
+
 	return &Config{
-		HostName:    getEnvValue("POD_HOSTNAME"),
-		NameSpace:   getEnvValue("NAMESPACE"),
-		ServiceName: getEnvValue("SERVICE_NAME"),
+		HostName:        getEnvValue("POD_HOSTNAME"),
+		NameSpace:       getEnvValue("NAMESPACE"),
+		ServiceName:     getEnvValue("SERVICE_NAME"),
+		StatefulSetName: getEnvValue("STATEFULSET_NAME"),
+		Replicas:        int32(replicas),
 
 		RootPassword: getEnvValue("MYSQL_ROOT_PASSWORD"),
 
@@ -120,6 +136,8 @@ func NewConfig() *Config {
 
 		AdmitDefeatHearbeatCount: int32(admitDefeatHearbeatCount),
 		ElectionTimeout:          int32(electionTimeout),
+
+		existMySQLData: existMySQLData,
 	}
 }
 
@@ -128,11 +146,11 @@ func (cfg *Config) buildExtraConfig(filePath string) (*ini.File, error) {
 	conf := ini.Empty()
 	sec := conf.Section("mysqld")
 
-	id, err := generateServerID(cfg.HostName)
+	ordinal, err := getOrdinal(cfg.HostName)
 	if err != nil {
 		return nil, err
 	}
-	if _, err := sec.NewKey("server-id", strconv.Itoa(id)); err != nil {
+	if _, err := sec.NewKey("server-id", strconv.Itoa(mysqlServerIDOffset+ordinal)); err != nil {
 		return nil, err
 	}
 
@@ -260,3 +278,79 @@ func (cfg *Config) buildClientConfig() (*ini.File, error) {
 
 	return conf, nil
 }
+
+func (cfg *Config) buildPostStart() ([]byte, error) {
+	ordinal, err := getOrdinal(cfg.HostName)
+	if err != nil {
+		return nil, err
+	}
+
+	nums := ordinal
+	if cfg.existMySQLData {
+		nums = int(cfg.Replicas)
+	}
+
+	host := fmt.Sprintf("%s.%s.%s", cfg.HostName, cfg.ServiceName, cfg.NameSpace)
+
+	str := fmt.Sprintf(`#!/bin/sh
+while true; do
+	info=$(curl -i -X GET -u root:%s http://%s:%d/v1/xenon/ping)
+	code=$(echo $info|grep "HTTP"|awk '{print $2}')
+	if [ "$code" -eq "200" ]; then
+		break
+	fi
+done
+`, cfg.RootPassword, host, utils.XenonPeerPort)
+
+	if !cfg.existMySQLData && ordinal == 0 {
+		str = fmt.Sprintf(`%s
+for i in $(seq 12); do
+	curl -i -X POST -u root:%s http://%s:%d/v1/raft/trytoleader
+	sleep 5
+	curl -i -X GET -u root:%s http://%s:%d/v1/raft/status | grep LEADER
+	if [ $? -eq 0 ] ; then
+		echo "trytoleader success"
+		break
+	fi
+	if [ $i -eq 12 ]; then
+		echo "wait trytoleader failed"
+	fi
+done
+`, str, cfg.RootPassword, host, utils.XenonPeerPort, cfg.RootPassword, host, utils.XenonPeerPort)
+	} else {
+		str = fmt.Sprintf(`%s
+i=0
+while [ $i -lt %d ]; do
+	if [ $i -ne %d ]; then
+		curl -i -X POST -d '{"address": "%s-'$i'.%s.%s:%d"}' -u root:%s http://%s:%d/v1/cluster/add
+		curl -i -X POST -d '{"address": "%s:%d"}' -u root:%s http://%s-$i.%s.%s:%d/v1/cluster/add
+	fi
+	i=$((i+1))
+done
+`, str, nums, ordinal, cfg.StatefulSetName, cfg.ServiceName, cfg.NameSpace, utils.XenonPort,
+			cfg.RootPassword, host, utils.XenonPeerPort, host, utils.XenonPort, cfg.RootPassword,
+			cfg.StatefulSetName, cfg.ServiceName, cfg.NameSpace, utils.XenonPeerPort)
+	}
+
+	return utils.StringToBytes(str), nil
+}
+
+// buildLeaderStart build the leader-start.sh.
+func (cfg *Config) buildLeaderStart() []byte {
+	str := fmt.Sprintf(`#!/usr/bin/env bash
+curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \
+--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \
+-d '[{"op": "replace", "path": "/metadata/labels/role", "value": "leader"}]'
+`, cfg.NameSpace)
+	return utils.StringToBytes(str)
+}
+
+// buildLeaderStop build the leader-stop.sh.
+func (cfg *Config) buildLeaderStop() []byte {
+	str := fmt.Sprintf(`#!/usr/bin/env bash
+curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \
+--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \
+-d '[{"op": "replace", "path": "/metadata/labels/role", "value": "follower"}]'
+`, cfg.NameSpace)
+	return utils.StringToBytes(str)
+}

sidecar/init.go

@@ -115,22 +115,28 @@ func runInitCommand(cfg *Config) error {
 		return fmt.Errorf("failed to save extra.cnf: %s", err)
 	}
 
-	// copy leader-start.sh from config-map to scripts mount.
-	leaderStartPath := path.Join(scriptsPath, "leader-start.sh")
-	if err = copyFile(path.Join(configMapPath, "leader-start.sh"), leaderStartPath); err != nil {
-		return fmt.Errorf("failed to copy scripts: %s", err)
+	// build post-start.sh.
+	bashPostStartPath := path.Join(scriptsPath, "post-start.sh")
+	bashPostStart, err := cfg.buildPostStart()
+	if err != nil {
+		return fmt.Errorf("failed to build post-start.sh: %s", err)
 	}
-	if err = os.Chmod(leaderStartPath, os.FileMode(0755)); err != nil {
-		return fmt.Errorf("failed to chmod scripts: %s", err)
+	if err = ioutil.WriteFile(bashPostStartPath, bashPostStart, os.FileMode(0755)); err != nil {
+		return fmt.Errorf("failed to write post-start.sh: %s", err)
 	}
 
-	// copy leader-stop.sh from config-map to scripts mount.
-	leaderStopPath := path.Join(scriptsPath, "leader-stop.sh")
-	if err = copyFile(path.Join(configMapPath, "leader-stop.sh"), leaderStopPath); err != nil {
-		return fmt.Errorf("failed to copy scripts: %s", err)
+	// build leader-start.sh.
+	bashLeaderStart := cfg.buildLeaderStart()
+	leaderStartPath := path.Join(scriptsPath, "leader-start.sh")
+	if err = ioutil.WriteFile(leaderStartPath, bashLeaderStart, os.FileMode(0755)); err != nil {
+		return fmt.Errorf("failed to write leader-start.sh: %s", err)
 	}
-	if err = os.Chmod(leaderStopPath, os.FileMode(0755)); err != nil {
-		return fmt.Errorf("failed to chmod scripts: %s", err)
+
+	// build leader-stop.sh.
+	bashLeaderStop := cfg.buildLeaderStop()
+	leaderStopPath := path.Join(scriptsPath, "leader-stop.sh")
+	if err = ioutil.WriteFile(leaderStopPath, bashLeaderStop, os.FileMode(0755)); err != nil {
+		return fmt.Errorf("failed to write leader-stop.sh: %s", err)
 	}
 
 	// for install tokudb.
@@ -152,17 +158,3 @@ func runInitCommand(cfg *Config) error {
 	log.Info("init command success")
 	return nil
 }
-
-// checkIfPathExists check if the path exists.
-func checkIfPathExists(path string) (bool, error) {
-	f, err := os.Open(path)
-	if os.IsNotExist(err) {
-		return false, nil
-	} else if err != nil {
-		log.Error(err, "failed to open file", "file", path)
-		return false, err
-	}
-
-	err = f.Close()
-	return true, err
-}

sidecar/util.go

@@ -102,8 +102,7 @@ func getEnvValue(key string) string {
 	return value
 }
 
-// Generate mysql server-id from pod ordinal index.
-func generateServerID(name string) (int, error) {
+func getOrdinal(name string) (int, error) {
 	idx := strings.LastIndexAny(name, "-")
 	if idx == -1 {
 		return -1, fmt.Errorf("failed to extract ordinal from hostname: %s", name)
@@ -114,5 +113,19 @@ func generateServerID(name string) (int, error) {
 		log.Error(err, "failed to extract ordinal form hostname", "hostname", name)
 		return -1, fmt.Errorf("failed to extract ordinal from hostname: %s", name)
 	}
-	return mysqlServerIDOffset + ordinal, nil
+	return ordinal, nil
+}
+
+// checkIfPathExists check if the path exists.
+func checkIfPathExists(path string) (bool, error) {
+	f, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil {
+		log.Error(err, "failed to open file", "file", path)
+		return false, err
+	}
+
+	err = f.Close()
+	return true, err
 }