From 41da038366ac5f5b6e9edc53b2c74d8ef4feaecf Mon Sep 17 00:00:00 2001 From: zhyass <34016424+zhyass@users.noreply.github.com> Date: Mon, 12 Jul 2021 17:58:56 +0800 Subject: [PATCH] *: add xenon post-start.sh #135 --- cluster/container/init_sidecar.go | 9 +++ cluster/container/xenon.go | 5 +- cluster/syncer/config_map.go | 22 +------ sidecar/config.go | 104 ++++++++++++++++++++++++++++-- sidecar/init.go | 44 ++++++------- sidecar/util.go | 19 +++++- 6 files changed, 144 insertions(+), 59 deletions(-) diff --git a/cluster/container/init_sidecar.go b/cluster/container/init_sidecar.go index f0300d7d1..d96bc72cf 100644 --- a/cluster/container/init_sidecar.go +++ b/cluster/container/init_sidecar.go @@ -17,6 +17,7 @@ limitations under the License. package container import ( + "fmt" "strconv" corev1 "k8s.io/api/core/v1" @@ -69,6 +70,14 @@ func (c *initSidecar) getEnvVars() []corev1.EnvVar { Name: "SERVICE_NAME", Value: c.GetNameForResource(utils.HeadlessSVC), }, + { + Name: "STATEFULSET_NAME", + Value: c.GetNameForResource(utils.StatefulSet), + }, + { + Name: "REPLICAS", + Value: fmt.Sprintf("%d", *c.Spec.Replicas), + }, { Name: "ADMIT_DEFEAT_HEARBEAT_COUNT", Value: strconv.Itoa(int(*c.Spec.XenonOpts.AdmitDefeatHearbeatCount)), diff --git a/cluster/container/xenon.go b/cluster/container/xenon.go index 959082b0f..4113869ad 100644 --- a/cluster/container/xenon.go +++ b/cluster/container/xenon.go @@ -17,8 +17,6 @@ limitations under the License. package container import ( - "fmt" - corev1 "k8s.io/api/core/v1" "github.com/radondb/radondb-mysql-kubernetes/cluster" @@ -55,11 +53,10 @@ func (c *xenon) getEnvVars() []corev1.EnvVar { // getLifecycle get the container lifecycle. func (c *xenon) getLifecycle() *corev1.Lifecycle { - arg := fmt.Sprintf("until (xenoncli xenon ping && xenoncli cluster add %s) > /dev/null 2>&1; do sleep 2; done", c.CreatePeers()) return &corev1.Lifecycle{ PostStart: &corev1.Handler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-c", arg}, + Command: []string{"sh", "-c", "/scripts/post-start.sh"}, }, }, } diff --git a/cluster/syncer/config_map.go b/cluster/syncer/config_map.go index 0395b650a..4ba3a7eb5 100644 --- a/cluster/syncer/config_map.go +++ b/cluster/syncer/config_map.go @@ -52,9 +52,7 @@ func NewConfigMapSyncer(cli client.Client, c *cluster.Cluster) syncer.Interface } cm.Data = map[string]string{ - "my.cnf": data, - "leader-start.sh": buildLeaderStart(c), - "leader-stop.sh": buildLeaderStop(c), + "my.cnf": data, } return nil @@ -117,21 +115,3 @@ func writeConfigs(cfg *ini.File) (string, error) { } return buf.String(), nil } - -// buildLeaderStart build the leader-start.sh. -func buildLeaderStart(c *cluster.Cluster) string { - return fmt.Sprintf(`#!/usr/bin/env bash -curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ ---cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ --d '[{"op": "replace", "path": "/metadata/labels/role", "value": "leader"}]' -`, c.Namespace) -} - -// buildLeaderStop build the leader-stop.sh. -func buildLeaderStop(c *cluster.Cluster) string { - return fmt.Sprintf(`#!/usr/bin/env bash -curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ ---cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ --d '[{"op": "replace", "path": "/metadata/labels/role", "value": "follower"}]' -`, c.Namespace) -} diff --git a/sidecar/config.go b/sidecar/config.go index 7788d411b..2a5202f95 100644 --- a/sidecar/config.go +++ b/sidecar/config.go @@ -34,6 +34,10 @@ type Config struct { NameSpace string // The name of the headless service. ServiceName string + // The name of the statefulset. + StatefulSetName string + // Replicas is the number of pods. + Replicas int32 // The password of the root user. RootPassword string @@ -70,6 +74,9 @@ type Config struct { AdmitDefeatHearbeatCount int32 // The parameter in xenon means election timeout(ms)。 ElectionTimeout int32 + + // Whether the MySQL data exists. + existMySQLData bool } // NewConfig returns a pointer to Config. @@ -80,6 +87,11 @@ func NewConfig() *Config { mysqlVersion, _ = semver.Parse(utils.MySQLDefaultVersion) } + replicas, err := strconv.ParseInt(getEnvValue("REPLICAS"), 10, 32) + if err != nil { + panic(err) + } + initTokuDB := false if len(getEnvValue("INIT_TOKUDB")) > 0 { initTokuDB = true @@ -94,10 +106,14 @@ func NewConfig() *Config { electionTimeout = 10000 } + existMySQLData, _ := checkIfPathExists(fmt.Sprintf("%s/mysql", dataPath)) + return &Config{ - HostName: getEnvValue("POD_HOSTNAME"), - NameSpace: getEnvValue("NAMESPACE"), - ServiceName: getEnvValue("SERVICE_NAME"), + HostName: getEnvValue("POD_HOSTNAME"), + NameSpace: getEnvValue("NAMESPACE"), + ServiceName: getEnvValue("SERVICE_NAME"), + StatefulSetName: getEnvValue("STATEFULSET_NAME"), + Replicas: int32(replicas), RootPassword: getEnvValue("MYSQL_ROOT_PASSWORD"), @@ -120,6 +136,8 @@ func NewConfig() *Config { AdmitDefeatHearbeatCount: int32(admitDefeatHearbeatCount), ElectionTimeout: int32(electionTimeout), + + existMySQLData: existMySQLData, } } @@ -128,11 +146,11 @@ func (cfg *Config) buildExtraConfig(filePath string) (*ini.File, error) { conf := ini.Empty() sec := conf.Section("mysqld") - id, err := generateServerID(cfg.HostName) + ordinal, err := getOrdinal(cfg.HostName) if err != nil { return nil, err } - if _, err := sec.NewKey("server-id", strconv.Itoa(id)); err != nil { + if _, err := sec.NewKey("server-id", strconv.Itoa(mysqlServerIDOffset+ordinal)); err != nil { return nil, err } @@ -260,3 +278,79 @@ func (cfg *Config) buildClientConfig() (*ini.File, error) { return conf, nil } + +func (cfg *Config) buildPostStart() ([]byte, error) { + ordinal, err := getOrdinal(cfg.HostName) + if err != nil { + return nil, err + } + + nums := ordinal + if cfg.existMySQLData { + nums = int(cfg.Replicas) + } + + host := fmt.Sprintf("%s.%s.%s", cfg.HostName, cfg.ServiceName, cfg.NameSpace) + + str := fmt.Sprintf(`#!/bin/sh +while true; do + info=$(curl -i -X GET -u root:%s http://%s:%d/v1/xenon/ping) + code=$(echo $info|grep "HTTP"|awk '{print $2}') + if [ "$code" -eq "200" ]; then + break + fi +done +`, cfg.RootPassword, host, utils.XenonPeerPort) + + if !cfg.existMySQLData && ordinal == 0 { + str = fmt.Sprintf(`%s +for i in $(seq 12); do + curl -i -X POST -u root:%s http://%s:%d/v1/raft/trytoleader + sleep 5 + curl -i -X GET -u root:%s http://%s:%d/v1/raft/status | grep LEADER + if [ $? -eq 0 ] ; then + echo "trytoleader success" + break + fi + if [ $i -eq 12 ]; then + echo "wait trytoleader failed" + fi +done +`, str, cfg.RootPassword, host, utils.XenonPeerPort, cfg.RootPassword, host, utils.XenonPeerPort) + } else { + str = fmt.Sprintf(`%s +i=0 +while [ $i -lt %d ]; do + if [ $i -ne %d ]; then + curl -i -X POST -d '{"address": "%s-'$i'.%s.%s:%d"}' -u root:%s http://%s:%d/v1/cluster/add + curl -i -X POST -d '{"address": "%s:%d"}' -u root:%s http://%s-$i.%s.%s:%d/v1/cluster/add + fi + i=$((i+1)) +done +`, str, nums, ordinal, cfg.StatefulSetName, cfg.ServiceName, cfg.NameSpace, utils.XenonPort, + cfg.RootPassword, host, utils.XenonPeerPort, host, utils.XenonPort, cfg.RootPassword, + cfg.StatefulSetName, cfg.ServiceName, cfg.NameSpace, utils.XenonPeerPort) + } + + return utils.StringToBytes(str), nil +} + +// buildLeaderStart build the leader-start.sh. +func (cfg *Config) buildLeaderStart() []byte { + str := fmt.Sprintf(`#!/usr/bin/env bash +curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ +--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ +-d '[{"op": "replace", "path": "/metadata/labels/role", "value": "leader"}]' +`, cfg.NameSpace) + return utils.StringToBytes(str) +} + +// buildLeaderStop build the leader-stop.sh. +func (cfg *Config) buildLeaderStop() []byte { + str := fmt.Sprintf(`#!/usr/bin/env bash +curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ +--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ +-d '[{"op": "replace", "path": "/metadata/labels/role", "value": "follower"}]' +`, cfg.NameSpace) + return utils.StringToBytes(str) +} diff --git a/sidecar/init.go b/sidecar/init.go index 5b2ca8524..d6024ece3 100644 --- a/sidecar/init.go +++ b/sidecar/init.go @@ -115,22 +115,28 @@ func runInitCommand(cfg *Config) error { return fmt.Errorf("failed to save extra.cnf: %s", err) } - // copy leader-start.sh from config-map to scripts mount. - leaderStartPath := path.Join(scriptsPath, "leader-start.sh") - if err = copyFile(path.Join(configMapPath, "leader-start.sh"), leaderStartPath); err != nil { - return fmt.Errorf("failed to copy scripts: %s", err) + // build post-start.sh. + bashPostStartPath := path.Join(scriptsPath, "post-start.sh") + bashPostStart, err := cfg.buildPostStart() + if err != nil { + return fmt.Errorf("failed to build post-start.sh: %s", err) } - if err = os.Chmod(leaderStartPath, os.FileMode(0755)); err != nil { - return fmt.Errorf("failed to chmod scripts: %s", err) + if err = ioutil.WriteFile(bashPostStartPath, bashPostStart, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to write post-start.sh: %s", err) } - // copy leader-stop.sh from config-map to scripts mount. - leaderStopPath := path.Join(scriptsPath, "leader-stop.sh") - if err = copyFile(path.Join(configMapPath, "leader-stop.sh"), leaderStopPath); err != nil { - return fmt.Errorf("failed to copy scripts: %s", err) + // build leader-start.sh. + bashLeaderStart := cfg.buildLeaderStart() + leaderStartPath := path.Join(scriptsPath, "leader-start.sh") + if err = ioutil.WriteFile(leaderStartPath, bashLeaderStart, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to write leader-start.sh: %s", err) } - if err = os.Chmod(leaderStopPath, os.FileMode(0755)); err != nil { - return fmt.Errorf("failed to chmod scripts: %s", err) + + // build leader-stop.sh. + bashLeaderStop := cfg.buildLeaderStop() + leaderStopPath := path.Join(scriptsPath, "leader-stop.sh") + if err = ioutil.WriteFile(leaderStopPath, bashLeaderStop, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to write leader-stop.sh: %s", err) } // for install tokudb. @@ -152,17 +158,3 @@ func runInitCommand(cfg *Config) error { log.Info("init command success") return nil } - -// checkIfPathExists check if the path exists. -func checkIfPathExists(path string) (bool, error) { - f, err := os.Open(path) - if os.IsNotExist(err) { - return false, nil - } else if err != nil { - log.Error(err, "failed to open file", "file", path) - return false, err - } - - err = f.Close() - return true, err -} diff --git a/sidecar/util.go b/sidecar/util.go index 4bdf077a9..7ce1df00a 100644 --- a/sidecar/util.go +++ b/sidecar/util.go @@ -102,8 +102,7 @@ func getEnvValue(key string) string { return value } -// Generate mysql server-id from pod ordinal index. -func generateServerID(name string) (int, error) { +func getOrdinal(name string) (int, error) { idx := strings.LastIndexAny(name, "-") if idx == -1 { return -1, fmt.Errorf("failed to extract ordinal from hostname: %s", name) @@ -114,5 +113,19 @@ func generateServerID(name string) (int, error) { log.Error(err, "failed to extract ordinal form hostname", "hostname", name) return -1, fmt.Errorf("failed to extract ordinal from hostname: %s", name) } - return mysqlServerIDOffset + ordinal, nil + return ordinal, nil +} + +// checkIfPathExists check if the path exists. +func checkIfPathExists(path string) (bool, error) { + f, err := os.Open(path) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + log.Error(err, "failed to open file", "file", path) + return false, err + } + + err = f.Close() + return true, err }