From 8d8784b1fc7a787763274a32eab5078e7ed272a3 Mon Sep 17 00:00:00 2001 From: zhyass <34016424+zhyass@users.noreply.github.com> Date: Mon, 12 Jul 2021 17:58:56 +0800 Subject: [PATCH] *: add xenon post-start.sh #135 --- cluster/container/init_sidecar.go | 4 ++ cluster/container/xenon.go | 5 +- cluster/syncer/config_map.go | 28 ++++------ sidecar/config.go | 88 +++++++++++++++++++++++++++++-- sidecar/init.go | 53 ++++++++++--------- sidecar/util.go | 19 +++++-- 6 files changed, 140 insertions(+), 57 deletions(-) diff --git a/cluster/container/init_sidecar.go b/cluster/container/init_sidecar.go index f0300d7d1..f25311fe6 100644 --- a/cluster/container/init_sidecar.go +++ b/cluster/container/init_sidecar.go @@ -69,6 +69,10 @@ func (c *initSidecar) getEnvVars() []corev1.EnvVar { Name: "SERVICE_NAME", Value: c.GetNameForResource(utils.HeadlessSVC), }, + { + Name: "STATEFULSET_NAME", + Value: c.GetNameForResource(utils.StatefulSet), + }, { Name: "ADMIT_DEFEAT_HEARBEAT_COUNT", Value: strconv.Itoa(int(*c.Spec.XenonOpts.AdmitDefeatHearbeatCount)), diff --git a/cluster/container/xenon.go b/cluster/container/xenon.go index 959082b0f..4113869ad 100644 --- a/cluster/container/xenon.go +++ b/cluster/container/xenon.go @@ -17,8 +17,6 @@ limitations under the License. package container import ( - "fmt" - corev1 "k8s.io/api/core/v1" "github.com/radondb/radondb-mysql-kubernetes/cluster" @@ -55,11 +53,10 @@ func (c *xenon) getEnvVars() []corev1.EnvVar { // getLifecycle get the container lifecycle. func (c *xenon) getLifecycle() *corev1.Lifecycle { - arg := fmt.Sprintf("until (xenoncli xenon ping && xenoncli cluster add %s) > /dev/null 2>&1; do sleep 2; done", c.CreatePeers()) return &corev1.Lifecycle{ PostStart: &corev1.Handler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-c", arg}, + Command: []string{"sh", "-c", "/scripts/post-start.sh"}, }, }, } diff --git a/cluster/syncer/config_map.go b/cluster/syncer/config_map.go index 0395b650a..1088609b0 100644 --- a/cluster/syncer/config_map.go +++ b/cluster/syncer/config_map.go @@ -52,9 +52,8 @@ func NewConfigMapSyncer(cli client.Client, c *cluster.Cluster) syncer.Interface } cm.Data = map[string]string{ - "my.cnf": data, - "leader-start.sh": buildLeaderStart(c), - "leader-stop.sh": buildLeaderStop(c), + "my.cnf": data, + "post-start.sh": buildPostStart(c), } return nil @@ -118,20 +117,11 @@ func writeConfigs(cfg *ini.File) (string, error) { return buf.String(), nil } -// buildLeaderStart build the leader-start.sh. -func buildLeaderStart(c *cluster.Cluster) string { - return fmt.Sprintf(`#!/usr/bin/env bash -curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ ---cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ --d '[{"op": "replace", "path": "/metadata/labels/role", "value": "leader"}]' -`, c.Namespace) -} - -// buildLeaderStop build the leader-stop.sh. -func buildLeaderStop(c *cluster.Cluster) string { - return fmt.Sprintf(`#!/usr/bin/env bash -curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ ---cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ --d '[{"op": "replace", "path": "/metadata/labels/role", "value": "follower"}]' -`, c.Namespace) +// buildPostStart build the post-start.sh. +func buildPostStart(c *cluster.Cluster) string { + return fmt.Sprintf(`#!/bin/sh +until (xenoncli xenon ping && xenoncli cluster add %s); do + sleep 2 +done +`, c.CreatePeers()) } diff --git a/sidecar/config.go b/sidecar/config.go index 2dbff0820..61f677278 100644 --- a/sidecar/config.go +++ b/sidecar/config.go @@ -34,6 +34,8 @@ type Config struct { NameSpace string // The name of the headless service. ServiceName string + // The name of the statefulset. + StatefulSetName string // The password of the root user. RootPassword string @@ -70,6 +72,9 @@ type Config struct { AdmitDefeatHearbeatCount int32 // The parameter in xenon means election timeout(ms)。 ElectionTimeout int32 + + // Whether the MySQL data exists. + existMySQLData bool } // NewConfig returns a pointer to Config. @@ -94,10 +99,13 @@ func NewConfig() *Config { electionTimeout = 10000 } + existMySQLData, _ := checkIfPathExists(fmt.Sprintf("%s/mysql", dataPath)) + return &Config{ - HostName: getEnvValue("POD_HOSTNAME"), - NameSpace: getEnvValue("NAMESPACE"), - ServiceName: getEnvValue("SERVICE_NAME"), + HostName: getEnvValue("POD_HOSTNAME"), + NameSpace: getEnvValue("NAMESPACE"), + ServiceName: getEnvValue("SERVICE_NAME"), + StatefulSetName: getEnvValue("STATEFULSET_NAME"), RootPassword: getEnvValue("MYSQL_ROOT_PASSWORD"), @@ -120,6 +128,8 @@ func NewConfig() *Config { AdmitDefeatHearbeatCount: int32(admitDefeatHearbeatCount), ElectionTimeout: int32(electionTimeout), + + existMySQLData: existMySQLData, } } @@ -128,11 +138,11 @@ func (cfg *Config) buildExtraConfig(filePath string) (*ini.File, error) { conf := ini.Empty() sec := conf.Section("mysqld") - id, err := generateServerID(cfg.HostName) + ordinal, err := getOrdinal(cfg.HostName) if err != nil { return nil, err } - if _, err := sec.NewKey("server-id", strconv.Itoa(id)); err != nil { + if _, err := sec.NewKey("server-id", strconv.Itoa(mysqlServerIDOffset+ordinal)); err != nil { return nil, err } @@ -254,3 +264,71 @@ func (cfg *Config) buildClientConfig() (*ini.File, error) { return conf, nil } + +func (cfg *Config) buildPostStart() ([]byte, error) { + ordinal, err := getOrdinal(cfg.HostName) + if err != nil { + return nil, err + } + host := fmt.Sprintf("%s.%s.%s", cfg.HostName, cfg.ServiceName, cfg.NameSpace) + + str := fmt.Sprintf(`#!/bin/sh +while true; do + info=$(curl -i -X GET -u root:%s http://%s:%d/v1/xenon/ping) + code=$(echo $info|grep "HTTP"|awk '{print $2}') + if [ "$code" -eq "200" ]; then + break + fi +done +`, cfg.RootPassword, host, utils.XenonPeerPort) + + if ordinal == 0 { + str = fmt.Sprintf(`%s +for i in $(seq 12); do + curl -i -X POST -u root:%s http://%s:%d/v1/raft/trytoleader + sleep 5 + curl -i -X GET -u root:%s http://%s:%d/v1/raft/status | grep LEADER + if [ $? -eq 0 ] ; then + echo "trytoleader success" + break + fi + if [ $i -eq 12 ]; then + echo "wait trytoleader failed" + fi +done +`, str, cfg.RootPassword, host, utils.XenonPeerPort, cfg.RootPassword, host, utils.XenonPeerPort) + } else { + str = fmt.Sprintf(`%s +i=0 +while [ $i -lt %d ]; do + curl -i -X POST -d '{"address": "%s-'$i'.%s.%s:%d"}' -u root:%s http://%s:%d/v1/cluster/add + curl -i -X POST -d '{"address": "%s:%d"}' -u root:%s http://%s-$i.%s.%s:%d/v1/cluster/add + i=$((i+1)) +done +`, str, ordinal, cfg.StatefulSetName, cfg.ServiceName, cfg.NameSpace, utils.XenonPort, cfg.RootPassword, + host, utils.XenonPeerPort, host, utils.XenonPort, cfg.RootPassword, cfg.StatefulSetName, + cfg.ServiceName, cfg.NameSpace, utils.XenonPeerPort) + } + + return utils.StringToBytes(str), nil +} + +// buildLeaderStart build the leader-start.sh. +func (cfg *Config) buildLeaderStart() []byte { + str := fmt.Sprintf(`#!/usr/bin/env bash +curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ +--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ +-d '[{"op": "replace", "path": "/metadata/labels/role", "value": "leader"}]' +`, cfg.NameSpace) + return utils.StringToBytes(str) +} + +// buildLeaderStop build the leader-stop.sh. +func (cfg *Config) buildLeaderStop() []byte { + str := fmt.Sprintf(`#!/usr/bin/env bash +curl -X PATCH -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" \ +--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/%s/pods/$HOSTNAME \ +-d '[{"op": "replace", "path": "/metadata/labels/role", "value": "follower"}]' +`, cfg.NameSpace) + return utils.StringToBytes(str) +} diff --git a/sidecar/init.go b/sidecar/init.go index eb414af10..1136ad16b 100644 --- a/sidecar/init.go +++ b/sidecar/init.go @@ -115,22 +115,37 @@ func runInitCommand(cfg *Config) error { return fmt.Errorf("failed to save extra.cnf: %s", err) } - // copy leader-start.sh from config-map to scripts mount. - leaderStartPath := path.Join(scriptsPath, "leader-start.sh") - if err = copyFile(path.Join(configMapPath, "leader-start.sh"), leaderStartPath); err != nil { - return fmt.Errorf("failed to copy scripts: %s", err) + // build post-start.sh. + bashPostStartPath := path.Join(scriptsPath, "post-start.sh") + if cfg.existMySQLData { + if err = copyFile(path.Join(configMapPath, "post-start.sh"), bashPostStartPath); err != nil { + return fmt.Errorf("failed to copy scripts: %s", err) + } + if err = os.Chmod(bashPostStartPath, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to chmod scripts: %s", err) + } + } else { + bashPostStart, err := cfg.buildPostStart() + if err != nil { + return fmt.Errorf("failed to build post-start.sh: %s", err) + } + if err = ioutil.WriteFile(bashPostStartPath, bashPostStart, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to write post-start.sh: %s", err) + } } - if err = os.Chmod(leaderStartPath, os.FileMode(0755)); err != nil { - return fmt.Errorf("failed to chmod scripts: %s", err) + + // build leader-start.sh. + bashLeaderStart := cfg.buildLeaderStart() + leaderStartPath := path.Join(scriptsPath, "leader-start.sh") + if err = ioutil.WriteFile(leaderStartPath, bashLeaderStart, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to write leader-start.sh: %s", err) } - // copy leader-stop.sh from config-map to scripts mount. + // build leader-stop.sh. + bashLeaderStop := cfg.buildLeaderStop() leaderStopPath := path.Join(scriptsPath, "leader-stop.sh") - if err = copyFile(path.Join(configMapPath, "leader-stop.sh"), leaderStopPath); err != nil { - return fmt.Errorf("failed to copy scripts: %s", err) - } - if err = os.Chmod(leaderStopPath, os.FileMode(0755)); err != nil { - return fmt.Errorf("failed to chmod scripts: %s", err) + if err = ioutil.WriteFile(leaderStopPath, bashLeaderStop, os.FileMode(0755)); err != nil { + return fmt.Errorf("failed to write leader-stop.sh: %s", err) } // for install tokudb. @@ -152,17 +167,3 @@ func runInitCommand(cfg *Config) error { log.Info("init command success") return nil } - -// checkIfPathExists check if the path exists. -func checkIfPathExists(path string) (bool, error) { - f, err := os.Open(path) - if os.IsNotExist(err) { - return false, nil - } else if err != nil { - log.Error(err, "failed to open file", "file", path) - return false, err - } - - err = f.Close() - return true, err -} diff --git a/sidecar/util.go b/sidecar/util.go index 4bdf077a9..7ce1df00a 100644 --- a/sidecar/util.go +++ b/sidecar/util.go @@ -102,8 +102,7 @@ func getEnvValue(key string) string { return value } -// Generate mysql server-id from pod ordinal index. -func generateServerID(name string) (int, error) { +func getOrdinal(name string) (int, error) { idx := strings.LastIndexAny(name, "-") if idx == -1 { return -1, fmt.Errorf("failed to extract ordinal from hostname: %s", name) @@ -114,5 +113,19 @@ func generateServerID(name string) (int, error) { log.Error(err, "failed to extract ordinal form hostname", "hostname", name) return -1, fmt.Errorf("failed to extract ordinal from hostname: %s", name) } - return mysqlServerIDOffset + ordinal, nil + return ordinal, nil +} + +// checkIfPathExists check if the path exists. +func checkIfPathExists(path string) (bool, error) { + f, err := os.Open(path) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + log.Error(err, "failed to open file", "file", path) + return false, err + } + + err = f.Close() + return true, err }