From 66f4b64e1b72898b41521b5ae82e3b50a41aa218 Mon Sep 17 00:00:00 2001 From: zhyass <34016424+zhyass@users.noreply.github.com> Date: Thu, 8 Jul 2021 15:59:16 +0800 Subject: [PATCH] *: add client.cnf for health check --- cluster/container/init_mysql.go | 5 - cluster/container/mysql.go | 19 ++-- config/samples/mysql_v1alpha1_cluster.yaml | 1 + sidecar/config.go | 113 ++++++++++++++++++++ sidecar/init.go | 117 ++++----------------- sidecar/util.go | 6 ++ utils/constants.go | 7 +- 7 files changed, 158 insertions(+), 110 deletions(-) diff --git a/cluster/container/init_mysql.go b/cluster/container/init_mysql.go index dd35e44cf..492017615 100644 --- a/cluster/container/init_mysql.go +++ b/cluster/container/init_mysql.go @@ -112,11 +112,6 @@ func (c *initMysql) getVolumeMounts() []corev1.VolumeMount { Name: utils.ConfVolumeName, MountPath: utils.ConfVolumeMountPath, }, - { - Name: utils.ConfMapVolumeName, - MountPath: utils.MyCnfMountPath, - SubPath: "my.cnf", - }, { Name: utils.DataVolumeName, MountPath: utils.DataVolumeMountPath, diff --git a/cluster/container/mysql.go b/cluster/container/mysql.go index 9d37b3be9..c233d8e56 100644 --- a/cluster/container/mysql.go +++ b/cluster/container/mysql.go @@ -17,6 +17,8 @@ limitations under the License. package container import ( + "fmt" + corev1 "k8s.io/api/core/v1" "github.com/radondb/radondb-mysql-kubernetes/cluster" @@ -86,7 +88,11 @@ func (c *mysql) getLivenessProbe() *corev1.Probe { return &corev1.Probe{ Handler: corev1.Handler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-c", "mysqladmin ping -uroot -p${MYSQL_ROOT_PASSWORD}"}, + Command: []string{ + "sh", + "-c", + fmt.Sprintf("mysqladmin --defaults-file=%s ping", utils.ConfClientPath), + }, }, }, InitialDelaySeconds: 30, @@ -102,7 +108,11 @@ func (c *mysql) getReadinessProbe() *corev1.Probe { return &corev1.Probe{ Handler: corev1.Handler{ Exec: &corev1.ExecAction{ - Command: []string{"sh", "-c", `mysql -uroot -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"`}, + Command: []string{ + "sh", + "-c", + fmt.Sprintf(`test $(mysql --defaults-file=%s -NB -e "SELECT 1") -eq 1`, utils.ConfClientPath), + }, }, }, InitialDelaySeconds: 10, @@ -120,11 +130,6 @@ func (c *mysql) getVolumeMounts() []corev1.VolumeMount { Name: utils.ConfVolumeName, MountPath: utils.ConfVolumeMountPath, }, - { - Name: utils.ConfMapVolumeName, - MountPath: utils.MyCnfMountPath, - SubPath: "my.cnf", - }, { Name: utils.DataVolumeName, MountPath: utils.DataVolumeMountPath, diff --git a/config/samples/mysql_v1alpha1_cluster.yaml b/config/samples/mysql_v1alpha1_cluster.yaml index 6cb69d0a6..fa7684bf9 100644 --- a/config/samples/mysql_v1alpha1_cluster.yaml +++ b/config/samples/mysql_v1alpha1_cluster.yaml @@ -8,6 +8,7 @@ spec: mysqlOpts: rootPassword: "" + rootHost: "127.0.0.1" user: qc_usr password: Qing@123 database: qingcloud diff --git a/sidecar/config.go b/sidecar/config.go index ca2feb142..2dbff0820 100644 --- a/sidecar/config.go +++ b/sidecar/config.go @@ -17,6 +17,7 @@ limitations under the License. package sidecar import ( + "fmt" "strconv" "github.com/blang/semver" @@ -141,3 +142,115 @@ func (cfg *Config) buildExtraConfig(filePath string) (*ini.File, error) { return conf, nil } + +// buildXenonConf build a config file for xenon. +func (cfg *Config) buildXenonConf() []byte { + pingTimeout := cfg.ElectionTimeout / cfg.AdmitDefeatHearbeatCount + heartbeatTimeout := cfg.ElectionTimeout / cfg.AdmitDefeatHearbeatCount + requestTimeout := cfg.ElectionTimeout / cfg.AdmitDefeatHearbeatCount + + version := "mysql80" + if cfg.MySQLVersion.Major == 5 { + if cfg.MySQLVersion.Minor == 6 { + version = "mysql56" + } else { + version = "mysql57" + } + } + + var masterSysVars, slaveSysVars string + if cfg.InitTokuDB { + masterSysVars = "tokudb_fsync_log_period=default;sync_binlog=default;innodb_flush_log_at_trx_commit=default" + slaveSysVars = "tokudb_fsync_log_period=1000;sync_binlog=1000;innodb_flush_log_at_trx_commit=1" + } else { + masterSysVars = "sync_binlog=default;innodb_flush_log_at_trx_commit=default" + slaveSysVars = "sync_binlog=1000;innodb_flush_log_at_trx_commit=1" + } + + hostName := fmt.Sprintf("%s.%s.%s", cfg.HostName, cfg.ServiceName, cfg.NameSpace) + + str := fmt.Sprintf(`{ + "log": { + "level": "INFO" + }, + "server": { + "endpoint": "%s:%d", + "peer-address": "%s:%d", + "enable-apis": true + }, + "replication": { + "passwd": "%s", + "user": "%s" + }, + "rpc": { + "request-timeout": %d + }, + "mysql": { + "admit-defeat-ping-count": 3, + "admin": "root", + "ping-timeout": %d, + "passwd": "%s", + "host": "localhost", + "version": "%s", + "master-sysvars": "%s", + "slave-sysvars": "%s", + "port": 3306, + "monitor-disabled": true + }, + "raft": { + "election-timeout": %d, + "admit-defeat-hearbeat-count": %d, + "heartbeat-timeout": %d, + "meta-datadir": "/var/lib/xenon/", + "leader-start-command": "/scripts/leader-start.sh", + "leader-stop-command": "/scripts/leader-stop.sh", + "semi-sync-degrade": true, + "purge-binlog-disabled": true, + "super-idle": false + } +} +`, hostName, utils.XenonPort, hostName, utils.XenonPeerPort, cfg.ReplicationPassword, cfg.ReplicationUser, requestTimeout, + pingTimeout, cfg.RootPassword, version, masterSysVars, slaveSysVars, cfg.ElectionTimeout, + cfg.AdmitDefeatHearbeatCount, heartbeatTimeout) + return utils.StringToBytes(str) +} + +// buildInitSql used to build init.sql. The file run after the mysql init. +func (cfg *Config) buildInitSql() []byte { + sql := fmt.Sprintf(`SET @@SESSION.SQL_LOG_BIN=0; +CREATE DATABASE IF NOT EXISTS %s; +DELETE FROM mysql.user WHERE user in ('%s', '%s', '%s', '%s'); +GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO '%s'@'%%' IDENTIFIED BY '%s'; +GRANT SELECT, PROCESS, REPLICATION CLIENT ON *.* TO '%s'@'%%' IDENTIFIED BY '%s'; +GRANT SUPER, PROCESS, RELOAD, CREATE, SELECT ON *.* TO '%s'@'%%' IDENTIFIED BY '%s'; +GRANT ALL ON %s.* TO '%s'@'%%' IDENTIFIED BY '%s'; +FLUSH PRIVILEGES; +`, cfg.Database, cfg.ReplicationUser, cfg.MetricsUser, cfg.OperatorUser, cfg.User, cfg.ReplicationUser, cfg.ReplicationPassword, + cfg.MetricsUser, cfg.MetricsPassword, cfg.OperatorUser, cfg.OperatorPassword, cfg.Database, cfg.User, cfg.Password) + + return utils.StringToBytes(sql) +} + +// buildClientConfig used to build client.conf. +func (cfg *Config) buildClientConfig() (*ini.File, error) { + conf := ini.Empty() + sec := conf.Section("client") + + if _, err := sec.NewKey("host", "127.0.0.1"); err != nil { + return nil, err + } + + if _, err := sec.NewKey("port", fmt.Sprintf("%d", utils.MysqlPort)); err != nil { + return nil, err + } + + if _, err := sec.NewKey("user", cfg.OperatorUser); err != nil { + return nil, err + } + + if _, err := sec.NewKey("password", cfg.OperatorPassword); err != nil { + return nil, err + } + + return conf, nil +} diff --git a/sidecar/init.go b/sidecar/init.go index 7b876f9ec..eb414af10 100644 --- a/sidecar/init.go +++ b/sidecar/init.go @@ -26,8 +26,6 @@ import ( "strconv" "github.com/spf13/cobra" - - "github.com/radondb/radondb-mysql-kubernetes/utils" ) // NewInitCommand return a pointer to cobra.Command. @@ -75,9 +73,24 @@ func runInitCommand(cfg *Config) error { } } - if err = os.Mkdir(configPath, os.FileMode(0755)); err != nil { + // copy appropriate my.cnf from config-map to config mount. + if err = copyFile(path.Join(configMapPath, "my.cnf"), path.Join(configPath, "my.cnf")); err != nil { + return fmt.Errorf("failed to copy my.cnf: %s", err) + } + + // build client.conf. + clientConfig, err := cfg.buildClientConfig() + if err != nil { + return fmt.Errorf("failed to build client.conf: %s", err) + } + // save client.conf to /etc/mysql. + if err := clientConfig.SaveTo(path.Join(clientConfPath)); err != nil { + return fmt.Errorf("failed to save client.conf: %s", err) + } + + if err = os.Mkdir(extraConfPath, os.FileMode(0755)); err != nil { if !os.IsExist(err) { - return fmt.Errorf("error mkdir %s: %s", configPath, err) + return fmt.Errorf("error mkdir %s: %s", extraConfPath, err) } } @@ -87,8 +100,8 @@ func runInitCommand(cfg *Config) error { } // build init.sql. - initSqlPath := path.Join(configPath, "init.sql") - if err = ioutil.WriteFile(initSqlPath, buildInitSql(cfg), 0644); err != nil { + initSqlPath := path.Join(extraConfPath, "init.sql") + if err = ioutil.WriteFile(initSqlPath, cfg.buildInitSql(), 0644); err != nil { return fmt.Errorf("failed to write init.sql: %s", err) } @@ -98,7 +111,7 @@ func runInitCommand(cfg *Config) error { return fmt.Errorf("failed to build extra.cnf: %s", err) } // save extra.cnf to conf.d. - if err := extraConfig.SaveTo(path.Join(configPath, "extra.cnf")); err != nil { + if err := extraConfig.SaveTo(path.Join(extraConfPath, "extra.cnf")); err != nil { return fmt.Errorf("failed to save extra.cnf: %s", err) } @@ -132,7 +145,7 @@ func runInitCommand(cfg *Config) error { // build xenon.json. xenonFilePath := path.Join(xenonPath, "xenon.json") - if err = ioutil.WriteFile(xenonFilePath, buildXenonConf(cfg), 0644); err != nil { + if err = ioutil.WriteFile(xenonFilePath, cfg.buildXenonConf(), 0644); err != nil { return fmt.Errorf("failed to write xenon.json: %s", err) } @@ -153,91 +166,3 @@ func checkIfPathExists(path string) (bool, error) { err = f.Close() return true, err } - -// buildXenonConf build a config file for xenon. -func buildXenonConf(cfg *Config) []byte { - pingTimeout := cfg.ElectionTimeout / cfg.AdmitDefeatHearbeatCount - heartbeatTimeout := cfg.ElectionTimeout / cfg.AdmitDefeatHearbeatCount - requestTimeout := cfg.ElectionTimeout / cfg.AdmitDefeatHearbeatCount - - version := "mysql80" - if cfg.MySQLVersion.Major == 5 { - if cfg.MySQLVersion.Minor == 6 { - version = "mysql56" - } else { - version = "mysql57" - } - } - - var masterSysVars, slaveSysVars string - if cfg.InitTokuDB { - masterSysVars = "tokudb_fsync_log_period=default;sync_binlog=default;innodb_flush_log_at_trx_commit=default" - slaveSysVars = "tokudb_fsync_log_period=1000;sync_binlog=1000;innodb_flush_log_at_trx_commit=1" - } else { - masterSysVars = "sync_binlog=default;innodb_flush_log_at_trx_commit=default" - slaveSysVars = "sync_binlog=1000;innodb_flush_log_at_trx_commit=1" - } - - hostName := fmt.Sprintf("%s.%s.%s", cfg.HostName, cfg.ServiceName, cfg.NameSpace) - - str := fmt.Sprintf(`{ - "log": { - "level": "INFO" - }, - "server": { - "endpoint": "%s:%d", - "peer-address": "%s:%d", - "enable-apis": true - }, - "replication": { - "passwd": "%s", - "user": "%s" - }, - "rpc": { - "request-timeout": %d - }, - "mysql": { - "admit-defeat-ping-count": 3, - "admin": "root", - "ping-timeout": %d, - "passwd": "%s", - "host": "localhost", - "version": "%s", - "master-sysvars": "%s", - "slave-sysvars": "%s", - "port": 3306, - "monitor-disabled": true - }, - "raft": { - "election-timeout": %d, - "admit-defeat-hearbeat-count": %d, - "heartbeat-timeout": %d, - "meta-datadir": "/var/lib/xenon/", - "leader-start-command": "/scripts/leader-start.sh", - "leader-stop-command": "/scripts/leader-stop.sh", - "semi-sync-degrade": true, - "purge-binlog-disabled": true, - "super-idle": false - } -} -`, hostName, utils.XenonPort, hostName, utils.XenonPeerPort, cfg.ReplicationPassword, cfg.ReplicationUser, requestTimeout, - pingTimeout, cfg.RootPassword, version, masterSysVars, slaveSysVars, cfg.ElectionTimeout, - cfg.AdmitDefeatHearbeatCount, heartbeatTimeout) - return utils.StringToBytes(str) -} - -// buildInitSql used to build init.sql. The file run after the mysql init. -func buildInitSql(cfg *Config) []byte { - sql := fmt.Sprintf(`SET @@SESSION.SQL_LOG_BIN=0; -CREATE DATABASE IF NOT EXISTS %s; -DELETE FROM mysql.user WHERE user in ('%s', '%s', '%s', '%s'); -GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO '%s'@'%%' IDENTIFIED BY '%s'; -GRANT SELECT, PROCESS, REPLICATION CLIENT ON *.* TO '%s'@'%%' IDENTIFIED BY '%s'; -GRANT SUPER, PROCESS, RELOAD, CREATE, SELECT ON *.* TO '%s'@'%%' IDENTIFIED BY '%s'; -GRANT ALL ON %s.* TO '%s'@'%%' IDENTIFIED BY '%s'; -FLUSH PRIVILEGES; -`, cfg.Database, cfg.ReplicationUser, cfg.MetricsUser, cfg.OperatorUser, cfg.User, cfg.ReplicationUser, cfg.ReplicationPassword, - cfg.MetricsUser, cfg.MetricsPassword, cfg.OperatorUser, cfg.OperatorPassword, cfg.Database, cfg.User, cfg.Password) - - return utils.StringToBytes(sql) -} diff --git a/sidecar/util.go b/sidecar/util.go index 3c6083416..4bdf077a9 100644 --- a/sidecar/util.go +++ b/sidecar/util.go @@ -37,6 +37,12 @@ var ( // configPath is the mysql configs path. configPath = utils.ConfVolumeMountPath + // clientConfPath is the client.cnf path. + clientConfPath = utils.ConfClientPath + + // extraConfPath is the mysql extra configs path. + extraConfPath = utils.ConfVolumeMountPath + "/conf.d" + // configMapPath is the mounted configmap. configMapPath = utils.ConfMapVolumeMountPath diff --git a/utils/constants.go b/utils/constants.go index dab22934d..f0580e8ad 100644 --- a/utils/constants.go +++ b/utils/constants.go @@ -74,8 +74,7 @@ const ( InitFileVolumeName = "init-mysql" // volumes mount path. - MyCnfMountPath = "/etc/mysql/my.cnf" - ConfVolumeMountPath = "/etc/mysql/conf.d" + ConfVolumeMountPath = "/etc/mysql" ConfMapVolumeMountPath = "/mnt/config-map" LogsVolumeMountPath = "/var/log/mysql" DataVolumeMountPath = "/var/lib/mysql" @@ -83,6 +82,10 @@ const ( ScriptsVolumeMountPath = "/scripts" XenonVolumeMountPath = "/etc/xenon" InitFileVolumeMountPath = "/docker-entrypoint-initdb.d" + + // The path to the client MySQL client configuration. + // The file used to liveness and readiness check. + ConfClientPath = "/etc/mysql/client.conf" ) // ResourceName is the type for aliasing resources that will be created.